diff options
| -rw-r--r-- | MANIFEST | 2 | ||||
| -rw-r--r-- | src/H5D.c | 46 | ||||
| -rw-r--r-- | src/H5Defl.c | 371 | ||||
| -rw-r--r-- | src/H5Dio.c | 479 | ||||
| -rw-r--r-- | src/H5Dmpio.c | 175 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 135 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 59 | ||||
| -rw-r--r-- | src/H5Dselect.c | 691 | ||||
| -rw-r--r-- | src/H5Fpkg.h | 17 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 17 | ||||
| -rw-r--r-- | src/H5Oefl.c | 343 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 24 | ||||
| -rw-r--r-- | src/H5S.c | 269 | ||||
| -rw-r--r-- | src/H5Smpio.c | 173 | ||||
| -rw-r--r-- | src/H5Spkg.h | 11 | ||||
| -rw-r--r-- | src/H5Sprivate.h | 59 | ||||
| -rw-r--r-- | src/H5Sselect.c | 670 | ||||
| -rw-r--r-- | src/H5T.c | 65 | ||||
| -rw-r--r-- | src/Makefile.in | 5 |
19 files changed, 1842 insertions, 1769 deletions
@@ -777,12 +777,14 @@ ./src/H5D.c ./src/H5Dcontig.c ./src/H5Dcompact.c +./src/H5Defl.c ./src/H5Dio.c ./src/H5Distore.c ./src/H5Dmpio.c ./src/H5Dprivate.h ./src/H5Dpublic.h ./src/H5Dpkg.h +./src/H5Dselect.c ./src/H5Dtest.c ./src/H5Ztrans.c ./src/H5E.c @@ -1316,7 +1316,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5Dget_space + * Function: H5Dget_space * * Purpose: Returns a copy of the file data space for a dataset. * @@ -2146,6 +2146,7 @@ H5D_create(H5G_entry_t *loc, const char *name, hid_t type_id, const H5S_t *space /* Initialize the shared dataset space */ if(NULL == (new_dset->shared = H5D_new(dcpl_id,TRUE,has_vl_type))) + HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* * Set the dataset's checked_filters flag to enable writing. @@ -2231,9 +2232,8 @@ H5D_create(H5G_entry_t *loc, const char *name, hid_t type_id, const H5S_t *space if(H5P_get(dc_plist, H5D_CRT_EXT_FILE_LIST_NAME, &new_dset->shared->efl) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't retrieve external file list") - /* Get the dataset's data storage method */ - if(H5P_get(dc_plist, H5D_CRT_LAYOUT_NAME, &(new_dset->shared->layout.type)) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't retrieve layout") + /* Set the dataset's data storage method */ + new_dset->shared->layout.type=dcpl_layout; } /* end if */ /* Check if this dataset is going into a parallel file and set space allocation time */ @@ -2273,16 +2273,16 @@ H5D_create(H5G_entry_t *loc, const char *name, hid_t type_id, const H5S_t *space } /* Set the I/O functions for this layout type */ - new_dset->shared->layout.readvv=H5O_efl_readvv; - new_dset->shared->layout.writevv=H5O_efl_writevv; + new_dset->shared->io_ops.readvv=H5D_efl_readvv; + new_dset->shared->io_ops.writevv=H5D_efl_writevv; } /* end if */ else { if (ndims>0 && max_dim[0]>dim[0]) HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, NULL, "extendible contiguous non-external dataset") /* Set the I/O functions for this layout type */ - new_dset->shared->layout.readvv=H5D_contig_readvv; - new_dset->shared->layout.writevv=H5D_contig_writevv; + new_dset->shared->io_ops.readvv=H5D_contig_readvv; + new_dset->shared->io_ops.writevv=H5D_contig_writevv; } /* end else */ /* Compute the total size of a chunk */ @@ -2332,8 +2332,8 @@ H5D_create(H5G_entry_t *loc, const char *name, hid_t type_id, const H5S_t *space new_dset->shared->layout.u.chunk.size *= new_dset->shared->layout.u.chunk.dim[u]; /* Set the I/O functions for this layout type */ - new_dset->shared->layout.readvv=H5D_istore_readvv; - new_dset->shared->layout.writevv=H5D_istore_writevv; + new_dset->shared->io_ops.readvv=H5D_istore_readvv; + new_dset->shared->io_ops.writevv=H5D_istore_writevv; /* Initialize the chunk cache for the dataset */ if(H5D_istore_init(file,new_dset)<0) @@ -2365,8 +2365,8 @@ H5D_create(H5G_entry_t *loc, const char *name, hid_t type_id, const H5S_t *space HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "compact dataset size is bigger than header message maximum size") /* Set the I/O functions for this layout type */ - new_dset->shared->layout.readvv=H5D_compact_readvv; - new_dset->shared->layout.writevv=H5D_compact_writevv; + new_dset->shared->io_ops.readvv=H5D_compact_readvv; + new_dset->shared->io_ops.writevv=H5D_compact_writevv; } /* end case */ break; @@ -2605,10 +2605,10 @@ H5D_open_oid(const H5G_entry_t *ent, hid_t dxpl_id) assert (ent); /* Allocate the dataset structure */ - /* (Set the 'vl_type' parameter to FALSE since it doesn't matter from here) */ if(NULL==(dataset=H5FL_CALLOC(H5D_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + /* (Set the 'vl_type' parameter to FALSE since it doesn't matter from here) */ if(NULL==(dataset->shared = H5D_new(H5P_DATASET_CREATE_DEFAULT,FALSE,FALSE))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") @@ -2671,8 +2671,8 @@ H5D_open_oid(const H5G_entry_t *ent, hid_t dxpl_id) } /* end if */ /* Set the I/O functions for this layout type */ - dataset->shared->layout.readvv=H5D_contig_readvv; - dataset->shared->layout.writevv=H5D_contig_writevv; + dataset->shared->io_ops.readvv=H5D_contig_readvv; + dataset->shared->io_ops.writevv=H5D_contig_writevv; break; case H5D_CHUNKED: @@ -2697,14 +2697,14 @@ H5D_open_oid(const H5G_entry_t *ent, hid_t dxpl_id) } /* Set the I/O functions for this layout type */ - dataset->shared->layout.readvv=H5D_istore_readvv; - dataset->shared->layout.writevv=H5D_istore_writevv; + dataset->shared->io_ops.readvv=H5D_istore_readvv; + dataset->shared->io_ops.writevv=H5D_istore_writevv; break; case H5D_COMPACT: /* Set the I/O functions for this layout type */ - dataset->shared->layout.readvv=H5D_compact_readvv; - dataset->shared->layout.writevv=H5D_compact_writevv; + dataset->shared->io_ops.readvv=H5D_compact_readvv; + dataset->shared->io_ops.writevv=H5D_compact_writevv; break; default: @@ -2782,8 +2782,8 @@ H5D_open_oid(const H5G_entry_t *ent, hid_t dxpl_id) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set external file list") /* Override the I/O functions for this layout type */ - dataset->shared->layout.readvv=H5O_efl_readvv; - dataset->shared->layout.writevv=H5O_efl_writevv; + dataset->shared->io_ops.readvv=H5D_efl_readvv; + dataset->shared->io_ops.writevv=H5D_efl_writevv; } /* end if */ } /* end if */ @@ -2863,7 +2863,7 @@ H5D_close(H5D_t *dataset) FUNC_ENTER_NOAPI(H5D_close, FAIL) /* check args */ - assert(dataset && dataset->ent.file); + assert(dataset && dataset->ent.file && dataset->shared); assert(dataset->shared->fo_count >0); /* Dump debugging info */ @@ -2903,7 +2903,7 @@ H5D_close(H5D_t *dataset) /* Free the buffer for the raw data for compact datasets */ dataset->shared->layout.u.compact.buf=H5MM_xfree(dataset->shared->layout.u.compact.buf); break; - + default: assert ("not implemented yet" && 0); #ifdef NDEBUG diff --git a/src/H5Defl.c b/src/H5Defl.c new file mode 100644 index 0000000..ab3e260 --- /dev/null +++ b/src/H5Defl.c @@ -0,0 +1,371 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Thursday, September 30, 2004 + */ + +#define H5D_PACKAGE /*suppress error about including H5Dpkg */ + +/* Pablo information */ +/* (Put before include files to avoid problems with inline functions) */ +#define PABLO_MASK H5D_efl_mask + +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ + +/* PRIVATE PROTOTYPES */ +static herr_t H5D_efl_read (const H5O_efl_t *efl, haddr_t addr, size_t size, + uint8_t *buf); +static herr_t H5D_efl_write(const H5O_efl_t *efl, haddr_t addr, size_t size, + const uint8_t *buf); + + +/*------------------------------------------------------------------------- + * Function: H5D_efl_read + * + * Purpose: Reads data from an external file list. It is an error to + * read past the logical end of file, but reading past the end + * of any particular member of the external file list results in + * zeros. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Robb Matzke + * Wednesday, March 4, 1998 + * + * Modifications: + * Robb Matzke, 1999-07-28 + * The ADDR argument is passed by value. + *------------------------------------------------------------------------- + */ +static herr_t +H5D_efl_read (const H5O_efl_t *efl, haddr_t addr, size_t size, uint8_t *buf) +{ + int fd=-1; + size_t to_read; +#ifndef NDEBUG + hsize_t tempto_read; +#endif /* NDEBUG */ + hsize_t skip=0; + haddr_t cur; + ssize_t n; + size_t u; /* Local index variable */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_efl_read); + + /* Check args */ + assert (efl && efl->nused>0); + assert (H5F_addr_defined (addr)); + assert (size < SIZET_MAX); + assert (buf || 0==size); + + /* Find the first efl member from which to read */ + for (u=0, cur=0; u<efl->nused; u++) { + if (H5O_EFL_UNLIMITED==efl->slot[u].size || addr < cur+efl->slot[u].size) { + skip = addr - cur; + break; + } + cur += efl->slot[u].size; + } + + /* Read the data */ + while (size) { + if (u>=efl->nused) + HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, "read past logical end of file"); + if (H5F_OVERFLOW_HSIZET2OFFT (efl->slot[u].offset+skip)) + HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed"); + if ((fd=HDopen (efl->slot[u].name, O_RDONLY, 0))<0) + HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file"); + if (HDlseek (fd, (off_t)(efl->slot[u].offset+skip), SEEK_SET)<0) + HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file"); +#ifndef NDEBUG + tempto_read = MIN(efl->slot[u].size-skip,(hsize_t)size); + H5_CHECK_OVERFLOW(tempto_read,hsize_t,size_t); + to_read = (size_t)tempto_read; +#else /* NDEBUG */ + to_read = MIN((size_t)(efl->slot[u].size-skip), size); +#endif /* NDEBUG */ + if ((n=HDread (fd, buf, to_read))<0) { + HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL, "read error in external raw data file"); + } else if ((size_t)n<to_read) { + HDmemset (buf+n, 0, to_read-n); + } + HDclose (fd); + fd = -1; + size -= to_read; + buf += to_read; + skip = 0; + u++; + } + +done: + if (fd>=0) + HDclose (fd); + + FUNC_LEAVE_NOAPI(ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5D_efl_write + * + * Purpose: Writes data to an external file list. It is an error to + * write past the logical end of file, but writing past the end + * of any particular member of the external file list just + * extends that file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Robb Matzke + * Wednesday, March 4, 1998 + * + * Modifications: + * Robb Matzke, 1999-07-28 + * The ADDR argument is passed by value. + *------------------------------------------------------------------------- + */ +static herr_t +H5D_efl_write (const H5O_efl_t *efl, haddr_t addr, size_t size, const uint8_t *buf) +{ + int fd=-1; + size_t to_write; +#ifndef NDEBUG + hsize_t tempto_write; +#endif /* NDEBUG */ + haddr_t cur; + hsize_t skip=0; + size_t u; /* Local index variable */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_efl_write); + + /* Check args */ + assert (efl && efl->nused>0); + assert (H5F_addr_defined (addr)); + assert (size < SIZET_MAX); + assert (buf || 0==size); + + /* Find the first efl member in which to write */ + for (u=0, cur=0; u<efl->nused; u++) { + if (H5O_EFL_UNLIMITED==efl->slot[u].size || addr < cur+efl->slot[u].size) { + skip = addr - cur; + break; + } + cur += efl->slot[u].size; + } + + /* Write the data */ + while (size) { + if (u>=efl->nused) + HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, "write past logical end of file"); + if (H5F_OVERFLOW_HSIZET2OFFT (efl->slot[u].offset+skip)) + HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed"); + if ((fd=HDopen (efl->slot[u].name, O_CREAT|O_RDWR, 0666))<0) { + if (HDaccess (efl->slot[u].name, F_OK)<0) { + HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL, "external raw data file does not exist"); + } else { + HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file"); + } + } + if (HDlseek (fd, (off_t)(efl->slot[u].offset+skip), SEEK_SET)<0) + HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file"); +#ifndef NDEBUG + tempto_write = MIN(efl->slot[u].size-skip,(hsize_t)size); + H5_CHECK_OVERFLOW(tempto_write,hsize_t,size_t); + to_write = (size_t)tempto_write; +#else /* NDEBUG */ + to_write = MIN((size_t)(efl->slot[u].size-skip), size); +#endif /* NDEBUG */ + if ((size_t)HDwrite (fd, buf, to_write)!=to_write) + HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL, "write error in external raw data file"); + HDclose (fd); + fd = -1; + size -= to_write; + buf += to_write; + skip = 0; + u++; + } + +done: + if (fd>=0) + HDclose (fd); + + FUNC_LEAVE_NOAPI(ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5D_efl_readvv + * + * Purpose: Reads data from an external file list. It is an error to + * read past the logical end of file, but reading past the end + * of any particular member of the external file list results in + * zeros. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Wednesday, May 7, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +ssize_t +H5D_efl_readvv(H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + void *_buf) +{ + const H5O_efl_t *efl=&(io_info->store->efl); /* Pointer to efl info */ + unsigned char *buf; /* Pointer to buffer to write */ + haddr_t addr; /* Actual address to read */ + size_t size; /* Size of sequence in bytes */ + size_t u; /* Counting variable */ + size_t v; /* Counting variable */ + ssize_t ret_value=0; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_efl_readvv, FAIL); + + /* Check args */ + assert (efl && efl->nused>0); + assert (_buf); + + /* Work through all the sequences */ + for(u=*dset_curr_seq, v=*mem_curr_seq; u<dset_max_nseq && v<mem_max_nseq; ) { + /* Choose smallest buffer to write */ + if(mem_len_arr[v]<dset_len_arr[u]) + size=mem_len_arr[v]; + else + size=dset_len_arr[u]; + + /* Compute offset on disk */ + addr=dset_offset_arr[u]; + + /* Compute offset in memory */ + buf = (unsigned char *)_buf + mem_offset_arr[v]; + + /* Read data */ + if (H5D_efl_read(efl, addr, size, buf)<0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); + + /* Update memory information */ + mem_len_arr[v]-=size; + mem_offset_arr[v]+=size; + if(mem_len_arr[v]==0) + v++; + + /* Update file information */ + dset_len_arr[u]-=size; + dset_offset_arr[u]+=size; + if(dset_len_arr[u]==0) + u++; + + /* Increment number of bytes copied */ + ret_value+=size; + } /* end for */ + + /* Update current sequence vectors */ + *dset_curr_seq=u; + *mem_curr_seq=v; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D_efl_readvv() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_efl_writevv + * + * Purpose: Writes data to an external file list. It is an error to + * write past the logical end of file, but writing past the end + * of any particular member of the external file list just + * extends that file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Friday, May 2, 2003 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +ssize_t +H5D_efl_writevv(H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + const void *_buf) +{ + const H5O_efl_t *efl=&(io_info->store->efl); /* Pointer to efl info */ + const unsigned char *buf; /* Pointer to buffer to write */ + haddr_t addr; /* Actual address to read */ + size_t size; /* Size of sequence in bytes */ + size_t u; /* Counting variable */ + size_t v; /* Counting variable */ + ssize_t ret_value=0; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_efl_writevv, FAIL); + + /* Check args */ + assert (efl && efl->nused>0); + assert (_buf); + + /* Work through all the sequences */ + for(u=*dset_curr_seq, v=*mem_curr_seq; u<dset_max_nseq && v<mem_max_nseq; ) { + /* Choose smallest buffer to write */ + if(mem_len_arr[v]<dset_len_arr[u]) + size=mem_len_arr[v]; + else + size=dset_len_arr[u]; + + /* Compute offset on disk */ + addr=dset_offset_arr[u]; + + /* Compute offset in memory */ + buf = (const unsigned char *)_buf + mem_offset_arr[v]; + + /* Write data */ + if (H5D_efl_write(efl, addr, size, buf)<0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); + + /* Update memory information */ + mem_len_arr[v]-=size; + mem_offset_arr[v]+=size; + if(mem_len_arr[v]==0) + v++; + + /* Update file information */ + dset_len_arr[u]-=size; + dset_offset_arr[u]+=size; + if(dset_len_arr[u]==0) + u++; + + /* Increment number of bytes copied */ + ret_value+=size; + } /* end for */ + + /* Update current sequence vectors */ + *dset_curr_seq=u; + *mem_curr_seq=v; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D_efl_writevv() */ + diff --git a/src/H5Dio.c b/src/H5Dio.c index 95c4c37..fbbf4dc 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -70,6 +70,8 @@ typedef struct fm_map { } fm_map; /* Local functions */ +static herr_t H5D_fill(const void *fill, const H5T_t *fill_type, void *buf, + const H5T_t *buf_type, const H5S_t *space, hid_t dxpl_id); static herr_t H5D_read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, hid_t dset_xfer_plist, void *buf/*out*/); @@ -77,24 +79,24 @@ static herr_t H5D_write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, hid_t dset_xfer_plist, const void *buf); static herr_t -H5D_contig_read(hsize_t nelmts, H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, H5T_path_t *tpath, H5S_conv_t *sconv, - const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, +H5D_contig_read(H5D_io_info_t *io_info, hsize_t nelmts, + const H5T_t *mem_type, const H5S_t *mem_space, + const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, void *buf/*out*/); static herr_t -H5D_contig_write(hsize_t nelmts, H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, H5T_path_t *tpath, H5S_conv_t *sconv, - const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, +H5D_contig_write(H5D_io_info_t *io_info, hsize_t nelmts, + const H5T_t *mem_type, const H5S_t *mem_space, + const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, const void *buf); static herr_t -H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, H5T_path_t *tpath, H5S_conv_t *sconv, - const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, +H5D_chunk_read(H5D_io_info_t *io_info, hsize_t nelmts, + const H5T_t *mem_type, const H5S_t *mem_space, + const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, void *buf/*out*/); static herr_t -H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, H5T_path_t *tpath, H5S_conv_t *sconv, - const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, +H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, + const H5T_t *mem_type, const H5S_t *mem_space, + const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, const void *buf); #ifdef H5_HAVE_PARALLEL static herr_t @@ -102,7 +104,19 @@ H5D_io_assist_mpio(hid_t dxpl_id, H5D_dxpl_cache_t *dxpl_cache, hbool_t *xfer_mode_changed); static herr_t H5D_io_restore_mpio(hid_t dxpl_id); -#endif /*H5_HAVE_PARALLEL*/ +static htri_t +H5D_get_collective_io_consensus(const H5F_t *file, + const htri_t local_opinion, + const unsigned flags); +#endif /* H5_HAVE_PARALLEL */ + +/* I/O info operations */ +static herr_t +H5D_ioinfo_init(H5D_t *dset, const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, + const H5S_t *mem_space, const H5S_t *file_space, + unsigned flags, hbool_t *use_par_opt_io, H5D_io_info_t *io_info); + +/* Chunk operations */ static herr_t H5D_create_chunk_map(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *file_space, const H5S_t *mem_space, fm_map *fm); static herr_t H5D_destroy_chunk_map(const fm_map *fm); @@ -113,9 +127,6 @@ static herr_t H5D_chunk_file_cb(void *elem, hid_t type_id, hsize_t ndims, hssize_t *coords, void *fm); static herr_t H5D_chunk_mem_cb(void *elem, hid_t type_id, hsize_t ndims, hssize_t *coords, void *fm); -static herr_t H5D_fill(const void *fill, const H5T_t *fill_type, void *buf, - const H5T_t *buf_type, const H5S_t *space, hid_t dxpl_id); - /* Declare a free list to manage blocks of single datatype element data */ H5FL_BLK_DEFINE(type_elem); @@ -639,12 +650,14 @@ H5D_read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, hsize_t nelmts; /*total number of elmts */ H5T_path_t *tpath = NULL; /*type conversion info */ const H5T_t *mem_type = NULL; /* Memory datatype */ - H5S_conv_t *sconv=NULL; /*space conversion funcs*/ + H5D_io_info_t io_info; /* Dataset I/O info */ hbool_t use_par_opt_io=FALSE; /* Whether the 'optimized' I/O routines with be parallel */ #ifdef H5_HAVE_PARALLEL hbool_t xfer_mode_changed=FALSE; /* Whether the transfer mode was changed */ +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY int prop_value,new_value; htri_t check_prop; +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ #endif /*H5_HAVE_PARALLEL*/ H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ H5D_dxpl_cache_t *dxpl_cache=&_dxpl_cache; /* Data transfer property cache */ @@ -756,9 +769,9 @@ H5D_read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, assert(0 && "Unhandled layout type!"); } /* end switch */ - /* Get dataspace functions */ - if (NULL==(sconv=H5S_find(dataset->ent.file, mem_space, file_space, sconv_flags, &use_par_opt_io, &dataset->shared->layout))) - HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert from file to memory data space") + /* Set up I/O operation */ + if(H5D_ioinfo_init(dataset,dxpl_cache,dxpl_id,mem_space,file_space,sconv_flags,&use_par_opt_io,&io_info)<0) + HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to set up I/O operation") #ifdef H5_HAVE_PARALLEL #ifdef H5_HAVE_INSTRUMENTED_LIBRARY @@ -791,13 +804,13 @@ H5D_read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, /* Determine correct I/O routine to invoke */ if(dataset->shared->layout.type!=H5D_CHUNKED) { - if(H5D_contig_read(nelmts, dataset, mem_type, mem_space, file_space, tpath, sconv, - dxpl_cache, dxpl_id, dataset->shared->type_id, mem_type_id, buf)<0) + if(H5D_contig_read(&io_info, nelmts, mem_type, mem_space, file_space, tpath, + dataset->shared->type_id, mem_type_id, buf)<0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data") } /* end if */ else { - if(H5D_chunk_read(nelmts, dataset, mem_type, mem_space, file_space, tpath, sconv, - dxpl_cache, dxpl_id, dataset->shared->type_id, mem_type_id, buf)<0) + if(H5D_chunk_read(&io_info, nelmts, mem_type, mem_space, file_space, tpath, + dataset->shared->type_id, mem_type_id, buf)<0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read data") } /* end else */ @@ -866,12 +879,14 @@ H5D_write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, hsize_t nelmts; /*total number of elmts */ H5T_path_t *tpath = NULL; /*type conversion info */ const H5T_t *mem_type = NULL; /* Memory datatype */ - H5S_conv_t *sconv=NULL; /*space conversion funcs*/ + H5D_io_info_t io_info; /* Dataset I/O info */ hbool_t use_par_opt_io=FALSE; /* Whether the 'optimized' I/O routines with be parallel */ #ifdef H5_HAVE_PARALLEL hbool_t xfer_mode_changed=FALSE; /* Whether the transfer mode was changed */ +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY int prop_value,new_value; htri_t check_prop; +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ #endif /*H5_HAVE_PARALLEL*/ H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ H5D_dxpl_cache_t *dxpl_cache=&_dxpl_cache; /* Data transfer property cache */ @@ -1003,9 +1018,9 @@ H5D_write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, assert(0 && "Unhandled layout type!"); } /* end switch */ - /* Get dataspace functions */ - if (NULL==(sconv=H5S_find(dataset->ent.file, mem_space, file_space, sconv_flags, &use_par_opt_io, &dataset->shared->layout))) - HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert from memory to file data space") + /* Set up I/O operation */ + if(H5D_ioinfo_init(dataset,dxpl_cache,dxpl_id,mem_space,file_space,sconv_flags,&use_par_opt_io,&io_info)<0) + HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to set up I/O operation") #ifdef H5_HAVE_PARALLEL #ifdef H5_HAVE_INSTRUMENTED_LIBRARY @@ -1014,20 +1029,20 @@ H5D_write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, a more general collective chunk IO algorithm is applied. */ - if(dataset->shared->layout.type == H5D_CHUNKED) { /*only check for chunking storage */ + if(dataset->shared->layout.type == H5D_CHUNKED) { /*only check for chunking storage */ - check_prop = H5Pexist(dxpl_id,H5D_XFER_COLL_CHUNK_NAME); - if(check_prop < 0) - HGOTO_ERROR(H5E_PLIST, H5E_UNSUPPORTED, FAIL, "unable to check property list"); - if(check_prop > 0) { - if(H5Pget(dxpl_id,H5D_XFER_COLL_CHUNK_NAME,&prop_value)<0) - HGOTO_ERROR(H5E_PLIST, H5E_UNSUPPORTED, FAIL, "unable to get property value"); - if(!use_par_opt_io) { - new_value = 0; - if(H5Pset(dxpl_id,H5D_XFER_COLL_CHUNK_NAME,&new_value)<0) - HGOTO_ERROR(H5E_PLIST, H5E_UNSUPPORTED, FAIL, "unable to set property value"); - } - } + check_prop = H5Pexist(dxpl_id,H5D_XFER_COLL_CHUNK_NAME); + if(check_prop < 0) + HGOTO_ERROR(H5E_PLIST, H5E_UNSUPPORTED, FAIL, "unable to check property list"); + if(check_prop > 0) { + if(H5Pget(dxpl_id,H5D_XFER_COLL_CHUNK_NAME,&prop_value)<0) + HGOTO_ERROR(H5E_PLIST, H5E_UNSUPPORTED, FAIL, "unable to get property value"); + if(!use_par_opt_io) { + new_value = 0; + if(H5Pset(dxpl_id,H5D_XFER_COLL_CHUNK_NAME,&new_value)<0) + HGOTO_ERROR(H5E_PLIST, H5E_UNSUPPORTED, FAIL, "unable to set property value"); + } + } } #endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ @@ -1038,13 +1053,13 @@ H5D_write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, /* Determine correct I/O routine to invoke */ if(dataset->shared->layout.type!=H5D_CHUNKED) { - if(H5D_contig_write(nelmts, dataset, mem_type, mem_space, file_space, tpath, sconv, - dxpl_cache, dxpl_id, mem_type_id, dataset->shared->type_id, buf)<0) + if(H5D_contig_write(&io_info, nelmts, mem_type, mem_space, file_space, tpath, + mem_type_id, dataset->shared->type_id, buf)<0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data") } /* end if */ else { - if(H5D_chunk_write(nelmts, dataset, mem_type, mem_space, file_space, tpath, sconv, - dxpl_cache, dxpl_id, mem_type_id, dataset->shared->type_id, buf)<0) + if(H5D_chunk_write(&io_info, nelmts, mem_type, mem_space, file_space, tpath, + mem_type_id, dataset->shared->type_id, buf)<0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write data") } /* end else */ @@ -1093,12 +1108,13 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_contig_read(hsize_t nelmts, H5D_t *dataset, +H5D_contig_read(H5D_io_info_t *io_info, hsize_t nelmts, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, H5T_path_t *tpath, H5S_conv_t *sconv, - const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, + const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, void *buf/*out*/) { + H5D_t *dataset=io_info->dset; /* Local pointer to dataset info */ + const H5D_dxpl_cache_t *dxpl_cache=io_info->dxpl_cache; /* Local pointer to dataset transfer info */ herr_t status; /*function return status*/ #ifdef H5S_DEBUG H5_timer_t timer; @@ -1120,7 +1136,6 @@ H5D_contig_read(hsize_t nelmts, H5D_t *dataset, hsize_t smine_start; /*strip mine start loc */ size_t n, smine_nelmts; /*elements per strip */ H5D_storage_t store; /*union of storage info for dataset */ - H5D_io_info_t io_info; /* Dataset I/O info */ herr_t ret_value = SUCCEED; /*return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_contig_read) @@ -1133,8 +1148,8 @@ H5D_contig_read(hsize_t nelmts, H5D_t *dataset, store.contig.dset_size=dataset->shared->layout.u.contig.size; } /* end if */ - /* Construct dataset I/O info */ - H5D_BUILD_IO_INFO(&io_info,dataset,dxpl_cache,dxpl_id,&store); + /* Set dataset storage for I/O info */ + io_info->store=&store; /* * If there is no type conversion then read directly into the @@ -1150,14 +1165,14 @@ H5D_contig_read(hsize_t nelmts, H5D_t *dataset, || dataset->shared->efl.nused>0 || 0 == nelmts || dataset->shared->layout.type==H5D_COMPACT); H5_CHECK_OVERFLOW(nelmts,hsize_t,size_t); - status = (sconv->read)(&io_info, dataset->shared->layout.readvv, + status = (io_info->ops.read)(io_info, (size_t)nelmts, H5T_get_size(dataset->shared->type), file_space, mem_space, buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[1].read_timer), &timer); - sconv->stats[1].read_nbytes += nelmts * H5T_get_size(dataset->shared->type); - sconv->stats[1].read_ncalls++; + H5_timer_end(&(io_info->stats->stats[1].read_timer), &timer); + io_info->stats->stats[1].read_nbytes += nelmts * H5T_get_size(dataset->shared->type); + io_info->stats->stats[1].read_ncalls++; #endif /* Check return value from optimized read */ @@ -1252,14 +1267,18 @@ H5D_contig_read(hsize_t nelmts, H5D_t *dataset, H5_timer_begin(&timer); #endif /* Sanity check that space is allocated, then read data from it */ - n = H5S_select_fgath(&io_info, dataset->shared->layout.readvv, + assert(((dataset->shared->layout.type==H5D_CONTIGUOUS && H5F_addr_defined(dataset->shared->layout.u.contig.addr)) + || (dataset->shared->layout.type==H5D_CHUNKED && H5F_addr_defined(dataset->shared->layout.u.chunk.addr))) + || dataset->shared->efl.nused>0 || + dataset->shared->layout.type==H5D_COMPACT); + n = H5D_select_fgath(io_info, file_space, &file_iter, smine_nelmts, tconv_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[1].gath_timer), &timer); - sconv->stats[1].gath_nbytes += n * src_type_size; - sconv->stats[1].gath_ncalls++; + H5_timer_end(&(io_info->stats->stats[1].gath_timer), &timer); + io_info->stats->stats[1].gath_nbytes += n * src_type_size; + io_info->stats->stats[1].gath_ncalls++; #endif if (n!=smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed") @@ -1268,12 +1287,12 @@ H5D_contig_read(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - n = H5S_select_mgath(buf, mem_space, &bkg_iter, + n = H5D_select_mgath(buf, mem_space, &bkg_iter, smine_nelmts, dxpl_cache, bkg_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[1].bkg_timer), &timer); - sconv->stats[1].bkg_nbytes += n * dst_type_size; - sconv->stats[1].bkg_ncalls++; + H5_timer_end(&(io_info->stats->stats[1].bkg_timer), &timer); + io_info->stats->stats[1].bkg_nbytes += n * dst_type_size; + io_info->stats->stats[1].bkg_ncalls++; #endif if (n!=smine_nelmts) HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed") @@ -1282,7 +1301,7 @@ H5D_contig_read(hsize_t nelmts, H5D_t *dataset, /* * Perform data type conversion. */ - if (H5T_convert(tpath, src_id, dst_id, smine_nelmts, 0, 0, tconv_buf, bkg_buf, dxpl_id)<0) + if (H5T_convert(tpath, src_id, dst_id, smine_nelmts, 0, 0, tconv_buf, bkg_buf, io_info->dxpl_id)<0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "data type conversion failed") /* Do the data transform after the conversion (since we're using type mem_type) */ @@ -1296,12 +1315,12 @@ H5D_contig_read(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - status = H5S_select_mscat(tconv_buf, mem_space, + status = H5D_select_mscat(tconv_buf, mem_space, &mem_iter, smine_nelmts, dxpl_cache, buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[1].scat_timer), &timer); - sconv->stats[1].scat_nbytes += smine_nelmts * dst_type_size; - sconv->stats[1].scat_ncalls++; + H5_timer_end(&(io_info->stats->stats[1].scat_timer), &timer); + io_info->stats->stats[1].scat_nbytes += smine_nelmts * dst_type_size; + io_info->stats->stats[1].scat_ncalls++; #endif if (status<0) HGOTO_ERROR (H5E_DATASET, H5E_READERROR, FAIL, "scatter failed") @@ -1351,12 +1370,13 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_contig_write(hsize_t nelmts, H5D_t *dataset, +H5D_contig_write(H5D_io_info_t *io_info, hsize_t nelmts, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, H5T_path_t *tpath, H5S_conv_t *sconv, - const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, + const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, const void *buf) { + H5D_t *dataset=io_info->dset; /* Local pointer to dataset info */ + const H5D_dxpl_cache_t *dxpl_cache=io_info->dxpl_cache; /* Local pointer to dataset transfer info */ herr_t status; /*function return status*/ #ifdef H5S_DEBUG H5_timer_t timer; @@ -1378,7 +1398,6 @@ H5D_contig_write(hsize_t nelmts, H5D_t *dataset, hsize_t smine_start; /*strip mine start loc */ size_t n, smine_nelmts; /*elements per strip */ H5D_storage_t store; /*union of storage info for dataset */ - H5D_io_info_t io_info; /* Dataset I/O info */ herr_t ret_value = SUCCEED; /*return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_contig_write) @@ -1391,8 +1410,8 @@ H5D_contig_write(hsize_t nelmts, H5D_t *dataset, store.contig.dset_size=dataset->shared->layout.u.contig.size; } /* end if */ - /* Construct dataset I/O info */ - H5D_BUILD_IO_INFO(&io_info,dataset,dxpl_cache,dxpl_id,&store); + /* Set dataset storage for I/O info */ + io_info->store=&store; /* * If there is no type conversion then write directly from the @@ -1403,14 +1422,14 @@ H5D_contig_write(hsize_t nelmts, H5D_t *dataset, H5_timer_begin(&timer); #endif H5_CHECK_OVERFLOW(nelmts,hsize_t,size_t); - status = (sconv->write)(&io_info, dataset->shared->layout.writevv, + status = (io_info->ops.write)(io_info, (size_t)nelmts, H5T_get_size(dataset->shared->type), file_space, mem_space, buf); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[0].write_timer), &timer); - sconv->stats[0].write_nbytes += nelmts * H5T_get_size(mem_type); - sconv->stats[0].write_ncalls++; + H5_timer_end(&(io_info->stats->stats[0].write_timer), &timer); + io_info->stats->stats[0].write_nbytes += nelmts * H5T_get_size(mem_type); + io_info->stats->stats[0].write_ncalls++; #endif /* Check return value from optimized write */ @@ -1508,12 +1527,12 @@ H5D_contig_write(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - n = H5S_select_mgath(buf, mem_space, &mem_iter, + n = H5D_select_mgath(buf, mem_space, &mem_iter, smine_nelmts, dxpl_cache, tconv_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[0].gath_timer), &timer); - sconv->stats[0].gath_nbytes += n * src_type_size; - sconv->stats[0].gath_ncalls++; + H5_timer_end(&(io_info->stats->stats[0].gath_timer), &timer); + io_info->stats->stats[0].gath_nbytes += n * src_type_size; + io_info->stats->stats[0].gath_ncalls++; #endif if (n!=smine_nelmts) HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed") @@ -1522,14 +1541,14 @@ H5D_contig_write(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - n = H5S_select_fgath(&io_info, dataset->shared->layout.readvv, + n = H5D_select_fgath(io_info, file_space, &bkg_iter, smine_nelmts, bkg_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[0].bkg_timer), &timer); - sconv->stats[0].bkg_nbytes += n * dst_type_size; - sconv->stats[0].bkg_ncalls++; + H5_timer_end(&(io_info->stats->stats[0].bkg_timer), &timer); + io_info->stats->stats[0].bkg_nbytes += n * dst_type_size; + io_info->stats->stats[0].bkg_ncalls++; #endif if (n!=smine_nelmts) HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed") @@ -1538,7 +1557,7 @@ H5D_contig_write(hsize_t nelmts, H5D_t *dataset, /* * Perform data type conversion. */ - if (H5T_convert(tpath, src_id, dst_id, smine_nelmts, 0, 0, tconv_buf, bkg_buf, dxpl_id)<0) + if (H5T_convert(tpath, src_id, dst_id, smine_nelmts, 0, 0, tconv_buf, bkg_buf, io_info->dxpl_id)<0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "data type conversion failed") /* Do the data transform after the type conversion (since we're using dataset->shared->type). */ @@ -1552,13 +1571,13 @@ H5D_contig_write(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - status = H5S_select_fscat(&io_info, dataset->shared->layout.writevv, + status = H5D_select_fscat(io_info, file_space, &file_iter, smine_nelmts, tconv_buf); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[0].scat_timer), &timer); - sconv->stats[0].scat_nbytes += smine_nelmts * dst_type_size; - sconv->stats[0].scat_ncalls++; + H5_timer_end(&(io_info->stats->stats[0].scat_timer), &timer); + io_info->stats->stats[0].scat_nbytes += smine_nelmts * dst_type_size; + io_info->stats->stats[0].scat_ncalls++; #endif if (status<0) HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "scatter failed") @@ -1607,12 +1626,13 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, +H5D_chunk_read(H5D_io_info_t *io_info, hsize_t nelmts, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, H5T_path_t *tpath, H5S_conv_t *sconv, - const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, + const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, void *buf/*out*/) { + H5D_t *dataset=io_info->dset; /* Local pointer to dataset info */ + const H5D_dxpl_cache_t *dxpl_cache=io_info->dxpl_cache; /* Local pointer to dataset transfer info */ fm_map fm; /* File<->memory mapping */ H5TB_NODE *chunk_node; /* Current node in chunk TBBT */ herr_t status; /*function return status*/ @@ -1636,7 +1656,6 @@ H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, uint8_t *tconv_buf = NULL; /*data type conv buffer */ uint8_t *bkg_buf = NULL; /*background buffer */ H5D_storage_t store; /*union of EFL and chunk pointer in file space */ - H5D_io_info_t io_info; /* Dataset I/O info */ herr_t ret_value = SUCCEED; /*return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_read) @@ -1645,8 +1664,8 @@ H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, if(H5D_create_chunk_map(dataset, mem_type, file_space, mem_space, &fm)<0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build chunk mapping") - /* Construct dataset I/O info */ - H5D_BUILD_IO_INFO(&io_info,dataset,dxpl_cache,dxpl_id,&store); + /* Set dataset storage for I/O info */ + io_info->store=&store; /* * If there is no type conversion then read directly into the @@ -1677,7 +1696,7 @@ H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, store.chunk.index = chunk_info->index; /* Perform the actual read operation */ - status = (sconv->read)(&io_info,dataset->shared->layout.readvv, + status = (io_info->ops.read)(io_info, chunk_info->chunk_points, H5T_get_size(dataset->shared->type), chunk_info->fspace, chunk_info->mspace, buf); @@ -1691,9 +1710,9 @@ H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, } /* end while */ #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[1].read_timer), &timer); - sconv->stats[1].read_nbytes += nelmts * H5T_get_size(dataset->shared->type); - sconv->stats[1].read_ncalls++; + H5_timer_end(&(io_info->stats->stats[1].read_timer), &timer); + io_info->stats->stats[1].read_nbytes += nelmts * H5T_get_size(dataset->shared->type); + io_info->stats->stats[1].read_ncalls++; #endif /* direct xfer accomplished successfully */ @@ -1802,14 +1821,14 @@ H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, assert(((dataset->shared->layout.type==H5D_CONTIGUOUS && H5F_addr_defined(dataset->shared->layout.u.contig.addr)) || (dataset->shared->layout.type==H5D_CHUNKED && H5F_addr_defined(dataset->shared->layout.u.chunk.addr))) || dataset->shared->efl.nused>0 || dataset->shared->layout.type==H5D_COMPACT); - n = H5S_select_fgath(&io_info, dataset->shared->layout.readvv, + n = H5D_select_fgath(io_info, chunk_info->fspace, &file_iter, smine_nelmts, tconv_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[1].gath_timer), &timer); - sconv->stats[1].gath_nbytes += n * src_type_size; - sconv->stats[1].gath_ncalls++; + H5_timer_end(&(io_info->stats->stats[1].gath_timer), &timer); + io_info->stats->stats[1].gath_nbytes += n * src_type_size; + io_info->stats->stats[1].gath_ncalls++; #endif if (n!=smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed") @@ -1818,12 +1837,12 @@ H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - n = H5S_select_mgath(buf, chunk_info->mspace, &bkg_iter, + n = H5D_select_mgath(buf, chunk_info->mspace, &bkg_iter, smine_nelmts, dxpl_cache, bkg_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[1].bkg_timer), &timer); - sconv->stats[1].bkg_nbytes += n * dst_type_size; - sconv->stats[1].bkg_ncalls++; + H5_timer_end(&(io_info->stats->stats[1].bkg_timer), &timer); + io_info->stats->stats[1].bkg_nbytes += n * dst_type_size; + io_info->stats->stats[1].bkg_ncalls++; #endif if (n!=smine_nelmts) HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed") @@ -1833,7 +1852,7 @@ H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, * Perform data type conversion. */ if (H5T_convert(tpath, src_id, dst_id, smine_nelmts, 0, 0, - tconv_buf, bkg_buf, dxpl_id)<0) + tconv_buf, bkg_buf, io_info->dxpl_id)<0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "data type conversion failed") /* Do the data transform after the conversion (since we're using type mem_type) */ @@ -1847,12 +1866,12 @@ H5D_chunk_read(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - status = H5S_select_mscat(tconv_buf, chunk_info->mspace, + status = H5D_select_mscat(tconv_buf, chunk_info->mspace, &mem_iter, smine_nelmts, dxpl_cache, buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[1].scat_timer), &timer); - sconv->stats[1].scat_nbytes += smine_nelmts * dst_type_size; - sconv->stats[1].scat_ncalls++; + H5_timer_end(&(io_info->stats->stats[1].scat_timer), &timer); + io_info->stats->stats[1].scat_nbytes += smine_nelmts * dst_type_size; + io_info->stats->stats[1].scat_ncalls++; #endif if (status<0) HGOTO_ERROR (H5E_DATASET, H5E_READERROR, FAIL, "scatter failed") @@ -1926,12 +1945,13 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, +H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, H5T_path_t *tpath, H5S_conv_t *sconv, - const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, + const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, const void *buf) { + H5D_t *dataset=io_info->dset; /* Local pointer to dataset info */ + const H5D_dxpl_cache_t *dxpl_cache=io_info->dxpl_cache; /* Local pointer to dataset transfer info */ fm_map fm; /* File<->memory mapping */ H5TB_NODE *chunk_node; /* Current node in chunk TBBT */ herr_t status; /*function return status*/ @@ -1955,7 +1975,6 @@ H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, uint8_t *tconv_buf = NULL; /*data type conv buffer */ uint8_t *bkg_buf = NULL; /*background buffer */ H5D_storage_t store; /*union of EFL and chunk pointer in file space */ - H5D_io_info_t io_info; /* Dataset I/O info */ herr_t ret_value = SUCCEED; /*return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_write) @@ -1964,8 +1983,8 @@ H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, if(H5D_create_chunk_map(dataset, mem_type, file_space, mem_space, &fm)<0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't build chunk mapping") - /* Construct dataset I/O info */ - H5D_BUILD_IO_INFO(&io_info,dataset,dxpl_cache,dxpl_id,&store); + /* Set dataset storage for I/O info */ + io_info->store=&store; /* * If there is no type conversion then write directly from the @@ -1990,7 +2009,7 @@ H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, store.chunk.index = chunk_info->index; /* Perform the actual write operation */ - status = (sconv->write)(&io_info, dataset->shared->layout.writevv, + status = (io_info->ops.write)(io_info, chunk_info->chunk_points, H5T_get_size(dataset->shared->type), chunk_info->fspace, chunk_info->mspace, buf); @@ -2004,9 +2023,9 @@ H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, } /* end while */ #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[0].write_timer), &timer); - sconv->stats[0].write_nbytes += nelmts * H5T_get_size(mem_type); - sconv->stats[0].write_ncalls++; + H5_timer_end(&(io_info->stats->stats[0].write_timer), &timer); + io_info->stats->stats[0].write_nbytes += nelmts * H5T_get_size(mem_type); + io_info->stats->stats[0].write_ncalls++; #endif /* direct xfer accomplished successfully */ @@ -2115,13 +2134,13 @@ H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - n = H5S_select_mgath(buf, chunk_info->mspace, &mem_iter, + n = H5D_select_mgath(buf, chunk_info->mspace, &mem_iter, smine_nelmts, dxpl_cache, tconv_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[1].gath_timer), &timer); - sconv->stats[1].gath_nbytes += n * src_type_size; - sconv->stats[1].gath_ncalls++; + H5_timer_end(&(io_info->stats->stats[1].gath_timer), &timer); + io_info->stats->stats[1].gath_nbytes += n * src_type_size; + io_info->stats->stats[1].gath_ncalls++; #endif if (n!=smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed") @@ -2130,14 +2149,14 @@ H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - n = H5S_select_fgath(&io_info, dataset->shared->layout.readvv, + n = H5D_select_fgath(io_info, chunk_info->fspace, &bkg_iter, smine_nelmts, bkg_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[0].bkg_timer), &timer); - sconv->stats[0].bkg_nbytes += n * dst_type_size; - sconv->stats[0].bkg_ncalls++; + H5_timer_end(&(io_info->stats->stats[0].bkg_timer), &timer); + io_info->stats->stats[0].bkg_nbytes += n * dst_type_size; + io_info->stats->stats[0].bkg_ncalls++; #endif if (n!=smine_nelmts) HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed") @@ -2147,7 +2166,7 @@ H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, * Perform data type conversion. */ if (H5T_convert(tpath, src_id, dst_id, smine_nelmts, 0, 0, - tconv_buf, bkg_buf, dxpl_id)<0) + tconv_buf, bkg_buf, io_info->dxpl_id)<0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "data type conversion failed") /* Do the data transform after the type conversion (since we're using dataset->shared->type) */ @@ -2161,14 +2180,14 @@ H5D_chunk_write(hsize_t nelmts, H5D_t *dataset, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - status = H5S_select_fscat(&io_info, dataset->shared->layout.writevv, + status = H5D_select_fscat(io_info, chunk_info->fspace, &file_iter, smine_nelmts, tconv_buf); #ifdef H5S_DEBUG - H5_timer_end(&(sconv->stats[0].scat_timer), &timer); - sconv->stats[0].scat_nbytes += n * dst_type_size; - sconv->stats[0].scat_ncalls++; + H5_timer_end(&(io_info->stats->stats[0].scat_timer), &timer); + io_info->stats->stats[0].scat_nbytes += n * dst_type_size; + io_info->stats->stats[0].scat_ncalls++; #endif if (status<0) HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "scatter failed") @@ -3128,3 +3147,189 @@ H5D_chunk_mem_cb(void UNUSED *elem, hid_t UNUSED type_id, hsize_t ndims, hssize_ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_chunk_mem_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5S_get_collective_io_consensus + * + * Purpose: Compare notes with all other processes involved in this I/O + * and see if all are go for collective I/O. + * + * If all are, return TRUE. + * + * If any process can't manage collective I/O, then collective + * I/O is impossible, and we return FALSE. + * + * If the flags indicate that collective I/O is impossible, + * skip the interprocess communication and just return FALSE. + * + * In any error is detected, return FAIL. + * + * Return: Success: TRUE or FALSE + * + * Failure: FAIL + * + * Programmer: JRM -- 8/30/04 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +#ifdef H5_HAVE_PARALLEL +static htri_t +H5D_get_collective_io_consensus(const H5F_t *file, + const htri_t local_opinion, + const unsigned flags) +{ + htri_t ret_value = FAIL; /* will update if successful */ + MPI_Comm comm; + int int_local_opinion; + int consensus; + int mpi_result; + + FUNC_ENTER_NOAPI_NOINIT(H5D_get_collective_io_consensus); + + HDassert ( ( local_opinion == TRUE ) || ( local_opinion == FALSE ) ); + + /* Don't do the interprocess communication unless the Parallel I/O + * conversion flag is set -- there may not be other processes to + * talk to. + */ + if ( ! ( flags & flags&H5S_CONV_PAR_IO_POSSIBLE ) ) { + + HGOTO_DONE(FALSE); + } + + comm = H5F_mpi_get_comm(file); + + if ( comm == MPI_COMM_NULL ) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, \ + "can't retrieve MPI communicator") + + if ( local_opinion == TRUE ) { + + int_local_opinion = 1; + + } else { + + int_local_opinion = 0; + } + + mpi_result = MPI_Allreduce((void *)(&int_local_opinion), + (void *)(&consensus), + 1, + MPI_INT, + MPI_LAND, + comm); + + if ( mpi_result != MPI_SUCCESS ) + HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_result) + + if ( consensus ) { + + ret_value = TRUE; + + } else { + + ret_value = FALSE; + } + +done: + + FUNC_LEAVE_NOAPI(ret_value); + +} /* H5D_get_collective_io_consensus() */ + +#endif /* H5_HAVE_PARALLEL */ + + +/*------------------------------------------------------------------------- + * Function: H5D_chunk_mem_cb + * + * Purpose: Routine for determining correct I/O operations for + * each I/O action. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, September 30, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_ioinfo_init(H5D_t *dset, const H5D_dxpl_cache_t *dxpl_cache, hid_t dxpl_id, + const H5S_t *mem_space, const H5S_t *file_space, + unsigned flags, hbool_t *use_par_opt_io, H5D_io_info_t *io_info) +{ +#ifdef H5_HAVE_PARALLEL + htri_t opt; /* Flag whether a selection is optimizable */ +#endif /* H5_HAVE_PARALLEL */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_ioinfo_init) + + /* check args */ + HDassert(dset); + HDassert(dset->ent.file); + HDassert(mem_space); + HDassert(file_space); + HDassert(use_par_opt_io); + HDassert(io_info); + + /* Set up "normal" I/O fields */ + io_info->dset=dset; + io_info->dxpl_cache=dxpl_cache; + io_info->dxpl_id=dxpl_id; + io_info->store=NULL; /* Set later in I/O routine? */ + + /* Set I/O operations to initial values */ + io_info->ops=dset->shared->io_ops; + +#ifdef H5_HAVE_PARALLEL + /* + * Check if we can set direct MPI-IO read/write functions + */ + opt=H5D_mpio_opt_possible(dset->ent.file,mem_space,file_space,flags,&dset->shared->layout); + if(opt==FAIL) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, NULL, "invalid check for direct IO dataspace "); + + opt = H5D_get_collective_io_consensus(dset->ent.file, opt, flags); + + if ( opt == FAIL ) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, NULL, \ + "check for collective I/O consensus failed."); + + /* Check if we can use the optimized parallel I/O routines */ + if(opt==TRUE) { + /* Set the pointers to the MPI-specific routines */ + io_info->ops.read = H5D_mpio_spaces_read; + io_info->ops.write = H5D_mpio_spaces_write; + + /* Indicate that the I/O will be parallel */ + *use_par_opt_io=TRUE; + } /* end if */ + else { + /* Indicate that the I/O will _NOT_ be parallel */ + *use_par_opt_io=FALSE; + +#endif /* H5_HAVE_PARALLEL */ + io_info->ops.read = H5D_select_read; + io_info->ops.write = H5D_select_write; +#ifdef H5_HAVE_PARALLEL + } /* end else */ +#endif /* H5_HAVE_PARALLEL */ + +#ifdef H5S_DEBUG + /* Get the information for the I/O statistics */ + if((io_info->stats=H5S_find(mem_space,file_space))==NULL) + HGOTO_ERROR(H5E_DATASET, H5E_BADSELECT, FAIL, "can't set up selection statistics"); +#endif /* H5S_DEBUG */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_ioinfo_init() */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index c5f227d..032dea9 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -47,6 +47,179 @@ H5D_mpio_spaces_xfer(H5D_io_info_t *io_info, size_t elmt_size, /*------------------------------------------------------------------------- + * Function: H5D_mpio_opt_possible + * + * Purpose: Checks if an direct I/O transfer is possible between memory and + * the file. + * + * Return: Success: Non-negative: TRUE or FALSE + * Failure: Negative + * + * Programmer: Quincey Koziol + * Wednesday, April 3, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +htri_t +H5D_mpio_opt_possible( const H5F_t *file, const H5S_t *mem_space, const H5S_t *file_space, const unsigned flags,const H5O_layout_t *layout) +{ + htri_t c1,c2; /* Flags whether a selection is optimizable */ + htri_t ret_value=TRUE; + + FUNC_ENTER_NOAPI(H5D_mpio_opt_possible, FAIL); + + /* Check args */ + assert(mem_space); + assert(file_space); + + /* Parallel I/O conversion flag must be set, if it is not collective IO, go to false. */ + if(!(flags&H5S_CONV_PAR_IO_POSSIBLE)) + HGOTO_DONE(FALSE); + + /* Check whether these are both simple or scalar dataspaces */ + if (!((H5S_SIMPLE==H5S_GET_EXTENT_TYPE(mem_space) || H5S_SCALAR==H5S_GET_EXTENT_TYPE(mem_space)) + && (H5S_SIMPLE==H5S_GET_EXTENT_TYPE(file_space) || H5S_SCALAR==H5S_GET_EXTENT_TYPE(file_space)))) + HGOTO_DONE(FALSE); + + /* Check whether both selections are "regular" */ + c1=H5S_SELECT_IS_REGULAR(file_space); + c2=H5S_SELECT_IS_REGULAR(mem_space); + if(c1==FAIL || c2==FAIL) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "invalid check for single selection blocks"); + if(c1==FALSE || c2==FALSE) + HGOTO_DONE(FALSE); + + /* Can't currently handle point selections */ + if (H5S_SEL_POINTS==H5S_GET_SELECT_TYPE(mem_space) || H5S_SEL_POINTS==H5S_GET_SELECT_TYPE(file_space)) + HGOTO_DONE(FALSE); + + /* Dataset storage must be contiguous or chunked */ + if ((flags&H5S_CONV_STORAGE_MASK)!=H5S_CONV_STORAGE_CONTIGUOUS && + (flags&H5S_CONV_STORAGE_MASK)!=H5S_CONV_STORAGE_CHUNKED) + HGOTO_DONE(FALSE); + + if ((flags&H5S_CONV_STORAGE_MASK)==H5S_CONV_STORAGE_CHUNKED) { + hsize_t chunk_dim[H5O_LAYOUT_NDIMS]; /* Chunk dimensions */ + hssize_t startf[H5S_MAX_RANK], /* Selection start bounds */ + endf[H5S_MAX_RANK]; /* Selection end bounds */ + unsigned dim_rankf; /* Number of dimensions of file dataspace */ + int pcheck_hyper,check_hyper, /* Flags for checking if selection is in one chunk */ + tnum_chunkf, /* Number of chunks selection overlaps */ + max_chunkf, /* Maximum number of chunks selection overlaps */ + min_chunkf, /* Minimum number of chunks selection overlaps */ + num_chunks_same; /* Flag indicating whether all processes have the same # of chunks to operate on */ + unsigned dim_chunks; /* Temporary number of chunks in a dimension */ + MPI_Comm comm; /* MPI communicator for file */ + int mpi_rank; /* Rank in MPI communicator */ + int mpi_code; /* MPI return code */ + unsigned u; /* Local index variable */ + + /* Getting MPI communicator and rank */ + if((comm = H5F_mpi_get_comm(file))==MPI_COMM_NULL) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator") + if((mpi_rank = H5F_mpi_get_rank(file))<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve MPI rank") + + /* Currently collective chunking storage + inside HDF5 is supported for either one of the following two cases: + 1. All the hyperslabs for one process is inside one chunk. + 2. For single hyperslab selection, the number of chunks that covered + the single selection for all processes should be equal. + KY, 2004/7/14 + */ + + /* Quincey, please read. + This is maybe redundant, I think only when both memory and file space be SCALAR + space, the collective IO can work. Otherwise, SELECT_POINT will be reached,collective + IO shouldn't work. + Please clarify and correct the code on the following, + Quincey said that it was probably okay if only one data space is SCALAR, + Still keep the code here until we added more tests later. + Kent */ + if(H5S_SCALAR==H5S_GET_EXTENT_TYPE(mem_space) || H5S_SCALAR ==H5S_GET_EXTENT_TYPE(file_space)) { + if(!(H5S_SCALAR==H5S_GET_EXTENT_TYPE(mem_space) && H5S_SCALAR ==H5S_GET_EXTENT_TYPE(file_space))) + HGOTO_DONE(FALSE) + else + HGOTO_DONE(TRUE) + } /* end if */ + + dim_rankf = H5S_GET_EXTENT_NDIMS(file_space); + + if(H5S_SELECT_BOUNDS(file_space,startf,endf)==FAIL) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE,FAIL, "invalid check for single selection blocks"); + + for(u=0; u < layout->u.chunk.ndims; u++) + chunk_dim[u] = layout->u.chunk.dim[u]; + + /* Case 1: check whether all hyperslab in this process is inside one chunk. + Note: we don't handle when starting point is less than zero since that may cover + two chunks. */ + + /*for file space checking*/ + pcheck_hyper = 1; + for (u=0; u<dim_rankf; u++) + if(endf[u]/chunk_dim[u]!=startf[u]/chunk_dim[u]) { + pcheck_hyper = 0; + break; + } + + if (MPI_SUCCESS != (mpi_code= MPI_Reduce(&pcheck_hyper,&check_hyper,1,MPI_INT,MPI_LAND,0,comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Reduce failed", mpi_code) + if (MPI_SUCCESS != (mpi_code= MPI_Bcast(&check_hyper,1,MPI_INT,0,comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code) + + /*if check_hyper is true, condition for collective IO case is fulfilled, no + need to do further test. */ + if(check_hyper) + HGOTO_DONE(TRUE); + + /* Case 2:Check whether the number of chunks that covered the single hyperslab is the same. + If not,no collective chunk IO. + KY, 2004/7/14 + */ + + c1 = H5S_SELECT_IS_SINGLE(file_space); + c2 = H5S_SELECT_IS_SINGLE(mem_space); + + if(c1==FAIL || c2 ==FAIL) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "invalid check for single selection blocks"); + if(c1==FALSE || c2 ==FALSE) + HGOTO_DONE(FALSE); + + /* Compute the number of chunks covered by the selection on this process */ + tnum_chunkf = 1; + for (u=0; u<dim_rankf; u++) { + dim_chunks = (endf[u]/chunk_dim[u]-startf[u]/chunk_dim[u])+1; + tnum_chunkf = dim_chunks*tnum_chunkf; + } + + /* Determine the minimum and maximum # of chunks for all processes */ + if (MPI_SUCCESS != (mpi_code= MPI_Reduce(&tnum_chunkf,&max_chunkf,1,MPI_INT,MPI_MAX,0,comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Reduce failed", mpi_code) + if (MPI_SUCCESS != (mpi_code= MPI_Reduce(&tnum_chunkf,&min_chunkf,1,MPI_INT,MPI_MIN,0,comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Reduce failed", mpi_code) + + /* Let the rank==0 process determine if the same number of chunks will be operated on by all processes */ + if(mpi_rank == 0) + num_chunks_same = (max_chunkf==min_chunkf); + + /* Broadcast the flag indicating the number of chunks are the same */ + if (MPI_SUCCESS != (mpi_code= MPI_Bcast(&num_chunks_same,1,MPI_INT,0,comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code) + + /* Can't handle case when number of chunks is different (yet) */ + if(!num_chunks_same) + HGOTO_DONE(FALSE); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5D_mpio_opt_possible() */ + + +/*------------------------------------------------------------------------- * Function: H5D_mpio_spaces_xfer * * Purpose: Use MPI-IO to transfer data efficiently @@ -209,7 +382,6 @@ done: */ herr_t H5D_mpio_spaces_read(H5D_io_info_t *io_info, - H5O_layout_readvv_func_t UNUSED op, size_t UNUSED nelmts, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, void *buf/*out*/) @@ -247,7 +419,6 @@ H5D_mpio_spaces_read(H5D_io_info_t *io_info, */ herr_t H5D_mpio_spaces_write(H5D_io_info_t *io_info, - H5O_layout_writevv_func_t UNUSED op, size_t UNUSED nelmts, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, const void *buf) diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 51ea060..a5bbeaf 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -57,6 +57,56 @@ /* Package Private Typedefs */ /****************************/ +/* + * If there is no data type conversion then it might be possible to + * transfer data points between application memory and the file in one + * step without going through the data type conversion buffer. + */ + +/* Read from file to application w/o intermediate scratch buffer */ +struct H5D_io_info_t; +typedef herr_t (*H5D_io_read_func_t)(struct H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + void *buf/*out*/); + + +/* Write directly from app buffer to file */ +typedef herr_t (*H5D_io_write_func_t)(struct H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + const void *buf); + +/* Function pointers for I/O on particular types of dataset layouts */ +typedef ssize_t (*H5D_io_readvv_func_t)(struct H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + void *buf); +typedef ssize_t (*H5D_io_writevv_func_t)(struct H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + const void *buf); + +/* Typedef for raw data I/O framework info */ +typedef struct H5D_io_ops_t { + H5D_io_read_func_t read; /* Direct I/O routine for reading */ + H5D_io_write_func_t write; /* Direct I/O routine for writing */ + H5D_io_readvv_func_t readvv; /* I/O routine for reading data */ + H5D_io_writevv_func_t writevv; /* I/O routine for writing data */ +} H5D_io_ops_t; + +/* Typedef for raw data I/O operation info */ +typedef struct H5D_io_info_t { + H5D_t *dset; /* Pointer to dataset being operated on */ + const H5D_dxpl_cache_t *dxpl_cache; /* Pointer to cache DXPL info */ + hid_t dxpl_id; /* Original DXPL ID */ + const H5D_storage_t *store; /* Dataset storage info */ + H5D_io_ops_t ops; /* I/O operation function pointers */ +#ifdef H5S_DEBUG + H5S_iostats_t *stats; /* I/O statistics */ +#endif /* H5S_DEBUG */ +} H5D_io_info_t; + /* The raw data chunk cache */ typedef struct H5D_rdcc_t { #ifdef H5D_ISTORE_DEBUG @@ -95,6 +145,7 @@ typedef struct H5D_shared_t { H5S_t *space; /* dataspace of this dataset */ hid_t dcpl_id; /* dataset creation property id */ H5D_dcpl_cache_t dcpl_cache; /* Cached DCPL values */ + H5D_io_ops_t io_ops; /* I/O operations */ H5O_layout_t layout; /* data layout */ hbool_t checked_filters;/* TRUE if dataset passes can_apply check */ @@ -112,7 +163,7 @@ typedef struct H5D_shared_t { * dataset in certain circumstances) */ H5D_rdcc_t chunk; /* Information about chunked data */ - }cache; + } cache; } H5D_shared_t; struct H5D_t { @@ -140,11 +191,54 @@ extern H5D_dxpl_cache_t H5D_def_dxpl_cache; H5_DLL herr_t H5D_alloc_storage (H5F_t *f, hid_t dxpl_id, H5D_t *dset, H5D_time_alloc_t time_alloc, hbool_t update_time, hbool_t full_overwrite); +/* Functions that perform serial I/O operations */ +H5_DLL herr_t H5D_select_fscat (H5D_io_info_t *io_info, + const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, + const void *_buf); +H5_DLL size_t H5D_select_fgath (H5D_io_info_t *io_info, + const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, + void *buf); +H5_DLL herr_t H5D_select_mscat (const void *_tscat_buf, + const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, + const H5D_dxpl_cache_t *dxpl_cache, void *_buf/*out*/); +H5_DLL size_t H5D_select_mgath (const void *_buf, + const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, + const H5D_dxpl_cache_t *dxpl_cache, void *_tgath_buf/*out*/); +H5_DLL herr_t H5D_select_read(H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + void *buf/*out*/); +H5_DLL herr_t H5D_select_write(H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + const void *buf/*out*/); + /* Functions that operate on contiguous storage */ H5_DLL herr_t H5D_contig_create(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout); H5_DLL herr_t H5D_contig_fill(H5D_t *dset, hid_t dxpl_id); +H5_DLL haddr_t H5D_contig_get_addr(const H5D_t *dset); +H5_DLL ssize_t H5D_contig_readvv(H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + void *buf); +H5_DLL ssize_t H5D_contig_writevv(H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + const void *buf); + +/* Functions that operate on compact dataset storage */ +H5_DLL ssize_t H5D_compact_readvv(H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_size_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_size_arr[], hsize_t mem_offset_arr[], + void *buf); +H5_DLL ssize_t H5D_compact_writevv(H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_size_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_size_arr[], hsize_t mem_offset_arr[], + const void *buf); /* Functions that operate on indexed storage */ +/* forward reference for collective-chunk IO use */ +struct H5D_istore_ud1_t; /*define in H5Distore.c*/ H5_DLL herr_t H5D_istore_init (const H5F_t *f, H5D_t *dset); H5_DLL herr_t H5D_istore_flush (H5D_t *dset, hid_t dxpl_id, unsigned flags); H5_DLL herr_t H5D_istore_create(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout); @@ -160,6 +254,45 @@ H5_DLL herr_t H5D_istore_dump_btree(H5F_t *f, hid_t dxpl_id, FILE *stream, unsig #ifdef H5D_ISTORE_DEBUG H5_DLL herr_t H5D_istore_stats (H5D_t *dset, hbool_t headers); #endif /* H5D_ISTORE_DEBUG */ +H5_DLL ssize_t H5D_istore_readvv(H5D_io_info_t *io_info, + size_t chunk_max_nseq, size_t *chunk_curr_seq, size_t chunk_len_arr[], hsize_t chunk_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + void *buf); +H5_DLL ssize_t H5D_istore_writevv(H5D_io_info_t *io_info, + size_t chunk_max_nseq, size_t *chunk_curr_seq, size_t chunk_len_arr[], hsize_t chunk_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + const void *buf); +H5_DLL haddr_t H5D_istore_get_addr(H5D_io_info_t *io_info, + struct H5D_istore_ud1_t *_udata); + +/* Functions that operate on external file list (efl) storage */ +H5_DLL ssize_t H5D_efl_readvv(H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + void *buf); +H5_DLL ssize_t H5D_efl_writevv(H5D_io_info_t *io_info, + size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], + size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], + const void *buf); + +#ifdef H5_HAVE_PARALLEL +/* MPI-IO function to read directly from app buffer to file rky980813 */ +H5_DLL herr_t H5D_mpio_spaces_read(H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const struct H5S_t *file_space, const struct H5S_t *mem_space, + void *buf/*out*/); + +/* MPI-IO function to write directly from app buffer to file rky980813 */ +H5_DLL herr_t H5D_mpio_spaces_write(H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const struct H5S_t *file_space, const struct H5S_t *mem_space, + const void *buf); + +/* MPI-IO function to check if a direct I/O transfer is possible between + * memory and the file */ +H5_DLL htri_t H5D_mpio_opt_possible(const H5F_t *file, const H5S_t *mem_space, + const H5S_t *file_space, const unsigned flags, const H5O_layout_t *layout); +#endif /* H5_HAVE_PARALLEL */ /* Testing functions */ #ifdef H5D_TESTING diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index a933204..6987c40 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -207,14 +207,6 @@ typedef struct H5D_dcpl_cache_t { H5D_fill_time_t fill_time; /* Fill time (H5D_CRT_FILL_TIME_NAME) */ } H5D_dcpl_cache_t; -/* Typedef for common raw data I/O operation info */ -typedef struct H5D_io_info_t { - H5D_t *dset; /* Pointer to dataset being operated on */ - const H5D_dxpl_cache_t *dxpl_cache; /* Pointer to cache DXPL info */ - hid_t dxpl_id; /* Original DXPL ID */ - const H5D_storage_t *store; /* Dataset storage info */ -} H5D_io_info_t; - /* Library-private functions defined in H5D package */ H5_DLL herr_t H5D_init(void); H5_DLL H5D_t *H5D_open(H5G_entry_t *ent, hid_t dxpl_id); @@ -236,62 +228,11 @@ H5_DLL herr_t H5D_get_dxpl_cache_real(hid_t dxpl_id, H5D_dxpl_cache_t *cache); /* Functions that operate on contiguous storage */ H5_DLL herr_t H5D_contig_delete(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout); -H5_DLL haddr_t H5D_contig_get_addr(const H5D_t *dset); -H5_DLL ssize_t H5D_contig_readvv(H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - void *buf); -H5_DLL ssize_t H5D_contig_writevv(H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - const void *buf); - -/* Functions that operate on compact dataset storage */ -H5_DLL ssize_t H5D_compact_readvv(H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_size_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_size_arr[], hsize_t mem_offset_arr[], - void *buf); -H5_DLL ssize_t H5D_compact_writevv(H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_size_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_size_arr[], hsize_t mem_offset_arr[], - const void *buf); - -/* forward reference for collective-chunk IO use */ -struct H5D_istore_ud1_t; /*define at H5Distore.c*/ /* Functions that operate on indexed storage */ H5_DLL herr_t H5D_istore_delete(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout); -H5_DLL ssize_t H5D_istore_readvv(H5D_io_info_t *io_info, - size_t chunk_max_nseq, size_t *chunk_curr_seq, size_t chunk_len_arr[], hsize_t chunk_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - void *buf); -H5_DLL ssize_t H5D_istore_writevv(H5D_io_info_t *io_info, - size_t chunk_max_nseq, size_t *chunk_curr_seq, size_t chunk_len_arr[], hsize_t chunk_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - const void *buf); -H5_DLL haddr_t H5D_istore_get_addr(H5D_io_info_t *io_info, - struct H5D_istore_ud1_t *_udata); H5_DLL herr_t H5D_istore_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int indent, int fwidth, int ndims); -#ifdef H5_HAVE_PARALLEL -/* Forward references */ -struct H5S_t; - -/* MPI-IO function to read directly from app buffer to file rky980813 */ -H5_DLL herr_t H5D_mpio_spaces_read(H5D_io_info_t *io_info, - H5O_layout_readvv_func_t op, - size_t nelmts, size_t elmt_size, - const struct H5S_t *file_space, const struct H5S_t *mem_space, - void *buf/*out*/); - -/* MPI-IO function to write directly from app buffer to file rky980813 */ -H5_DLL herr_t H5D_mpio_spaces_write(H5D_io_info_t *io_info, - H5O_layout_writevv_func_t op, - size_t nelmts, size_t elmt_size, - const struct H5S_t *file_space, const struct H5S_t *mem_space, - const void *buf); -#endif /* H5_HAVE_PARALLEL */ - #endif diff --git a/src/H5Dselect.c b/src/H5Dselect.c new file mode 100644 index 0000000..a12823e --- /dev/null +++ b/src/H5Dselect.c @@ -0,0 +1,691 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Programmer: Quincey Koziol <koziol@ncsa.uiuc.ued> + * Thursday, September 30, 2004 + * + * Purpose: Dataspace I/O functions. + */ + +#define H5D_PACKAGE /*suppress error about including H5Dpkg */ + +/* Pablo information */ +/* (Put before include files to avoid problems with inline functions) */ +#define PABLO_MASK H5D_select_mask + +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ + +/* Declare a free list to manage sequences of size_t */ +H5FL_SEQ_DEFINE_STATIC(size_t); + +/* Declare a free list to manage sequences of hsize_t */ +H5FL_SEQ_DEFINE_STATIC(hsize_t); + + +/*------------------------------------------------------------------------- + * Function: H5D_select_fscat + * + * Purpose: Scatters dataset elements from the type conversion buffer BUF + * to the file F where the data points are arranged according to + * the file dataspace FILE_SPACE and stored according to + * LAYOUT and EFL. Each element is ELMT_SIZE bytes. + * The caller is requesting that NELMTS elements are copied. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, June 20, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_select_fscat (H5D_io_info_t *io_info, + const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, + const void *_buf) +{ + const uint8_t *buf=_buf; /* Alias for pointer arithmetic */ + hsize_t _off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets */ + hsize_t *off=NULL; /* Pointer to sequence offsets */ + hsize_t mem_off; /* Offset in memory */ + size_t mem_curr_seq; /* "Current sequence" in memory */ + size_t dset_curr_seq; /* "Current sequence" in dataset */ + size_t _len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths */ + size_t *len=NULL; /* Array to store sequence lengths */ + size_t orig_mem_len, mem_len; /* Length of sequence in memory */ + size_t nseq; /* Number of sequences generated */ + size_t nelem; /* Number of elements used in sequences */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_select_fscat, FAIL); + + /* Check args */ + assert (io_info); + assert (space); + assert (iter); + assert (nelmts>0); + assert (_buf); + assert(TRUE==H5P_isa_class(io_info->dxpl_id,H5P_DATASET_XFER)); + + /* Allocate the vector I/O arrays */ + if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if((len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); + if((off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); + } /* end if */ + else { + len=_len; + off=_off; + } /* end else */ + + /* Loop until all elements are written */ + while(nelmts>0) { + /* Get list of sequences for selection to write */ + if(H5S_SELECT_GET_SEQ_LIST(space,H5S_GET_SEQ_LIST_SORTED,iter,io_info->dxpl_cache->vec_size,nelmts,&nseq,&nelem,off,len)<0) + HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); + + /* Reset the current sequence information */ + mem_curr_seq=dset_curr_seq=0; + orig_mem_len=mem_len=nelem*iter->elmt_size; + mem_off=0; + + /* Write sequence list out */ + if ((*io_info->ops.writevv)(io_info, nseq, &dset_curr_seq, len, off, 1, &mem_curr_seq, &mem_len, &mem_off, buf)<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_WRITEERROR, FAIL, "write error"); + + /* Update buffer */ + buf += orig_mem_len; + + /* Decrement number of elements left to process */ + nelmts -= nelem; + } /* end while */ + +done: + if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if(len!=NULL) + H5FL_SEQ_FREE(size_t,len); + if(off!=NULL) + H5FL_SEQ_FREE(hsize_t,off); + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value); +} /* H5D_select_fscat() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_select_fgath + * + * Purpose: Gathers data points from file F and accumulates them in the + * type conversion buffer BUF. The LAYOUT argument describes + * how the data is stored on disk and EFL describes how the data + * is organized in external files. ELMT_SIZE is the size in + * bytes of a datum which this function treats as opaque. + * FILE_SPACE describes the dataspace of the dataset on disk + * and the elements that have been selected for reading (via + * hyperslab, etc). This function will copy at most NELMTS + * elements. + * + * Return: Success: Number of elements copied. + * Failure: 0 + * + * Programmer: Quincey Koziol + * Monday, June 24, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +size_t +H5D_select_fgath (H5D_io_info_t *io_info, + const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, + void *_buf/*out*/) +{ + uint8_t *buf=_buf; /* Alias for pointer arithmetic */ + hsize_t _off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets */ + hsize_t *off=NULL; /* Pointer to sequence offsets */ + hsize_t mem_off; /* Offset in memory */ + size_t mem_curr_seq; /* "Current sequence" in memory */ + size_t dset_curr_seq; /* "Current sequence" in dataset */ + size_t _len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths */ + size_t *len=NULL; /* Pointer to sequence lengths */ + size_t orig_mem_len, mem_len; /* Length of sequence in memory */ + size_t nseq; /* Number of sequences generated */ + size_t nelem; /* Number of elements used in sequences */ + size_t ret_value=nelmts; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_select_fgath, 0); + + /* Check args */ + assert (io_info); + assert (io_info->dset); + assert (io_info->store); + assert (space); + assert (iter); + assert (nelmts>0); + assert (_buf); + + /* Allocate the vector I/O arrays */ + if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if((len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "can't allocate I/O length vector array"); + if((off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "can't allocate I/O offset vector array"); + } /* end if */ + else { + len=_len; + off=_off; + } /* end else */ + + /* Loop until all elements are read */ + while(nelmts>0) { + /* Get list of sequences for selection to read */ + if(H5S_SELECT_GET_SEQ_LIST(space,H5S_GET_SEQ_LIST_SORTED,iter,io_info->dxpl_cache->vec_size,nelmts,&nseq,&nelem,off,len)<0) + HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, 0, "sequence length generation failed"); + + /* Reset the current sequence information */ + mem_curr_seq=dset_curr_seq=0; + orig_mem_len=mem_len=nelem*iter->elmt_size; + mem_off=0; + + /* Read sequence list in */ + if ((*io_info->ops.readvv)(io_info, nseq, &dset_curr_seq, len, off, 1, &mem_curr_seq, &mem_len, &mem_off, buf)<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_READERROR, 0, "read error"); + + /* Update buffer */ + buf += orig_mem_len; + + /* Decrement number of elements left to process */ + nelmts -= nelem; + } /* end while */ + +done: + if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if(len!=NULL) + H5FL_SEQ_FREE(size_t,len); + if(off!=NULL) + H5FL_SEQ_FREE(hsize_t,off); + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value); +} /* H5D_select_fgath() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_select_mscat + * + * Purpose: Scatters NELMTS data points from the scatter buffer + * TSCAT_BUF to the application buffer BUF. Each element is + * ELMT_SIZE bytes and they are organized in application memory + * according to SPACE. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Monday, July 8, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_select_mscat (const void *_tscat_buf, const H5S_t *space, + H5S_sel_iter_t *iter, size_t nelmts, const H5D_dxpl_cache_t *dxpl_cache, + void *_buf/*out*/) +{ + uint8_t *buf=(uint8_t *)_buf; /* Get local copies for address arithmetic */ + const uint8_t *tscat_buf=(const uint8_t *)_tscat_buf; + hsize_t _off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets */ + hsize_t *off=NULL; /* Pointer to sequence offsets */ + size_t _len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths */ + size_t *len=NULL; /* Pointer to sequence lengths */ + size_t curr_len; /* Length of bytes left to process in sequence */ + size_t nseq; /* Number of sequences generated */ + size_t curr_seq; /* Current sequence being processed */ + size_t nelem; /* Number of elements used in sequences */ + herr_t ret_value=SUCCEED; /* Number of elements scattered */ + + FUNC_ENTER_NOAPI(H5D_select_mscat, FAIL); + + /* Check args */ + assert (tscat_buf); + assert (space); + assert (iter); + assert (nelmts>0); + assert (buf); + + /* Allocate the vector I/O arrays */ + if(dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if((len = H5FL_SEQ_MALLOC(size_t,dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); + if((off = H5FL_SEQ_MALLOC(hsize_t,dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); + } /* end if */ + else { + len=_len; + off=_off; + } /* end else */ + + /* Loop until all elements are written */ + while(nelmts>0) { + /* Get list of sequences for selection to write */ + if(H5S_SELECT_GET_SEQ_LIST(space,0,iter,dxpl_cache->vec_size,nelmts,&nseq,&nelem,off,len)<0) + HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, 0, "sequence length generation failed"); + + /* Loop, while sequences left to process */ + for(curr_seq=0; curr_seq<nseq; curr_seq++) { + /* Get the number of bytes in sequence */ + curr_len=len[curr_seq]; + + HDmemcpy(buf+off[curr_seq],tscat_buf,curr_len); + + /* Advance offset in destination buffer */ + tscat_buf+=curr_len; + } /* end for */ + + /* Decrement number of elements left to process */ + nelmts -= nelem; + } /* end while */ + +done: + if(dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if(len!=NULL) + H5FL_SEQ_FREE(size_t,len); + if(off!=NULL) + H5FL_SEQ_FREE(hsize_t,off); + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value); +} /* H5D_select_mscat() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_select_mgath + * + * Purpose: Gathers dataset elements from application memory BUF and + * copies them into the gather buffer TGATH_BUF. + * Each element is ELMT_SIZE bytes and arranged in application + * memory according to SPACE. + * The caller is requesting that at most NELMTS be gathered. + * + * Return: Success: Number of elements copied. + * Failure: 0 + * + * Programmer: Quincey Koziol + * Monday, June 24, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +size_t +H5D_select_mgath (const void *_buf, const H5S_t *space, + H5S_sel_iter_t *iter, size_t nelmts, const H5D_dxpl_cache_t *dxpl_cache, + void *_tgath_buf/*out*/) +{ + const uint8_t *buf=(const uint8_t *)_buf; /* Get local copies for address arithmetic */ + uint8_t *tgath_buf=(uint8_t *)_tgath_buf; + hsize_t _off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets */ + hsize_t *off=NULL; /* Pointer to sequence offsets */ + size_t _len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths */ + size_t *len=NULL; /* Pointer to sequence lengths */ + size_t curr_len; /* Length of bytes left to process in sequence */ + size_t nseq; /* Number of sequences generated */ + size_t curr_seq; /* Current sequence being processed */ + size_t nelem; /* Number of elements used in sequences */ + size_t ret_value=nelmts; /* Number of elements gathered */ + + FUNC_ENTER_NOAPI(H5D_select_mgath, 0); + + /* Check args */ + assert (buf); + assert (space); + assert (iter); + assert (nelmts>0); + assert (tgath_buf); + + /* Allocate the vector I/O arrays */ + if(dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if((len = H5FL_SEQ_MALLOC(size_t,dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "can't allocate I/O length vector array"); + if((off = H5FL_SEQ_MALLOC(hsize_t,dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "can't allocate I/O offset vector array"); + } /* end if */ + else { + len=_len; + off=_off; + } /* end else */ + + /* Loop until all elements are written */ + while(nelmts>0) { + /* Get list of sequences for selection to write */ + if(H5S_SELECT_GET_SEQ_LIST(space,0,iter,dxpl_cache->vec_size,nelmts,&nseq,&nelem,off,len)<0) + HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, 0, "sequence length generation failed"); + + /* Loop, while sequences left to process */ + for(curr_seq=0; curr_seq<nseq; curr_seq++) { + /* Get the number of bytes in sequence */ + curr_len=len[curr_seq]; + + HDmemcpy(tgath_buf,buf+off[curr_seq],curr_len); + + /* Advance offset in gather buffer */ + tgath_buf+=curr_len; + } /* end for */ + + /* Decrement number of elements left to process */ + nelmts -= nelem; + } /* end while */ + +done: + if(dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if(len!=NULL) + H5FL_SEQ_FREE(size_t,len); + if(off!=NULL) + H5FL_SEQ_FREE(hsize_t,off); + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value); +} /* H5D_select_mgath() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_select_read + * + * Purpose: Reads directly from file into application memory. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Tuesday, July 23, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_select_read(H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + void *buf/*out*/) +{ + H5S_sel_iter_t mem_iter; /* Memory selection iteration info */ + hbool_t mem_iter_init=0; /* Memory selection iteration info has been initialized */ + H5S_sel_iter_t file_iter; /* File selection iteration info */ + hbool_t file_iter_init=0; /* File selection iteration info has been initialized */ + hsize_t _mem_off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets in memory */ + hsize_t *mem_off=NULL; /* Pointer to sequence offsets in memory */ + hsize_t _file_off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets in the file */ + hsize_t *file_off=NULL; /* Pointer to sequence offsets in the file */ + size_t _mem_len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths in memory */ + size_t *mem_len=NULL; /* Pointer to sequence lengths in memory */ + size_t _file_len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths in the file */ + size_t *file_len=NULL; /* Pointer to sequence lengths in the file */ + size_t mem_nseq; /* Number of sequences generated in the file */ + size_t file_nseq; /* Number of sequences generated in memory */ + size_t mem_nelem; /* Number of elements used in memory sequences */ + size_t file_nelem; /* Number of elements used in file sequences */ + size_t curr_mem_seq; /* Current memory sequence to operate on */ + size_t curr_file_seq; /* Current file sequence to operate on */ + ssize_t tmp_file_len; /* Temporary number of bytes in file sequence */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_select_read, FAIL); + + /* Check args */ + assert(io_info); + assert(io_info->dset); + assert(io_info->dxpl_cache); + assert(io_info->store); + assert(buf); + assert(TRUE==H5P_isa_class(io_info->dxpl_id,H5P_DATASET_XFER)); + + /* Initialize file iterator */ + if (H5S_select_iter_init(&file_iter, file_space, elmt_size)<0) + HGOTO_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator"); + file_iter_init=1; /* File selection iteration info has been initialized */ + + /* Initialize memory iterator */ + if (H5S_select_iter_init(&mem_iter, mem_space, elmt_size)<0) + HGOTO_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator"); + mem_iter_init=1; /* Memory selection iteration info has been initialized */ + + /* Allocate the vector I/O arrays */ + if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if((mem_len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); + if((mem_off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); + if((file_len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); + if((file_off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); + } /* end if */ + else { + mem_len=_mem_len; + mem_off=_mem_off; + file_len=_file_len; + file_off=_file_off; + } /* end else */ + + /* Initialize sequence counts */ + curr_mem_seq=curr_file_seq=0; + mem_nseq=file_nseq=0; + + /* Loop, until all bytes are processed */ + while(nelmts>0) { + /* Check if more file sequences are needed */ + if(curr_file_seq>=file_nseq) { + /* Get sequences for file selection */ + if(H5S_SELECT_GET_SEQ_LIST(file_space,H5S_GET_SEQ_LIST_SORTED,&file_iter,io_info->dxpl_cache->vec_size,nelmts,&file_nseq,&file_nelem,file_off,file_len)<0) + HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); + + /* Start at the beginning of the sequences again */ + curr_file_seq=0; + } /* end if */ + + /* Check if more memory sequences are needed */ + if(curr_mem_seq>=mem_nseq) { + /* Get sequences for memory selection */ + if(H5S_SELECT_GET_SEQ_LIST(mem_space,0,&mem_iter,io_info->dxpl_cache->vec_size,nelmts,&mem_nseq,&mem_nelem,mem_off,mem_len)<0) + HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); + + /* Start at the beginning of the sequences again */ + curr_mem_seq=0; + } /* end if */ + + /* Read file sequences into current memory sequence */ + if ((tmp_file_len=(*io_info->ops.readvv)(io_info, + file_nseq, &curr_file_seq, file_len, file_off, + mem_nseq, &curr_mem_seq, mem_len, mem_off, + buf))<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_READERROR, FAIL, "read error"); + + /* Decrement number of elements left to process */ + assert((tmp_file_len%elmt_size)==0); + nelmts-=(tmp_file_len/elmt_size); + } /* end while */ + +done: + /* Release file selection iterator */ + if(file_iter_init) { + if (H5S_SELECT_ITER_RELEASE(&file_iter)<0) + HDONE_ERROR (H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator"); + } /* end if */ + + /* Release memory selection iterator */ + if(mem_iter_init) { + if (H5S_SELECT_ITER_RELEASE(&mem_iter)<0) + HDONE_ERROR (H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator"); + } /* end if */ + + /* Free vector arrays */ + if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if(file_len!=NULL) + H5FL_SEQ_FREE(size_t,file_len); + if(file_off!=NULL) + H5FL_SEQ_FREE(hsize_t,file_off); + if(mem_len!=NULL) + H5FL_SEQ_FREE(size_t,mem_len); + if(mem_off!=NULL) + H5FL_SEQ_FREE(hsize_t,mem_off); + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D_select_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_select_write + * + * Purpose: Writes directly from application memory into a file + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Tuesday, July 23, 2002 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_select_write(H5D_io_info_t *io_info, + size_t nelmts, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + const void *buf/*out*/) +{ + H5S_sel_iter_t mem_iter; /* Memory selection iteration info */ + hbool_t mem_iter_init=0; /* Memory selection iteration info has been initialized */ + H5S_sel_iter_t file_iter; /* File selection iteration info */ + hbool_t file_iter_init=0; /* File selection iteration info has been initialized */ + hsize_t _mem_off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets in memory */ + hsize_t *mem_off=NULL; /* Pointer to sequence offsets in memory */ + hsize_t _file_off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets in the file */ + hsize_t *file_off=NULL; /* Pointer to sequence offsets in the file */ + size_t _mem_len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths in memory */ + size_t *mem_len=NULL; /* Pointer to sequence lengths in memory */ + size_t _file_len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths in the file */ + size_t *file_len=NULL; /* Pointer to sequence lengths in the file */ + size_t mem_nseq; /* Number of sequences generated in the file */ + size_t file_nseq; /* Number of sequences generated in memory */ + size_t mem_nelem; /* Number of elements used in memory sequences */ + size_t file_nelem; /* Number of elements used in file sequences */ + size_t curr_mem_seq; /* Current memory sequence to operate on */ + size_t curr_file_seq; /* Current file sequence to operate on */ + ssize_t tmp_file_len; /* Temporary number of bytes in file sequence */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_select_write, FAIL); + + /* Check args */ + assert(io_info); + assert(io_info->dset); + assert(io_info->store); + assert(TRUE==H5P_isa_class(io_info->dxpl_id,H5P_DATASET_XFER)); + assert(buf); + + /* Allocate the vector I/O arrays */ + if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if((mem_len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); + if((mem_off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); + if((file_len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); + if((file_off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); + } /* end if */ + else { + mem_len=_mem_len; + mem_off=_mem_off; + file_len=_file_len; + file_off=_file_off; + } /* end else */ + + /* Initialize file iterator */ + if (H5S_select_iter_init(&file_iter, file_space, elmt_size)<0) + HGOTO_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator"); + file_iter_init=1; /* File selection iteration info has been initialized */ + + /* Initialize memory iterator */ + if (H5S_select_iter_init(&mem_iter, mem_space, elmt_size)<0) + HGOTO_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator"); + mem_iter_init=1; /* Memory selection iteration info has been initialized */ + + /* Initialize sequence counts */ + curr_mem_seq=curr_file_seq=0; + mem_nseq=file_nseq=0; + + /* Loop, until all bytes are processed */ + while(nelmts>0) { + /* Check if more file sequences are needed */ + if(curr_file_seq>=file_nseq) { + /* Get sequences for file selection */ + if(H5S_SELECT_GET_SEQ_LIST(file_space,H5S_GET_SEQ_LIST_SORTED,&file_iter,io_info->dxpl_cache->vec_size,nelmts,&file_nseq,&file_nelem,file_off,file_len)<0) + HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); + + /* Start at the beginning of the sequences again */ + curr_file_seq=0; + } /* end if */ + + /* Check if more memory sequences are needed */ + if(curr_mem_seq>=mem_nseq) { + /* Get sequences for memory selection */ + if(H5S_SELECT_GET_SEQ_LIST(mem_space,0,&mem_iter,io_info->dxpl_cache->vec_size,nelmts,&mem_nseq,&mem_nelem,mem_off,mem_len)<0) + HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); + + /* Start at the beginning of the sequences again */ + curr_mem_seq=0; + } /* end if */ + + /* Write memory sequences into file sequences */ + if ((tmp_file_len=(*io_info->ops.writevv)(io_info, + file_nseq, &curr_file_seq, file_len, file_off, + mem_nseq, &curr_mem_seq, mem_len, mem_off, + buf))<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_WRITEERROR, FAIL, "write error"); + + /* Decrement number of elements left to process */ + assert((tmp_file_len%elmt_size)==0); + nelmts-=(tmp_file_len/elmt_size); + } /* end while */ + +done: + /* Release file selection iterator */ + if(file_iter_init) { + if (H5S_SELECT_ITER_RELEASE(&file_iter)<0) + HDONE_ERROR (H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator"); + } /* end if */ + + /* Release memory selection iterator */ + if(mem_iter_init) { + if (H5S_SELECT_ITER_RELEASE(&mem_iter)<0) + HDONE_ERROR (H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator"); + } /* end if */ + + /* Free vector arrays */ + if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { + if(file_len!=NULL) + H5FL_SEQ_FREE(size_t,file_len); + if(file_off!=NULL) + H5FL_SEQ_FREE(hsize_t,file_off); + if(mem_len!=NULL) + H5FL_SEQ_FREE(size_t,mem_len); + if(mem_off!=NULL) + H5FL_SEQ_FREE(hsize_t,mem_off); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D_select_write() */ + diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 24c980c..7997524 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -71,23 +71,6 @@ #define H5F_ACC_PUBLIC_FLAGS 0x00ffu /* - * Macros that check for overflows. These are somewhat dangerous to fiddle - * with. - */ -#if (H5_SIZEOF_SIZE_T >= H5_SIZEOF_OFF_T) -# define H5F_OVERFLOW_SIZET2OFFT(X) \ - ((size_t)(X)>=(size_t)((size_t)1<<(8*sizeof(off_t)-1))) -#else -# define H5F_OVERFLOW_SIZET2OFFT(X) 0 -#endif -#if (H5_SIZEOF_HSIZE_T >= H5_SIZEOF_OFF_T) -# define H5F_OVERFLOW_HSIZET2OFFT(X) \ - ((hsize_t)(X)>=(hsize_t)((hsize_t)1<<(8*sizeof(off_t)-1))) -#else -# define H5F_OVERFLOW_HSIZET2OFFT(X) 0 -#endif - -/* * Define the structure to store the file information for HDF5 files. One of * these structures is allocated per file, not per H5Fopen(). That is, set of * H5F_t structs can all point to the same H5F_file_t struct. The `nrefs' diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index c02a86a..71f00a1 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -260,6 +260,23 @@ typedef struct H5F_t H5F_t; case 2: UINT16DECODE(p,l); break; \ } +/* + * Macros that check for overflows. These are somewhat dangerous to fiddle + * with. + */ +#if (H5_SIZEOF_SIZE_T >= H5_SIZEOF_OFF_T) +# define H5F_OVERFLOW_SIZET2OFFT(X) \ + ((size_t)(X)>=(size_t)((size_t)1<<(8*sizeof(off_t)-1))) +#else +# define H5F_OVERFLOW_SIZET2OFFT(X) 0 +#endif +#if (H5_SIZEOF_HSIZE_T >= H5_SIZEOF_OFF_T) +# define H5F_OVERFLOW_HSIZET2OFFT(X) \ + ((hsize_t)(X)>=(hsize_t)((hsize_t)1<<(8*sizeof(off_t)-1))) +#else +# define H5F_OVERFLOW_HSIZET2OFFT(X) 0 +#endif + /* ========= File Creation properties ============ */ /* Definitions for the size of the file user block in bytes */ #define H5F_CRT_USER_BLOCK_NAME "block_size" diff --git a/src/H5Oefl.c b/src/H5Oefl.c index 7e753ed..c795956 100644 --- a/src/H5Oefl.c +++ b/src/H5Oefl.c @@ -17,7 +17,6 @@ * Tuesday, November 25, 1997 */ -#define H5F_PACKAGE /*suppress error about including H5Fpkg */ #define H5O_PACKAGE /*suppress error about including H5Opkg */ /* Pablo information */ @@ -25,9 +24,8 @@ #define PABLO_MASK H5O_efl_mask #include "H5private.h" /* Generic Functions */ -#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ -#include "H5Fpkg.h" /* File access */ +#include "H5Fprivate.h" /* File access */ #include "H5HLprivate.h" /* Local Heaps */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ @@ -40,10 +38,6 @@ static size_t H5O_efl_size(H5F_t *f, const void *_mesg); static herr_t H5O_efl_reset(void *_mesg); static herr_t H5O_efl_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, FILE * stream, int indent, int fwidth); -static herr_t H5O_efl_read (const H5O_efl_t *efl, haddr_t addr, size_t size, - uint8_t *buf); -static herr_t H5O_efl_write(const H5O_efl_t *efl, haddr_t addr, size_t size, - const uint8_t *buf); /* This message derives from H5O */ const H5O_class_t H5O_EFL[1] = {{ @@ -424,341 +418,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5O_efl_read - * - * Purpose: Reads data from an external file list. It is an error to - * read past the logical end of file, but reading past the end - * of any particular member of the external file list results in - * zeros. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Wednesday, March 4, 1998 - * - * Modifications: - * Robb Matzke, 1999-07-28 - * The ADDR argument is passed by value. - *------------------------------------------------------------------------- - */ -static herr_t -H5O_efl_read (const H5O_efl_t *efl, haddr_t addr, size_t size, uint8_t *buf) -{ - int fd=-1; - size_t to_read; -#ifndef NDEBUG - hsize_t tempto_read; -#endif /* NDEBUG */ - hsize_t skip=0; - haddr_t cur; - ssize_t n; - size_t u; /* Local index variable */ - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5O_efl_read); - - /* Check args */ - assert (efl && efl->nused>0); - assert (H5F_addr_defined (addr)); - assert (size < SIZET_MAX); - assert (buf || 0==size); - - /* Find the first efl member from which to read */ - for (u=0, cur=0; u<efl->nused; u++) { - if (H5O_EFL_UNLIMITED==efl->slot[u].size || addr < cur+efl->slot[u].size) { - skip = addr - cur; - break; - } - cur += efl->slot[u].size; - } - - /* Read the data */ - while (size) { - if (u>=efl->nused) - HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, "read past logical end of file"); - if (H5F_OVERFLOW_HSIZET2OFFT (efl->slot[u].offset+skip)) - HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed"); - if ((fd=HDopen (efl->slot[u].name, O_RDONLY, 0))<0) - HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file"); - if (HDlseek (fd, (off_t)(efl->slot[u].offset+skip), SEEK_SET)<0) - HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file"); -#ifndef NDEBUG - tempto_read = MIN(efl->slot[u].size-skip,(hsize_t)size); - H5_CHECK_OVERFLOW(tempto_read,hsize_t,size_t); - to_read = (size_t)tempto_read; -#else /* NDEBUG */ - to_read = MIN((size_t)(efl->slot[u].size-skip), size); -#endif /* NDEBUG */ - if ((n=HDread (fd, buf, to_read))<0) { - HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL, "read error in external raw data file"); - } else if ((size_t)n<to_read) { - HDmemset (buf+n, 0, to_read-n); - } - HDclose (fd); - fd = -1; - size -= to_read; - buf += to_read; - skip = 0; - u++; - } - -done: - if (fd>=0) - HDclose (fd); - - FUNC_LEAVE_NOAPI(ret_value); -} - - -/*------------------------------------------------------------------------- - * Function: H5O_efl_write - * - * Purpose: Writes data to an external file list. It is an error to - * write past the logical end of file, but writing past the end - * of any particular member of the external file list just - * extends that file. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Wednesday, March 4, 1998 - * - * Modifications: - * Robb Matzke, 1999-07-28 - * The ADDR argument is passed by value. - *------------------------------------------------------------------------- - */ -static herr_t -H5O_efl_write (const H5O_efl_t *efl, haddr_t addr, size_t size, const uint8_t *buf) -{ - int fd=-1; - size_t to_write; -#ifndef NDEBUG - hsize_t tempto_write; -#endif /* NDEBUG */ - haddr_t cur; - hsize_t skip=0; - size_t u; /* Local index variable */ - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5O_efl_write); - - /* Check args */ - assert (efl && efl->nused>0); - assert (H5F_addr_defined (addr)); - assert (size < SIZET_MAX); - assert (buf || 0==size); - - /* Find the first efl member in which to write */ - for (u=0, cur=0; u<efl->nused; u++) { - if (H5O_EFL_UNLIMITED==efl->slot[u].size || addr < cur+efl->slot[u].size) { - skip = addr - cur; - break; - } - cur += efl->slot[u].size; - } - - /* Write the data */ - while (size) { - if (u>=efl->nused) - HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, "write past logical end of file"); - if (H5F_OVERFLOW_HSIZET2OFFT (efl->slot[u].offset+skip)) - HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed"); - if ((fd=HDopen (efl->slot[u].name, O_CREAT|O_RDWR, 0666))<0) { - if (HDaccess (efl->slot[u].name, F_OK)<0) { - HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL, "external raw data file does not exist"); - } else { - HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file"); - } - } - if (HDlseek (fd, (off_t)(efl->slot[u].offset+skip), SEEK_SET)<0) - HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file"); -#ifndef NDEBUG - tempto_write = MIN(efl->slot[u].size-skip,(hsize_t)size); - H5_CHECK_OVERFLOW(tempto_write,hsize_t,size_t); - to_write = (size_t)tempto_write; -#else /* NDEBUG */ - to_write = MIN((size_t)(efl->slot[u].size-skip), size); -#endif /* NDEBUG */ - if ((size_t)HDwrite (fd, buf, to_write)!=to_write) - HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL, "write error in external raw data file"); - HDclose (fd); - fd = -1; - size -= to_write; - buf += to_write; - skip = 0; - u++; - } - -done: - if (fd>=0) - HDclose (fd); - - FUNC_LEAVE_NOAPI(ret_value); -} - - -/*------------------------------------------------------------------------- - * Function: H5O_efl_readvv - * - * Purpose: Reads data from an external file list. It is an error to - * read past the logical end of file, but reading past the end - * of any particular member of the external file list results in - * zeros. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Wednesday, May 7, 2003 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -ssize_t -H5O_efl_readvv(H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - void *_buf) -{ - const H5O_efl_t *efl=&(io_info->store->efl); /* Pointer to efl info */ - unsigned char *buf; /* Pointer to buffer to write */ - haddr_t addr; /* Actual address to read */ - size_t size; /* Size of sequence in bytes */ - size_t u; /* Counting variable */ - size_t v; /* Counting variable */ - ssize_t ret_value=0; /* Return value */ - - FUNC_ENTER_NOAPI(H5O_efl_readvv, FAIL); - - /* Check args */ - assert (efl && efl->nused>0); - assert (_buf); - - /* Work through all the sequences */ - for(u=*dset_curr_seq, v=*mem_curr_seq; u<dset_max_nseq && v<mem_max_nseq; ) { - /* Choose smallest buffer to write */ - if(mem_len_arr[v]<dset_len_arr[u]) - size=mem_len_arr[v]; - else - size=dset_len_arr[u]; - - /* Compute offset on disk */ - addr=dset_offset_arr[u]; - - /* Compute offset in memory */ - buf = (unsigned char *)_buf + mem_offset_arr[v]; - - /* Read data */ - if (H5O_efl_read(efl, addr, size, buf)<0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); - - /* Update memory information */ - mem_len_arr[v]-=size; - mem_offset_arr[v]+=size; - if(mem_len_arr[v]==0) - v++; - - /* Update file information */ - dset_len_arr[u]-=size; - dset_offset_arr[u]+=size; - if(dset_len_arr[u]==0) - u++; - - /* Increment number of bytes copied */ - ret_value+=size; - } /* end for */ - - /* Update current sequence vectors */ - *dset_curr_seq=u; - *mem_curr_seq=v; - -done: - FUNC_LEAVE_NOAPI(ret_value); -} /* end H5O_efl_readvv() */ - - -/*------------------------------------------------------------------------- - * Function: H5O_efl_writevv - * - * Purpose: Writes data to an external file list. It is an error to - * write past the logical end of file, but writing past the end - * of any particular member of the external file list just - * extends that file. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Friday, May 2, 2003 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -ssize_t -H5O_efl_writevv(H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - const void *_buf) -{ - const H5O_efl_t *efl=&(io_info->store->efl); /* Pointer to efl info */ - const unsigned char *buf; /* Pointer to buffer to write */ - haddr_t addr; /* Actual address to read */ - size_t size; /* Size of sequence in bytes */ - size_t u; /* Counting variable */ - size_t v; /* Counting variable */ - ssize_t ret_value=0; /* Return value */ - - FUNC_ENTER_NOAPI(H5O_efl_writevv, FAIL); - - /* Check args */ - assert (efl && efl->nused>0); - assert (_buf); - - /* Work through all the sequences */ - for(u=*dset_curr_seq, v=*mem_curr_seq; u<dset_max_nseq && v<mem_max_nseq; ) { - /* Choose smallest buffer to write */ - if(mem_len_arr[v]<dset_len_arr[u]) - size=mem_len_arr[v]; - else - size=dset_len_arr[u]; - - /* Compute offset on disk */ - addr=dset_offset_arr[u]; - - /* Compute offset in memory */ - buf = (const unsigned char *)_buf + mem_offset_arr[v]; - - /* Write data */ - if (H5O_efl_write(efl, addr, size, buf)<0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); - - /* Update memory information */ - mem_len_arr[v]-=size; - mem_offset_arr[v]+=size; - if(mem_len_arr[v]==0) - v++; - - /* Update file information */ - dset_len_arr[u]-=size; - dset_offset_arr[u]+=size; - if(dset_len_arr[u]==0) - u++; - - /* Increment number of bytes copied */ - ret_value+=size; - } /* end for */ - - /* Update current sequence vectors */ - *dset_curr_seq=u; - *mem_curr_seq=v; - -done: - FUNC_LEAVE_NOAPI(ret_value); -} /* end H5O_efl_writevv() */ - - -/*------------------------------------------------------------------------- * Function: H5O_efl_debug * * Purpose: Prints debugging info for a message. diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 347c1c3..f70443a 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -35,9 +35,7 @@ #include "H5Spublic.h" /* Dataspace functions */ /* Private headers needed by this file */ -#include "H5private.h" /* Generic functions */ #include "H5HGprivate.h" /* Global heap functions */ -#include "H5RCprivate.h" /* Reference counted object functions */ #include "H5Tprivate.h" /* Datatype functions */ #include "H5Zprivate.h" /* I/O pipeline filters */ @@ -143,18 +141,6 @@ typedef struct H5O_layout_compact_t { void *buf; /* Buffer for compact dataset */ } H5O_layout_compact_t; -/* Function pointers for I/O on particular types of dataset layouts */ -/* (Forward declare some structs/unions to avoid #include problems) */ -struct H5D_io_info_t; -typedef ssize_t (*H5O_layout_readvv_func_t)(struct H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - void *buf); -typedef ssize_t (*H5O_layout_writevv_func_t)(struct H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - const void *buf); - typedef struct H5O_layout_t { H5D_layout_t type; /* Type of layout */ unsigned version; /* Version of message */ @@ -163,8 +149,6 @@ typedef struct H5O_layout_t { H5O_layout_chunk_t chunk; /* Information for chunked layout */ H5O_layout_compact_t compact; /* Information for compact layout */ } u; - H5O_layout_readvv_func_t readvv; /* I/O routine for reading data */ - H5O_layout_writevv_func_t writevv; /* I/O routine for writing data */ } H5O_layout_t; /* Enable reading/writing "bogus" messages */ @@ -279,14 +263,6 @@ H5_DLL size_t H5O_layout_meta_size(H5F_t *f, const void *_mesg); /* EFL operators */ H5_DLL hsize_t H5O_efl_total_size(H5O_efl_t *efl); -H5_DLL ssize_t H5O_efl_readvv(struct H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - void *buf); -H5_DLL ssize_t H5O_efl_writevv(struct H5D_io_info_t *io_info, - size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], - size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], - const void *buf); /* Fill value operators */ H5_DLL herr_t H5O_fill_convert(void *_fill, H5T_t *type, hid_t dxpl_id); @@ -43,24 +43,17 @@ static htri_t H5S_is_simple(const H5S_t *sdim); static herr_t H5S_encode(H5S_t *obj, unsigned char *buf, size_t *nalloc); static H5S_t *H5S_decode(const unsigned char *buf); -#ifdef H5_HAVE_PARALLEL -htri_t H5S_get_collective_io_consensus(const H5F_t *file, - const htri_t local_opinion, - const unsigned flags); -#endif /* H5_HAVE_PARALLEL */ - - #ifdef H5S_DEBUG /* Names of the selection names, for debugging */ static const char *H5S_sel_names[]={ "none", "point", "hyperslab", "all" }; -#endif /* H5S_DEBUG */ /* The path table, variable length */ -static H5S_conv_t **H5S_conv_g = NULL; -static size_t H5S_aconv_g = 0; /*entries allocated*/ -static size_t H5S_nconv_g = 0; /*entries used*/ +static H5S_iostats_t **H5S_iostats_g = NULL; +static size_t H5S_aiostats_g = 0; /*entries allocated*/ +static size_t H5S_niostats_g = 0; /*entries used*/ +#endif /* H5S_DEBUG */ #ifdef H5_HAVE_PARALLEL /* Global vars whose value can be set from environment variable also */ @@ -138,14 +131,13 @@ done: int H5S_term_interface(void) { - size_t i; int n=0; - #ifdef H5S_DEBUG + size_t i; int j, nprints=0; - H5S_conv_t *path=NULL; + H5S_iostats_t *path=NULL; char buf[256]; -#endif +#endif /* H5S_DEBUG */ FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5S_term_interface); @@ -158,8 +150,8 @@ H5S_term_interface(void) * Print statistics about each conversion path. */ if (H5DEBUG(S)) { - for (i=0; i<H5S_nconv_g; i++) { - path = H5S_conv_g[i]; + for (i=0; i<H5S_niostats_g; i++) { + path = H5S_iostats_g[i]; for (j=0; j<2; j++) { if (0==path->stats[j].gath_ncalls && 0==path->stats[j].scat_ncalls && @@ -268,16 +260,18 @@ H5S_term_interface(void) } } } -#endif +#endif /* H5S_DEBUG */ /* Free data types */ H5I_dec_type_ref(H5I_DATASPACE); +#ifdef H5S_DEBUG /* Clear/free conversion table */ - for (i=0; i<H5S_nconv_g; i++) - H5MM_xfree(H5S_conv_g[i]); - H5S_conv_g = H5MM_xfree(H5S_conv_g); - H5S_nconv_g = H5S_aconv_g = 0; + for (i=0; i<H5S_niostats_g; i++) + H5MM_xfree(H5S_iostats_g[i]); + H5S_iostats_g = H5MM_xfree(H5S_iostats_g); + H5S_niostats_g = H5S_aiostats_g = 0; +#endif /* H5S_DEBUG */ /* Shut down interface */ H5_interface_initialize_g = 0; @@ -1401,103 +1395,7 @@ done: FUNC_LEAVE_NOAPI(ret_value); } - -/*------------------------------------------------------------------------- - * Function: H5S_get_collective_io_consensus - * - * Purpose: Compare notes with all other processes involved in this I/O - * and see if all are go for collective I/O. - * - * If all are, return TRUE. - * - * If any process can't manage collective I/O, then collective - * I/O is impossible, and we return FALSE. - * - * If the flags indicate that collective I/O is impossible, - * skip the interprocess communication and just return FALSE. - * - * In any error is detected, return FAIL. - * - * Return: Success: TRUE or FALSE - * - * Failure: FAIL - * - * Programmer: JRM -- 8/30/04 - * - * Modifications: - * - * None. - * - *------------------------------------------------------------------------- - */ - -#ifdef H5_HAVE_PARALLEL -htri_t -H5S_get_collective_io_consensus(const H5F_t *file, - const htri_t local_opinion, - const unsigned flags) -{ - htri_t ret_value = FAIL; /* will update if successful */ - MPI_Comm comm; - int int_local_opinion; - int consensus; - int mpi_result; - - FUNC_ENTER_NOAPI(H5S_get_collective_io_consensus, NULL); - - HDassert ( ( local_opinion == TRUE ) || ( local_opinion == FALSE ) ); - - /* Don't do the interprocess communication unless the Parallel I/O - * conversion flag is set -- there may not be other processes to - * talk to. - */ - if ( ! ( flags & flags&H5S_CONV_PAR_IO_POSSIBLE ) ) { - - HGOTO_DONE(FALSE); - } - - comm = H5F_mpi_get_comm(file); - - if ( comm == MPI_COMM_NULL ) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, \ - "can't retrieve MPI communicator") - - if ( local_opinion == TRUE ) { - - int_local_opinion = 1; - - } else { - - int_local_opinion = 0; - } - - mpi_result = MPI_Allreduce((void *)(&int_local_opinion), - (void *)(&consensus), - 1, - MPI_INT, - MPI_LAND, - comm); - - if ( mpi_result != MPI_SUCCESS ) - HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_result) - - if ( consensus ) { - - ret_value = TRUE; - - } else { - - ret_value = FALSE; - } - -done: - - FUNC_LEAVE_NOAPI(ret_value); - -} /* H5S_get_collective_io_consensus() */ - -#endif /* H5_HAVE_PARALLEL */ - +#ifdef H5S_DEBUG /*------------------------------------------------------------------------- * Function: H5S_find @@ -1527,41 +1425,18 @@ done: * along with other data whose scope is the conversion path (like path * statistics). * - * John Mainzer, 8/30/04 - * Modified code to check with all other processes that have the - * file open before OKing collective I/O. + * John Mainzer, 8/30/04 + * Modified code to check with all other processes that have the + * file open before OKing collective I/O. * *------------------------------------------------------------------------- */ -H5S_conv_t * -H5S_find (const H5F_t -#ifndef H5_HAVE_PARALLEL -UNUSED -#endif/* H5_HAVE_PARALLEL*/ -*file, -const H5S_t *mem_space, const H5S_t *file_space, unsigned -#ifndef H5_HAVE_PARALLEL -UNUSED -#endif /* H5_HAVE_PARALLEL */ -flags, hbool_t -#ifndef H5_HAVE_PARALLEL -UNUSED -#endif /* H5_HAVE_PARALLEL */ -*use_par_opt_io, -#ifndef H5_HAVE_PARALLEL -UNUSED -#endif -const H5O_layout_t *layout - -) +H5S_iostats_t * +H5S_find (const H5S_t *mem_space, const H5S_t *file_space) { - H5S_conv_t *path=NULL; /* Space conversion path */ -#ifdef H5_HAVE_PARALLEL - htri_t opt; /* Flag whether a selection is optimizable */ -#endif /* H5_HAVE_PARALLEL */ - size_t i; /* Index variable */ - H5S_conv_t *ret_value; /* Return value */ - + H5S_iostats_t *path=NULL; /* Space conversion path */ + size_t u; /* Index variable */ + H5S_iostats_t *ret_value; /* Return value */ FUNC_ENTER_NOAPI(H5S_find, NULL); @@ -1577,47 +1452,10 @@ const H5O_layout_t *layout * Is this path already present in the data space conversion path table? * If so then return a pointer to that entry. */ - for (i=0; i<H5S_nconv_g; i++) { - if (H5S_conv_g[i]->ftype==H5S_GET_SELECT_TYPE(file_space) && - H5S_conv_g[i]->mtype==H5S_GET_SELECT_TYPE(mem_space)) { - -#ifdef H5_HAVE_PARALLEL - /* - * Check if we can set direct MPI-IO read/write functions - */ - opt=H5S_mpio_opt_possible(file,mem_space,file_space,flags,layout); - if(opt==FAIL) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, NULL, "invalid check for direct IO dataspace "); - - opt = H5S_get_collective_io_consensus(file, opt, flags); - - if ( opt == FAIL ) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, NULL, \ - "check for collective I/O consensus failed."); - - /* Check if we can use the optimized parallel I/O routines */ - if(opt==TRUE) { - /* Set the pointers to the MPI-specific routines */ - H5S_conv_g[i]->read = H5D_mpio_spaces_read; - H5S_conv_g[i]->write = H5D_mpio_spaces_write; - - /* Indicate that the I/O will be parallel */ - *use_par_opt_io=TRUE; - } /* end if */ - else { - /* Indicate that the I/O will _NOT_ be parallel */ - *use_par_opt_io=FALSE; - -#endif /* H5_HAVE_PARALLEL */ - H5S_conv_g[i]->read = H5S_select_read; - H5S_conv_g[i]->write = H5S_select_write; -#ifdef H5_HAVE_PARALLEL - } /* end else */ -#endif /* H5_HAVE_PARALLEL */ - - HGOTO_DONE(H5S_conv_g[i]); - } - } + for (u=0; u<H5S_niostats_g; u++) + if (H5S_iostats_g[u]->ftype==H5S_GET_SELECT_TYPE(file_space) && + H5S_iostats_g[u]->mtype==H5S_GET_SELECT_TYPE(mem_space)) + HGOTO_DONE(H5S_iostats_g[u]); /* * The path wasn't found. Create a new path. @@ -1629,53 +1467,19 @@ const H5O_layout_t *layout path->ftype = H5S_GET_SELECT_TYPE(file_space); path->mtype = H5S_GET_SELECT_TYPE(mem_space); -#ifdef H5_HAVE_PARALLEL - /* - * Check if we can set direct MPI-IO read/write functions - */ - opt=H5S_mpio_opt_possible(file,mem_space,file_space,flags,layout); - if(opt==FAIL) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, NULL, "invalid check for direct IO dataspace "); - - opt = H5S_get_collective_io_consensus(file, opt, flags); - - if ( opt == FAIL ) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, NULL, \ - "check for collective I/O consensus failed."); - - /* Check if we can use the optimized parallel I/O routines */ - if(opt==TRUE) { - /* Set the pointers to the MPI-specific routines */ - path->read = H5D_mpio_spaces_read; - path->write = H5D_mpio_spaces_write; - - /* Indicate that the I/O will be parallel */ - *use_par_opt_io=TRUE; - } /* end if */ - else { - /* Indicate that the I/O will _NOT_ be parallel */ - *use_par_opt_io=FALSE; - -#endif /* H5_HAVE_PARALLEL */ - path->read = H5S_select_read; - path->write = H5S_select_write; -#ifdef H5_HAVE_PARALLEL - } /* end else */ -#endif /* H5_HAVE_PARALLEL */ - /* * Add the new path to the table. */ - if (H5S_nconv_g>=H5S_aconv_g) { - size_t n = MAX(10, 2*H5S_aconv_g); - H5S_conv_t **p = H5MM_realloc(H5S_conv_g, n*sizeof(H5S_conv_g[0])); + if (H5S_niostats_g>=H5S_aiostats_g) { + size_t n = MAX(10, 2*H5S_aiostats_g); + H5S_iostats_t **p = H5MM_realloc(H5S_iostats_g, n*sizeof(H5S_iostats_g[0])); if (NULL==p) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for data space conversion path table"); - H5S_aconv_g = n; - H5S_conv_g = p; + H5S_aiostats_g = n; + H5S_iostats_g = p; } /* end if */ - H5S_conv_g[H5S_nconv_g++] = path; + H5S_iostats_g[H5S_niostats_g++] = path; /* Set the return value */ ret_value=path; @@ -1687,7 +1491,8 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value); -} +} /* end H5S_find() */ +#endif /* H5S_DEBUG */ /*------------------------------------------------------------------------- diff --git a/src/H5Smpio.c b/src/H5Smpio.c index 0a9a7bf..dab77ee 100644 --- a/src/H5Smpio.c +++ b/src/H5Smpio.c @@ -539,177 +539,4 @@ H5S_mpio_space_type( const H5S_t *space, size_t elmt_size, done: FUNC_LEAVE_NOAPI(ret_value); } - - -/*------------------------------------------------------------------------- - * Function: H5S_mpio_opt_possible - * - * Purpose: Checks if an direct I/O transfer is possible between memory and - * the file. - * - * Return: Success: Non-negative: TRUE or FALSE - * Failure: Negative - * - * Programmer: Quincey Koziol - * Wednesday, April 3, 2002 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -htri_t -H5S_mpio_opt_possible( const H5F_t *file, const H5S_t *mem_space, const H5S_t *file_space, const unsigned flags,const H5O_layout_t *layout) -{ - htri_t c1,c2; /* Flags whether a selection is optimizable */ - htri_t ret_value=TRUE; - - FUNC_ENTER_NOAPI(H5S_mpio_opt_possible, FAIL); - - /* Check args */ - assert(mem_space); - assert(file_space); - - /* Parallel I/O conversion flag must be set, if it is not collective IO, go to false. */ - if(!(flags&H5S_CONV_PAR_IO_POSSIBLE)) - HGOTO_DONE(FALSE); - - /* Check whether these are both simple or scalar dataspaces */ - if (!((H5S_SIMPLE==H5S_GET_EXTENT_TYPE(mem_space) || H5S_SCALAR==H5S_GET_EXTENT_TYPE(mem_space)) - && (H5S_SIMPLE==H5S_GET_EXTENT_TYPE(file_space) || H5S_SCALAR==H5S_GET_EXTENT_TYPE(file_space)))) - HGOTO_DONE(FALSE); - - /* Check whether both selections are "regular" */ - c1=H5S_SELECT_IS_REGULAR(file_space); - c2=H5S_SELECT_IS_REGULAR(mem_space); - if(c1==FAIL || c2==FAIL) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "invalid check for single selection blocks"); - if(c1==FALSE || c2==FALSE) - HGOTO_DONE(FALSE); - - /* Can't currently handle point selections */ - if (H5S_SEL_POINTS==H5S_GET_SELECT_TYPE(mem_space) || H5S_SEL_POINTS==H5S_GET_SELECT_TYPE(file_space)) - HGOTO_DONE(FALSE); - - /* Dataset storage must be contiguous or chunked */ - if ((flags&H5S_CONV_STORAGE_MASK)!=H5S_CONV_STORAGE_CONTIGUOUS && - (flags&H5S_CONV_STORAGE_MASK)!=H5S_CONV_STORAGE_CHUNKED) - HGOTO_DONE(FALSE); - - if ((flags&H5S_CONV_STORAGE_MASK)==H5S_CONV_STORAGE_CHUNKED) { - hsize_t chunk_dim[H5O_LAYOUT_NDIMS]; /* Chunk dimensions */ - hssize_t startf[H5S_MAX_RANK], /* Selection start bounds */ - endf[H5S_MAX_RANK]; /* Selection end bounds */ - unsigned dim_rankf; /* Number of dimensions of file dataspace */ - int pcheck_hyper,check_hyper, /* Flags for checking if selection is in one chunk */ - tnum_chunkf, /* Number of chunks selection overlaps */ - max_chunkf, /* Maximum number of chunks selection overlaps */ - min_chunkf, /* Minimum number of chunks selection overlaps */ - num_chunks_same; /* Flag indicating whether all processes have the same # of chunks to operate on */ - unsigned dim_chunks; /* Temporary number of chunks in a dimension */ - MPI_Comm comm; /* MPI communicator for file */ - int mpi_rank; /* Rank in MPI communicator */ - int mpi_code; /* MPI return code */ - unsigned u; /* Local index variable */ - - /* Getting MPI communicator and rank */ - if((comm = H5F_mpi_get_comm(file))==MPI_COMM_NULL) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator") - if((mpi_rank = H5F_mpi_get_rank(file))<0) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve MPI rank") - - /* Currently collective chunking storage - inside HDF5 is supported for either one of the following two cases: - 1. All the hyperslabs for one process is inside one chunk. - 2. For single hyperslab selection, the number of chunks that covered - the single selection for all processes should be equal. - KY, 2004/7/14 - */ - - /* Quincey, please read. - This is maybe redundant, I think only when both memory and file space be SCALAR - space, the collective IO can work. Otherwise, SELECT_POINT will be reached,collective - IO shouldn't work. - Please clarify and correct the code on the following, - Quincey said that it was probably okay if only one data space is SCALAR, - Still keep the code here until we added more tests later. - Kent */ - if(H5S_SCALAR==mem_space->extent.type || H5S_SCALAR ==file_space->extent.type) { - if(!(H5S_SCALAR==mem_space->extent.type && H5S_SCALAR ==file_space->extent.type)) - HGOTO_DONE(FALSE) - else - HGOTO_DONE(TRUE) - } /* end if */ - - dim_rankf = file_space->extent.rank; - - if(H5S_SELECT_BOUNDS(file_space,startf,endf)==FAIL) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE,FAIL, "invalid check for single selection blocks"); - - for(u=0; u < layout->u.chunk.ndims; u++) - chunk_dim[u] = layout->u.chunk.dim[u]; - - /* Case 1: check whether all hyperslab in this process is inside one chunk. - Note: we don't handle when starting point is less than zero since that may cover - two chunks. */ - - /*for file space checking*/ - pcheck_hyper = 1; - for (u=0; u<dim_rankf; u++) - if(endf[u]/chunk_dim[u]!=startf[u]/chunk_dim[u]) { - pcheck_hyper = 0; - break; - } - - if (MPI_SUCCESS != (mpi_code= MPI_Reduce(&pcheck_hyper,&check_hyper,1,MPI_INT,MPI_LAND,0,comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Reduce failed", mpi_code) - if (MPI_SUCCESS != (mpi_code= MPI_Bcast(&check_hyper,1,MPI_INT,0,comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code) - - /*if check_hyper is true, condition for collective IO case is fulfilled, no - need to do further test. */ - if(check_hyper) - HGOTO_DONE(TRUE); - - /* Case 2:Check whether the number of chunks that covered the single hyperslab is the same. - If not,no collective chunk IO. - KY, 2004/7/14 - */ - - c1 = H5S_SELECT_IS_SINGLE(file_space); - c2 = H5S_SELECT_IS_SINGLE(mem_space); - - if(c1==FAIL || c2 ==FAIL) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "invalid check for single selection blocks"); - if(c1==FALSE || c2 ==FALSE) - HGOTO_DONE(FALSE); - - /* Compute the number of chunks covered by the selection on this process */ - tnum_chunkf = 1; - for (u=0; u<dim_rankf; u++) { - dim_chunks = (endf[u]/chunk_dim[u]-startf[u]/chunk_dim[u])+1; - tnum_chunkf = dim_chunks*tnum_chunkf; - } - - /* Determine the minimum and maximum # of chunks for all processes */ - if (MPI_SUCCESS != (mpi_code= MPI_Reduce(&tnum_chunkf,&max_chunkf,1,MPI_INT,MPI_MAX,0,comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Reduce failed", mpi_code) - if (MPI_SUCCESS != (mpi_code= MPI_Reduce(&tnum_chunkf,&min_chunkf,1,MPI_INT,MPI_MIN,0,comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Reduce failed", mpi_code) - - /* Let the rank==0 process determine if the same number of chunks will be operated on by all processes */ - if(mpi_rank == 0) - num_chunks_same = (max_chunkf==min_chunkf); - - /* Broadcast the flag indicating the number of chunks are the same */ - if (MPI_SUCCESS != (mpi_code= MPI_Bcast(&num_chunks_same,1,MPI_INT,0,comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code) - - /* Can't handle case when number of chunks is different (yet) */ - if(!num_chunks_same) - HGOTO_DONE(FALSE); - } - -done: - FUNC_LEAVE_NOAPI(ret_value); -} /* H5S_mpio_opt_possible() */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Spkg.h b/src/H5Spkg.h index ce93770..dc67f81 100644 --- a/src/H5Spkg.h +++ b/src/H5Spkg.h @@ -36,9 +36,6 @@ #define H5S_VALID_MAX 0x01 #define H5S_VALID_PERM 0x02 -/* Flags for "get_seq_list" methods */ -#define H5S_GET_SEQ_LIST_SORTED 0x0001 - /* * Dataspace extent information */ @@ -223,14 +220,6 @@ H5_DLL herr_t H5S_extent_copy(H5S_extent_t *dst, const H5S_extent_t *src); /* Operations on selections */ -#ifdef H5_HAVE_PARALLEL -/* MPI-IO function to check if a direct I/O transfer is possible between - * memory and the file */ -H5_DLL htri_t H5S_mpio_opt_possible(const H5F_t *file, const H5S_t *mem_space, - const H5S_t *file_space, const unsigned flags, const H5O_layout_t *layout); - -#endif /* H5_HAVE_PARALLEL */ - /* Testing functions */ #ifdef H5S_TESTING H5_DLL htri_t H5S_select_shape_same_test(hid_t sid1, hid_t sid2); diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 4879bfd..a0fdf09 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -37,6 +37,9 @@ #define H5S_CONV_STORAGE_CHUNKED 0x0004 /* i.e. '2' */ #define H5S_CONV_STORAGE_MASK 0x0006 +/* Flags for "get_seq_list" methods */ +#define H5S_GET_SEQ_LIST_SORTED 0x0001 + /* Forward references of package typedefs */ typedef struct H5S_t H5S_t; typedef struct H5S_pnt_node_t H5S_pnt_node_t; @@ -106,32 +109,11 @@ typedef struct H5S_sel_iter_t { } u; } H5S_sel_iter_t; -typedef struct H5S_conv_t { +#ifdef H5S_DEBUG +typedef struct H5S_iostats_t { H5S_sel_type ftype; H5S_sel_type mtype; - /* - * If there is no data type conversion then it might be possible to - * transfer data points between application memory and the file in one - * step without going through the data type conversion buffer. - */ - - /* Read from file to application w/o intermediate scratch buffer */ - herr_t (*read)(H5D_io_info_t *io_info, - H5O_layout_readvv_func_t op, - size_t nelmts, size_t elmt_size, - const H5S_t *file_space, const H5S_t *mem_space, - void *buf/*out*/); - - - /* Write directly from app buffer to file */ - herr_t (*write)(H5D_io_info_t *io_info, - H5O_layout_writevv_func_t op, - size_t nelmts, size_t elmt_size, - const H5S_t *file_space, const H5S_t *mem_space, - const void *buf); - -#ifdef H5S_DEBUG struct { H5_timer_t scat_timer; /*time spent scattering */ hsize_t scat_nbytes; /*scatter throughput */ @@ -149,8 +131,8 @@ typedef struct H5S_conv_t { hsize_t write_nbytes; /*total bytes written */ hsize_t write_ncalls; /*number of calls */ } stats[2]; /* 0=output, 1=input */ +} H5S_iostats_t; #endif -} H5S_conv_t; /* If the module using this macro is allowed access to the private variables, access them directly */ #ifdef H5S_PACKAGE @@ -206,8 +188,9 @@ typedef struct H5S_conv_t { /* Operations on dataspaces */ H5_DLL H5S_t *H5S_copy(const H5S_t *src, hbool_t share_selection); H5_DLL herr_t H5S_close(H5S_t *ds); -H5_DLL H5S_conv_t *H5S_find(const H5F_t *file,const H5S_t *mem_space, const H5S_t *file_space, - unsigned flags, hbool_t *use_par_opt_io,const H5O_layout_t *layout ); +#ifdef H5S_DEBUG +H5_DLL H5S_iostats_t *H5S_find(const H5S_t *mem_space, const H5S_t *file_space); +#endif /* H5S_DEBUG */ H5_DLL H5S_class_t H5S_get_simple_extent_type(const H5S_t *ds); H5_DLL hssize_t H5S_get_simple_extent_npoints(const H5S_t *ds); H5_DLL hsize_t H5S_get_npoints_max(const H5S_t *ds); @@ -234,30 +217,6 @@ H5_DLL herr_t H5S_select_iterate(void *buf, hid_t type_id, const H5S_t *space, H5D_operator_t op, void *operator_data); H5_DLL herr_t H5S_select_fill(void *fill, size_t fill_size, const H5S_t *space, void *buf); -H5_DLL herr_t H5S_select_fscat (H5D_io_info_t *io_info, - H5O_layout_writevv_func_t op, - const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const void *_buf); -H5_DLL size_t H5S_select_fgath (H5D_io_info_t *io_info, - H5O_layout_readvv_func_t op, - const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - void *buf); -H5_DLL herr_t H5S_select_mscat (const void *_tscat_buf, - const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, - const H5D_dxpl_cache_t *dxpl_cache, void *_buf/*out*/); -H5_DLL size_t H5S_select_mgath (const void *_buf, - const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, - const H5D_dxpl_cache_t *dxpl_cache, void *_tgath_buf/*out*/); -H5_DLL herr_t H5S_select_read(H5D_io_info_t *io_info, - H5O_layout_readvv_func_t op, - size_t nelmts, size_t elmt_size, - const H5S_t *file_space, const H5S_t *mem_space, - void *buf/*out*/); -H5_DLL herr_t H5S_select_write(H5D_io_info_t *io_info, - H5O_layout_writevv_func_t op, - size_t nelmts, size_t elmt_size, - const H5S_t *file_space, const H5S_t *mem_space, - const void *buf/*out*/); H5_DLL htri_t H5S_select_valid(const H5S_t *space); H5_DLL hssize_t H5S_get_select_npoints(const H5S_t *space); H5_DLL herr_t H5S_get_select_bounds(const H5S_t *space, hssize_t *start, hssize_t *end); diff --git a/src/H5Sselect.c b/src/H5Sselect.c index af3b7c6..9ff1ac0 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -25,11 +25,9 @@ #define PABLO_MASK H5S_select_mask #include "H5private.h" /* Generic Functions */ -#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ -#include "H5Oprivate.h" /* Object headers */ #include "H5Spkg.h" /* Dataspaces */ #include "H5Vprivate.h" /* Vector and array functions */ @@ -40,15 +38,6 @@ static htri_t H5S_select_iter_has_next_block (const H5S_sel_iter_t *iter); static herr_t H5S_select_iter_next_block(H5S_sel_iter_t *iter); #endif /* LATER */ -/* Declare external the free list for hssize_t arrays */ -H5FL_ARR_EXTERN(hssize_t); - -/* Declare a free list to manage sequences of size_t */ -H5FL_SEQ_DEFINE_STATIC(size_t); - -/* Declare a free list to manage sequences of hsize_t */ -H5FL_SEQ_DEFINE_STATIC(hsize_t); - /* Declare a free list to manage blocks of single datatype element data */ H5FL_BLK_EXTERN(type_elem); @@ -1518,662 +1507,3 @@ done: FUNC_LEAVE_NOAPI(ret_value); } /* H5S_select_fill() */ - -/*------------------------------------------------------------------------- - * Function: H5S_select_fscat - * - * Purpose: Scatters dataset elements from the type conversion buffer BUF - * to the file F where the data points are arranged according to - * the file dataspace FILE_SPACE and stored according to - * LAYOUT and EFL. Each element is ELMT_SIZE bytes. - * The caller is requesting that NELMTS elements are copied. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Thursday, June 20, 2002 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -herr_t -H5S_select_fscat (H5D_io_info_t *io_info, - H5O_layout_writevv_func_t op, - const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, - const void *_buf) -{ - const uint8_t *buf=_buf; /* Alias for pointer arithmetic */ - hsize_t _off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets */ - hsize_t *off=NULL; /* Pointer to sequence offsets */ - hsize_t mem_off; /* Offset in memory */ - size_t mem_curr_seq; /* "Current sequence" in memory */ - size_t dset_curr_seq; /* "Current sequence" in dataset */ - size_t _len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths */ - size_t *len=NULL; /* Array to store sequence lengths */ - size_t orig_mem_len, mem_len; /* Length of sequence in memory */ - size_t nseq; /* Number of sequences generated */ - size_t nelem; /* Number of elements used in sequences */ - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5S_select_fscat, FAIL); - - /* Check args */ - assert (io_info); - assert (op); - assert (space); - assert (iter); - assert (nelmts>0); - assert (_buf); - assert(TRUE==H5P_isa_class(io_info->dxpl_id,H5P_DATASET_XFER)); - - /* Allocate the vector I/O arrays */ - if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if((len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); - if((off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); - } /* end if */ - else { - len=_len; - off=_off; - } /* end else */ - - /* Loop until all elements are written */ - while(nelmts>0) { - /* Get list of sequences for selection to write */ - if(H5S_SELECT_GET_SEQ_LIST(space,H5S_GET_SEQ_LIST_SORTED,iter,io_info->dxpl_cache->vec_size,nelmts,&nseq,&nelem,off,len)<0) - HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); - - /* Reset the current sequence information */ - mem_curr_seq=dset_curr_seq=0; - orig_mem_len=mem_len=nelem*iter->elmt_size; - mem_off=0; - - /* Write sequence list out */ - if ((*op)(io_info, nseq, &dset_curr_seq, len, off, 1, &mem_curr_seq, &mem_len, &mem_off, buf)<0) - HGOTO_ERROR(H5E_DATASPACE, H5E_WRITEERROR, FAIL, "write error"); - - /* Update buffer */ - buf += orig_mem_len; - - /* Decrement number of elements left to process */ - nelmts -= nelem; - } /* end while */ - -done: - if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if(len!=NULL) - H5FL_SEQ_FREE(size_t,len); - if(off!=NULL) - H5FL_SEQ_FREE(hsize_t,off); - } /* end if */ - FUNC_LEAVE_NOAPI(ret_value); -} /* H5S_select_fscat() */ - - -/*------------------------------------------------------------------------- - * Function: H5S_select_fgath - * - * Purpose: Gathers data points from file F and accumulates them in the - * type conversion buffer BUF. The LAYOUT argument describes - * how the data is stored on disk and EFL describes how the data - * is organized in external files. ELMT_SIZE is the size in - * bytes of a datum which this function treats as opaque. - * FILE_SPACE describes the dataspace of the dataset on disk - * and the elements that have been selected for reading (via - * hyperslab, etc). This function will copy at most NELMTS - * elements. - * - * Return: Success: Number of elements copied. - * Failure: 0 - * - * Programmer: Quincey Koziol - * Monday, June 24, 2002 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -size_t -H5S_select_fgath (H5D_io_info_t *io_info, - H5O_layout_readvv_func_t op, - const H5S_t *space, H5S_sel_iter_t *iter, size_t nelmts, - void *_buf/*out*/) -{ - uint8_t *buf=_buf; /* Alias for pointer arithmetic */ - hsize_t _off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets */ - hsize_t *off=NULL; /* Pointer to sequence offsets */ - hsize_t mem_off; /* Offset in memory */ - size_t mem_curr_seq; /* "Current sequence" in memory */ - size_t dset_curr_seq; /* "Current sequence" in dataset */ - size_t _len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths */ - size_t *len=NULL; /* Pointer to sequence lengths */ - size_t orig_mem_len, mem_len; /* Length of sequence in memory */ - size_t nseq; /* Number of sequences generated */ - size_t nelem; /* Number of elements used in sequences */ - size_t ret_value=nelmts; /* Return value */ - - FUNC_ENTER_NOAPI(H5S_select_fgath, 0); - - /* Check args */ - assert (io_info); - assert (io_info->dset); - assert (io_info->store); - assert (space); - assert (iter); - assert (nelmts>0); - assert (_buf); - - /* Allocate the vector I/O arrays */ - if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if((len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "can't allocate I/O length vector array"); - if((off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "can't allocate I/O offset vector array"); - } /* end if */ - else { - len=_len; - off=_off; - } /* end else */ - - /* Loop until all elements are read */ - while(nelmts>0) { - /* Get list of sequences for selection to read */ - if(H5S_SELECT_GET_SEQ_LIST(space,H5S_GET_SEQ_LIST_SORTED,iter,io_info->dxpl_cache->vec_size,nelmts,&nseq,&nelem,off,len)<0) - HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, 0, "sequence length generation failed"); - - /* Reset the current sequence information */ - mem_curr_seq=dset_curr_seq=0; - orig_mem_len=mem_len=nelem*iter->elmt_size; - mem_off=0; - - /* Read sequence list in */ - if ((*op)(io_info, nseq, &dset_curr_seq, len, off, 1, &mem_curr_seq, &mem_len, &mem_off, buf)<0) - HGOTO_ERROR(H5E_DATASPACE, H5E_READERROR, 0, "read error"); - - /* Update buffer */ - buf += orig_mem_len; - - /* Decrement number of elements left to process */ - nelmts -= nelem; - } /* end while */ - -done: - if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if(len!=NULL) - H5FL_SEQ_FREE(size_t,len); - if(off!=NULL) - H5FL_SEQ_FREE(hsize_t,off); - } /* end if */ - FUNC_LEAVE_NOAPI(ret_value); -} /* H5S_select_fgath() */ - - -/*------------------------------------------------------------------------- - * Function: H5S_select_mscat - * - * Purpose: Scatters NELMTS data points from the scatter buffer - * TSCAT_BUF to the application buffer BUF. Each element is - * ELMT_SIZE bytes and they are organized in application memory - * according to SPACE. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Monday, July 8, 2002 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -herr_t -H5S_select_mscat (const void *_tscat_buf, const H5S_t *space, - H5S_sel_iter_t *iter, size_t nelmts, const H5D_dxpl_cache_t *dxpl_cache, - void *_buf/*out*/) -{ - uint8_t *buf=(uint8_t *)_buf; /* Get local copies for address arithmetic */ - const uint8_t *tscat_buf=(const uint8_t *)_tscat_buf; - hsize_t _off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets */ - hsize_t *off=NULL; /* Pointer to sequence offsets */ - size_t _len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths */ - size_t *len=NULL; /* Pointer to sequence lengths */ - size_t curr_len; /* Length of bytes left to process in sequence */ - size_t nseq; /* Number of sequences generated */ - size_t curr_seq; /* Current sequence being processed */ - size_t nelem; /* Number of elements used in sequences */ - herr_t ret_value=SUCCEED; /* Number of elements scattered */ - - FUNC_ENTER_NOAPI(H5S_select_mscat, FAIL); - - /* Check args */ - assert (tscat_buf); - assert (space); - assert (iter); - assert (nelmts>0); - assert (buf); - - /* Allocate the vector I/O arrays */ - if(dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if((len = H5FL_SEQ_MALLOC(size_t,dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); - if((off = H5FL_SEQ_MALLOC(hsize_t,dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); - } /* end if */ - else { - len=_len; - off=_off; - } /* end else */ - - /* Loop until all elements are written */ - while(nelmts>0) { - /* Get list of sequences for selection to write */ - if(H5S_SELECT_GET_SEQ_LIST(space,0,iter,dxpl_cache->vec_size,nelmts,&nseq,&nelem,off,len)<0) - HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, 0, "sequence length generation failed"); - - /* Loop, while sequences left to process */ - for(curr_seq=0; curr_seq<nseq; curr_seq++) { - /* Get the number of bytes in sequence */ - curr_len=len[curr_seq]; - - HDmemcpy(buf+off[curr_seq],tscat_buf,curr_len); - - /* Advance offset in destination buffer */ - tscat_buf+=curr_len; - } /* end for */ - - /* Decrement number of elements left to process */ - nelmts -= nelem; - } /* end while */ - -done: - if(dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if(len!=NULL) - H5FL_SEQ_FREE(size_t,len); - if(off!=NULL) - H5FL_SEQ_FREE(hsize_t,off); - } /* end if */ - FUNC_LEAVE_NOAPI(ret_value); -} /* H5S_select_mscat() */ - - -/*------------------------------------------------------------------------- - * Function: H5S_select_mgath - * - * Purpose: Gathers dataset elements from application memory BUF and - * copies them into the gather buffer TGATH_BUF. - * Each element is ELMT_SIZE bytes and arranged in application - * memory according to SPACE. - * The caller is requesting that at most NELMTS be gathered. - * - * Return: Success: Number of elements copied. - * Failure: 0 - * - * Programmer: Quincey Koziol - * Monday, June 24, 2002 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -size_t -H5S_select_mgath (const void *_buf, const H5S_t *space, - H5S_sel_iter_t *iter, size_t nelmts, const H5D_dxpl_cache_t *dxpl_cache, - void *_tgath_buf/*out*/) -{ - const uint8_t *buf=(const uint8_t *)_buf; /* Get local copies for address arithmetic */ - uint8_t *tgath_buf=(uint8_t *)_tgath_buf; - hsize_t _off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets */ - hsize_t *off=NULL; /* Pointer to sequence offsets */ - size_t _len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths */ - size_t *len=NULL; /* Pointer to sequence lengths */ - size_t curr_len; /* Length of bytes left to process in sequence */ - size_t nseq; /* Number of sequences generated */ - size_t curr_seq; /* Current sequence being processed */ - size_t nelem; /* Number of elements used in sequences */ - size_t ret_value=nelmts; /* Number of elements gathered */ - - FUNC_ENTER_NOAPI(H5S_select_mgath, 0); - - /* Check args */ - assert (buf); - assert (space); - assert (iter); - assert (nelmts>0); - assert (tgath_buf); - - /* Allocate the vector I/O arrays */ - if(dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if((len = H5FL_SEQ_MALLOC(size_t,dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "can't allocate I/O length vector array"); - if((off = H5FL_SEQ_MALLOC(hsize_t,dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "can't allocate I/O offset vector array"); - } /* end if */ - else { - len=_len; - off=_off; - } /* end else */ - - /* Loop until all elements are written */ - while(nelmts>0) { - /* Get list of sequences for selection to write */ - if(H5S_SELECT_GET_SEQ_LIST(space,0,iter,dxpl_cache->vec_size,nelmts,&nseq,&nelem,off,len)<0) - HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, 0, "sequence length generation failed"); - - /* Loop, while sequences left to process */ - for(curr_seq=0; curr_seq<nseq; curr_seq++) { - /* Get the number of bytes in sequence */ - curr_len=len[curr_seq]; - - HDmemcpy(tgath_buf,buf+off[curr_seq],curr_len); - - /* Advance offset in gather buffer */ - tgath_buf+=curr_len; - } /* end for */ - - /* Decrement number of elements left to process */ - nelmts -= nelem; - } /* end while */ - -done: - if(dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if(len!=NULL) - H5FL_SEQ_FREE(size_t,len); - if(off!=NULL) - H5FL_SEQ_FREE(hsize_t,off); - } /* end if */ - FUNC_LEAVE_NOAPI(ret_value); -} /* H5S_select_mgath() */ - - -/*------------------------------------------------------------------------- - * Function: H5S_select_read - * - * Purpose: Reads directly from file into application memory. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Tuesday, July 23, 2002 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -herr_t -H5S_select_read(H5D_io_info_t *io_info, - H5O_layout_readvv_func_t op, - size_t nelmts, size_t elmt_size, - const H5S_t *file_space, const H5S_t *mem_space, - void *buf/*out*/) -{ - H5S_sel_iter_t mem_iter; /* Memory selection iteration info */ - hbool_t mem_iter_init=0; /* Memory selection iteration info has been initialized */ - H5S_sel_iter_t file_iter; /* File selection iteration info */ - hbool_t file_iter_init=0; /* File selection iteration info has been initialized */ - hsize_t _mem_off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets in memory */ - hsize_t *mem_off=NULL; /* Pointer to sequence offsets in memory */ - hsize_t _file_off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets in the file */ - hsize_t *file_off=NULL; /* Pointer to sequence offsets in the file */ - size_t _mem_len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths in memory */ - size_t *mem_len=NULL; /* Pointer to sequence lengths in memory */ - size_t _file_len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths in the file */ - size_t *file_len=NULL; /* Pointer to sequence lengths in the file */ - size_t mem_nseq; /* Number of sequences generated in the file */ - size_t file_nseq; /* Number of sequences generated in memory */ - size_t mem_nelem; /* Number of elements used in memory sequences */ - size_t file_nelem; /* Number of elements used in file sequences */ - size_t curr_mem_seq; /* Current memory sequence to operate on */ - size_t curr_file_seq; /* Current file sequence to operate on */ - ssize_t tmp_file_len; /* Temporary number of bytes in file sequence */ - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5S_select_read, FAIL); - - /* Check args */ - assert(io_info); - assert(io_info->dset); - assert(io_info->dxpl_cache); - assert(io_info->store); - assert(buf); - assert(TRUE==H5P_isa_class(io_info->dxpl_id,H5P_DATASET_XFER)); - - /* Initialize file iterator */ - if (H5S_select_iter_init(&file_iter, file_space, elmt_size)<0) - HGOTO_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator"); - file_iter_init=1; /* File selection iteration info has been initialized */ - - /* Initialize memory iterator */ - if (H5S_select_iter_init(&mem_iter, mem_space, elmt_size)<0) - HGOTO_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator"); - mem_iter_init=1; /* Memory selection iteration info has been initialized */ - - /* Allocate the vector I/O arrays */ - if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if((mem_len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); - if((mem_off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); - if((file_len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); - if((file_off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); - } /* end if */ - else { - mem_len=_mem_len; - mem_off=_mem_off; - file_len=_file_len; - file_off=_file_off; - } /* end else */ - - /* Initialize sequence counts */ - curr_mem_seq=curr_file_seq=0; - mem_nseq=file_nseq=0; - - /* Loop, until all bytes are processed */ - while(nelmts>0) { - /* Check if more file sequences are needed */ - if(curr_file_seq>=file_nseq) { - /* Get sequences for file selection */ - if(H5S_SELECT_GET_SEQ_LIST(file_space,H5S_GET_SEQ_LIST_SORTED,&file_iter,io_info->dxpl_cache->vec_size,nelmts,&file_nseq,&file_nelem,file_off,file_len)<0) - HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); - - /* Start at the beginning of the sequences again */ - curr_file_seq=0; - } /* end if */ - - /* Check if more memory sequences are needed */ - if(curr_mem_seq>=mem_nseq) { - /* Get sequences for memory selection */ - if(H5S_SELECT_GET_SEQ_LIST(mem_space,0,&mem_iter,io_info->dxpl_cache->vec_size,nelmts,&mem_nseq,&mem_nelem,mem_off,mem_len)<0) - HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); - - /* Start at the beginning of the sequences again */ - curr_mem_seq=0; - } /* end if */ - - /* Read file sequences into current memory sequence */ - if ((tmp_file_len=(*op)(io_info, - file_nseq, &curr_file_seq, file_len, file_off, - mem_nseq, &curr_mem_seq, mem_len, mem_off, - buf))<0) - HGOTO_ERROR(H5E_DATASPACE, H5E_READERROR, FAIL, "read error"); - - /* Decrement number of elements left to process */ - assert((tmp_file_len%elmt_size)==0); - nelmts-=(tmp_file_len/elmt_size); - } /* end while */ - -done: - /* Release file selection iterator */ - if(file_iter_init) { - if (H5S_SELECT_ITER_RELEASE(&file_iter)<0) - HDONE_ERROR (H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator"); - } /* end if */ - - /* Release memory selection iterator */ - if(mem_iter_init) { - if (H5S_SELECT_ITER_RELEASE(&mem_iter)<0) - HDONE_ERROR (H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator"); - } /* end if */ - - /* Free vector arrays */ - if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if(file_len!=NULL) - H5FL_SEQ_FREE(size_t,file_len); - if(file_off!=NULL) - H5FL_SEQ_FREE(hsize_t,file_off); - if(mem_len!=NULL) - H5FL_SEQ_FREE(size_t,mem_len); - if(mem_off!=NULL) - H5FL_SEQ_FREE(hsize_t,mem_off); - } /* end if */ - FUNC_LEAVE_NOAPI(ret_value); -} /* end H5S_select_read() */ - - -/*------------------------------------------------------------------------- - * Function: H5S_select_write - * - * Purpose: Writes directly from application memory into a file - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Tuesday, July 23, 2002 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -herr_t -H5S_select_write(H5D_io_info_t *io_info, - H5O_layout_writevv_func_t op, - size_t nelmts, size_t elmt_size, - const H5S_t *file_space, const H5S_t *mem_space, - const void *buf/*out*/) -{ - H5S_sel_iter_t mem_iter; /* Memory selection iteration info */ - hbool_t mem_iter_init=0; /* Memory selection iteration info has been initialized */ - H5S_sel_iter_t file_iter; /* File selection iteration info */ - hbool_t file_iter_init=0; /* File selection iteration info has been initialized */ - hsize_t _mem_off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets in memory */ - hsize_t *mem_off=NULL; /* Pointer to sequence offsets in memory */ - hsize_t _file_off[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence offsets in the file */ - hsize_t *file_off=NULL; /* Pointer to sequence offsets in the file */ - size_t _mem_len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths in memory */ - size_t *mem_len=NULL; /* Pointer to sequence lengths in memory */ - size_t _file_len[H5D_XFER_HYPER_VECTOR_SIZE_DEF]; /* Array to store sequence lengths in the file */ - size_t *file_len=NULL; /* Pointer to sequence lengths in the file */ - size_t mem_nseq; /* Number of sequences generated in the file */ - size_t file_nseq; /* Number of sequences generated in memory */ - size_t mem_nelem; /* Number of elements used in memory sequences */ - size_t file_nelem; /* Number of elements used in file sequences */ - size_t curr_mem_seq; /* Current memory sequence to operate on */ - size_t curr_file_seq; /* Current file sequence to operate on */ - ssize_t tmp_file_len; /* Temporary number of bytes in file sequence */ - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5S_select_write, FAIL); - - /* Check args */ - assert(io_info); - assert(io_info->dset); - assert(io_info->store); - assert(TRUE==H5P_isa_class(io_info->dxpl_id,H5P_DATASET_XFER)); - assert(buf); - - /* Allocate the vector I/O arrays */ - if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if((mem_len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); - if((mem_off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); - if((file_len = H5FL_SEQ_MALLOC(size_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); - if((file_off = H5FL_SEQ_MALLOC(hsize_t,io_info->dxpl_cache->vec_size))==NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); - } /* end if */ - else { - mem_len=_mem_len; - mem_off=_mem_off; - file_len=_file_len; - file_off=_file_off; - } /* end else */ - - /* Initialize file iterator */ - if (H5S_select_iter_init(&file_iter, file_space, elmt_size)<0) - HGOTO_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator"); - file_iter_init=1; /* File selection iteration info has been initialized */ - - /* Initialize memory iterator */ - if (H5S_select_iter_init(&mem_iter, mem_space, elmt_size)<0) - HGOTO_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator"); - mem_iter_init=1; /* Memory selection iteration info has been initialized */ - - /* Initialize sequence counts */ - curr_mem_seq=curr_file_seq=0; - mem_nseq=file_nseq=0; - - /* Loop, until all bytes are processed */ - while(nelmts>0) { - /* Check if more file sequences are needed */ - if(curr_file_seq>=file_nseq) { - /* Get sequences for file selection */ - if(H5S_SELECT_GET_SEQ_LIST(file_space,H5S_GET_SEQ_LIST_SORTED,&file_iter,io_info->dxpl_cache->vec_size,nelmts,&file_nseq,&file_nelem,file_off,file_len)<0) - HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); - - /* Start at the beginning of the sequences again */ - curr_file_seq=0; - } /* end if */ - - /* Check if more memory sequences are needed */ - if(curr_mem_seq>=mem_nseq) { - /* Get sequences for memory selection */ - if(H5S_SELECT_GET_SEQ_LIST(mem_space,0,&mem_iter,io_info->dxpl_cache->vec_size,nelmts,&mem_nseq,&mem_nelem,mem_off,mem_len)<0) - HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed"); - - /* Start at the beginning of the sequences again */ - curr_mem_seq=0; - } /* end if */ - - /* Write memory sequences into file sequences */ - if ((tmp_file_len=(*op)(io_info, - file_nseq, &curr_file_seq, file_len, file_off, - mem_nseq, &curr_mem_seq, mem_len, mem_off, - buf))<0) - HGOTO_ERROR(H5E_DATASPACE, H5E_WRITEERROR, FAIL, "write error"); - - /* Decrement number of elements left to process */ - assert((tmp_file_len%elmt_size)==0); - nelmts-=(tmp_file_len/elmt_size); - } /* end while */ - -done: - /* Release file selection iterator */ - if(file_iter_init) { - if (H5S_SELECT_ITER_RELEASE(&file_iter)<0) - HDONE_ERROR (H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator"); - } /* end if */ - - /* Release memory selection iterator */ - if(mem_iter_init) { - if (H5S_SELECT_ITER_RELEASE(&mem_iter)<0) - HDONE_ERROR (H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator"); - } /* end if */ - - /* Free vector arrays */ - if(io_info->dxpl_cache->vec_size!=H5D_XFER_HYPER_VECTOR_SIZE_DEF) { - if(file_len!=NULL) - H5FL_SEQ_FREE(size_t,file_len); - if(file_off!=NULL) - H5FL_SEQ_FREE(hsize_t,file_off); - if(mem_len!=NULL) - H5FL_SEQ_FREE(size_t,mem_len); - if(mem_off!=NULL) - H5FL_SEQ_FREE(hsize_t,mem_off); - } /* end if */ - - FUNC_LEAVE_NOAPI(ret_value); -} /* end H5S_select_write() */ - @@ -29,17 +29,17 @@ /* (Put before include files to avoid problems with inline functions) */ #define PABLO_MASK H5T_mask -#include "H5private.h" /*generic functions */ -#include "H5Dprivate.h" /*datasets (for H5Tcopy) */ -#include "H5Eprivate.h" /*error handling */ +#include "H5private.h" /*generic functions */ +#include "H5Dprivate.h" /*datasets (for H5Tcopy) */ +#include "H5Eprivate.h" /*error handling */ #include "H5Fpkg.h" /* File */ -#include "H5FLprivate.h" /* Free Lists */ -#include "H5FOprivate.h" /* File objects */ -#include "H5Gprivate.h" /*groups */ -#include "H5Iprivate.h" /*ID functions */ -#include "H5MMprivate.h" /*memory management */ -#include "H5Pprivate.h" /* Property Lists */ -#include "H5Tpkg.h" /*data-type functions */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5FOprivate.h" /* File objects */ +#include "H5Gprivate.h" /*groups */ +#include "H5Iprivate.h" /*ID functions */ +#include "H5MMprivate.h" /*memory management */ +#include "H5Pprivate.h" /* Property Lists */ +#include "H5Tpkg.h" /*data-type functions */ /* Check for header needed for SGI floating-point code */ #ifdef H5_HAVE_SYS_FPU_H @@ -1520,10 +1520,9 @@ H5Topen(hid_t loc_id, const char *name) HGOTO_ERROR (H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register named data type"); done: - if(ret_value<0) { + if(ret_value<0) if(type!=NULL) - H5T_close (type); - } /* end if */ + H5T_close(type); FUNC_LEAVE_API(ret_value); } @@ -2808,7 +2807,7 @@ H5T_create(H5T_class_t type, size_t size) case H5T_COMPOUND: if (NULL==(dt = H5FL_CALLOC(H5T_t))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); - if (NULL==(dt->shared = H5FL_CALLOC(H5T_shared_t))) + if (NULL==(dt->shared = H5FL_CALLOC(H5T_shared_t))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); dt->shared->type = type; @@ -2956,6 +2955,8 @@ H5T_open (H5G_entry_t *ent, hid_t dxpl_id) } else { + shared_fo->fo_count++; + if(NULL == (dt = H5FL_CALLOC(H5T_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "can't allocate space for datatype") @@ -2964,8 +2965,6 @@ H5T_open (H5G_entry_t *ent, hid_t dxpl_id) /* Shallow copy (take ownership) of the group entry object */ if(H5G_ent_copy(&(dt->ent),ent,H5G_COPY_SHALLOW)<0) HGOTO_ERROR (H5E_DATATYPE, H5E_CANTCOPY, NULL, "can't copy group entry") - - shared_fo->fo_count++; } ret_value = dt; @@ -2978,6 +2977,8 @@ done: } H5FL_FREE(H5T_t, dt); } + if(shared_fo) + shared_fo->fo_count--; } FUNC_LEAVE_NOAPI(ret_value); } @@ -3015,16 +3016,17 @@ H5T_open_oid (H5G_entry_t *ent, hid_t dxpl_id) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") if (H5O_open (ent)<0) - HGOTO_ERROR (H5E_DATATYPE, H5E_CANTOPENOBJ, NULL, "unable to open named data type"); + HGOTO_ERROR (H5E_DATATYPE, H5E_CANTOPENOBJ, NULL, "unable to open named data type"); /* The fourth argument to H5O_read is dt because we've already CALLOC'ed memory for it */ if (NULL==(dt=H5O_read (ent, H5O_DTYPE_ID, 0, dt, dxpl_id))) - HGOTO_ERROR (H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to load type message from object header"); + HGOTO_ERROR (H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to load type message from object header"); /* Mark the type as named and open */ dt->shared->state = H5T_STATE_OPEN; + /* Shallow copy (take ownership) of the group entry object */ H5G_ent_copy(&(dt->ent),ent,H5G_COPY_SHALLOW); - + /* Set return value */ ret_value=dt; @@ -3098,7 +3100,7 @@ H5T_copy(const H5T_t *old_dt, H5T_copy_t method) new_dt->ent = old_dt->ent; *(new_dt->shared) = *(old_dt->shared); new_dt->shared->fo_count = 1; - + /* Copy parent information */ if (new_dt->shared->parent) new_dt->shared->parent = H5T_copy(new_dt->shared->parent, method); @@ -3265,7 +3267,7 @@ H5T_copy(const H5T_t *old_dt, H5T_copy_t method) /* Deep copy of the symbol table entry */ if (H5G_ent_copy(&(new_dt->ent), &(old_dt->ent),H5G_COPY_DEEP)<0) HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, NULL, "unable to copy entry"); - + /* Set return value */ ret_value=new_dt; @@ -3331,7 +3333,7 @@ done: * Function: H5T_free * * Purpose: Frees all memory associated with a datatype, but does not - * free the H5T_t or H5D_shared_t structures (which should + * free the H5T_t or H5T_shared_t structures (which should * be done in H5T_close). * * Return: Non-negative on success/Negative on failure @@ -3359,10 +3361,11 @@ H5T_free(H5T_t *dt) */ if (H5T_STATE_OPEN==dt->shared->state) { assert (H5F_addr_defined(dt->ent.header)); + /* Remove the datatype from the list of opened objects in the file */ if(H5FO_delete(dt->ent.file, H5AC_dxpl_id, dt->ent.header)<0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTRELEASE, FAIL, "can't remove datatype from list of open objects") if (H5O_close(&(dt->ent))<0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to close data type object header"); + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to close data type object header"); dt->shared->state = H5T_STATE_NAMED; } @@ -3457,6 +3460,9 @@ H5T_close(H5T_t *dt) H5G_free_ent_name(&(dt->ent)); } + /* Free the datatype struct */ + H5FL_FREE(H5T_t,dt); + done: FUNC_LEAVE_NOAPI(ret_value); } @@ -3552,7 +3558,7 @@ H5T_set_size(H5T_t *dt, size_t size) if((num_membs = H5T_get_nmembers(dt))<0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to get number of members"); - for(i=0; i<num_membs; i++) { + for(i=0; i<(unsigned)num_membs; i++) { memb_offset = H5T_get_member_offset(dt, i); if(memb_offset > max_offset) { max_offset = memb_offset; @@ -4883,8 +4889,15 @@ H5T_print_stats(H5T_path_t UNUSED * path, int UNUSED * nprint/*in,out*/) "----------", "-----", "-----", "----", "------", "-------", "---------"); } - nbytes = MAX (H5T_get_size (path->src), - H5T_get_size (path->dst)); + if(path->src && path->dst) + nbytes = MAX (H5T_get_size (path->src), + H5T_get_size (path->dst)); + else if(path->src) + nbytes = H5T_get_size (path->src); + else if(path->dst) + nbytes = H5T_get_size (path->dst); + else + nbytes = 0; nbytes *= path->stats.nelmts; H5_bandwidth(bandwidth, (double)nbytes, path->stats.timer.etime); diff --git a/src/Makefile.in b/src/Makefile.in index 98c8927..c980764 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -30,8 +30,9 @@ LIB=libhdf5.la DISTCLEAN=libhdf5.settings ## Source and object files for the library (lexicographically)... -LIB_SRC=H5.c H5A.c H5AC.c H5B.c H5C.c H5D.c H5Dcontig.c H5Dcompact.c H5Dio.c \ - H5Distore.c H5Dmpio.c H5Dtest.c H5E.c H5F.c H5Fdbg.c H5FD.c \ +LIB_SRC=H5.c H5A.c H5AC.c H5B.c H5C.c H5D.c H5Dcontig.c H5Dcompact.c H5Defl.c \ + H5Dio.c H5Distore.c H5Dmpio.c H5Dselect.c H5Dtest.c H5E.c H5F.c \ + H5Fdbg.c H5FD.c \ H5FDcore.c H5FDfamily.c H5FDfphdf5.c H5FDgass.c H5FDlog.c H5FDmpi.c \ H5FDmpio.c H5FDmpiposix.c H5FDmulti.c H5FDsec2.c H5FDsrb.c \ H5FDstdio.c H5FDstream.c H5FL.c H5FO.c H5FP.c H5FPclient.c \ |