diff options
| -rw-r--r-- | src/H5Dcontig.c | 130 | ||||
| -rw-r--r-- | src/H5Dseq.c | 50 | ||||
| -rw-r--r-- | src/H5FD.c | 2 | ||||
| -rw-r--r-- | src/H5FDlog.c | 16 | ||||
| -rw-r--r-- | src/H5FDlog.h | 8 | ||||
| -rw-r--r-- | src/H5FDsec2.h | 8 | ||||
| -rw-r--r-- | src/H5Farray.c | 26 | ||||
| -rw-r--r-- | src/H5Fcontig.c | 130 | ||||
| -rw-r--r-- | src/H5Fpkg.h | 4 | ||||
| -rw-r--r-- | src/H5Fseq.c | 50 |
10 files changed, 290 insertions, 134 deletions
diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index d24e086..e08707e 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -43,7 +43,7 @@ static intn interface_initialize_g = 0; *------------------------------------------------------------------------- */ herr_t -H5F_contig_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxpl_id, +H5F_contig_read(H5F_t *f, hsize_t max_data, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxpl_id, void *buf/*out*/) { haddr_t abs_eoa; /* Absolute end of file address */ @@ -129,7 +129,7 @@ H5F_contig_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxp rel_eoa=abs_eoa-f->shared->base_addr; /* Compute the size of the sieve buffer */ - f->shared->sieve_size=MIN(rel_eoa-addr,f->shared->sieve_buf_size); + f->shared->sieve_size=MIN(rel_eoa-addr,MIN(max_data,f->shared->sieve_buf_size)); /* Read the new sieve buffer */ if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { @@ -176,7 +176,7 @@ H5F_contig_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxp rel_eoa=abs_eoa-f->shared->base_addr; /* Compute the size of the sieve buffer */ - f->shared->sieve_size=MIN(rel_eoa-addr,f->shared->sieve_buf_size); + f->shared->sieve_size=MIN(rel_eoa-addr,MIN(max_data,f->shared->sieve_buf_size)); /* Read the new sieve buffer */ if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { @@ -221,7 +221,7 @@ H5F_contig_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxp *------------------------------------------------------------------------- */ herr_t -H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, +H5F_contig_write(H5F_t *f, hsize_t max_data, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxpl_id, const void *buf) { haddr_t abs_eoa; /* Absolute end of file address */ @@ -239,7 +239,7 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if(f->shared->sieve_buf) { haddr_t sieve_start, sieve_end; /* Start & end locations of sieve buffer */ haddr_t contig_end; /* End locations of block to write */ - hsize_t sieve_size; /* size of sieve buffer */ + hsize_t sieve_size; /* size of sieve buffer */ /* Stash local copies of these value */ sieve_start=f->shared->sieve_loc; @@ -285,50 +285,85 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if (H5F_block_write(f, type, addr, size, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); - } + } /* end if */ } /* end if */ /* Element size fits within the buffer size */ else { - /* Flush the sieve buffer if it's dirty */ - if(f->shared->sieve_dirty) { - /* Write to file */ - if (H5F_block_write(f, H5FD_MEM_DRAW, sieve_start, sieve_size, dxpl_id, f->shared->sieve_buf)<0) { - HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "block write failed"); - } - - /* Reset sieve buffer dirty flag */ - f->shared->sieve_dirty=0; + /* Check if it is possible to (exactly) prepend or append to existing (dirty) sieve buffer */ + if(((addr+size)==sieve_start || addr==sieve_end) && + (size+sieve_size)<=f->shared->sieve_buf_size && + f->shared->sieve_dirty) { + /* Prepend to existing sieve buffer */ + if((addr+size)==sieve_start) { + /* Move existing sieve information to correct location */ + assert(sieve_size==(hsize_t)((size_t)sieve_size)); /*check for overflow*/ + HDmemmove(f->shared->sieve_buf+size,f->shared->sieve_buf,(size_t)sieve_size); + + /* Copy in new information (must be first in sieve buffer) */ + assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ + HDmemcpy(f->shared->sieve_buf,buf,(size_t)size); + + /* Adjust sieve location */ + f->shared->sieve_loc=addr; + + } /* end if */ + /* Append to existing sieve buffer */ + else { + /* Copy in new information */ + assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ + HDmemcpy(f->shared->sieve_buf+sieve_size,buf,(size_t)size); + } /* end else */ + + /* Adjust sieve size */ + f->shared->sieve_size += size; + } /* end if */ + /* Can't add the new data onto the existing sieve buffer */ + else { + /* Flush the sieve buffer if it's dirty */ + if(f->shared->sieve_dirty) { + /* Write to file */ + if (H5F_block_write(f, H5FD_MEM_DRAW, sieve_start, sieve_size, dxpl_id, f->shared->sieve_buf)<0) { + HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, + "block write failed"); + } /* end if */ - /* Determine the new sieve buffer size & location */ - f->shared->sieve_loc=addr; + /* Reset sieve buffer dirty flag */ + f->shared->sieve_dirty=0; + } /* end if */ - /* Make certain we don't read off the end of the file */ - if (HADDR_UNDEF==(abs_eoa=H5FD_get_eoa(f->shared->lf))) { - HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, - "unable to determine file size"); - } + /* Determine the new sieve buffer size & location */ + f->shared->sieve_loc=addr; - /* Adjust absolute EOA address to relative EOA address */ - rel_eoa=abs_eoa-f->shared->base_addr; + /* Make certain we don't read off the end of the file */ + if (HADDR_UNDEF==(abs_eoa=H5FD_get_eoa(f->shared->lf))) { + HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, + "unable to determine file size"); + } /* end if */ - /* Compute the size of the sieve buffer */ - f->shared->sieve_size=MIN(rel_eoa-addr,f->shared->sieve_buf_size); + /* Adjust absolute EOA address to relative EOA address */ + rel_eoa=abs_eoa-f->shared->base_addr; - /* Read the new sieve buffer */ - if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { - HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, - "block read failed"); - } + /* Compute the size of the sieve buffer */ + f->shared->sieve_size=MIN(rel_eoa-addr,MIN(max_data,f->shared->sieve_buf_size)); - /* Grab the data out of the buffer (must be first piece of data in buffer) */ - assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ - HDmemcpy(f->shared->sieve_buf,buf,(size_t)size); + /* Check if there is any point in reading the data from the file */ + if(f->shared->sieve_size>size) { + /* Read the new sieve buffer */ + if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { + HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, + "block read failed"); + } /* end if */ + } /* end if */ + + /* Grab the data out of the buffer (must be first piece of data in buffer) */ + assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ + HDmemcpy(f->shared->sieve_buf,buf,(size_t)size); - /* Set sieve buffer dirty flag */ - f->shared->sieve_dirty=1; + /* Set sieve buffer dirty flag */ + f->shared->sieve_dirty=1; + } /* end else */ } /* end else */ } /* end else */ } /* end if */ @@ -347,7 +382,7 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if (NULL==(f->shared->sieve_buf=H5MM_malloc((size_t)f->shared->sieve_buf_size))) { HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed"); - } + } /* end if */ /* Determine the new sieve buffer size & location */ f->shared->sieve_loc=addr; @@ -356,19 +391,22 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if (HADDR_UNDEF==(abs_eoa=H5FD_get_eoa(f->shared->lf))) { HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to determine file size"); - } + } /* end if */ /* Adjust absolute EOA address to relative EOA address */ rel_eoa=abs_eoa-f->shared->base_addr; /* Compute the size of the sieve buffer */ - f->shared->sieve_size=MIN(rel_eoa-addr,f->shared->sieve_buf_size); + f->shared->sieve_size=MIN(rel_eoa-addr,MIN(max_data,f->shared->sieve_buf_size)); - /* Read the new sieve buffer */ - if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { - HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, - "block read failed"); - } + /* Check if there is any point in reading the data from the file */ + if(f->shared->sieve_size>size) { + /* Read the new sieve buffer */ + if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { + HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, + "block read failed"); + } /* end if */ + } /* end if */ /* Grab the data out of the buffer (must be first piece of data in buffer) */ assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ @@ -383,7 +421,7 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if (H5F_block_write(f, type, addr, size, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); - } + } /* end if */ } /* end else */ FUNC_LEAVE(SUCCEED); diff --git a/src/H5Dseq.c b/src/H5Dseq.c index 4c77052..c6e29a1 100644 --- a/src/H5Dseq.c +++ b/src/H5Dseq.c @@ -57,8 +57,8 @@ static intn interface_initialize_g = 0; */ herr_t H5F_seq_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, - const struct H5O_pline_t *pline, const H5O_fill_t *fill, - const struct H5O_efl_t *efl, const H5S_t *file_space, size_t elmt_size, + const struct H5O_pline_t *pline, const H5O_fill_t *fill, + const struct H5O_efl_t *efl, const H5S_t *file_space, size_t elmt_size, hsize_t seq_len, hsize_t file_offset, void *buf/*out*/) { hsize_t dset_dims[H5O_LAYOUT_NDIMS]; /* dataspace dimensions */ @@ -67,7 +67,9 @@ H5F_seq_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, hsize_t down_size[H5O_LAYOUT_NDIMS]; /* Cumulative yperslab sizes (in elements) */ hsize_t acc; /* Accumulator for hyperslab sizes (in elements) */ intn ndims; + hsize_t max_data = 0; /*bytes in dataset */ haddr_t addr=0; /*address in file */ + uintn u; /*counters */ intn i,j; /*counters */ #ifdef H5_HAVE_PARALLEL H5FD_mpio_xfer_t xfer_mode=H5FD_MPIO_INDEPENDENT; @@ -118,6 +120,13 @@ H5F_seq_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, addr = 0; } else { addr = layout->addr; + + /* Compute the size of the dataset in bytes */ + for(u=0, max_data=1; u<layout->ndims; u++) + max_data *= layout->dim[u]; + + /* Adjust the maximum size of the data by the offset into it */ + max_data -= file_offset; } addr += file_offset; @@ -150,18 +159,18 @@ H5F_seq_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, #endif /* Read directly from file if the dataset is in an external file */ - /* Note: We can't use data sieve buffers for datasets in external files - * because the 'addr' of all external files is set to 0 (above) and - * all datasets in external files would alias to the same set of - * file offsets, totally mixing up the data sieve buffer information. -QAK - */ if (efl && efl->nused>0) { + /* Note: We can't use data sieve buffers for datasets in external files + * because the 'addr' of all external files is set to 0 (above) and + * all datasets in external files would alias to the same set of + * file offsets, totally mixing up the data sieve buffer information. -QAK + */ if (H5O_efl_read(f, efl, addr, seq_len, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, "external data read failed"); } } else { - if (H5F_contig_read(f, H5FD_MEM_DRAW, addr, seq_len, dxpl_id, buf)<0) { + if (H5F_contig_read(f, max_data, H5FD_MEM_DRAW, addr, seq_len, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, "block read failed"); } @@ -491,8 +500,8 @@ printf("%s: partial_size=%lu, coords[%d]=%ld, hslab_size[%d]=%ld\n",FUNC,(unsign */ herr_t H5F_seq_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, - const struct H5O_pline_t *pline, const H5O_fill_t *fill, - const struct H5O_efl_t *efl, const H5S_t *file_space, size_t elmt_size, + const struct H5O_pline_t *pline, const H5O_fill_t *fill, + const struct H5O_efl_t *efl, const H5S_t *file_space, size_t elmt_size, hsize_t seq_len, hsize_t file_offset, const void *buf) { hsize_t dset_dims[H5O_LAYOUT_NDIMS]; /* dataspace dimensions */ @@ -501,7 +510,9 @@ H5F_seq_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, hsize_t down_size[H5O_LAYOUT_NDIMS]; /* Cumulative hyperslab sizes (in elements) */ hsize_t acc; /* Accumulator for hyperslab sizes (in elements) */ intn ndims; + hsize_t max_data = 0; /*bytes in dataset */ haddr_t addr; /*address in file */ + uintn u; /*counters */ intn i,j; /*counters */ #ifdef H5_HAVE_PARALLEL H5FD_mpio_xfer_t xfer_mode=H5FD_MPIO_INDEPENDENT; @@ -552,6 +563,13 @@ H5F_seq_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, addr = 0; } else { addr = layout->addr; + + /* Compute the size of the dataset in bytes */ + for(u=0, max_data=1; u<layout->ndims; u++) + max_data *= layout->dim[u]; + + /* Adjust the maximum size of the data by the offset into it */ + max_data -= file_offset; } addr += file_offset; @@ -584,18 +602,18 @@ H5F_seq_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, #endif /* Write directly to file if the dataset is in an external file */ - /* Note: We can't use data sieve buffers for datasets in external files - * because the 'addr' of all external files is set to 0 (above) and - * all datasets in external files would alias to the same set of - * file offsets, totally mixing up the data sieve buffer information. -QAK - */ if (efl && efl->nused>0) { + /* Note: We can't use data sieve buffers for datasets in external files + * because the 'addr' of all external files is set to 0 (above) and + * all datasets in external files would alias to the same set of + * file offsets, totally mixing up the data sieve buffer information. -QAK + */ if (H5O_efl_write(f, efl, addr, seq_len, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "external data write failed"); } } else { - if (H5F_contig_write(f, H5FD_MEM_DRAW, addr, seq_len, dxpl_id, buf)<0) { + if (H5F_contig_write(f, max_data, H5FD_MEM_DRAW, addr, seq_len, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); } @@ -2315,7 +2315,7 @@ H5FD_write(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t s /* New piece of metadata doesn't adjoin or overlap the existing accumulator */ else { /* Write out the existing metadata accumulator, with dispatch to driver */ - if ((file->cls->write)(file, type, dxpl_id, file->accum_loc, file->accum_size, file->meta_accum)<0) + if ((file->cls->write)(file, H5FD_MEM_DEFAULT, dxpl_id, file->accum_loc, file->accum_size, file->meta_accum)<0) HRETURN_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver write request failed"); /* Cache the new piece of metadata */ diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 86b3159..e025456 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -30,7 +30,7 @@ #endif /* MAX */ /* The size of the buffer to track allocation requests */ -#define TRACK_BUFFER 5000000 +#define TRACK_BUFFER 15000000 /* The driver identification number, initialized at runtime */ static hid_t H5FD_LOG_g = 0; @@ -677,8 +677,8 @@ H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hsize_t size) FUNC_ENTER(H5FD_log_alloc, HADDR_UNDEF); - addr = file->eoa; - file->eoa += size; + addr = file->eoa; + file->eoa += size; #ifdef QAK printf("%s: flavor=%s, size=%lu\n",FUNC,flavors[type],(unsigned long)size); @@ -912,10 +912,12 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, hadd FUNC_ENTER(H5FD_log_write, FAIL); assert(file && file->pub.cls); + assert(size>0); assert(buf); /* Verify that we are writing out the type of data we allocated in this location */ - assert(type==file->flavor[addr]); + assert(type==H5FD_MEM_DEFAULT || type==file->flavor[addr] || file->flavor[addr]==H5FD_MEM_DEFAULT); + assert(type==H5FD_MEM_DEFAULT || type==file->flavor[(addr+size)-1] || file->flavor[(addr+size)-1]==H5FD_MEM_DEFAULT); /* Check for overflow conditions */ if (HADDR_UNDEF==addr) @@ -939,8 +941,12 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, hadd HDfprintf(file->logfp,"Seek: From %10a To %10a\n",file->pos,addr); /* Log information about the write */ - if(file->fa.verbosity>0) + if(file->fa.verbosity>0) { + /* Check if this is the first write into a "default" section, grabbed by the metadata agregation algorithm */ + if(file->flavor[addr]==H5FD_MEM_DEFAULT) + HDmemset(&file->flavor[addr],type,(size_t)size); HDfprintf(file->logfp,"%10a-%10a (%10lu bytes) Written, flavor=%s\n",addr,addr+size-1,(unsigned long)size,flavors[file->flavor[addr]]); + } /* end if */ } /* Seek to the correct location */ diff --git a/src/H5FDlog.h b/src/H5FDlog.h index 7a2930d..6703cf7 100644 --- a/src/H5FDlog.h +++ b/src/H5FDlog.h @@ -14,8 +14,16 @@ #define H5FD_LOG (H5FD_log_init()) +#ifdef __cplusplus +extern "C" { +#endif + __DLL__ hid_t H5FD_log_init(void); __DLL__ herr_t H5Pset_fapl_log(hid_t fapl_id, char *logfile, int verbosity); +#ifdef __cplusplus +} +#endif + #endif diff --git a/src/H5FDsec2.h b/src/H5FDsec2.h index fb7774a..e1484d1 100644 --- a/src/H5FDsec2.h +++ b/src/H5FDsec2.h @@ -14,7 +14,15 @@ #define H5FD_SEC2 (H5FD_sec2_init()) +#ifdef __cplusplus +extern "C" { +#endif + __DLL__ hid_t H5FD_sec2_init(void); __DLL__ herr_t H5Pset_fapl_sec2(hid_t fapl_id); +#ifdef __cplusplus +} +#endif + #endif diff --git a/src/H5Farray.c b/src/H5Farray.c index e2e71ab..cde7359 100644 --- a/src/H5Farray.c +++ b/src/H5Farray.c @@ -140,6 +140,7 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, hsize_t idx[H5O_LAYOUT_NDIMS]; /*multi-dim counter */ size_t mem_start; /*byte offset to start */ hsize_t file_start; /*byte offset to start */ + hsize_t max_data = 0; /*bytes in dataset */ hsize_t elmt_size = 1; /*bytes per element */ size_t nelmts, z; /*number of elements */ uintn ndims; /*stride dimensionality */ @@ -247,6 +248,13 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, addr = 0; } else { addr = layout->addr; + + /* Compute the size of the dataset in bytes */ + for(u=0, max_data=1; u<layout->ndims; u++) + max_data *= layout->dim[u]; + + /* Adjust the maximum size of the data by the offset into it */ + max_data -= file_start; } addr += file_start; buf += mem_start; @@ -313,7 +321,7 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, "external data read failed"); } } else { - if (H5F_contig_read(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) { + if (H5F_contig_read(f, max_data, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, "block read failed"); } @@ -324,6 +332,9 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, addr += file_stride[j]; buf += mem_stride[j]; + /* Adjust the maximum size of the data by the offset into it */ + max_data -= file_stride[j]; + if (--idx[j]) carray = FALSE; else @@ -411,6 +422,7 @@ H5F_arr_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, hsize_t idx[H5O_LAYOUT_NDIMS]; /*multi-dim counter */ hsize_t mem_start; /*byte offset to start */ hsize_t file_start; /*byte offset to start */ + hsize_t max_data = 0; /*bytes in dataset */ hsize_t elmt_size = 1; /*bytes per element */ size_t nelmts, z; /*number of elements */ uintn ndims; /*dimensionality */ @@ -518,6 +530,13 @@ H5F_arr_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, addr = 0; } else { addr = layout->addr; + + /* Compute the size of the dataset in bytes */ + for(u=0, max_data=1; u<layout->ndims; u++) + max_data *= layout->dim[u]; + + /* Adjust the maximum size of the data by the offset into it */ + max_data -= file_start; } addr += file_start; buf += mem_start; @@ -561,7 +580,7 @@ H5F_arr_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, "external data write failed"); } } else { - if (H5F_contig_write(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) { + if (H5F_contig_write(f, max_data, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); } @@ -572,6 +591,9 @@ H5F_arr_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, addr += file_stride[j]; buf += mem_stride[j]; + /* Adjust the maximum size of the data by the offset into it */ + max_data -= file_stride[j]; + if (--idx[j]) carray = FALSE; else diff --git a/src/H5Fcontig.c b/src/H5Fcontig.c index d24e086..e08707e 100644 --- a/src/H5Fcontig.c +++ b/src/H5Fcontig.c @@ -43,7 +43,7 @@ static intn interface_initialize_g = 0; *------------------------------------------------------------------------- */ herr_t -H5F_contig_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxpl_id, +H5F_contig_read(H5F_t *f, hsize_t max_data, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxpl_id, void *buf/*out*/) { haddr_t abs_eoa; /* Absolute end of file address */ @@ -129,7 +129,7 @@ H5F_contig_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxp rel_eoa=abs_eoa-f->shared->base_addr; /* Compute the size of the sieve buffer */ - f->shared->sieve_size=MIN(rel_eoa-addr,f->shared->sieve_buf_size); + f->shared->sieve_size=MIN(rel_eoa-addr,MIN(max_data,f->shared->sieve_buf_size)); /* Read the new sieve buffer */ if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { @@ -176,7 +176,7 @@ H5F_contig_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxp rel_eoa=abs_eoa-f->shared->base_addr; /* Compute the size of the sieve buffer */ - f->shared->sieve_size=MIN(rel_eoa-addr,f->shared->sieve_buf_size); + f->shared->sieve_size=MIN(rel_eoa-addr,MIN(max_data,f->shared->sieve_buf_size)); /* Read the new sieve buffer */ if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { @@ -221,7 +221,7 @@ H5F_contig_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxp *------------------------------------------------------------------------- */ herr_t -H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, +H5F_contig_write(H5F_t *f, hsize_t max_data, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxpl_id, const void *buf) { haddr_t abs_eoa; /* Absolute end of file address */ @@ -239,7 +239,7 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if(f->shared->sieve_buf) { haddr_t sieve_start, sieve_end; /* Start & end locations of sieve buffer */ haddr_t contig_end; /* End locations of block to write */ - hsize_t sieve_size; /* size of sieve buffer */ + hsize_t sieve_size; /* size of sieve buffer */ /* Stash local copies of these value */ sieve_start=f->shared->sieve_loc; @@ -285,50 +285,85 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if (H5F_block_write(f, type, addr, size, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); - } + } /* end if */ } /* end if */ /* Element size fits within the buffer size */ else { - /* Flush the sieve buffer if it's dirty */ - if(f->shared->sieve_dirty) { - /* Write to file */ - if (H5F_block_write(f, H5FD_MEM_DRAW, sieve_start, sieve_size, dxpl_id, f->shared->sieve_buf)<0) { - HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "block write failed"); - } - - /* Reset sieve buffer dirty flag */ - f->shared->sieve_dirty=0; + /* Check if it is possible to (exactly) prepend or append to existing (dirty) sieve buffer */ + if(((addr+size)==sieve_start || addr==sieve_end) && + (size+sieve_size)<=f->shared->sieve_buf_size && + f->shared->sieve_dirty) { + /* Prepend to existing sieve buffer */ + if((addr+size)==sieve_start) { + /* Move existing sieve information to correct location */ + assert(sieve_size==(hsize_t)((size_t)sieve_size)); /*check for overflow*/ + HDmemmove(f->shared->sieve_buf+size,f->shared->sieve_buf,(size_t)sieve_size); + + /* Copy in new information (must be first in sieve buffer) */ + assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ + HDmemcpy(f->shared->sieve_buf,buf,(size_t)size); + + /* Adjust sieve location */ + f->shared->sieve_loc=addr; + + } /* end if */ + /* Append to existing sieve buffer */ + else { + /* Copy in new information */ + assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ + HDmemcpy(f->shared->sieve_buf+sieve_size,buf,(size_t)size); + } /* end else */ + + /* Adjust sieve size */ + f->shared->sieve_size += size; + } /* end if */ + /* Can't add the new data onto the existing sieve buffer */ + else { + /* Flush the sieve buffer if it's dirty */ + if(f->shared->sieve_dirty) { + /* Write to file */ + if (H5F_block_write(f, H5FD_MEM_DRAW, sieve_start, sieve_size, dxpl_id, f->shared->sieve_buf)<0) { + HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, + "block write failed"); + } /* end if */ - /* Determine the new sieve buffer size & location */ - f->shared->sieve_loc=addr; + /* Reset sieve buffer dirty flag */ + f->shared->sieve_dirty=0; + } /* end if */ - /* Make certain we don't read off the end of the file */ - if (HADDR_UNDEF==(abs_eoa=H5FD_get_eoa(f->shared->lf))) { - HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, - "unable to determine file size"); - } + /* Determine the new sieve buffer size & location */ + f->shared->sieve_loc=addr; - /* Adjust absolute EOA address to relative EOA address */ - rel_eoa=abs_eoa-f->shared->base_addr; + /* Make certain we don't read off the end of the file */ + if (HADDR_UNDEF==(abs_eoa=H5FD_get_eoa(f->shared->lf))) { + HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, + "unable to determine file size"); + } /* end if */ - /* Compute the size of the sieve buffer */ - f->shared->sieve_size=MIN(rel_eoa-addr,f->shared->sieve_buf_size); + /* Adjust absolute EOA address to relative EOA address */ + rel_eoa=abs_eoa-f->shared->base_addr; - /* Read the new sieve buffer */ - if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { - HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, - "block read failed"); - } + /* Compute the size of the sieve buffer */ + f->shared->sieve_size=MIN(rel_eoa-addr,MIN(max_data,f->shared->sieve_buf_size)); - /* Grab the data out of the buffer (must be first piece of data in buffer) */ - assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ - HDmemcpy(f->shared->sieve_buf,buf,(size_t)size); + /* Check if there is any point in reading the data from the file */ + if(f->shared->sieve_size>size) { + /* Read the new sieve buffer */ + if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { + HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, + "block read failed"); + } /* end if */ + } /* end if */ + + /* Grab the data out of the buffer (must be first piece of data in buffer) */ + assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ + HDmemcpy(f->shared->sieve_buf,buf,(size_t)size); - /* Set sieve buffer dirty flag */ - f->shared->sieve_dirty=1; + /* Set sieve buffer dirty flag */ + f->shared->sieve_dirty=1; + } /* end else */ } /* end else */ } /* end else */ } /* end if */ @@ -347,7 +382,7 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if (NULL==(f->shared->sieve_buf=H5MM_malloc((size_t)f->shared->sieve_buf_size))) { HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed"); - } + } /* end if */ /* Determine the new sieve buffer size & location */ f->shared->sieve_loc=addr; @@ -356,19 +391,22 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if (HADDR_UNDEF==(abs_eoa=H5FD_get_eoa(f->shared->lf))) { HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to determine file size"); - } + } /* end if */ /* Adjust absolute EOA address to relative EOA address */ rel_eoa=abs_eoa-f->shared->base_addr; /* Compute the size of the sieve buffer */ - f->shared->sieve_size=MIN(rel_eoa-addr,f->shared->sieve_buf_size); + f->shared->sieve_size=MIN(rel_eoa-addr,MIN(max_data,f->shared->sieve_buf_size)); - /* Read the new sieve buffer */ - if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { - HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, - "block read failed"); - } + /* Check if there is any point in reading the data from the file */ + if(f->shared->sieve_size>size) { + /* Read the new sieve buffer */ + if (H5F_block_read(f, type, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) { + HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, + "block read failed"); + } /* end if */ + } /* end if */ /* Grab the data out of the buffer (must be first piece of data in buffer) */ assert(size==(hsize_t)((size_t)size)); /*check for overflow*/ @@ -383,7 +421,7 @@ H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, if (H5F_block_write(f, type, addr, size, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); - } + } /* end if */ } /* end else */ FUNC_LEAVE(SUCCEED); diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 3c7c142..763a592 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -199,9 +199,9 @@ __DLL__ herr_t H5F_istore_allocate (H5F_t *f, hid_t dxpl_id, const struct H5O_fill_t *fill); /* Functions that operate on contiguous storage wrt boot block */ -__DLL__ herr_t H5F_contig_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, +__DLL__ herr_t H5F_contig_read(H5F_t *f, hsize_t max_data, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxpl_id, void *_buf/*out*/); -__DLL__ herr_t H5F_contig_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, +__DLL__ herr_t H5F_contig_write(H5F_t *f, hsize_t max_data, H5FD_mem_t type, haddr_t addr, hsize_t size, hid_t dxpl_id, const void *buf); #endif diff --git a/src/H5Fseq.c b/src/H5Fseq.c index 4c77052..c6e29a1 100644 --- a/src/H5Fseq.c +++ b/src/H5Fseq.c @@ -57,8 +57,8 @@ static intn interface_initialize_g = 0; */ herr_t H5F_seq_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, - const struct H5O_pline_t *pline, const H5O_fill_t *fill, - const struct H5O_efl_t *efl, const H5S_t *file_space, size_t elmt_size, + const struct H5O_pline_t *pline, const H5O_fill_t *fill, + const struct H5O_efl_t *efl, const H5S_t *file_space, size_t elmt_size, hsize_t seq_len, hsize_t file_offset, void *buf/*out*/) { hsize_t dset_dims[H5O_LAYOUT_NDIMS]; /* dataspace dimensions */ @@ -67,7 +67,9 @@ H5F_seq_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, hsize_t down_size[H5O_LAYOUT_NDIMS]; /* Cumulative yperslab sizes (in elements) */ hsize_t acc; /* Accumulator for hyperslab sizes (in elements) */ intn ndims; + hsize_t max_data = 0; /*bytes in dataset */ haddr_t addr=0; /*address in file */ + uintn u; /*counters */ intn i,j; /*counters */ #ifdef H5_HAVE_PARALLEL H5FD_mpio_xfer_t xfer_mode=H5FD_MPIO_INDEPENDENT; @@ -118,6 +120,13 @@ H5F_seq_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, addr = 0; } else { addr = layout->addr; + + /* Compute the size of the dataset in bytes */ + for(u=0, max_data=1; u<layout->ndims; u++) + max_data *= layout->dim[u]; + + /* Adjust the maximum size of the data by the offset into it */ + max_data -= file_offset; } addr += file_offset; @@ -150,18 +159,18 @@ H5F_seq_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, #endif /* Read directly from file if the dataset is in an external file */ - /* Note: We can't use data sieve buffers for datasets in external files - * because the 'addr' of all external files is set to 0 (above) and - * all datasets in external files would alias to the same set of - * file offsets, totally mixing up the data sieve buffer information. -QAK - */ if (efl && efl->nused>0) { + /* Note: We can't use data sieve buffers for datasets in external files + * because the 'addr' of all external files is set to 0 (above) and + * all datasets in external files would alias to the same set of + * file offsets, totally mixing up the data sieve buffer information. -QAK + */ if (H5O_efl_read(f, efl, addr, seq_len, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, "external data read failed"); } } else { - if (H5F_contig_read(f, H5FD_MEM_DRAW, addr, seq_len, dxpl_id, buf)<0) { + if (H5F_contig_read(f, max_data, H5FD_MEM_DRAW, addr, seq_len, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL, "block read failed"); } @@ -491,8 +500,8 @@ printf("%s: partial_size=%lu, coords[%d]=%ld, hslab_size[%d]=%ld\n",FUNC,(unsign */ herr_t H5F_seq_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, - const struct H5O_pline_t *pline, const H5O_fill_t *fill, - const struct H5O_efl_t *efl, const H5S_t *file_space, size_t elmt_size, + const struct H5O_pline_t *pline, const H5O_fill_t *fill, + const struct H5O_efl_t *efl, const H5S_t *file_space, size_t elmt_size, hsize_t seq_len, hsize_t file_offset, const void *buf) { hsize_t dset_dims[H5O_LAYOUT_NDIMS]; /* dataspace dimensions */ @@ -501,7 +510,9 @@ H5F_seq_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, hsize_t down_size[H5O_LAYOUT_NDIMS]; /* Cumulative hyperslab sizes (in elements) */ hsize_t acc; /* Accumulator for hyperslab sizes (in elements) */ intn ndims; + hsize_t max_data = 0; /*bytes in dataset */ haddr_t addr; /*address in file */ + uintn u; /*counters */ intn i,j; /*counters */ #ifdef H5_HAVE_PARALLEL H5FD_mpio_xfer_t xfer_mode=H5FD_MPIO_INDEPENDENT; @@ -552,6 +563,13 @@ H5F_seq_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, addr = 0; } else { addr = layout->addr; + + /* Compute the size of the dataset in bytes */ + for(u=0, max_data=1; u<layout->ndims; u++) + max_data *= layout->dim[u]; + + /* Adjust the maximum size of the data by the offset into it */ + max_data -= file_offset; } addr += file_offset; @@ -584,18 +602,18 @@ H5F_seq_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout, #endif /* Write directly to file if the dataset is in an external file */ - /* Note: We can't use data sieve buffers for datasets in external files - * because the 'addr' of all external files is set to 0 (above) and - * all datasets in external files would alias to the same set of - * file offsets, totally mixing up the data sieve buffer information. -QAK - */ if (efl && efl->nused>0) { + /* Note: We can't use data sieve buffers for datasets in external files + * because the 'addr' of all external files is set to 0 (above) and + * all datasets in external files would alias to the same set of + * file offsets, totally mixing up the data sieve buffer information. -QAK + */ if (H5O_efl_write(f, efl, addr, seq_len, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "external data write failed"); } } else { - if (H5F_contig_write(f, H5FD_MEM_DRAW, addr, seq_len, dxpl_id, buf)<0) { + if (H5F_contig_write(f, max_data, H5FD_MEM_DRAW, addr, seq_len, dxpl_id, buf)<0) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); } |