From 775bd09974b4681672bce3c1a10948bad5d482da Mon Sep 17 00:00:00 2001 From: Raymond Lu Date: Mon, 16 Oct 2006 17:31:27 -0500 Subject: [svn-r12766] Made two changes to Direct I/O VFD: first added 3 parameters to H5Pset_fapl_direct to control memory boundary, file block size, and maximal copy buffer size; second in H5FD_direct_write and H5FD_direct_read, the library checks whether data buffer is aligned. If it is, then write and read the data directly instead of making a copy buffer. --- src/H5FDdirect.c | 291 ++++++++++++++++++++++++++++++++----------------------- src/H5FDdirect.h | 3 +- test/h5test.c | 5 +- test/vfd.c | 5 +- 4 files changed, 176 insertions(+), 128 deletions(-) diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index 36dae03..3815cc1 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -144,8 +144,11 @@ typedef struct H5FD_direct_t { HADDR_UNDEF==(A)+(Z) || \ (file_offset_t)((A)+(Z))<(file_offset_t)(A)) -/* Hard-code file system block size */ -#define FBSIZE 4096 +/* Global variables for memory boundary, file block size, and maximal copy buffer size. + * They can be changed through function H5Pset_fapl_direct. */ +hsize_t _boundary = 4096; +hsize_t _fbsize = 4096; +hsize_t _cbsize = 128*1024*1024; /* Prototypes */ static H5FD_t *H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, @@ -299,7 +302,7 @@ H5FD_direct_term(void) *------------------------------------------------------------------------- */ herr_t -H5Pset_fapl_direct(hid_t fapl_id) +H5Pset_fapl_direct(hid_t fapl_id, hsize_t boundary, hsize_t block_size, hsize_t cbuf_size) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value; @@ -310,6 +313,13 @@ H5Pset_fapl_direct(hid_t fapl_id) if(NULL == (plist = H5P_object_verify(fapl_id,H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if(boundary != 0) + _boundary = boundary; + if(block_size != 0) + _fbsize = block_size; + if(cbuf_size != 0) + _cbsize = cbuf_size; + ret_value= H5P_set_driver(plist, H5FD_DIRECT, NULL); done: @@ -726,59 +736,79 @@ H5FD_direct_read(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, ha if (addr+size>file->eoa) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow") - /* memory page size needed for the Direct IO option. Make a bigger - * buffer for aligned I/O. */ - mem_page_size = getpagesize(); - alloc_size = (size / FBSIZE + 1) * FBSIZE + FBSIZE; - if (posix_memalign(©_buf, mem_page_size, alloc_size) != 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "posix_memalign failed") - memset(copy_buf, 0, alloc_size); - - /* look for the aligned position for reading the data */ - if(file_seek(file->fd, (file_offset_t)(addr - addr % FBSIZE), SEEK_SET) < 0) - HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") - - /* - * Read the aligned data in file first, being careful of interrupted system calls, - * partial results, and the end of the file. - */ - p1 = copy_buf; - while(alloc_size > 0) { - do { - nbytes = HDread(file->fd, p1, alloc_size); - } while(-1==nbytes && EINTR==errno); - - if (-1==nbytes) /* error */ - HSYS_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file read failed") - if (0==nbytes) { - /* end of file but not end of format address space */ - break; - } else { - assert(nbytes>0); - assert((size_t)nbytes<=alloc_size); - H5_CHECK_OVERFLOW(nbytes,ssize_t,size_t); - H5_CHECK_OVERFLOW(nbytes,ssize_t,haddr_t); - alloc_size -= (size_t)nbytes; - p1 = (unsigned char*)p1 + nbytes; - } + if((addr%_fbsize==0) && (size%_fbsize==0) && ((haddr_t)buf%_fbsize==0)) { + while (size>0) { + do { + nbytes = HDread(file->fd, buf, size); + } while (-1==nbytes && EINTR==errno); + if (-1==nbytes) /* error */ + HSYS_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file read failed") + if (0==nbytes) { + /* end of file but not end of format address space */ + HDmemset(buf, 0, size); + break; + } + assert(nbytes>=0); + assert((size_t)nbytes<=size); + H5_CHECK_OVERFLOW(nbytes,ssize_t,size_t); + size -= (size_t)nbytes; + H5_CHECK_OVERFLOW(nbytes,ssize_t,haddr_t); + addr += (haddr_t)nbytes; + buf = (char*)buf + nbytes; + } + } else { + /* allocate memory needed for the Direct IO option. Make a bigger + * buffer for aligned I/O. */ + alloc_size = (size / _fbsize + 1) * _fbsize + _fbsize; + if (posix_memalign(©_buf, _boundary, alloc_size) != 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "posix_memalign failed") + memset(copy_buf, 0, alloc_size); + + /* look for the aligned position for reading the data */ + if(file_seek(file->fd, (file_offset_t)(addr - addr % _fbsize), SEEK_SET) < 0) + HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") + + /* + * Read the aligned data in file first, being careful of interrupted system calls, + * partial results, and the end of the file. + */ + p1 = copy_buf; + while(alloc_size > 0) { + do { + nbytes = HDread(file->fd, p1, alloc_size); + } while(-1==nbytes && EINTR==errno); + + if (-1==nbytes) /* error */ + HSYS_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file read failed") + if (0==nbytes) { + /* end of file but not end of format address space */ + break; + } else { + assert(nbytes>0); + assert((size_t)nbytes<=alloc_size); + H5_CHECK_OVERFLOW(nbytes,ssize_t,size_t); + H5_CHECK_OVERFLOW(nbytes,ssize_t,haddr_t); + alloc_size -= (size_t)nbytes; + p1 = (unsigned char*)p1 + nbytes; + } + } + + /*look for the right position to copy the data and copy the data + *to the original buffer.*/ + p2 = (unsigned char*)copy_buf + addr % _fbsize; + memcpy(buf, p2, size); + + /*update address and buffer*/ + addr += (haddr_t)size; + buf = (unsigned char*)buf + size; + + HDfree(copy_buf); } - /*look for the right position to copy the data and copy the data - *to the original buffer.*/ - p2 = (unsigned char*)copy_buf + addr % FBSIZE; - memcpy(buf, p2, size); - - /*update address and buffer*/ - addr += (haddr_t)size; - buf = (unsigned char*)buf + size; - /* Update current position */ file->pos = addr; file->op = OP_READ; - if(((addr % FBSIZE != 0) || (size % FBSIZE != 0)) && copy_buf) - HDfree(copy_buf); - done: if(ret_value<0) { /* Reset last file I/O information */ @@ -831,87 +861,102 @@ H5FD_direct_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, h if (addr+size>file->eoa) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow") - /* memory page size needed for the Direct IO option. Make a - * bigger buffer for aligned I/O - */ - mem_page_size = getpagesize(); - alloc_size = (size / FBSIZE + 1) * FBSIZE + FBSIZE; - if (posix_memalign(©_buf, mem_page_size, alloc_size) != 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "posix_memalign failed") - memset(copy_buf, 0, alloc_size); - - /* look for the right position for reading the data */ - if(file_seek(file->fd, (file_offset_t)(addr - addr % FBSIZE), SEEK_SET) < 0) - HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") - - /* - * Read the aligned data first, being careful of interrupted system calls, - * partial results, and the end of the file. - */ - p2 = copy_buf; - dup_size = alloc_size; - while(dup_size > 0) { - do { - nbytes = read(file->fd, p2, dup_size); - } while (-1==nbytes && EINTR==errno); - if (-1==nbytes) /* error */ - HSYS_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file read failed") - if (0==nbytes) { - /* end of file but not end of format address space */ - break; - } else { - assert(nbytes>0); - assert((size_t)nbytes<=dup_size); - H5_CHECK_OVERFLOW(nbytes,ssize_t,size_t); - H5_CHECK_OVERFLOW(nbytes,ssize_t,haddr_t); - dup_size -= (size_t)nbytes; - p2 = (unsigned char*)p2 + nbytes; - } - } - - /*append or copy the data to be written to the aligned buffer.*/ - p1 = (unsigned char*)copy_buf + addr % FBSIZE; - memcpy(p1, buf, size); - - /*look for the aligned position for writing the data*/ - if(file_seek(file->fd, (file_offset_t)(addr - addr % FBSIZE), SEEK_SET) < 0) - HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") - - /* - * Write the data, being careful of interrupted system calls and partial write. - * It doesn't truncate the extra data introduced by alignment because that step - * is done in H5FD_direct_flush. - */ - p2 = copy_buf; - while (alloc_size>0) { - do { - nbytes = HDwrite(file->fd, p2, alloc_size); - } while (-1==nbytes && EINTR==errno); - - if (-1==nbytes) { /* error */ - HSYS_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") - } - assert(nbytes>0); - assert((size_t)nbytes<=alloc_size); - H5_CHECK_OVERFLOW(nbytes,ssize_t,size_t); - H5_CHECK_OVERFLOW(nbytes,ssize_t,haddr_t); - alloc_size -= (size_t)nbytes; - p2 = (unsigned char*)p2 + nbytes; + if((addr%_fbsize==0) && (size%_fbsize==0) && ((haddr_t)buf%_fbsize==0)) { + while (size>0) { + do { + nbytes = HDwrite(file->fd, buf, size); + } while (-1==nbytes && EINTR==errno); + if (-1==nbytes) /* error */ + HSYS_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") + assert(nbytes>0); + assert((size_t)nbytes<=size); + H5_CHECK_OVERFLOW(nbytes,ssize_t,size_t); + size -= (size_t)nbytes; + H5_CHECK_OVERFLOW(nbytes,ssize_t,haddr_t); + addr += (haddr_t)nbytes; + buf = (const char*)buf + nbytes; + } + } else { + /* allocate memory needed for the Direct IO option. Make a + * bigger buffer for aligned I/O + */ + alloc_size = (size / _fbsize + 1) * _fbsize + _fbsize; + if (posix_memalign(©_buf, _boundary, alloc_size) != 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "posix_memalign failed") + memset(copy_buf, 0, alloc_size); + + /* look for the right position for reading the data */ + if(file_seek(file->fd, (file_offset_t)(addr - addr % _fbsize), SEEK_SET) < 0) + HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") + + /* + * Read the aligned data first, being careful of interrupted system calls, + * partial results, and the end of the file. + */ + p2 = copy_buf; + dup_size = alloc_size; + while(dup_size > 0) { + do { + nbytes = read(file->fd, p2, dup_size); + } while (-1==nbytes && EINTR==errno); + if (-1==nbytes) /* error */ + HSYS_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file read failed") + if (0==nbytes) { + /* end of file but not end of format address space */ + break; + } else { + assert(nbytes>0); + assert((size_t)nbytes<=dup_size); + H5_CHECK_OVERFLOW(nbytes,ssize_t,size_t); + H5_CHECK_OVERFLOW(nbytes,ssize_t,haddr_t); + dup_size -= (size_t)nbytes; + p2 = (unsigned char*)p2 + nbytes; + } + } + + /*append or copy the data to be written to the aligned buffer.*/ + p1 = (unsigned char*)copy_buf + addr % _fbsize; + memcpy(p1, buf, size); + + /*look for the aligned position for writing the data*/ + if(file_seek(file->fd, (file_offset_t)(addr - addr % _fbsize), SEEK_SET) < 0) + HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") + + /* + * Write the data, being careful of interrupted system calls and partial write. + * It doesn't truncate the extra data introduced by alignment because that step + * is done in H5FD_direct_flush. + */ + p2 = copy_buf; + while (alloc_size>0) { + do { + nbytes = HDwrite(file->fd, p2, alloc_size); + } while (-1==nbytes && EINTR==errno); + + if (-1==nbytes) { /* error */ + HSYS_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") + } + assert(nbytes>0); + assert((size_t)nbytes<=alloc_size); + H5_CHECK_OVERFLOW(nbytes,ssize_t,size_t); + H5_CHECK_OVERFLOW(nbytes,ssize_t,haddr_t); + alloc_size -= (size_t)nbytes; + p2 = (unsigned char*)p2 + nbytes; + } + + /*Update the address and size*/ + addr += (haddr_t)size; + buf = (const char*)buf + size; + + HDfree(copy_buf); } - /*Update the address and size*/ - addr += (haddr_t)size; - buf = (const char*)buf + size; - /* Update current position and eof */ file->pos = addr; file->op = OP_WRITE; if (file->pos>file->eof) file->eof = file->pos; - if(((addr % FBSIZE != 0) || (size % FBSIZE != 0)) && copy_buf) - HDfree(copy_buf); - done: if(ret_value<0) { /* Reset last file I/O information */ diff --git a/src/H5FDdirect.h b/src/H5FDdirect.h index 25ff3de..4e4d821 100644 --- a/src/H5FDdirect.h +++ b/src/H5FDdirect.h @@ -36,7 +36,8 @@ extern "C" { H5_DLL hid_t H5FD_direct_init(void); H5_DLL void H5FD_direct_term(void); -H5_DLL herr_t H5Pset_fapl_direct(hid_t fapl_id); +H5_DLL herr_t H5Pset_fapl_direct(hid_t fapl_id, hsize_t alignment, hsize_t block_size, + hsize_t cbuf_size); #ifdef __cplusplus } diff --git a/test/h5test.c b/test/h5test.c index ad85a8f..6c9e8ee 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -579,8 +579,9 @@ h5_fileaccess(void) return -1; } else if (!HDstrcmp(name, "direct")) { #ifdef H5_HAVE_DIRECT - /* Linux direct read() and write() system calls */ - if (H5Pset_fapl_direct(fapl)<0) return -1; + /* Linux direct read() and write() system calls. Set memory boundary, file block size, + * and copy buffer size to the default values. */ + if (H5Pset_fapl_direct(fapl, 0, 0, 0)<0) return -1; #endif } else { /* Unknown driver */ diff --git a/test/vfd.c b/test/vfd.c index 952ed4d..e42f643 100644 --- a/test/vfd.c +++ b/test/vfd.c @@ -151,9 +151,10 @@ test_direct(void) return 0; #else /*H5_HAVE_DIRECT*/ - /* Set property list and file name for SEC2 driver. */ + /* Set property list and file name for Direct driver. Set memory alignment boundary + * and file block size to 512 which is the minimum for Linux 2.6. */ fapl = h5_fileaccess(); - if(H5Pset_fapl_direct(fapl)<0) + if(H5Pset_fapl_direct(fapl, 512, 4096, 64*1024*1024)<0) TEST_ERROR; h5_fixname(FILENAME[4], fapl, filename, sizeof filename); -- cgit v0.12