From 61952803b31af3761bac13490081a216f625bf97 Mon Sep 17 00:00:00 2001 From: Scott Wegner Date: Wed, 13 Jun 2007 15:59:38 -0500 Subject: [svn-r13862] Although still not completely ready, this fixes many problems in the new Windows file driver related to writing large buffers (4GB+). sec2 is still set as the default driver on Windows. Tested: .NET on WinXP and 2005 on WinXP x64 --- src/H5FDwindows.c | 111 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 47 deletions(-) diff --git a/src/H5FDwindows.c b/src/H5FDwindows.c index 7a08d33..43afcef 100644 --- a/src/H5FDwindows.c +++ b/src/H5FDwindows.c @@ -42,6 +42,8 @@ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ +#ifdef H5_HAVE_WINDOWS + /* The driver identification number, initialized at runtime */ static hid_t H5FD_WINDOWS_g = 0; @@ -51,6 +53,11 @@ static hid_t H5FD_WINDOWS_g = 0; #define OP_WRITE 2 /* + * This is the max number of bytes that will be read or written by the file driver. + * Values above 2^31-1 may cause trouble with Windows. Default is 2^30. + */ +#define IO_BUF_SIZE 1073741824 +/* * The description of a file belonging to this driver. The `eoa' and `eof' * determine the amount of hdf5 address space in use and the high-water mark * of the file (the current size of the underlying file). The `pos' @@ -68,7 +75,7 @@ typedef struct H5FD_windows_t { * .NET doesn't support our 64-bit safe stdio functions, * so we will use io.h functions instead. */ -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO int fd; /*the file descriptor */ #else FILE *fp; /*the file handle */ @@ -318,12 +325,11 @@ H5FD_windows_open(const char *name, unsigned flags, hid_t UNUSED fapl_id, struct _BY_HANDLE_FILE_INFORMATION fileinfo; H5FD_t *ret_value; struct _stati64 sb; -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO int o_flags; #else unsigned write_access = 0; - __int64 x; -#endif /* _MSC_VER */ +#endif /* WINDOWS_USE_STDIO */ FUNC_ENTER_NOAPI(H5FD_windows_open, NULL) @@ -338,7 +344,7 @@ H5FD_windows_open(const char *name, unsigned flags, hid_t UNUSED fapl_id, if (ADDR_OVERFLOW(maxaddr)) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr") -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO /* Build the open flags */ o_flags = (H5F_ACC_RDWR & flags) ? O_RDWR : O_RDONLY; if (H5F_ACC_TRUNC & flags) o_flags |= O_TRUNC; @@ -346,14 +352,14 @@ H5FD_windows_open(const char *name, unsigned flags, hid_t UNUSED fapl_id, if (H5F_ACC_EXCL & flags) o_flags |= O_EXCL; /* Open the file */ - if ((fd=HDopen(name, o_flags, 0666))<0) + if ((fd=_open(name, o_flags, 0666))<0) HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file") if (_fstati64(fd, &sb) == -1) HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file") -#else /* _MSC_VER < 1400 */ +#else /* WINDOWS_USE_STDIO */ /* Translate our flags into a mode, and open the file */ - if (access(name, F_OK) < 0) { + if (_access_s(name, F_OK)) { if ((flags & H5F_ACC_CREAT) && (flags & H5F_ACC_RDWR)) { f = fopen(name, "wb+"); write_access=1; /* Note the write access */ @@ -363,20 +369,20 @@ H5FD_windows_open(const char *name, unsigned flags, hid_t UNUSED fapl_id, } else if ((flags & H5F_ACC_CREAT) && (flags & H5F_ACC_EXCL)) { HSYS_GOTO_ERROR(H5E_FILE, H5E_FILEEXISTS, NULL, "file exists but CREAT and EXCL were specified") } else if (flags & H5F_ACC_RDWR) { - if (flags & H5F_ACC_TRUNC) + if (flags & H5F_ACC_TRUNC) f = fopen(name, "wb+"); - else + else f = fopen(name, "rb+"); write_access=1; /* Note the write access */ - } else { + } else f = fopen(name, "rb"); - } + if (!f) - HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "fopen failed") + HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "fsopen failed") if (_stati64(name, &sb) == -1) HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to stat file") -#endif /* _MSC_VER < 1400 */ +#endif /* WINDOWS_USE_STDIO */ /* Create the new file struct */ if (NULL==(file=H5FL_CALLOC(H5FD_windows_t))) @@ -386,14 +392,14 @@ H5FD_windows_open(const char *name, unsigned flags, hid_t UNUSED fapl_id, file->pos = HADDR_UNDEF; file->op = OP_UNKNOWN; -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO file->fd = fd; #else file->fp = f; if((fd = _fileno(f)) == -1) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to get file descriptor for file") file->write_access=write_access; /* Note the write_access for later */ -#endif /* _MSC_VER < 1400 */ +#endif /* WINDOWS_USE_STDIO */ if( (filehandle = (HANDLE)_get_osfhandle(fd)) == INVALID_HANDLE_VALUE) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to get file handle for file") @@ -408,13 +414,13 @@ H5FD_windows_open(const char *name, unsigned flags, hid_t UNUSED fapl_id, done: if(ret_value==NULL) { -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO if(fd>=0) _close(fd); #else if(f) fclose(file->fp); -#endif /* _MSC_VER */ +#endif /* WINDOWS_USE_STDIO */ } /* end if */ FUNC_LEAVE_NOAPI(ret_value) @@ -445,11 +451,11 @@ H5FD_windows_close(H5FD_t *_file) FUNC_ENTER_NOAPI(H5FD_windows_close, FAIL) -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO if (_close(file->fd)<0) #else if (fclose(file->fp)<0) -#endif /* _MSC_VER */ +#endif /* WINDOWS_USE_STDIO */ HSYS_GOTO_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL, "unable to close file") H5FL_FREE(H5FD_windows_t,file); @@ -670,11 +676,11 @@ H5FD_windows_get_handle(H5FD_t *_file, hid_t UNUSED fapl, void** file_handle) if(!file_handle) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO *file_handle = &(file->fd); #else *file_handle = &(file->fp); -#endif /* _MSC_VER */ +#endif /* WINDOWS_USE_STDIO */ if(*file_handle==NULL) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "get handle failed") @@ -757,7 +763,7 @@ H5FD_windows_read(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, h size_t size, void *buf/*out*/) { H5FD_windows_t *file = (H5FD_windows_t*)_file; - ssize_t nbytes; + int nbytes; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5FD_windows_read, FAIL) @@ -775,11 +781,11 @@ H5FD_windows_read(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, h /* Seek to the correct location */ if ((addr!=file->pos || OP_READ!=file->op)) -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO if (_lseeki64(file->fd, addr, SEEK_SET) == -1) { #else - if (fseeki64(file->fp, addr, SEEK_SET) == -1) { -#endif /* _MSC_VER */ + if (_fseeki64(file->fp, addr, SEEK_SET) == -1) { +#endif /* WINDOWS_USE_STDIO */ file->op = OP_UNKNOWN; file->pos = HADDR_UNDEF; HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") @@ -790,9 +796,9 @@ H5FD_windows_read(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, h * and the end of the file. */ while (size>0) { -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO do { - nbytes = _read(file->fd, buf, size); + nbytes = _read(file->fd, buf, (unsigned)(size <= IO_BUF_SIZE ? size: IO_BUF_SIZE)); } while (-1==nbytes && EINTR==errno); if (-1==nbytes) /* error */ HSYS_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file read failed") @@ -803,7 +809,7 @@ H5FD_windows_read(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, h } #else do { - nbytes = fread(buf,(size_t)1,size,file->fp); + nbytes = fread(buf,(size_t)1,(size <= IO_BUF_SIZE ? size: IO_BUF_SIZE),file->fp); } while (!nbytes && EINTR==errno); if(!nbytes) { if (ferror(file->fp)) /* error */ @@ -814,7 +820,7 @@ H5FD_windows_read(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, h break; } } -#endif /* _MSC_VER */ +#endif /* WINDOWS_USE_STDIO */ assert(nbytes>=0); assert((size_t)nbytes<=size); H5_CHECK_OVERFLOW(nbytes,ssize_t,size_t); @@ -883,11 +889,11 @@ H5FD_windows_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, /* Seek to the correct location */ if ((addr!=file->pos || OP_WRITE!=file->op)) { -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO if (_lseeki64(file->fd, addr, SEEK_SET) == -1) { #else if (_fseeki64(file->fp, addr, SEEK_SET) == -1) { -#endif /* _MSC_VER */ +#endif /* WINDOWS_USE_STDIO */ file->op = OP_UNKNOWN; file->pos = HADDR_UNDEF; HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") @@ -898,17 +904,20 @@ H5FD_windows_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, * Write the data, being careful of interrupted system calls and partial * results */ - while (size>0) { + while (size>0) { do { -#if _MSC_VER < 1400 - nbytes = _write(file->fd, buf, size); +#ifndef WINDOWS_USE_STDIO + nbytes = _write(file->fd, buf, (unsigned)(size <= IO_BUF_SIZE ? size: IO_BUF_SIZE)); } while (-1==nbytes && EINTR==errno); if (-1==nbytes) /* error */ #else - nbytes = fwrite(buf, 1, size,file->fp); - } while (!nbytes && EINTR==errno); + + + /* Write 1GB or less at a time */ + nbytes = fwrite(buf, 1, (size <= IO_BUF_SIZE ? size: IO_BUF_SIZE),file->fp); + } while (!nbytes && EINTR==errno); if (!nbytes) /* error */ -#endif /* _MSC_VER */ +#endif /* WINDOWS_USE_STDIO */ HSYS_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") assert(nbytes>0); assert((size_t)nbytes<=size); @@ -917,7 +926,7 @@ H5FD_windows_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, H5_CHECK_OVERFLOW(nbytes,ssize_t,haddr_t); addr += (haddr_t)nbytes; buf = (const char*)buf + nbytes; - } + } /* Update current position and eof */ file->pos = addr; @@ -959,37 +968,43 @@ H5FD_windows_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned closing) { H5FD_windows_t *file = (H5FD_windows_t*)_file; herr_t ret_value=SUCCEED; /* Return value */ -#if _MSC_VER <= 1400 +#ifndef WINDOWS_USE_STDIO LARGE_INTEGER li; HANDLE filehandle; -#endif /* _MSC_VER */ +#else + int fd; +#endif /* WINDOWS_USE_STDIO */ FUNC_ENTER_NOAPI(H5FD_windows_flush, FAIL) assert(file); -#if _MSC_VER < 1400 +#ifndef WINDOWS_USE_STDIO /* Extend the file to make sure it's large enough */ if( (filehandle = (HANDLE)_get_osfhandle(file->fd)) == INVALID_HANDLE_VALUE) HGOTO_ERROR(H5E_FILE, H5E_FILEOPEN, FAIL, "unable to get file handle for file") li.QuadPart = file->eoa; - if(SetFilePointerEx(filehandle, li, NULL, FILE_BEGIN) == 0) - HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly") + (void)SetFilePointer((HANDLE)filehandle,li.LowPart,&li.HighPart,FILE_BEGIN); if(SetEndOfFile(filehandle) == 0) HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly") -#else /* _MSC_VER +#else /* WINDOWS_USE_STDIO */ /* Only try to flush if we have write access */ if(!file->write_access) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush without write access") + if((fd = _fileno(file->fp)) == -1) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to get file descriptor for file") + if(_chsize_s(fd, file->eoa)) + HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly") + /* Flush */ - if(!closing) { + if(!closing) if (fflush(file->fp) == EOF) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "fflush failed") -#endif /* _MSC_VER */ +#endif /* WINDOWS_USE_STDIO */ /* Update the eof value */ file->eof = file->eoa; @@ -1002,3 +1017,5 @@ H5FD_windows_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned closing) done: FUNC_LEAVE_NOAPI(ret_value) } + +#endif /* H5_HAVE_WINDOWS */ \ No newline at end of file -- cgit v0.12