summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDana Robinson <derobins@hdfgroup.org>2012-03-21 22:11:05 (GMT)
committerDana Robinson <derobins@hdfgroup.org>2012-03-21 22:11:05 (GMT)
commit0d2f2074cb64114fdd8aea5164e114d08d621f1b (patch)
tree95fab910517cc359efa0fdb5e96351ada1ed0473
parent0d118ea88a384bb18afa87a8ae068ceb99dc812c (diff)
downloadhdf5-0d2f2074cb64114fdd8aea5164e114d08d621f1b.zip
hdf5-0d2f2074cb64114fdd8aea5164e114d08d621f1b.tar.gz
hdf5-0d2f2074cb64114fdd8aea5164e114d08d621f1b.tar.bz2
[svn-r22117] Fix for HDFFV-7916 (Windows/POSIX correctness issues in the core VFD) and HDFFV-7603 (core VFD has trouble with 2GB+ files on Windows).
Propagates the SEC2 driver fixes from HDF5 1.8.8 to the core VFD (mainly concerning the backing store). These fixes also conveniently fixed 7603 as well. Tested on: 64-bit Windows 7 jam koala ostrich
-rw-r--r--src/H5FDcore.c271
1 files changed, 164 insertions, 107 deletions
diff --git a/src/H5FDcore.c b/src/H5FDcore.c
index 64cb1bb..940d4c7 100644
--- a/src/H5FDcore.c
+++ b/src/H5FDcore.c
@@ -38,8 +38,24 @@
/* The driver identification number, initialized at runtime */
static hid_t H5FD_CORE_g = 0;
-/*
- * The description of a file belonging to this driver. The `eoa' and `eof'
+/* Since Windows doesn't follow the rest of the world when it comes
+ * to POSIX I/O types, some typedefs and constants are needed to avoid
+ * making the code messy with #ifdefs.
+ * NOTE: These are only used when writing data to the backing store on
+ * file close.
+ */
+#ifdef H5_HAVE_WIN32_API
+typedef unsigned int h5_core_io_t;
+typedef int h5_core_io_ret_t;
+static int H5_CORE_MAX_IO_BYTES_g = INT_MAX;
+#else
+/* Unix, everyone else */
+typedef size_t h5_core_io_t;
+typedef ssize_t h5_core_io_ret_t;
+static size_t H5_CORE_MAX_IO_BYTES_g = SSIZET_MAX;
+#endif /* H5_HAVE_WIN32_API */
+
+/* The description of a file belonging to this driver. The `eoa' and `eof'
* determine the amount of hdf5 address space in use and the high-water mark
* of the file (the current size of the underlying memory).
*/
@@ -54,8 +70,7 @@ typedef struct H5FD_core_t {
int fd; /*backing store file descriptor */
/* Information for determining uniqueness of a file with a backing store */
#ifndef H5_HAVE_WIN32_API
- /*
- * On most systems the combination of device and i-node number uniquely
+ /* On most systems the combination of device and i-node number uniquely
* identify a file.
*/
dev_t device; /*file device number */
@@ -65,18 +80,26 @@ typedef struct H5FD_core_t {
ino_t inode; /*file i-node number */
#endif /*H5_VMS*/
#else
- /*
- * On H5_HAVE_WIN32_API the low-order word of a unique identifier associated with the
- * file and the volume serial number uniquely identify a file. This number
- * (which, both? -rpm) may change when the system is restarted or when the
- * file is opened. After a process opens a file, the identifier is
- * constant until the file is closed. An application can use this
- * identifier and the volume serial number to determine whether two
- * handles refer to the same file.
+ /* Files in windows are uniquely identified by the volume serial
+ * number and the file index (both low and high parts).
+ *
+ * There are caveats where these numbers can change, especially
+ * on FAT file systems. On NTFS, however, a file should keep
+ * those numbers the same until renamed or deleted (though you
+ * can use ReplaceFile() on NTFS to keep the numbers the same
+ * while renaming).
+ *
+ * See the MSDN "BY_HANDLE_FILE_INFORMATION Structure" entry for
+ * more information.
+ *
+ * http://msdn.microsoft.com/en-us/library/aa363788(v=VS.85).aspx
*/
- DWORD fileindexlo;
- DWORD fileindexhi;
-#endif
+ DWORD nFileIndexLow;
+ DWORD nFileIndexHigh;
+ DWORD dwVolumeSerialNumber;
+
+ HANDLE hFile; /* Native windows file handle */
+#endif /* H5_HAVE_WIN32_API */
hbool_t dirty; /*changes not saved? */
} H5FD_core_t;
@@ -89,8 +112,7 @@ typedef struct H5FD_core_fapl_t {
/* Allocate memory in multiples of this size by default */
#define H5FD_CORE_INCREMENT 8192
-/*
- * These macros check for overflow of various quantities. These macros
+/* These macros check for overflow of various quantities. These macros
* assume that file_offset_t is signed and haddr_t and size_t are unsigned.
*
* ADDR_OVERFLOW: Checks whether a file address of type `haddr_t'
@@ -200,8 +222,6 @@ H5FD_core_init_interface(void)
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
hid_t
@@ -258,15 +278,6 @@ H5FD_core_term(void)
* Programmer: Robb Matzke
* Thursday, February 19, 1998
*
- * Modifications:
- * Robb Matzke, 1999-10-19
- * Added the BACKING_STORE argument. If set then the entire file
- * contents are flushed to a file with the same name as this
- * core file.
- *
- * Raymond Lu, 2001-10-25
- * Changed the file access list to the new generic list.
- *
*-------------------------------------------------------------------------
*/
herr_t
@@ -305,14 +316,6 @@ done:
* Programmer: Robb Matzke
* Tuesday, August 10, 1999
*
- * Modifications:
- * Robb Matzke, 1999-10-19
- * Added the BACKING_STORE argument.
- *
- * Raymond Lu
- * 2001-10-25
- * Changed file access list to the new generic property list.
- *
*-------------------------------------------------------------------------
*/
herr_t
@@ -355,8 +358,6 @@ done:
* Programmer: Robb Matzke
* Friday, August 13, 1999
*
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
static void *
@@ -396,16 +397,6 @@ done:
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
- * Modifications:
- * Robb Matzke, 1999-10-19
- * The backing store file is created and opened if specified.
- *
- * Raymond Lu, 2006-11-30
- * Enabled the driver to read an existing file depending on
- * the setting of the backing_store and file open flags.
- *
- * Allen Byrne, 2008-1-23
- * changed if of fapl_id to assert
*-------------------------------------------------------------------------
*/
static H5FD_t *
@@ -417,7 +408,6 @@ H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id,
H5FD_core_fapl_t *fa=NULL;
H5P_genplist_t *plist; /* Property list pointer */
#ifdef H5_HAVE_WIN32_API
- HFILE filehandle;
struct _BY_HANDLE_FILE_INFORMATION fileinfo;
#endif
h5_stat_t sb;
@@ -461,8 +451,7 @@ H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id,
if(name && *name)
file->name = H5MM_xstrdup(name);
- /*
- * The increment comes from either the file access property list or the
+ /* The increment comes from either the file access property list or the
* default value. But if the file access property list was zero then use
* the default value instead.
*/
@@ -474,10 +463,16 @@ H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id,
if(fd >= 0) {
/* Retrieve information for determining uniqueness of file */
#ifdef H5_HAVE_WIN32_API
- filehandle = _get_osfhandle(fd);
- (void)GetFileInformationByHandle((HANDLE)filehandle, &fileinfo);
- file->fileindexhi = fileinfo.nFileIndexHigh;
- file->fileindexlo = fileinfo.nFileIndexLow;
+ file->hFile = (HANDLE)_get_osfhandle(fd);
+ if(INVALID_HANDLE_VALUE == file->hFile)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to get Windows file handle")
+
+ if(!GetFileInformationByHandle((HANDLE)file->hFile, &fileinfo))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to get Windows file information")
+
+ file->nFileIndexHigh = fileinfo.nFileIndexHigh;
+ file->nFileIndexLow = fileinfo.nFileIndexLow;
+ file->dwVolumeSerialNumber = fileinfo.dwVolumeSerialNumber;
#else /* H5_HAVE_WIN32_API */
file->device = sb.st_dev;
#ifdef H5_VMS
@@ -496,7 +491,7 @@ H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id,
size_t size;
/* Retrieve file size */
- size = (size_t)sb.st_size;
+ size = (size_t)sb.st_size;
/* Check if we should allocate the memory buffer and read in existing data */
if(size) {
@@ -507,9 +502,40 @@ H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id,
/* Set up data structures */
file->eof = size;
- /* Read in existing data */
- if(HDread(file->fd, file->mem, size) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read file")
+ /* Read in existing data, being careful of interrupted system calls,
+ * partial results, and the end of the file.
+ */
+ while(size > 0) {
+
+ h5_core_io_t bytes_in = 0; /* # of bytes to read */
+ h5_core_io_ret_t bytes_read = -1; /* # of bytes actually read */
+
+ /* Trying to read more bytes than the return type can handle is
+ * undefined behavior in POSIX.
+ */
+ if(size > H5_CORE_MAX_IO_BYTES_g)
+ bytes_in = H5_CORE_MAX_IO_BYTES_g;
+ else
+ bytes_in = (h5_core_io_t)size;
+
+ do {
+ bytes_read = HDread(file->fd, file->mem, bytes_in);
+ } while(-1 == bytes_read && EINTR == errno);
+
+ if(-1 == bytes_read) { /* error */
+ int myerrno = errno;
+ time_t mytime = HDtime(NULL);
+ HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR);
+
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "file read failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', file->mem = %p, size = %lu, offset = %llu", HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), file->mem, (unsigned long)size, (unsigned long long)myoffset);
+ } /* end if */
+
+ HDassert(bytes_read >= 0);
+ HDassert((size_t)bytes_read <= size);
+
+ size -= (size_t)bytes_read;
+
+ } /* end while */
} /* end if */
} /* end if */
@@ -578,13 +604,6 @@ done:
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
- * Modifications:
- * Neil Fortner
- * Tuesday, March 9, 2010
- * Modified function to compare low level file information if
- * a backing store is opened for both files, similar to the
- * sec2 file driver.
- *
*-------------------------------------------------------------------------
*/
static int
@@ -599,11 +618,14 @@ H5FD_core_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
if(f1->fd >= 0 && f2->fd >= 0) {
/* Compare low level file information for backing store */
#ifdef H5_HAVE_WIN32_API
- if (f1->fileindexhi < f2->fileindexhi) HGOTO_DONE(-1)
- if (f1->fileindexhi > f2->fileindexhi) HGOTO_DONE(1)
+ if(f1->dwVolumeSerialNumber < f2->dwVolumeSerialNumber) HGOTO_DONE(-1)
+ if(f1->dwVolumeSerialNumber > f2->dwVolumeSerialNumber) HGOTO_DONE(1)
- if (f1->fileindexlo < f2->fileindexlo) HGOTO_DONE(-1)
- if (f1->fileindexlo > f2->fileindexlo) HGOTO_DONE(1)
+ if(f1->nFileIndexHigh < f2->nFileIndexHigh) HGOTO_DONE(-1)
+ if(f1->nFileIndexHigh > f2->nFileIndexHigh) HGOTO_DONE(1)
+
+ if(f1->nFileIndexLow < f2->nFileIndexLow) HGOTO_DONE(-1)
+ if(f1->nFileIndexLow > f2->nFileIndexLow) HGOTO_DONE(1)
#else
#ifdef H5_DEV_T_IS_SCALAR
@@ -702,11 +724,6 @@ H5FD_core_query(const H5FD_t * _file, unsigned long *flags /* out */)
* Programmer: Robb Matzke
* Monday, August 2, 1999
*
- * Modifications:
- * Raymond Lu
- * 21 Dec. 2006
- * Added the parameter TYPE. It's only used for MULTI driver.
- *
*-------------------------------------------------------------------------
*/
static haddr_t
@@ -734,11 +751,6 @@ H5FD_core_get_eoa(const H5FD_t *_file, H5FD_mem_t UNUSED type)
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
- * Modifications:
- * Raymond Lu
- * 21 Dec. 2006
- * Added the parameter TYPE. It's only used for MULTI driver.
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -775,8 +787,6 @@ done:
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
static haddr_t
@@ -800,8 +810,6 @@ H5FD_core_get_eof(const H5FD_t *_file)
* Programmer: Raymond Lu
* Sept. 16, 2002
*
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -867,8 +875,6 @@ done:
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
/* ARGSUSED */
@@ -881,8 +887,8 @@ H5FD_core_read(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, hadd
FUNC_ENTER_NOAPI_NOINIT
- assert(file && file->pub.cls);
- assert(buf);
+ HDassert(file && file->pub.cls);
+ HDassert(buf);
/* Check for overflow conditions */
if (HADDR_UNDEF == addr)
@@ -934,8 +940,6 @@ done:
* Programmer: Robb Matzke
* Thursday, July 29, 1999
*
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
/* ARGSUSED */
@@ -1005,10 +1009,6 @@ done:
* Programmer: Robb Matzke
* Friday, October 15, 1999
*
- * Modifications:
- * Raymond Lu, 2006-11-30
- * Added a condition check for backing store flag, for an
- * existing file can be opened for read and write now.
*-------------------------------------------------------------------------
*/
/* ARGSUSED */
@@ -1027,19 +1027,40 @@ H5FD_core_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned UNUSED closing)
if (0!=HDlseek(file->fd, (off_t)0, SEEK_SET))
HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "error seeking in backing store")
+
+ while (size > 0) {
- while (size) {
- ssize_t n;
-
- H5_CHECK_OVERFLOW(size,hsize_t,size_t);
- n = HDwrite(file->fd, ptr, (size_t)size);
- if (n<0 && EINTR==errno)
- continue;
- if (n<0)
- HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "error writing backing store")
- ptr += (size_t)n;
- size -= (size_t)n;
- }
+ h5_core_io_t bytes_in = 0; /* # of bytes to write */
+ h5_core_io_ret_t bytes_wrote = -1; /* # of bytes written */
+
+ /* Trying to write more bytes than the return type can handle is
+ * undefined behavior in POSIX.
+ */
+ if(size > H5_CORE_MAX_IO_BYTES_g)
+ bytes_in = H5_CORE_MAX_IO_BYTES_g;
+ else
+ bytes_in = (h5_core_io_t)size;
+
+ do {
+ bytes_wrote = HDwrite(file->fd, ptr, bytes_in);
+ } while(-1 == bytes_wrote && EINTR == errno);
+
+ if(-1 == bytes_wrote) { /* error */
+ int myerrno = errno;
+ time_t mytime = HDtime(NULL);
+ HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR);
+
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to backing store failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', ptr = %p, size = %lu, offset = %llu", HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), ptr, (unsigned long)size, (unsigned long long)myoffset);
+ } /* end if */
+
+ HDassert(bytes_wrote > 0);
+ HDassert((size_t)bytes_wrote <= size);
+
+ size -= (size_t)bytes_wrote;
+ ptr = (unsigned char *)ptr + bytes_wrote;
+
+ } /* end while */
+
file->dirty = FALSE;
}
@@ -1094,6 +1115,43 @@ if(file->eof < new_eof)
/* Update backing store, if using it */
if(file->fd >= 0 && file->backing_store) {
+#ifdef H5_HAVE_WIN32_API
+ LARGE_INTEGER li; /* 64-bit (union) integer for SetFilePointer() call */
+ DWORD dwPtrLow; /* Low-order pointer bits from SetFilePointer()
+ * Only used as an error code here.
+ */
+ DWORD dwError; /* DWORD error code from GetLastError() */
+ BOOL bError; /* Boolean error flag */
+
+ /* Windows uses this odd QuadPart union for 32/64-bit portability */
+ li.QuadPart = (__int64)file->eoa;
+
+ /* Extend the file to make sure it's large enough.
+ *
+ * Since INVALID_SET_FILE_POINTER can technically be a valid return value
+ * from SetFilePointer(), we also need to check GetLastError().
+ */
+ dwPtrLow = SetFilePointer(file->hFile, li.LowPart, &li.HighPart, FILE_BEGIN);
+ if(INVALID_SET_FILE_POINTER == dwPtrLow) {
+ dwError = GetLastError();
+ if(dwError != NO_ERROR )
+ HGOTO_ERROR(H5E_FILE, H5E_FILEOPEN, FAIL, "unable to set file pointer")
+ }
+
+ bError = SetEndOfFile(file->hFile);
+ if(0 == bError)
+ HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly")
+#else /* H5_HAVE_WIN32_API */
+#ifdef H5_VMS
+ /* Reset seek offset to the beginning of the file, so that the file isn't
+ * re-extended later. This may happen on Open VMS. */
+ if(-1 == HDlseek(file->fd, (HDoff_t)0, SEEK_SET))
+ HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position")
+#endif
+ if(-1 == HDftruncate(file->fd, (HDoff_t)file->eoa))
+ HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly")
+#endif /* H5_HAVE_WIN32_API */
+
#ifdef H5_VMS
/* Reset seek offset to the beginning of the file, so that the file isn't
* re-extended later. This may happen on Open VMS. */
@@ -1112,4 +1170,3 @@ if(file->eof < new_eof)
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_core_truncate() */
-