[svn-r21501] - Propagated Windows VFD tweaks to SEC2 and log VFDs.

- Created typedefs and #defines to make POSIX I/O type-safe on Windows and true POSIX platforms.

- Added pre-checks and removed post-checks for POSIX I/O sizes.  When the number of bytes to be sent exceeds the maximum return value, the operation is split into smaller subsets.

- General code tidying and cleanup.

Tested on: jam (64-bit linux), jam + threadsafe, jam + parallel, Windows 7 (64- and 32-bit), freedom (64-bt BSD).

1 files changed, 170 insertions, 109 deletions

diff --git a/src/H5FDlog.c b/src/H5FDlog.c
index d9b379f..6fe38a4 100644
--- a/src/H5FDlog.c
+++ b/src/H5FDlog.c
@@ -43,6 +43,21 @@
 /* The driver identification number, initialized at runtime */
 static hid_t H5FD_LOG_g = 0;
 
+/* Since Windows doesn't follow the rest of the world when it comes
+ * to POSIX I/O types, some typedefs and constants are needed to avoid
+ * making the code messy with #ifdefs.
+ */
+#ifdef H5_HAVE_WIN32_API
+typedef unsigned int    h5_log_io_t;
+typedef int             h5_log_io_ret_t;
+static int H5_LOG_MAX_IO_BYTES_g = INT_MAX;
+#else
+/* Unix, everyone else */
+typedef size_t          h5_log_io_t;
+typedef ssize_t         h5_log_io_ret_t;
+static size_t H5_LOG_MAX_IO_BYTES_g = SSIZET_MAX;
+#endif /* H5_HAVE_WIN32_API */
+
 /* Driver-specific file access properties */
 typedef struct H5FD_log_fapl_t {
     char *logfile;              /* Allocated log file name */
@@ -65,11 +80,10 @@ static const char *flavors[]={
     "H5FD_MEM_OHDR",
 };
 
-/*
- * The description of a file belonging to this driver. The `eoa' and `eof'
+/* The description of a file belonging to this driver. The `eoa' and `eof'
  * determine the amount of hdf5 address space in use and the high-water mark
- * of the file (the current size of the underlying Unix file). The `pos'
- * value is used to eliminate file position updates when they would be a
+ * of the file (the current size of the underlying filesystem file). The
+ * `pos' value is used to eliminate file position updates when they would be a
  * no-op. Unfortunately we've found systems that use separate file position
  * indicators for reading and writing so the lseek can only be eliminated if
  * the current operation is the same as the previous operation.  When opening
@@ -78,69 +92,71 @@ static const char *flavors[]={
  * occurs), and `op' will be set to H5F_OP_UNKNOWN.
  */
 typedef struct H5FD_log_t {
-    H5FD_t	pub;			/*public stuff, must be first	*/
-    int		fd;			/*the unix file			*/
-    haddr_t	eoa;			/*end of allocated region	*/
-    haddr_t	eof;			/*end of file; current file size*/
-    haddr_t	pos;			/*current file I/O position	*/
-    H5FD_file_op_t	op;		/*last operation		*/
-    char	filename[H5FD_MAX_FILENAME_LEN];     /* Copy of file name from open operation */
+    H5FD_t          pub;    /* public stuff, must be first      */
+    int             fd;     /* the unix file                    */
+    haddr_t         eoa;    /* end of allocated region          */
+    haddr_t         eof;    /* end of file; current file size   */
+    haddr_t         pos;    /* current file I/O position        */
+    H5FD_file_op_t  op;     /* last operation                   */
+    char            filename[H5FD_MAX_FILENAME_LEN];    /* Copy of file name from open operation */
 #ifndef H5_HAVE_WIN32_API
-    /*
-     * On most systems the combination of device and i-node number uniquely
-     * identify a file.
+    /* On most systems the combination of device and i-node number uniquely
+     * identify a file.  Note that Cygwin, MinGW and other Windows POSIX
+     * environments have the stat function (which fakes inodes)
+     * and will use the 'device + inodes' scheme as opposed to the
+     * Windows code further below.
      */
-    dev_t	device;			/*file device number		*/
+    dev_t           device;         /* file device number   */
 #ifdef H5_VMS
-    ino_t	inode[3];		/*file i-node number		*/
+    ino_t           inode[3];       /* file i-node number   */
 #else
-    ino_t	inode;			/*file i-node number		*/
+    ino_t           inode;          /* file i-node number   */
 #endif /*H5_VMS*/
 #else
-    /*
-     * On H5_HAVE_WIN32_API the low-order word of a unique identifier associated with the
-     * file and the volume serial number uniquely identify a file. This number
-     * (which, both? -rpm) may change when the system is restarted or when the
-     * file is opened. After a process opens a file, the identifier is
-     * constant until the file is closed. An application can use this
-     * identifier and the volume serial number to determine whether two
-     * handles refer to the same file.
+    /* Files in windows are uniquely identified by the volume serial
+     * number and the file index (both low and high parts).
+     *
+     * There are caveats where these numbers can change, especially
+     * on FAT file systems.  On NTFS, however, a file should keep
+     * those numbers the same until renamed or deleted (though you
+     * can use ReplaceFile() on NTFS to keep the numbers the same
+     * while renaming).
+     *
+     * See the MSDN "BY_HANDLE_FILE_INFORMATION Structure" entry for
+     * more information.
+     *
+     * http://msdn.microsoft.com/en-us/library/aa363788(v=VS.85).aspx
      */
-    DWORD fileindexlo;
-    DWORD fileindexhi;
-#endif
-
-    /* Information from properties set by 'h5repart' tool */
-    hbool_t     fam_to_sec2;    /* Whether to eliminate the family driver info
-                                 * and convert this file to a single file */
+    DWORD           nFileIndexLow;
+    DWORD           nFileIndexHigh;
+    DWORD           dwVolumeSerialNumber;
+    
+    HANDLE          hFile;      /* Native windows file handle */
+#endif  /* H5_HAVE_WIN32_API */
+
+    /* Information from properties set by 'h5repart' tool
+     *
+     * Whether to eliminate the family driver info and convert this file to
+     * a single file
+     */
+    hbool_t     fam_to_sec2;
 
     /* Fields for tracking I/O operations */
-    unsigned char *nread;   /* Number of reads from a file location */
-    unsigned char *nwrite;  /* Number of write to a file location */
-    unsigned char *flavor;  /* Flavor of information written to file location */
-    unsigned long long total_read_ops;  /* Total number of read operations */
-    unsigned long long total_write_ops; /* Total number of write operations */
-    unsigned long long total_seek_ops;  /* Total number of seek operations */
-    unsigned long long total_truncate_ops; /* Total number of truncate operations */
-    double      total_read_time;    /* Total time spent in read operations */
-    double      total_write_time;   /* Total time spent in write operations */
-    double      total_seek_time;    /* Total time spent in seek operations */
-    size_t      iosize;     /* Size of I/O information buffers */
-    FILE       *logfp;      /* Log file pointer */
-    H5FD_log_fapl_t fa;	    /* Driver-specific file access properties*/
+    unsigned char       *nread;                 /* Number of reads from a file location             */
+    unsigned char       *nwrite;                /* Number of write to a file location               */
+    unsigned char       *flavor;                /* Flavor of information written to file location   */
+    unsigned long long  total_read_ops;         /* Total number of read operations                  */
+    unsigned long long  total_write_ops;        /* Total number of write operations                 */
+    unsigned long long  total_seek_ops;         /* Total number of seek operations                  */
+    unsigned long long  total_truncate_ops;     /* Total number of truncate operations              */
+    double              total_read_time;        /* Total time spent in read operations              */
+    double              total_write_time;       /* Total time spent in write operations             */
+    double              total_seek_time;        /* Total time spent in seek operations              */
+    size_t              iosize;                 /* Size of I/O information buffers                  */
+    FILE                *logfp;                 /* Log file pointer                                 */
+    H5FD_log_fapl_t     fa;                     /* Driver-specific file access properties           */
 } H5FD_log_t;
 
-
-/*
- * This driver supports systems that have the lseek64() function by defining
- * some macros here so we don't have to have conditional compilations later
- * throughout the code.
- *
- * HDoff_t:	The datatype for file offsets, the second argument of
- *		the lseek() or lseek64() call.
- *
- */
-
 /*
  * These macros check for overflow of various quantities.  These macros
  * assume that HDoff_t is signed and haddr_t and size_t are unsigned.
@@ -306,8 +322,7 @@ H5FD_log_term(void)
  * Function:	H5Pset_fapl_log
  *
  * Purpose:	Modify the file access property list to use the H5FD_LOG
- *		driver defined in this source file.  There are no driver
- *		specific properties.
+ *		driver defined in this source file.
  *
  * Return:	Non-negative on success/Negative on failure
  *
@@ -319,9 +334,9 @@ H5FD_log_term(void)
 herr_t
 H5Pset_fapl_log(hid_t fapl_id, const char *logfile, unsigned long long flags, size_t buf_size)
 {
-    H5FD_log_fapl_t	fa;     /* File access property list information */
-    H5P_genplist_t *plist;      /* Property list pointer */
-    herr_t ret_value;
+    H5FD_log_fapl_t     fa;         /* File access property list information */
+    H5P_genplist_t      *plist;     /* Property list pointer */
+    herr_t              ret_value;  /* Return value */
 
     FUNC_ENTER_API(H5Pset_fapl_log, FAIL)
     H5TRACE4("e", "i*sULz", fapl_id, logfile, flags, buf_size);
@@ -330,7 +345,7 @@ H5Pset_fapl_log(hid_t fapl_id, const char *logfile, unsigned long long flags, si
     if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
 
-    /* Deep copy the logfile string */
+    /* Deep copy the log filename */
     fa.logfile = H5MM_xstrdup(logfile);
 
     fa.flags = flags;
@@ -456,7 +471,7 @@ H5FD_log_fapl_free(void *_fa)
 /*-------------------------------------------------------------------------
  * Function:	H5FD_log_open
  *
- * Purpose:	Create and/or opens a Unix file as an HDF5 file.
+ * Purpose:	Create and/or opens a file as an HDF5 file.
  *
  * Return:	Success:	A pointer to a new file data structure. The
  *				public fields will be initialized by the
@@ -477,7 +492,6 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
     int		fd = (-1);      /* File descriptor */
     int		o_flags;        /* Flags for open() call */
 #ifdef H5_HAVE_WIN32_API
-    HFILE filehandle;
     struct _BY_HANDLE_FILE_INFORMATION fileinfo;
 #endif
 #ifdef H5_HAVE_GETTIMEOFDAY
@@ -574,10 +588,16 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
     file->pos = HADDR_UNDEF;
     file->op = OP_UNKNOWN;
 #ifdef H5_HAVE_WIN32_API
-    filehandle = _get_osfhandle(fd);
-    (void)GetFileInformationByHandle((HANDLE)filehandle, &fileinfo);
-    file->fileindexhi = fileinfo.nFileIndexHigh;
-    file->fileindexlo = fileinfo.nFileIndexLow;
+    file->hFile = (HANDLE)_get_osfhandle(fd);
+    if(INVALID_HANDLE_VALUE == file->hFile)
+        HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to get Windows file handle")
+
+    if(!GetFileInformationByHandle((HANDLE)file->hFile, &fileinfo))
+        HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to get Windows file information")
+
+    file->nFileIndexHigh = fileinfo.nFileIndexHigh;
+    file->nFileIndexLow = fileinfo.nFileIndexLow;
+    file->dwVolumeSerialNumber = fileinfo.dwVolumeSerialNumber;
 #else /* H5_HAVE_WIN32_API */
     file->device = sb.st_dev;
 #ifdef H5_VMS
@@ -659,7 +679,7 @@ done:
 /*-------------------------------------------------------------------------
  * Function:	H5FD_log_close
  *
- * Purpose:	Closes a Unix file.
+ * Purpose:	Closes an HDF5 file.
  *
  * Return:	Success:	0
  *		Failure:	-1, file not closed.
@@ -829,12 +849,14 @@ H5FD_log_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
     FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5FD_log_cmp)
 
 #ifdef H5_HAVE_WIN32_API
-    if(f1->fileindexhi < f2->fileindexhi) HGOTO_DONE(-1)
-    if(f1->fileindexhi > f2->fileindexhi) HGOTO_DONE(1)
+    if(f1->dwVolumeSerialNumber < f2->dwVolumeSerialNumber) HGOTO_DONE(-1)
+    if(f1->dwVolumeSerialNumber > f2->dwVolumeSerialNumber) HGOTO_DONE(1)
 
-    if(f1->fileindexlo < f2->fileindexlo) HGOTO_DONE(-1)
-    if(f1->fileindexlo > f2->fileindexlo) HGOTO_DONE(1)
+    if(f1->nFileIndexHigh < f2->nFileIndexHigh) HGOTO_DONE(-1)
+    if(f1->nFileIndexHigh > f2->nFileIndexHigh) HGOTO_DONE(1)
 
+    if(f1->nFileIndexLow < f2->nFileIndexLow) HGOTO_DONE(-1)
+    if(f1->nFileIndexLow > f2->nFileIndexLow) HGOTO_DONE(1)
 #else
 #ifdef H5_DEV_T_IS_SCALAR
     if(f1->device < f2->device) HGOTO_DONE(-1)
@@ -1014,11 +1036,11 @@ H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t UNUSED type, haddr_t addr)
  * Function:	H5FD_log_get_eof
  *
  * Purpose:	Returns the end-of-file marker, which is the greater of
- *		either the Unix end-of-file or the HDF5 end-of-address
+ *		either the filesystem end-of-file or the HDF5 end-of-address
  *		markers.
  *
  * Return:	Success:	End of file address, the first address past
- *				the end of the "file", either the Unix file
+ *				the end of the "file", either the filesystem file
  *				or the HDF5 file.
  *		Failure:	HADDR_UNDEF
  *
@@ -1091,7 +1113,6 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t UNUSED dxpl_id, haddr_t addr
 	       size_t size, void *buf/*out*/)
 {
     H5FD_log_t		*file = (H5FD_log_t *)_file;
-    ssize_t		nbytes;
     size_t              orig_size = size; /* Save the original size for later */
     haddr_t             orig_addr = addr;
 #ifdef H5_HAVE_GETTIMEOFDAY
@@ -1178,10 +1199,23 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t UNUSED dxpl_id, haddr_t addr
         HDgettimeofday(&timeval_start, NULL);
 #endif /* H5_HAVE_GETTIMEOFDAY */
     while(size > 0) {
+
+        h5_log_io_t         bytes_in        = 0;    /* # of bytes to read       */
+        h5_log_io_ret_t     bytes_read      = -1;   /* # of bytes actually read */ 
+
+        /* Trying to read more bytes than the return type can handle is
+         * undefined behavior in POSIX.
+         */
+        if(size > H5_LOG_MAX_IO_BYTES_g)
+            bytes_in = H5_LOG_MAX_IO_BYTES_g;
+        else
+            bytes_in = (h5_log_io_t)size;
+
         do {
-            nbytes = HDread(file->fd, buf, size);
-        } while(-1 == nbytes && EINTR == errno);
-        if(-1 == nbytes) { /* error */
+            bytes_read = HDread(file->fd, buf, bytes_in);
+        } while(-1 == bytes_read && EINTR == errno);
+
+        if(-1 == bytes_read) { /* error */
             int myerrno = errno;
             time_t mytime = HDtime(NULL);
             HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR);
@@ -1191,18 +1225,20 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t UNUSED dxpl_id, haddr_t addr
 
             HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file read failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', buf = %p, size = %lu, offset = %llu", HDctime(&mytime), file->filename, file->fd, myerrno, HDstrerror(myerrno), buf, (unsigned long)size, (unsigned long long)myoffset);
         } /* end if */
-        if(0 == nbytes) {
+
+        if(0 == bytes_read) {
             /* end of file but not end of format address space */
             HDmemset(buf, 0, size);
             break;
         } /* end if */
-        HDassert(nbytes >= 0);
-        HDassert((size_t)nbytes <= size);
-        H5_CHECK_OVERFLOW(nbytes, ssize_t, size_t);
-        size -= (size_t)nbytes;
-        H5_CHECK_OVERFLOW(nbytes, ssize_t, haddr_t);
-        addr += (haddr_t)nbytes;
-        buf = (char *)buf + nbytes;
+
+        HDassert(bytes_read >= 0);
+        HDassert((size_t)bytes_read <= size);
+        
+        size -= (size_t)bytes_read;
+        addr += (haddr_t)bytes_read;
+        buf = (char *)buf + bytes_read;
+
     } /* end while */
 #ifdef H5_HAVE_GETTIMEOFDAY
     if(file->fa.flags & H5FD_LOG_TIME_READ)
@@ -1278,7 +1314,6 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t UNUSED dxpl_id, haddr_t add
 		size_t size, const void *buf)
 {
     H5FD_log_t		*file = (H5FD_log_t *)_file;
-    ssize_t		nbytes;
     size_t              orig_size = size; /* Save the original size for later */
     haddr_t             orig_addr = addr;
 #ifdef H5_HAVE_GETTIMEOFDAY
@@ -1370,10 +1405,23 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t UNUSED dxpl_id, haddr_t add
         HDgettimeofday(&timeval_start, NULL);
 #endif /* H5_HAVE_GETTIMEOFDAY */
     while(size > 0) {
+
+        h5_log_io_t         bytes_in        = 0;    /* # of bytes to write  */
+        h5_log_io_ret_t     bytes_wrote     = -1;   /* # of bytes written   */ 
+
+        /* Trying to write more bytes than the return type can handle is
+         * undefined behavior in POSIX.
+         */
+        if(size > H5_LOG_MAX_IO_BYTES_g)
+            bytes_in = H5_LOG_MAX_IO_BYTES_g;
+        else
+            bytes_in = (h5_log_io_t)size;
+
         do {
-            nbytes = HDwrite(file->fd, buf, size);
-        } while(-1 == nbytes && EINTR == errno);
-        if(-1 == nbytes) { /* error */
+            bytes_wrote = HDwrite(file->fd, buf, bytes_in);
+        } while(-1 == bytes_wrote && EINTR == errno);
+
+        if(-1 == bytes_wrote) { /* error */
             int myerrno = errno;
             time_t mytime = HDtime(NULL);
             HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR);
@@ -1383,13 +1431,13 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t UNUSED dxpl_id, haddr_t add
 
             HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', buf = %p, size = %lu, offset = %llu", HDctime(&mytime), file->filename, file->fd, myerrno, HDstrerror(myerrno), buf, (unsigned long)size, (unsigned long long)myoffset);
         } /* end if */
-        HDassert(nbytes > 0);
-        HDassert((size_t)nbytes <= size);
-        H5_CHECK_OVERFLOW(nbytes, ssize_t, size_t);
-        size -= (size_t)nbytes;
-        H5_CHECK_OVERFLOW(nbytes, ssize_t, haddr_t);
-        addr += (haddr_t)nbytes;
-        buf = (const char *)buf + nbytes;
+
+        HDassert(bytes_wrote > 0);
+        HDassert((size_t)bytes_wrote <= size);
+
+        size -= (size_t)bytes_wrote;
+        addr += (haddr_t)bytes_wrote;
+        buf = (const char *)buf + bytes_wrote;
     } /* end while */
 #ifdef H5_HAVE_GETTIMEOFDAY
     if(file->fa.flags & H5FD_LOG_TIME_WRITE)
@@ -1478,17 +1526,30 @@ H5FD_log_truncate(H5FD_t *_file, hid_t UNUSED dxpl_id, hbool_t UNUSED closing)
     /* Extend the file to make sure it's large enough */
     if(!H5F_addr_eq(file->eoa, file->eof)) {
 #ifdef H5_HAVE_WIN32_API
-        HFILE filehandle;   /* Windows file handle */
-        LARGE_INTEGER li;   /* 64-bit integer for SetFilePointer() call */
-
-        /* Map the posix file handle to a Windows file handle */
-        filehandle = _get_osfhandle(file->fd);
-
-        /* Translate 64-bit integers into form Windows wants */
-        /* [This algorithm is from the Windows documentation for SetFilePointer()] */
-        li.QuadPart = (LONGLONG)file->eoa;
-        (void)SetFilePointer((HANDLE)filehandle, li.LowPart, &li.HighPart, FILE_BEGIN);
-        if(SetEndOfFile((HANDLE)filehandle) == 0)
+        LARGE_INTEGER   li;         /* 64-bit (union) integer for SetFilePointer() call */
+        DWORD           dwPtrLow;   /* Low-order pointer bits from SetFilePointer()
+                                     * Only used as an error code here.
+                                     */
+        DWORD           dwError;    /* DWORD error code from GetLastError() */
+        BOOL            bError;     /* Boolean error flag */
+
+        /* Windows uses this odd QuadPart union for 32/64-bit portability */
+        li.QuadPart = (__int64)file->eoa;
+
+        /* Extend the file to make sure it's large enough.
+         *
+         * Since INVALID_SET_FILE_POINTER can technically be a valid return value
+         * from SetFilePointer(), we also need to check GetLastError().
+         */
+        dwPtrLow = SetFilePointer(file->hFile, li.LowPart, &li.HighPart, FILE_BEGIN);
+        if(INVALID_SET_FILE_POINTER == dwPtrLow) {
+            dwError = GetLastError();
+            if(dwError != NO_ERROR )
+                HGOTO_ERROR(H5E_FILE, H5E_FILEOPEN, FAIL, "unable to set file pointer")
+        }
+
+        bError = SetEndOfFile(file->hFile);
+        if(0 == bError)
             HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly")
 #else /* H5_HAVE_WIN32_API */
 #ifdef H5_VMS