summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--src/H5FDmpio.c198
-rw-r--r--src/H5FDprivate.h3
-rw-r--r--src/H5Fint.c11
3 files changed, 212 insertions, 0 deletions
diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c
index f594d8e..3fe6ed9 100644
--- a/src/H5FDmpio.c
+++ b/src/H5FDmpio.c
@@ -56,6 +56,7 @@ static char H5FD_mpi_native_g[] = "native";
* library to determine whether the file is empty, truncated, or okay. The MPIO
* driver doesn't bother to keep it updated since it's an expensive operation.
*/
+#if 0 /* original version */ /* JRM */
typedef struct H5FD_mpio_t {
H5FD_t pub; /*public stuff, must be first */
MPI_File f; /*MPIO file handle */
@@ -68,6 +69,26 @@ typedef struct H5FD_mpio_t {
haddr_t last_eoa; /* Last known end-of-address marker */
haddr_t local_eof; /* Local end-of-file address for each process */
} H5FD_mpio_t;
+#else /* modified version */ /* JRM */
+typedef struct H5FD_mpio_t {
+ H5FD_t pub; /*public stuff, must be first */
+ MPI_File f; /*MPIO file handle */
+ MPI_Comm comm; /*communicator */
+ MPI_Info info; /*file information */
+ int mpi_rank; /* This process's rank */
+ int mpi_size; /* Total number of processes */
+ haddr_t eof; /*end-of-file marker */
+ haddr_t eoa; /*end-of-address marker */
+ haddr_t last_eoa; /* Last known end-of-address marker */
+ haddr_t local_eof; /* Local end-of-file address for each process */
+ herr_t do_pre_trunc_barrier; /* hack to allow us to skip */
+ /* unnecessary barriers in */
+ /* H5FD_mpio_trucate() without a VFD */
+ /* API change. This should be removed */
+ /* as soon as be make the necessary */
+ /* VFD API change. */
+} H5FD_mpio_t;
+#endif /* modified version */ /* JRM */
/* Private Prototypes */
@@ -1039,6 +1060,11 @@ H5FD_mpio_open(const char *name, unsigned flags, hid_t fapl_id,
file->eof = H5FD_mpi_MPIOff_to_haddr(size);
file->local_eof = file->eof;
+#if 1 /* JRM */
+ /* Mark initial barriers in H5FD_mpio_truncate() as necessary */
+ file->do_pre_trunc_barrier = TRUE;
+#endif /* JRM */
+
/* Set return value */
ret_value=(H5FD_t*)file;
@@ -1930,6 +1956,7 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_mpio_flush() */
+#if 0 /* original version */
/*-------------------------------------------------------------------------
* Function: H5FD_mpio_truncate
@@ -1996,6 +2023,177 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_mpio_truncate() */
+#else /* modified versin */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_mark_pre_trunc_barrier_unecessary
+ *
+ * Purpose: Hack to allow us to avoid most unnecessary barriers
+ * prior in H5FD_mpio_truncate().
+ *
+ * This function should be deleted when next we modify the
+ * VFD interface. This change should allow us to tell the
+ * truncate function to omit the initial barrier if no
+ * file I/O has occurred since the last barrier.
+ *
+ * Return: void
+ *
+ *
+ * Programmer: John Mainzer
+ * 12/14/17
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+void
+H5FD_mpio_mark_pre_trunc_barrier_unecessary(H5FD_t *_file)
+{
+ H5FD_mpio_t *file = (H5FD_mpio_t*)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(file);
+ HDassert(H5FD_MPIO == file->pub.driver_id);
+
+ file->do_pre_trunc_barrier = FALSE;
+
+ FUNC_LEAVE_NOAPI_VOID
+
+} /* end H5FD_mpio_mark_pre_trunc_barrier_unecessary() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_mpio_truncate
+ *
+ * Purpose: Make certain the file's size matches it's allocated size
+ *
+ * This is a little sticky in the mpio case, as it is not
+ * easy for us to track the current EOF by extracting it from
+ * write calls.
+ *
+ * Instead, we first check to see if the eoa has changed since
+ * the last call to this function. If it has, we call
+ * MPI_File_get_size() to determine the current EOF, and
+ * only call MPI_File_set_size() if this value disagrees
+ * with the current eoa.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: Quincey Koziol
+ * January 31, 2008
+ *
+ * Changes: Heavily reworked to avoid unnecessary MPI_File_set_size()
+ * calls. The hope is that these calls are superfluous in the
+ * typical case, allowing us to avoid truncates most of the
+ * time.
+ *
+ * The basic idea is to query the file system to get the
+ * current eof, and only truncate if the file systems
+ * conception of the eof disagrees with our eoa.
+ *
+ * JRM -- 10/27/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_mpio_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing)
+{
+ H5FD_mpio_t *file = (H5FD_mpio_t*)_file;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#ifdef H5FDmpio_DEBUG
+ if(H5FD_mpio_Debug[(int)'t'])
+ HDfprintf(stdout, "Entering %s\n", FUNC);
+#endif
+ HDassert(file);
+ HDassert(H5FD_MPIO == file->pub.driver_id);
+
+ if ( !H5F_addr_eq(file->eoa, file->last_eoa) ) {
+
+ int mpi_code; /* mpi return code */
+ MPI_Offset size;
+ MPI_Offset needed_eof;
+
+ /* In principle, it is possible for the size returned by the
+ * call to MPI_File_get_size() to depend on whether writes from
+ * all proceeses have completed at the time process 0 makes the
+ * call.
+ *
+ * In practice, most (all?) truncate calls will come after a barrier
+ * and with no interviening writes to the file (with the possible
+ * exception of sueprblock / superblock extension message updates).
+ *
+ * Unfortunately, the current VFD API doesn't let us pass in a
+ * flag indicating whether this particular call is unnecessary.
+ * To work around this, I have added the new function
+ * H5FD_mpio_mark_pre_trunc_barrier_unecessary() allow us to
+ * set a flag in H5FD_mpio_t indicating that we can skip the
+ * barrier.
+ *
+ * This is a pretty ugly hack, but until we revise the VFD API,
+ * it is about the best we can do.
+ */
+ if (file->do_pre_trunc_barrier) {
+ if (MPI_SUCCESS!= (mpi_code=MPI_Barrier(file->comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed(1)", mpi_code)
+ }
+
+ /* Only processor p0 will get the filesize and broadcast it. */
+ if (file->mpi_rank == 0) {
+ if (MPI_SUCCESS != (mpi_code=MPI_File_get_size(file->f, &size)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_get_size failed", mpi_code)
+ } /* end if */
+
+ /* Broadcast file size */
+ if(MPI_SUCCESS != (mpi_code = MPI_Bcast(&size, (int)sizeof(MPI_Offset),
+ MPI_BYTE, 0, file->comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code)
+
+ if(H5FD_mpi_haddr_to_MPIOff(file->eoa, &needed_eof) < 0)
+ HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, \
+ "cannot convert from haddr_t to MPI_Offset")
+
+ if (size != needed_eof) /* eoa != eof. Set eof to eoa */ {
+
+ /* Extend the file's size */
+ if(MPI_SUCCESS != (mpi_code=MPI_File_set_size(file->f, needed_eof)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_File_set_size failed", mpi_code)
+
+ /* In general, we must wait until all processes have finished
+ * the truncate before any process can continue, since it is
+ * possible that a process would write at the end of the
+ * file, and this write would be discarded by the truncate.
+ *
+ * While this is an issue for a user initiated flush, it may
+ * not be an issue at file close. If so, we may be able to
+ * optimize out the following barrier in that case.
+ */
+ if(MPI_SUCCESS != (mpi_code = MPI_Barrier(file->comm)))
+ HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code)
+ }
+
+ /* Update the 'last' eoa value */
+ file->last_eoa = file->eoa;
+ } /* end if */
+
+done:
+ file->do_pre_trunc_barrier = TRUE;
+
+#ifdef H5FDmpio_DEBUG
+ if(H5FD_mpio_Debug[(int)'t'])
+ HDfprintf(stdout, "Leaving %s\n", FUNC);
+#endif
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_mpio_truncate() */
+
+#endif /* modified version */
+
/*-------------------------------------------------------------------------
* Function: H5FD_mpio_mpi_rank
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index e758951..f7be327 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -203,6 +203,9 @@ H5_DLL int H5FD_mpi_get_rank(const H5FD_t *file);
H5_DLL int H5FD_mpi_get_size(const H5FD_t *file);
H5_DLL MPI_Comm H5FD_mpi_get_comm(const H5FD_t *_file);
H5_DLL herr_t H5FD_get_mpi_info(H5FD_t *file, void** file_info);
+#if 1 /* JRM */
+H5_DLL void H5FD_mpio_mark_pre_trunc_barrier_unecessary(H5FD_t *_file);
+#endif /* JRM */
#endif /* H5_HAVE_PARALLEL */
#endif /* !_H5FDprivate_H */
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 8212eb5..25df964 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -1532,6 +1532,17 @@ H5F__flush_phase2(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closi
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush metadata cache")
/* Truncate the file to the current allocated size */
+#if 1 /* JRM */
+#ifdef H5_HAVE_PARALLEL
+ if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) {
+ /* Since we just returned from a call to H5AC_flush(), we just
+ * passed through a barrier. Hence we can skip the barrier on
+ * entry to the mpio file driver call below.
+ */
+ H5FD_mpio_mark_pre_trunc_barrier_unecessary(f->shared->lf);
+ }
+#endif /* H5_HAVE_PARALLEL */
+#endif /* JRM */
if(H5FD_truncate(f->shared->lf, meta_dxpl_id, closing) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed")