[svn-r6371] Purpose:

    Update
Description:
    Changes:

        - Added support for the server dumping metadata writes to a
          client.
        - Some of the code wasn't handling allocated buffers correctly
          (freeing twice).
        - Modified server so that it handles metadata only.
Platforms tested:
    Linux

4 files changed, 597 insertions, 280 deletions

diff --git a/src/H5FP.c b/src/H5FP.c
index f45c70e..16e210b 100644
--- a/src/H5FP.c
+++ b/src/H5FP.c
@@ -213,19 +213,13 @@ done:
 /*
  * Function:    H5FP_send_metadata
  * Purpose:     Send a string of metadata to a process.
- *
- *              NOTE: You should never call this function directly!!
- *              There's special setup for sending a string to a processor
- *              which needs to occur first. The H5FP_request_* and
- *              H5FP_reply_* functions take care of this for you.
- *
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 30. July, 2002
  * Modifications:
  */
 herr_t
-H5FP_send_metadata(const char *mdata, int len, int rank)
+H5FP_send_metadata(const char *mdata, int len, int to)
 {
     herr_t ret_value = SUCCEED;
 
@@ -235,7 +229,7 @@ H5FP_send_metadata(const char *mdata, int len, int rank)
     assert(len);
 
     /* casts the CONST away: Okay */
-    if (MPI_Send((void *)mdata, len, MPI_BYTE, rank, H5FP_TAG_METADATA, H5FP_SAP_COMM)
+    if (MPI_Send((void *)mdata, len, MPI_BYTE, to, H5FP_TAG_METADATA, H5FP_SAP_COMM)
             != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Send failed");
 
@@ -244,6 +238,46 @@ done:
 }
 
 /*
+ * Function:    H5FP_read_metadata
+ * Purpose:     Read a string of metadata from process FROM.
+ * Return:      Success:    SUCCEED
+ *              Failure:    FAIL
+ * Programmer:  Bill Wendling, 31. January, 2003
+ * Modifications:
+ */
+herr_t
+H5FP_read_metadata(char **mdata, int len, int from)
+{
+    MPI_Status status;
+    herr_t ret_value = SUCCEED;
+    int mrc;
+
+    FUNC_ENTER_NOAPI(H5FP_read_metadata, FAIL);
+
+    /* check args */
+    assert(mdata);
+
+    /*
+     * There is metadata associated with this request. Get it as a
+     * string (requires another read).
+     */
+    if ((*mdata = (char *)HDmalloc((size_t)len + 1)) == NULL)
+        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "out of memory");
+
+    HDmemset(*mdata, 0, (size_t)len + 1);
+
+    if ((mrc = MPI_Recv(*mdata, len, MPI_BYTE, from, H5FP_TAG_METADATA,
+                        H5FP_SAP_COMM, &status)) != MPI_SUCCESS) {
+        HDfree(*mdata);
+        *mdata = NULL;
+        HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
+    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value);
+}
+
+/*
  *===----------------------------------------------------------------------===
  *                            Private Functions
  *===----------------------------------------------------------------------===
@@ -271,7 +305,7 @@ H5FP_commit_sap_datatypes(void)
     FUNC_ENTER_NOAPI(H5FP_commit_sap_datatypes, FAIL);
 
     /* Commit the H5FP_request_t datatype */
-    block_length[0] = 8;
+    block_length[0] = 9;
     block_length[1] = 1;
     block_length[2] = sizeof(req.oid);
     old_types[0] = MPI_INT;
@@ -306,18 +340,14 @@ H5FP_commit_sap_datatypes(void)
     /* Commit the H5FP_read_t datatype */
     block_length[0] = 6;
     block_length[1] = 1;
-    block_length[2] = 1;
     old_types[0] = MPI_INT;
     old_types[1] = HADDR_AS_MPI_TYPE;
-    old_types[2] = MPI_LONG_LONG;
     MPI_Address(&sap_read.req_id, &displs[0]);
     MPI_Address(&sap_read.addr, &displs[1]);
-    MPI_Address(&sap_read.size, &displs[2]);
-    displs[2] -= displs[1];
     displs[1] -= displs[0];
     displs[0] -= displs[0];
 
-    if (MPI_Type_struct(3, block_length, displs, old_types,
+    if (MPI_Type_struct(2, block_length, displs, old_types,
                         &H5FP_read_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed");
 
diff --git a/src/H5FPclient.c b/src/H5FPclient.c
index d1a029c..0d0b09e 100644
--- a/src/H5FPclient.c
+++ b/src/H5FPclient.c
@@ -19,6 +19,7 @@
 #include "H5ACprivate.h"        /* Metadata Cache                       */
 #include "H5Dprivate.h"         /* Dataset Functions                    */
 #include "H5Eprivate.h"         /* Error Handling                       */
+#include "H5Fprivate.h"         /* Files                                */
 #include "H5Gpkg.h"             /* Group functions                      */
 #include "H5Iprivate.h"         /* ID Functions                         */
 #include "H5MMprivate.h"        /* Memory allocation                    */
@@ -39,8 +40,13 @@ static int interface_initialize_g = 0;
 
 /* local functions */
 static unsigned H5FP_gen_request_id(void);
+static herr_t H5FP_dump_to_file(H5F_t *file, H5FP_read *sap_read);
 
-/** Public Library (non-API) Functions **/
+/*
+ *===----------------------------------------------------------------------===
+ *                    Public Library (non-API) Functions
+ *===----------------------------------------------------------------------===
+ */
 
 /*
  * Function:    H5FP_request_open
@@ -78,6 +84,10 @@ H5FP_request_open(const char *mdata, int md_size, H5FP_obj_t obj_type,
     HDmemset(&mpi_status, 0, sizeof(MPI_Status));
 
     if (H5FP_my_rank == H5FP_capt_rank) {
+        /*
+         * The captain process sends the information about the file to
+         * the SAP.
+         */
         HDmemset(&req, 0, sizeof(req));
         req.req_type = H5FP_REQ_OPEN;
         req.req_id = H5FP_gen_request_id();
@@ -98,9 +108,6 @@ H5FP_request_open(const char *mdata, int md_size, H5FP_obj_t obj_type,
                         H5FP_TAG_FILE_ID, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
-    printf("%u: New File ID == %u\n", H5FP_my_rank, *file_id);
-    fflush(stdout);
-
 done:
     *req_id = req.req_id;
     FUNC_LEAVE_NOAPI(ret_value);
@@ -127,9 +134,11 @@ H5FP_request_lock(unsigned file_id, unsigned char *obj_oid,
 
     FUNC_ENTER_NOAPI(H5FP_request_lock, FAIL);
 
+    /* check args */
     assert(obj_oid);
     assert(req_id);
     assert(status);
+
     HDmemset(&req, 0, sizeof(req));
 
     *status = H5FP_STATUS_OK;
@@ -166,6 +175,9 @@ H5FP_request_lock(unsigned file_id, unsigned char *obj_oid,
     }
 
 done:
+    if (ret_value == FAIL)
+        *status = H5FP_STATUS_LOCK_FAILED;
+
     *req_id = req.req_id;
     FUNC_LEAVE_NOAPI(ret_value);
 }
@@ -177,8 +189,8 @@ done:
  *              returned in a pointer supplied by the user. The status
  *              of the SAP is returned to the user in the supplied STATUS
  *              pointer.
- * Return:      Success: The request ID.
- *              Failure: -1
+ * Return:      Success:    SUCCEED
+ *              Failure:    FAIL
  * Programmer:  Bill Wendling, 30. July, 2002
  * Modifications:
  */
@@ -192,6 +204,7 @@ H5FP_request_release_lock(unsigned file_id, unsigned char *obj_oid,
 
     FUNC_ENTER_NOAPI(H5FP_request_release_lock, FAIL);
 
+    /* check args */
     assert(obj_oid);
     assert(req_id);
     assert(status);
@@ -236,64 +249,95 @@ done:
 }
 
 /*
- * Function:    H5FP_request_write_metadata
- * Purpose:     Tell the SAP that we want to change a piece of metadata
- *              associated with the file. The request ID is returned in a
- *              pointer supplied by the user.
+ * Function:    H5FP_request_read_metadata
+ * Purpose:     Read a piece of metadata from the SAP. That is, if the
+ *              SAP has access to that metadata. If not, then we'll need
+ *              to read it from disk.
+ *
+ *              This function has the potential of causing the process to
+ *              act as a dumper for the SAP's metadata. Places which call
+ *              this function and check the STATUS variable should take
+ *              this into account.
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
 herr_t
-H5FP_request_write_metadata(unsigned file_id, unsigned char *obj_oid,
-                            H5FP_obj_t obj_type, H5AC_subid_t type_id,
-                            haddr_t addr, int mdata_size, const char *mdata,
-                            unsigned *req_id, H5FP_status_t *status)
+H5FP_request_read_metadata(H5F_t *file, unsigned file_id, H5FP_obj_t obj_type,
+                           H5AC_subid_t type_id, haddr_t addr, size_t size,
+                           uint8_t **buf, unsigned *req_id,
+                           H5FP_status_t *status)
 {
-    H5FP_reply sap_reply;
-    MPI_Status mpi_status;
     H5FP_request req;
+    H5FP_read sap_read;     /* metadata info read from the SAP's cache */
     herr_t ret_value = SUCCEED;
+    MPI_Status mpi_status;
     int mrc;
 
-    FUNC_ENTER_NOAPI(H5FP_request_change, FAIL);
+    FUNC_ENTER_NOAPI(H5FP_request_read_metadata, FAIL);
 
-    assert(mdata);
-    assert(obj_oid);
+    /* check args */
+    assert(file);
+    assert(buf);
     assert(req_id);
     assert(status);
 
     HDmemset(&req, 0, sizeof(req));
 
-    H5FP_COPY_OID(req.oid, obj_oid);
-    req.req_type = H5FP_REQ_WRITE;
+    *status = H5FP_STATUS_OK;
+    req.req_type = H5FP_REQ_READ;
     req.req_id = H5FP_gen_request_id();
-    req.proc_rank = H5FP_my_rank;
     req.file_id = file_id;
+    req.proc_rank = H5FP_my_rank;
     req.obj_type = obj_type;
     req.type_id = type_id;
     req.addr = addr;
-    req.md_size = mdata_size;
 
     if ((mrc = MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
-    /* The first MPI_Send will have been sent before this one will be read. */
-    if (H5FP_send_metadata(mdata, mdata_size, (int)H5FP_sap_rank) != SUCCEED)
-        HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL, "can't send metadata to server");
-
     HDmemset(&mpi_status, 0, sizeof(mpi_status));
 
-    if ((mrc = MPI_Recv(&sap_reply, 1, H5FP_reply_t, (int)H5FP_sap_rank,
-                        H5FP_TAG_REPLY, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
+    if ((mrc = MPI_Recv(&sap_read, 1, H5FP_read_t, (int)H5FP_sap_rank, H5FP_TAG_READ,
+                        H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
 
-    *status = sap_reply.status;
+    *status = sap_read.status;
 
-    if (sap_reply.status != H5FP_STATUS_OK)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCHANGE, FAIL, "can't write metadata to server");
+    switch (*status) {
+    case H5FP_STATUS_OK:
+        /* use the info in the H5FP_read_t structure to update the metadata */
+        HDmemset(*buf, '\0', size);
+        HDmemset(&mpi_status, 0, sizeof(mpi_status));
+
+        if ((mrc = MPI_Recv(*buf, (int)sap_read.md_size, MPI_BYTE, (int)H5FP_sap_rank,
+                            H5FP_TAG_METADATA, H5FP_SAP_COMM,
+                            &mpi_status)) != MPI_SUCCESS)
+            HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
+
+        break;
+    case H5FP_STATUS_DUMPING:
+        /*
+         * Collect the metadata updates from the SAP and write them to
+         * the file. We fall through because at this point the metadata
+         * won't be cached on the server anymore.
+         */
+        if (H5FP_dump_to_file(file, &sap_read) == FAIL)
+            HGOTO_ERROR(H5E_FPHDF5, H5E_WRITEERROR, FAIL,
+                        "can't write metadata update to file");
+
+        /* FALLTHROUGH */
+    case H5FP_STATUS_MDATA_NOT_CACHED:
+        /*
+         * The metadata wasn't in the SAP's cache. Should read from disk
+         * now.
+         */
+        break;
+    default:
+        break;
+    }
 
 done:
     *req_id = req.req_id;
@@ -301,74 +345,103 @@ done:
 }
 
 /*
- * Function:    H5FP_request_read_metadata
- * Purpose:     Read a piece of metadata from the SAP. That is, if the
- *              SAP has access to that metadata. If not, then we'll need
- *              to read it from disk.
+ * Function:    H5FP_request_write_metadata
+ * Purpose:     Tell the SAP that we want to change a piece of metadata
+ *              associated with the file. The request ID is returned in a
+ *              pointer supplied by the user.
+ *
+ *              This function has the potential of causing the process to
+ *              act as a dumper for the SAP's metadata. Places which call
+ *              this function and check the STATUS variable should take
+ *              this into account.
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
 herr_t
-H5FP_request_read_metadata(unsigned file_id, H5FP_obj_t obj_type,
-                           H5AC_subid_t type_id, haddr_t addr, size_t size,
-                           uint8_t **buf, unsigned *req_id,
-                           H5FP_status_t *status)
+H5FP_request_write_metadata(H5F_t *file, unsigned file_id, uint8_t *obj_oid,
+                            H5FP_obj_t obj_type, H5AC_subid_t type_id,
+                            haddr_t addr, int mdata_size, const char *mdata,
+                            unsigned *req_id, H5FP_status_t *status)
 {
-    H5FP_request req;
+    H5FP_reply sap_reply;
     H5FP_read sap_read;     /* metadata info read from the SAP's cache */
-    herr_t ret_value = SUCCEED;
     MPI_Status mpi_status;
+    H5FP_request req;
+    herr_t ret_value = SUCCEED;
     int mrc;
 
-    FUNC_ENTER_NOAPI(H5FP_request_read_metadata, FAIL);
+    FUNC_ENTER_NOAPI(H5FP_request_change, FAIL);
 
+    /* check args */
+    assert(file);
+    assert(mdata);
+    assert(obj_oid);
     assert(req_id);
     assert(status);
 
     HDmemset(&req, 0, sizeof(req));
 
-    *status = H5FP_STATUS_OK;
-    req.req_type = H5FP_REQ_READ;
+    H5FP_COPY_OID(req.oid, obj_oid);
+    req.req_type = H5FP_REQ_WRITE;
     req.req_id = H5FP_gen_request_id();
-    req.file_id = file_id;
     req.proc_rank = H5FP_my_rank;
+    req.file_id = file_id;
     req.obj_type = obj_type;
     req.type_id = type_id;
     req.addr = addr;
+    req.md_size = mdata_size;
 
     if ((mrc = MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
+    /* The first MPI_Send will have been sent before this one will be read. */
+    if (H5FP_send_metadata(mdata, mdata_size, (int)H5FP_sap_rank) != SUCCEED)
+        HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL, "can't send metadata to server");
+
     HDmemset(&mpi_status, 0, sizeof(mpi_status));
 
-    if ((mrc = MPI_Recv(&sap_read, 1, H5FP_read_t, (int)H5FP_sap_rank, H5FP_TAG_READ,
-                        H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
+    if ((mrc = MPI_Recv(&sap_reply, 1, H5FP_reply_t, (int)H5FP_sap_rank,
+                        H5FP_TAG_REPLY, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
 
-    if (sap_read.status == H5FP_STATUS_OK) {
-        /* use the info in the H5FP_read_t structure to update the metadata */
-        memset(*buf, '\0', size);
-        HDmemset(&mpi_status, 0, sizeof(mpi_status));
+    *status = sap_reply.status;
 
-        if ((mrc = MPI_Recv(*buf, sap_read.md_size, MPI_BYTE, (int)H5FP_sap_rank,
-                            H5FP_TAG_METADATA, H5FP_SAP_COMM,
-                            &mpi_status)) != MPI_SUCCESS)
-            HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
-    } else {
+    switch (*status) {
+    case H5FP_STATUS_DUMPING:
         /*
-         * FIXME!!!
+         * Collect the metadata updates from the SAP and write them to
+         * the file. The function which sends us the dumping data sends
+         * it to us as an H5FP_read object instead of the H5FP_reply
+         * object we got above. So we need this "extra" read.
          *
-         * The metadata wasn't in the SAP's cache. Should read from disk
-         * now.
+         * FIXME: This is probably too much of a hack and could be fixed
+         * for read/write/closing instances...
          */
+        if ((mrc = MPI_Recv(&sap_read, 1, H5FP_read_t, (int)H5FP_sap_rank,
+                            H5FP_TAG_READ, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS) {
+            *status = H5FP_STATUS_DUMPING_FAILED;
+            HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
+        }
+
+        if (H5FP_dump_to_file(file, &sap_read) == FAIL) {
+            *status = H5FP_STATUS_DUMPING_FAILED;
+            HGOTO_ERROR(H5E_FPHDF5, H5E_WRITEERROR, FAIL,
+                        "can't write metadata update to file");
+        }
+
+        break;
+    case H5FP_STATUS_OK:
+        /* Nothing to do... */
+        break;
+    default:
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCHANGE, FAIL, "can't write metadata to server");
     }
 
 done:
     *req_id = req.req_id;
-    *status = sap_read.status;
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
@@ -378,20 +451,32 @@ done:
  *              made on the file and then close the file. The request ID
  *              is returned in a pointer passed to the function by the
  *              user.
+ *
+ *              This function has the potential of causing the process to
+ *              act as a dumper for the SAP's metadata. Places which call
+ *              this function and check the STATUS variable should take
+ *              this into account.
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
 herr_t
-H5FP_request_close(unsigned file_id, unsigned *req_id)
+H5FP_request_close(H5F_t *file, unsigned file_id, unsigned *req_id,
+                   H5FP_status_t *status)
 {
+    H5FP_reply sap_reply;
+    H5FP_read sap_read;     /* metadata info read from the SAP's cache */
     H5FP_request req;
+    MPI_Status mpi_status;
     int ret_value = SUCCEED, mrc;
 
     FUNC_ENTER_NOAPI(H5FP_request_close, FAIL);
 
+    /* check args */
+    assert(file);
     assert(req_id);
+    assert(status);
 
     HDmemset(&req, 0, sizeof(req));
 
@@ -404,9 +489,47 @@ H5FP_request_close(unsigned file_id, unsigned *req_id)
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
-    *req_id = req.req_id;
+    HDmemset(&mpi_status, 0, sizeof(mpi_status));
+
+    if ((mrc = MPI_Recv(&sap_reply, 1, H5FP_reply_t, (int)H5FP_sap_rank,
+                        H5FP_TAG_REPLY, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
+
+    *status = sap_reply.status;
+
+    switch (*status) {
+    case H5FP_STATUS_DUMPING:
+        /*
+         * Collect the metadata updates from the SAP and write them to
+         * the file. The function which sends us the dumping data sends
+         * it to us as an H5FP_read object instead of the H5FP_reply
+         * object we got above. So we need this "extra" read.
+         *
+         * FIXME: This is probably too much of a hack and could be fixed
+         * for read/write/closing instances...
+         */
+        if ((mrc = MPI_Recv(&sap_read, 1, H5FP_read_t, (int)H5FP_sap_rank,
+                            H5FP_TAG_READ, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS) {
+            *status = H5FP_STATUS_DUMPING_FAILED;
+            HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
+        }
+
+        if (H5FP_dump_to_file(file, &sap_read) == FAIL) {
+            *status = H5FP_STATUS_DUMPING_FAILED;
+            HGOTO_ERROR(H5E_FPHDF5, H5E_WRITEERROR, FAIL,
+                        "can't write metadata update to file");
+        }
+
+        break;
+    case H5FP_STATUS_OK:
+        /* Nothing to do... */
+        break;
+    default:
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCHANGE, FAIL, "can't write metadata to server");
+    }
 
 done:
+    *req_id = req.req_id;
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
@@ -428,9 +551,55 @@ static unsigned
 H5FP_gen_request_id()
 {
     static unsigned int i = 0;
-
     FUNC_ENTER_NOINIT(H5FP_gen_request_id);
     FUNC_LEAVE_NOAPI(i++);
 }
 
+/*
+ * Function:    H5FP_dump_to_file
+ * Purpose:     Dump the updated metadata to the file. 
+ * Return:      Success:    SUCCEED
+ *              Failure:    FAIL
+ * Programmer:  Bill Wendling, 03. February 2003
+ * Modifications:
+ */
+static herr_t
+H5FP_dump_to_file(H5F_t *file, H5FP_read *sap_read)
+{
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOINIT(H5FP_dump_to_file);
+
+    /* check args */
+    assert(file);
+
+    /*
+     * At this point, we've received a message saying that the SAP is
+     * dumping info to us. There's a metadata read waiting for us right
+     * now...
+     */
+    do {
+        MPI_Status mpi_status;
+        int mrc;
+        char *mdata;
+
+        if (H5FP_read_metadata(&mdata, (int)sap_read->md_size,
+                               (int)H5FP_sap_rank) != FAIL) {
+            /* FIXME: Write to the file with this metadata */
+            HDfree(mdata);
+        } else {
+            /* FIXME: Error */
+        }
+
+        HDmemset(&mpi_status, 0, sizeof(mpi_status));
+
+        if ((mrc = MPI_Recv(sap_read, 1, H5FP_read_t, (int)H5FP_sap_rank, H5FP_TAG_READ,
+                            H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
+            HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
+    } while (sap_read->status != H5FP_STATUS_DUMPING_FINISHED);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value);
+}
+
 #endif  /* H5_HAVE_FPHDF5 */
diff --git a/src/H5FPprivate.h b/src/H5FPprivate.h
index 1e5c3aa..dcd79c2 100644
--- a/src/H5FPprivate.h
+++ b/src/H5FPprivate.h
@@ -136,6 +136,11 @@ typedef enum sap_status {
     /* For read requests */
     H5FP_STATUS_MDATA_NOT_CACHED,
 
+    /* For dumping data to client */
+    H5FP_STATUS_DUMPING,
+    H5FP_STATUS_DUMPING_FINISHED,
+    H5FP_STATUS_DUMPING_FAILED,
+
     /* Out of memory error */
     H5FP_STATUS_OOM,
 
@@ -156,8 +161,9 @@ typedef struct {
     unsigned        file_id;    /* SAP's file ID for the specific file      */
     H5FP_obj_t      obj_type;   /* Type of the object                       */
     H5FP_lock_t     rw_lock;    /* Indicates read or write lock             */
+    H5FD_mem_t      mem_type;   /* Type of memory updated, if req'd         */
     H5AC_subid_t    type_id;    /* Type of metadata                         */
-    int             md_size;    /* Size of the metadata sent in next msg    */
+    unsigned        md_size;    /* Size of the metadata sent in next msg    */
     haddr_t         addr;       /* Address of the metadata                  */
     unsigned char   oid[H5R_OBJ_REF_BUF_SIZE]; /* Buffer to store OID of object */
 } H5FP_request;
@@ -185,9 +191,8 @@ typedef struct {
     H5FP_status_t   status;     /* Status of the request                    */
     H5FD_mem_t      mem_type;   /* Type of memory updated, if req'd         */
     H5AC_subid_t    type_id;    /* Type of metadata                         */
-    int             md_size;    /* Size of the metadata sent in next msg    */
+    unsigned        md_size;    /* Size of the metadata sent in next msg    */
     haddr_t         addr;       /* Address of the metadata                  */
-    hsize_t         size;       /* Size of memory updated, if req'd         */
 } H5FP_read;
 
 extern MPI_Datatype H5FP_read_t; /* MPI datatype for the H5FP_read obj      */
@@ -209,7 +214,8 @@ extern "C" {
 #endif  /* __cplusplus */
 
 /* NOTE: Don't use this function explicitly!! */
-extern herr_t H5FP_send_metadata(const char *mdata, int len, int rank);
+extern herr_t H5FP_send_metadata(const char *mdata, int len, int to);
+extern herr_t H5FP_read_metadata(char **mdata, int len, int from);
 
 /* Start the SAP */
 extern herr_t H5FP_sap_receive_loop(void);
@@ -223,15 +229,18 @@ extern herr_t H5FP_request_lock(unsigned sap_file_id, unsigned char *mdata,
 extern herr_t H5FP_request_release_lock(unsigned sap_file_id, unsigned char *mdata,
                                         int last, unsigned *req_id,
                                         H5FP_status_t *status);
-extern herr_t H5FP_request_write_metadata(unsigned sap_file_id, unsigned char *obj_oid,
-                                          H5FP_obj_t obj_type, H5AC_subid_t type_id,
-                                          haddr_t addr, int mdata_len, const char *mdata,
-                                          unsigned *req_id, H5FP_status_t *status);
-extern herr_t H5FP_request_read_metadata(unsigned sap_file_id, H5FP_obj_t obj_type,
-                                         H5AC_subid_t type_id, haddr_t addr,
-                                         size_t size, uint8_t **buf, unsigned *req_id,
+extern herr_t H5FP_request_read_metadata(H5F_t *file, unsigned sap_file_id,
+                                         H5FP_obj_t obj_type, H5AC_subid_t type_id,
+                                         haddr_t addr, size_t size,
+                                         uint8_t **buf, unsigned *req_id,
                                          H5FP_status_t *status);
-extern herr_t H5FP_request_close(unsigned sap_file_id, unsigned *req_id);
+extern herr_t H5FP_request_write_metadata(H5F_t *file, unsigned sap_file_id,
+                                          unsigned char *obj_oid, H5FP_obj_t obj_type,
+                                          H5AC_subid_t type_id, haddr_t addr,
+                                          int mdata_len, const char *mdata,
+                                          unsigned *req_id, H5FP_status_t *status);
+extern herr_t H5FP_request_close(H5F_t *file, unsigned sap_file_id, unsigned *req_id,
+                                 H5FP_status_t *status);
 
 #ifdef __cplusplus
 }
diff --git a/src/H5FPserver.c b/src/H5FPserver.c
index e073feb..6f44933 100644
--- a/src/H5FPserver.c
+++ b/src/H5FPserver.c
@@ -53,6 +53,7 @@ typedef struct {
 } H5FP_object_lock;
 
 typedef struct {
+    H5FD_mem_t      mem_type;   /* type of memory updated, if req'd         */
     H5AC_subid_t    type_id;    /* id of the type of this metadata          */
     H5FP_obj_t      obj_type;   /* type of object modified                  */
     haddr_t         addr;       /* address of the metadata                  */
@@ -64,11 +65,18 @@ typedef struct {
     unsigned        file_id;    /* the file id the SAP keeps per file       */
     char           *filename;   /* the filename - of dubious use            */
     int             closing;    /* we're closing the file - no more changes */
+    unsigned        num_mods;   /* number of mdata writes outstanding       */
     H5TB_TREE      *mod_tree;   /* a tree of metadata updates done          */
     H5TB_TREE      *locks;      /* a tree of locks on objects in the file   */
 } H5FP_file_info;
 
-static H5TB_TREE *fs_tree;
+/*
+ * This marks the point at which we want to dump all of the metadata
+ * write to a process so that that process can write them to the file.
+ */
+#define H5FP_MDATA_CACHE_HIGHWATER_MARK     1024
+
+static H5TB_TREE *file_info_tree;
 
 /* local functions */
 static herr_t H5FP_sap_receive(H5FP_request *req, int source, int tag, char **buf);
@@ -77,14 +85,18 @@ static herr_t H5FP_sap_receive(H5FP_request *req, int source, int tag, char **bu
 static unsigned H5FP_gen_sap_file_id(void);
 
     /* local functions for handling object locks */
-static int H5FP_object_lock_cmp(H5FP_object_lock *o1, H5FP_object_lock *o2, int cmparg);
+static int H5FP_object_lock_cmp(H5FP_object_lock *o1,
+                                H5FP_object_lock *o2,
+                                int cmparg);
 static H5FP_object_lock *H5FP_new_object_lock(const unsigned char *oid,
                                               unsigned rank,
                                               H5FP_obj_t obj_type,
                                               H5FP_lock_t rw_lock);
 static herr_t H5FP_free_object_lock(H5FP_object_lock *ol);
-static H5FP_object_lock *H5FP_find_object_lock(H5FP_file_info *fs, unsigned char *oid);
-static herr_t H5FP_remove_object_lock_from_list(H5FP_file_info *fs, H5FP_object_lock *ol);
+static H5FP_object_lock *H5FP_find_object_lock(H5FP_file_info *info,
+                                               unsigned char *oid);
+static herr_t H5FP_remove_object_lock_from_list(H5FP_file_info *info,
+                                                H5FP_object_lock *ol);
 
     /* local file information handling functions */
 static herr_t H5FP_add_new_file_info_to_list(unsigned file_id, char *filename);
@@ -92,21 +104,26 @@ static int H5FP_file_info_cmp(H5FP_file_info *k1, H5FP_file_info *k2, int cmparg
 static H5FP_file_info *H5FP_new_file_info_node(unsigned file_id, char *filename);
 static H5FP_file_info *H5FP_find_file_info(unsigned file_id);
 static herr_t H5FP_remove_file_id_from_list(unsigned file_id);
-static herr_t H5FP_free_file_info_node(H5FP_file_info *fs);
+static herr_t H5FP_free_file_info_node(H5FP_file_info *info);
 
     /* local file modification structure handling functions */
 static H5FP_mdata_mod *H5FP_new_file_mod_node(unsigned rank,
+                                              H5FD_mem_t mem_type,
                                               H5AC_subid_t type_id,
                                               haddr_t addr,
-                                              int md_size,
+                                              unsigned md_size,
                                               char *metadata);
-static herr_t H5FP_free_mod_node(H5FP_mdata_mod *fs);
+static herr_t H5FP_free_mod_node(H5FP_mdata_mod *info);
 
     /* local request handling functions */
-static herr_t H5FP_sap_handle_open_request(H5FP_request req, char *mdata, int md_size);
+static herr_t H5FP_sap_handle_open_request(H5FP_request req,
+                                           char *mdata,
+                                           unsigned md_size);
 static herr_t H5FP_sap_handle_lock_request(H5FP_request req);
 static herr_t H5FP_sap_handle_release_lock_request(H5FP_request req);
-static herr_t H5FP_sap_handle_write_request(H5FP_request req, char *mdata, int md_size);
+static herr_t H5FP_sap_handle_write_request(H5FP_request req,
+                                            char *mdata,
+                                            unsigned md_size);
 static herr_t H5FP_sap_handle_read_request(H5FP_request req);
 static herr_t H5FP_sap_handle_close_request(H5FP_request req);
 
@@ -136,8 +153,8 @@ H5FP_sap_receive_loop(void)
     FUNC_ENTER_NOAPI(H5FP_sap_receive_loop, FAIL);
 
     /* Create the file structure tree. */
-    if ((fs_tree = H5TB_dmake((H5TB_cmp_t)H5FP_file_info_cmp,
-                              sizeof(H5FP_file_info), FALSE)) == NULL)
+    if ((file_info_tree = H5TB_dmake((H5TB_cmp_t)H5FP_file_info_cmp,
+                                     sizeof(H5FP_file_info), FALSE)) == NULL)
         HGOTO_ERROR(H5E_FPHDF5, H5E_CANTMAKETREE, FAIL, "cannot make TBBT tree");
 
     for (;;) {
@@ -178,6 +195,9 @@ H5FP_sap_receive_loop(void)
             HGOTO_ERROR(H5E_FPHDF5, H5E_ARGS, FAIL, "invalid request type");
         }
 
+        /*
+         * If the above calls didn't succeed, free the buffer
+         */
         if (hrc != SUCCEED)
             HDfree(buf);
     }
@@ -191,8 +211,9 @@ done:
  * Purpose:     Receive a message from SOURCE with the given TAG. The REQ
  *              object passed in as a pointer is filled by the MPI_Recv
  *              function.
- * Return:      Pointer to string passed in, if one was sent. As well as
- *              the SAP_request object.
+ * Return:      Success:    Pointer to string passed in, if one was sent.
+ *                          As well as the SAP_request object.
+ *              Failure:    FAIL
  * Programmer:  Bill Wendling, 17. September, 2002
  * Modifications:
  */
@@ -211,23 +232,9 @@ H5FP_sap_receive(H5FP_request *req, int source, int tag, char **buf)
                         H5FP_SAP_COMM, &status)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
 
-    if (buf && req->md_size) {
-        /*
-         * There is metadata associated with this request. Get it as a
-         * string (requires another read).
-         */
-        if ((*buf = (char *)HDmalloc((size_t)req->md_size + 1)) == NULL)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "out of memory");
-
-        HDmemset(*buf, 0, (size_t)req->md_size + 1);
-
-        if ((mrc = MPI_Recv(*buf, req->md_size, MPI_BYTE, (int)req->proc_rank,
-                            H5FP_TAG_METADATA, H5FP_SAP_COMM, &status)) != MPI_SUCCESS) {
-            HDfree(*buf);
-            *buf = NULL;
-            HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
-        }
-    }
+    if (buf && req->md_size)
+        if (H5FP_read_metadata(buf, (int)req->md_size, (int)req->proc_rank) == FAIL)
+            HGOTO_ERROR(H5E_FPHDF5, H5E_CANTRECV, FAIL, "can't read metadata from process");
 
 done:
     FUNC_LEAVE_NOAPI(ret_value);
@@ -240,9 +247,10 @@ done:
  * Programmer:  Bill Wendling, 09. September 2002
  * Modifications:
  */
-static int H5FP_object_lock_cmp(H5FP_object_lock *o1,
-                                H5FP_object_lock *o2,
-                                int UNUSED cmparg)
+static int
+H5FP_object_lock_cmp(H5FP_object_lock *o1,
+                     H5FP_object_lock *o2,
+                     int UNUSED cmparg)
 {
     FUNC_ENTER_NOINIT(H5FP_object_lock_cmp);
     assert(o1);
@@ -304,27 +312,28 @@ H5FP_free_object_lock(H5FP_object_lock *ol)
 /*
  * Function:    H5FP_find_object_lock
  * Purpose:     Find the object lock for the given OID if there is one.
- * Return:      Pointer to the object or NULL.
+ * Return:      Success:    Pointer to the object
+ *              Failure:    NULL
  * Programmer:  Bill Wendling, 09. September 2002
  * Modifications:
  */
 static H5FP_object_lock *
-H5FP_find_object_lock(H5FP_file_info *fs, unsigned char *oid)
+H5FP_find_object_lock(H5FP_file_info *info, unsigned char *oid)
 {
     H5FP_object_lock *ret_value = NULL;
 
     FUNC_ENTER_NOINIT(H5FP_find_object_lock);
 
-    assert(fs);
+    assert(info);
     assert(oid);
 
-    if (fs->locks && fs->locks->root) {
+    if (info->locks && info->locks->root) {
         H5TB_NODE *node;
         H5FP_object_lock ol;
 
         H5FP_COPY_OID(ol.oid, oid); /* This is the key field for the TBBT */
 
-        if ((node = H5TB_dfind(fs->locks, (void *)&ol, NULL)) == NULL)
+        if ((node = H5TB_dfind(info->locks, (void *)&ol, NULL)) == NULL)
             HGOTO_ERROR(H5E_FPHDF5, H5E_NOTFOUND, NULL, "lock not found");
 
         ret_value = (H5FP_object_lock *)node->data;
@@ -344,7 +353,7 @@ done:
  * Modifications:
  */
 static herr_t
-H5FP_remove_object_lock_from_list(H5FP_file_info *fs,
+H5FP_remove_object_lock_from_list(H5FP_file_info *info,
                                   H5FP_object_lock *ol)
 {
     H5TB_NODE *node;
@@ -352,8 +361,8 @@ H5FP_remove_object_lock_from_list(H5FP_file_info *fs,
 
     FUNC_ENTER_NOINIT(H5FP_remove_object_lock_from_list);
 
-    if ((node = H5TB_dfind(fs->locks, (void *)ol, NULL)) != NULL) {
-        H5FP_free_object_lock(H5TB_rem(&fs->locks->root, node, NULL));
+    if ((node = H5TB_dfind(info->locks, (void *)ol, NULL)) != NULL) {
+        H5FP_free_object_lock(H5TB_rem(&info->locks->root, node, NULL));
         ret_value = SUCCEED;
     }
 
@@ -382,19 +391,18 @@ H5FP_file_mod_cmp(H5FP_mdata_mod *k1,
  * Function:    H5FP_free_mod_node
  * Purpose:     Helper function to free up an SAP_FILE_MOD node and all
  *              of the malloced space it has.
- * Return:      Success:    SUCCEED
- *              Failure:    Doesn't fail
+ * Return:      SUCCEED (doesn't fail)
  * Programmer:  Bill Wendling, 31. July, 2002
  * Modifications:
  */
 static herr_t
-H5FP_free_mod_node(H5FP_mdata_mod *fs)
+H5FP_free_mod_node(H5FP_mdata_mod *info)
 {
     FUNC_ENTER_NOINIT(H5FP_free_mod_node);
 
-    if (fs) {
-        HDfree(fs->metadata);
-        HDfree(fs);
+    if (info) {
+        HDfree(info->metadata);
+        HDfree(info);
     }
 
     FUNC_LEAVE_NOAPI(SUCCEED);
@@ -405,24 +413,24 @@ H5FP_free_mod_node(H5FP_mdata_mod *fs)
  * Purpose:     Create a new sap_file_mod node and initialize it. This
  *              object now has responsibility for freeing the metadata
  *              information.
- * Return:      Success: Pointer to new sap_file_mod structure.
- *              Failure: NULL
+ * Return:      Success:    Pointer to new sap_file_mod structure.
+ *              Failure:    NULL
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
 static H5FP_mdata_mod *
-H5FP_new_file_mod_node(unsigned UNUSED rank, H5AC_subid_t type_id,
-                       haddr_t addr, int md_size, char *metadata)
+H5FP_new_file_mod_node(unsigned UNUSED rank, H5FD_mem_t mem_type,
+                       H5AC_subid_t type_id, haddr_t addr, unsigned md_size,
+                       char *metadata)
 {
     H5FP_mdata_mod *ret_value = NULL;
 
     FUNC_ENTER_NOINIT(H5FP_new_file_mod_node);
 
-    ret_value = (H5FP_mdata_mod *)HDmalloc(sizeof(H5FP_mdata_mod));
-
-    if (!ret_value)
+    if ((ret_value = (H5FP_mdata_mod *)HDmalloc(sizeof(H5FP_mdata_mod))) == NULL)
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "out of memory");
 
+    ret_value->mem_type = mem_type;
     ret_value->addr = addr;
     ret_value->type_id = type_id;
     ret_value->md_size = md_size;
@@ -444,9 +452,9 @@ done:
  * Modifications:
  */
 static herr_t
-H5FP_add_file_mod_to_list(H5FP_file_info *fs, H5AC_subid_t type_id,
-                          haddr_t addr, unsigned rank, int md_size,
-                          char *metadata)
+H5FP_add_file_mod_to_list(H5FP_file_info *info, H5FD_mem_t mem_type, 
+                          H5AC_subid_t type_id, haddr_t addr, unsigned rank,
+                          unsigned md_size, char *metadata)
 {
     H5FP_mdata_mod *fm, mod;
     H5TB_NODE *node;
@@ -454,11 +462,11 @@ H5FP_add_file_mod_to_list(H5FP_file_info *fs, H5AC_subid_t type_id,
 
     FUNC_ENTER_NOINIT(H5FP_add_file_mod_to_list);
 
-    assert(fs);
+    assert(info);
 
     mod.addr = addr;    /* This is the key field for the TBBT */
 
-    if ((node = H5TB_dfind(fs->mod_tree, (void *)&mod, NULL)) != NULL) {
+    if ((node = H5TB_dfind(info->mod_tree, (void *)&mod, NULL)) != NULL) {
         /*
          * The metadata is in the cache already. All we have to do is
          * replace what's there. The addr and type should be the same.
@@ -469,9 +477,9 @@ H5FP_add_file_mod_to_list(H5FP_file_info *fs, H5AC_subid_t type_id,
         fm->metadata = metadata;
         fm->md_size = md_size;
         ret_value = SUCCEED;
-    } else if ((fm = H5FP_new_file_mod_node(rank, type_id, addr,
+    } else if ((fm = H5FP_new_file_mod_node(rank, mem_type, type_id, addr,
                                             md_size, metadata)) != NULL) {
-        if (!H5TB_dins(fs->mod_tree, (void *)fm, NULL))
+        if (!H5TB_dins(info->mod_tree, (void *)fm, NULL))
             HGOTO_ERROR(H5E_FPHDF5, H5E_CANTINSERT, FAIL,
                         "can't insert modification into tree");
 
@@ -483,52 +491,23 @@ done:
 }
 
 /*
- * Function:    H5FP_remove_mod_from_list
- * Purpose:     Remove a modification from the list of modifications on
- *              this file.
- * Return:      Success:    SUCCEED
- *              Failure:    FAIL
- * Programmer:  Bill Wendling, 31. July, 2002
- * Modifications:
- */
-static herr_t
-H5FP_remove_mod_from_list(H5FP_file_info *fs, H5FP_mdata_mod *mod)
-{
-    H5TB_NODE *node;
-    herr_t ret_value = FAIL;
-
-    FUNC_ENTER_NOINIT(H5FP_remove_mod_from_list);
-
-    if ((node = H5TB_dfind(fs->mod_tree, (void *)mod, NULL)) != NULL) {
-        H5FP_free_mod_node(H5TB_rem(&fs->mod_tree->root, node, NULL));
-        ret_value = SUCCEED;
-    }
-
-    FUNC_LEAVE_NOAPI(ret_value);
-}
-
-/*
  * Function:    H5FP_free_file_info_node
  * Purpose:     Helper function to free up an SAP_FILE_STRUCT node and all
  *              of the malloced space it has.
- *
- *              Note: It is an error to call this function before all
- *              modifications have been synced with all processes in the
- *              H5FP_SAP_COMM.
  * Return:      SUCCEED (never fails)
  * Programmer:  Bill Wendling, 31. July, 2002
  * Modifications:
  */
 static herr_t
-H5FP_free_file_info_node(H5FP_file_info *fs)
+H5FP_free_file_info_node(H5FP_file_info *info)
 {
     FUNC_ENTER_NOINIT(H5FP_free_file_info_node);
 
-    if (fs) {
-        H5TB_dfree(fs->mod_tree, (void (*)(void*))H5FP_free_mod_node, NULL);
-        H5TB_dfree(fs->locks, (void (*)(void*))H5FP_free_object_lock, NULL);
-        HDfree(fs->filename);
-        HDfree(fs);
+    if (info) {
+        H5TB_dfree(info->mod_tree, (void (*)(void*))H5FP_free_mod_node, NULL);
+        H5TB_dfree(info->locks, (void (*)(void*))H5FP_free_object_lock, NULL);
+        HDfree(info->filename);
+        HDfree(info);
     }
 
     FUNC_LEAVE_NOAPI(SUCCEED);
@@ -571,6 +550,7 @@ H5FP_new_file_info_node(unsigned file_id, char *filename)
     ret_value->file_id = file_id;
     ret_value->filename = filename;
     ret_value->closing = FALSE;
+    ret_value->num_mods = 0;
 
     if ((ret_value->mod_tree = H5TB_dmake((H5TB_cmp_t)H5FP_file_mod_cmp,
                                           sizeof(H5FP_mdata_mod), FALSE)) == NULL) {
@@ -605,12 +585,12 @@ H5FP_find_file_info(unsigned file_id)
 
     FUNC_ENTER_NOINIT(H5FP_find_file_info);
 
-    if (fs_tree && fs_tree->root) {
+    if (file_info_tree && file_info_tree->root) {
         H5FP_file_info s;
 
         s.file_id = file_id;    /* This is the key field for the TBBT */
 
-        if ((node = H5TB_dfind(fs_tree, (void *)&s, NULL)) != NULL)
+        if ((node = H5TB_dfind(file_info_tree, (void *)&s, NULL)) != NULL)
             ret_value = (H5FP_file_info *)node->data;
     }
 
@@ -628,15 +608,17 @@ H5FP_find_file_info(unsigned file_id)
 static herr_t
 H5FP_add_new_file_info_to_list(unsigned file_id, char *filename)
 {
-    H5FP_file_info *fs;
+    H5FP_file_info *info;
     herr_t ret_value = FAIL;
 
     FUNC_ENTER_NOINIT(H5FP_add_new_file_info_to_list);
 
-    if ((fs = H5FP_new_file_info_node(file_id, filename)) != NULL) {
-        if (!H5TB_dins(fs_tree, (void *)fs, NULL))
+    if ((info = H5FP_new_file_info_node(file_id, filename)) != NULL) {
+        if (!H5TB_dins(file_info_tree, (void *)info, NULL)) {
+            H5FP_free_file_info_node(info);
             HGOTO_ERROR(H5E_FPHDF5, H5E_CANTINSERT, FAIL,
                         "can't insert file structure into tree");
+        }
 
         ret_value = SUCCEED;
     }
@@ -669,8 +651,8 @@ H5FP_remove_file_id_from_list(unsigned file_id)
 
     s.file_id = file_id;    /* This is the key field for the TBBT */
 
-    if ((node = H5TB_dfind(fs_tree, (void *)&s, NULL)) != NULL) {
-        H5FP_free_file_info_node(H5TB_rem(&fs_tree->root, node, NULL));
+    if ((node = H5TB_dfind(file_info_tree, (void *)&s, NULL)) != NULL) {
+        H5FP_free_file_info_node(H5TB_rem(&file_info_tree->root, node, NULL));
         ret_value = SUCCEED;
     }
 
@@ -683,6 +665,14 @@ H5FP_remove_file_id_from_list(unsigned file_id)
  *===----------------------------------------------------------------------===
  */
 
+/*
+ * Function:    H5FP_send_reply
+ * Purpose:     Send an H5FP_reply message to process TO.
+ * Return:      Success:    SUCCEED
+ *              Failure:    FAIL
+ * Programmer:  Bill Wendling, 31. July, 2002
+ * Modifications:
+ */
 static herr_t
 H5FP_send_reply(unsigned to, unsigned req_id, unsigned file_id, H5FP_status_t status)
 {
@@ -705,6 +695,76 @@ done:
 }
 
 /*
+ * Function:    H5FP_dump
+ * Purpose:     Dump all metadata writes to a process so that that
+ *              process will write them to the file.
+ * Return:      Success:    SUCCEED
+ *              Failure:    FAIL
+ * Programmer:  Bill Wendling, 31. July, 2002
+ * Modifications:
+ */
+static herr_t
+H5FP_dump(H5FP_file_info *info, unsigned to, unsigned req_id, unsigned file_id)
+{
+    H5FP_read r;
+    H5TB_NODE *node;
+    herr_t ret_value = SUCCEED;
+    int mrc;
+
+    FUNC_ENTER_NOINIT(H5FP_dump);
+
+    assert(info);
+
+    if (!info->mod_tree)
+        /* Nothing to write to the file */
+        HGOTO_DONE(SUCCEED);
+
+    r.req_id = req_id;
+    r.file_id = file_id;
+    r.status = H5FP_STATUS_DUMPING;
+
+    node = H5TB_first(info->mod_tree->root);
+
+    while (node) {
+        H5FP_mdata_mod *m = (H5FP_mdata_mod *)node->data;
+
+        r.mem_type = m->mem_type;
+        r.type_id = m->type_id;
+        r.addr = m->addr;
+        r.md_size = m->md_size;
+
+        if ((mrc = MPI_Send(&r, 1, H5FP_read_t, (int)to, H5FP_TAG_REPLY,
+                            H5FP_SAP_COMM)) != MPI_SUCCESS)
+            HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
+
+        if (H5FP_send_metadata(m->metadata, (int)m->md_size, (int)to) != SUCCEED)
+            HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL,
+                        "can't dump metadata to client");
+
+        node = H5TB_next(node);
+    }
+
+    /* Tell the receiving process that we're finished... */
+    r.mem_type = 0;
+    r.type_id = 0;
+    r.addr = 0;
+    r.md_size = 0;
+    r.status = H5FP_STATUS_DUMPING_FINISHED;
+
+    if ((mrc = MPI_Send(&r, 1, H5FP_read_t, (int)to, H5FP_TAG_REPLY,
+                        H5FP_SAP_COMM)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
+
+    /* Free up the modification tree */
+    H5TB_dfree(info->mod_tree, (void (*)(void*))H5FP_free_mod_node, NULL);
+    info->mod_tree = NULL;
+    info->num_mods = 0;
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value);
+}
+
+/*
  *===----------------------------------------------------------------------===
  *                    Functions to handle SAP requests
  *===----------------------------------------------------------------------===
@@ -735,7 +795,7 @@ H5FP_gen_sap_file_id()
  * Modifications:
  */
 static herr_t
-H5FP_sap_handle_open_request(H5FP_request req, char *mdata, int UNUSED md_size)
+H5FP_sap_handle_open_request(H5FP_request req, char *mdata, unsigned UNUSED md_size)
 {
     herr_t ret_value = SUCCEED;
     int mrc;
@@ -746,11 +806,9 @@ H5FP_sap_handle_open_request(H5FP_request req, char *mdata, int UNUSED md_size)
         unsigned new_file_id = H5FP_gen_sap_file_id();
         int i;
 
-        if (H5FP_add_new_file_info_to_list(new_file_id, mdata) != SUCCEED) {
-            HDfree(mdata);
+        if (H5FP_add_new_file_info_to_list(new_file_id, mdata) != SUCCEED)
             HGOTO_ERROR(H5E_FPHDF5, H5E_CANTINSERT, FAIL,
                         "can't insert new file structure to list");
-        }
 
         /* broadcast the file id to all processes */
         /*
@@ -801,12 +859,12 @@ static herr_t
 H5FP_sap_handle_lock_request(H5FP_request req)
 {
     struct lock_group {
-        unsigned char oid[sizeof(req.oid)];
-        unsigned file_id;
-        unsigned locked;
-        H5FP_lock_t rw_lock;
-        H5FP_file_info *fs;
-        H5FP_object_lock *lock;
+        unsigned char       oid[sizeof(req.oid)];
+        unsigned            file_id;
+        unsigned            locked;
+        H5FP_lock_t         rw_lock;
+        H5FP_file_info     *info;
+        H5FP_object_lock   *lock;
     } *oids;
     unsigned list_size = 2; /* the size of the "oids" list */
     H5FP_status_t exit_state = H5FP_STATUS_LOCK_ACQUIRED;
@@ -859,19 +917,18 @@ H5FP_sap_handle_lock_request(H5FP_request req)
      * not, the it's an error and we need to return.
      */
     for (j = 0; j <= i; ++j) {
-        if ((oids[j].fs = H5FP_find_file_info(oids[j].file_id)) == NULL) {
-            /* FIXME: the file ID is invalid */
+        if ((oids[j].info = H5FP_find_file_info(oids[j].file_id)) == NULL) {
             exit_state = H5FP_STATUS_BAD_FILE_ID;
             HGOTO_DONE(FAIL);
         }
         
-        if (oids[j].fs->closing) {
+        if (oids[j].info->closing) {
             /* we're closing the file - don't accept anymore locks */
             exit_state = H5FP_STATUS_FILE_CLOSING;
             HGOTO_DONE(FAIL);
         }
 
-        oids[j].lock = H5FP_find_object_lock(oids[j].fs, oids[j].oid);
+        oids[j].lock = H5FP_find_object_lock(oids[j].info, oids[j].oid);
 
         /*
          * Don't panic!
@@ -893,7 +950,7 @@ H5FP_sap_handle_lock_request(H5FP_request req)
 
     /*
      * Actually acquire the locks. This shouldn't fail because of the
-     * previous check. The only thing which can likely occur is an
+     * previous checks. The only thing which can likely occur is an
      * out-of-memory error.
      */
     for (j = 0; j <= i; ++j) {
@@ -934,7 +991,7 @@ H5FP_sap_handle_lock_request(H5FP_request req)
                                                           req.obj_type, req.rw_lock);
 
             if (lock) {
-                if (!H5TB_dins(oids[j].fs->locks, (void *)lock, NULL)) {
+                if (!H5TB_dins(oids[j].info->locks, (void *)lock, NULL)) {
                     H5FP_free_object_lock(lock);
                     exit_state = H5FP_STATUS_LOCK_FAILED;
                     ret_value = FAIL;
@@ -970,7 +1027,7 @@ rollback:
             if (oids[j].lock) {
                 if (oids[j].lock->owned_rank == req.proc_rank) {
                     if (--oids[j].lock->ref_count == 0) {
-                        H5FP_remove_object_lock_from_list(oids[j].fs, oids[j].lock);
+                        H5FP_remove_object_lock_from_list(oids[j].info, oids[j].lock);
                     }
                 } else {
                     /* CATASTROPHIC FAILURE!!! */
@@ -1007,10 +1064,10 @@ static herr_t
 H5FP_sap_handle_release_lock_request(H5FP_request req)
 {
     struct release_group {
-        unsigned char oid[sizeof(req.oid)];
-        unsigned file_id;
-        H5FP_file_info *fs;
-        H5FP_object_lock *lock;
+        unsigned char       oid[sizeof(req.oid)];
+        unsigned            file_id;
+        H5FP_file_info     *info;
+        H5FP_object_lock   *lock;
     } *oids;
     unsigned list_size = 2; /* the size of the "oids" list */
     H5FP_status_t exit_state = H5FP_STATUS_LOCK_RELEASED;
@@ -1060,13 +1117,12 @@ H5FP_sap_handle_release_lock_request(H5FP_request req)
      * will help keep us from being in a catastrophic state.
      */
     for (j = 0; j <= i; ++j) {
-        if ((oids[j].fs = H5FP_find_file_info(oids[j].file_id)) == NULL) {
-            /* FIXME: the file ID is invalid */
+        if ((oids[j].info = H5FP_find_file_info(oids[j].file_id)) == NULL) {
             exit_state = H5FP_STATUS_BAD_FILE_ID;
             HGOTO_DONE(FAIL);
         }
 
-        oids[j].lock = H5FP_find_object_lock(oids[j].fs, oids[j].oid);
+        oids[j].lock = H5FP_find_object_lock(oids[j].info, oids[j].oid);
 
         if (!oids[j].lock || oids[j].lock->owned_rank != req.proc_rank) {
             exit_state = H5FP_STATUS_BAD_LOCK;
@@ -1083,7 +1139,7 @@ H5FP_sap_handle_release_lock_request(H5FP_request req)
         if (oids[j].lock) {
             if (oids[j].lock->owned_rank == req.proc_rank) {
                 if (--oids[j].lock->ref_count == 0)
-                    H5FP_remove_object_lock_from_list(oids[j].fs, oids[j].lock);
+                    H5FP_remove_object_lock_from_list(oids[j].info, oids[j].lock);
             } else {
                 /* AAAIIIIEEE!!! */
                 exit_state = H5FP_STATUS_CATASTROPHIC;
@@ -1103,61 +1159,6 @@ done:
 }
 
 /*
- * Function:    H5FP_sap_handle_write_request
- * Purpose:     Handle a request to write a piece of metadata in the
- *              file.
- * Return:      Success:    SUCCEED
- *              Failure:    FAIL
- * Programmer:  Bill Wendling, 06. August, 2002
- * Modifications:
- */
-static herr_t
-H5FP_sap_handle_write_request(H5FP_request req, char *mdata, int md_size)
-{
-    H5FP_file_info *fs;
-    H5FP_status_t exit_state = H5FP_STATUS_OK;
-    herr_t ret_value = SUCCEED;
-
-    FUNC_ENTER_NOINIT(H5FP_sap_handle_write_request);
-
-    if ((fs = H5FP_find_file_info(req.file_id)) != NULL) {
-        if (fs->closing) {
-            /* we're closing the file - don't accept anymore changes */
-            exit_state = H5FP_STATUS_FILE_CLOSING;
-            ret_value = FAIL;
-        } else {
-            /* handle the change request */
-            H5FP_object_lock *lock = H5FP_find_object_lock(fs, req.oid);
-
-            if (!lock || lock->owned_rank != req.proc_rank
-                    || lock->rw_lock != H5FP_LOCK_WRITE) {
-                /*
-                 * There isn't a write lock or we don't own the write
-                 * lock on this OID
-                 */
-                exit_state = H5FP_STATUS_NO_LOCK;
-                ret_value = FAIL;
-            } else if (H5FP_add_file_mod_to_list(fs, req.type_id, req.addr,
-                                                 req.proc_rank, md_size,
-                                                 mdata) != SUCCEED) {
-                exit_state = H5FP_STATUS_OOM;
-                ret_value = FAIL;
-            }
-        }
-    } else {
-        /* error: there isn't a file opened to change */
-        exit_state = H5FP_STATUS_BAD_FILE_ID;
-        ret_value = FAIL;
-    }
-
-    if (ret_value == FAIL)
-        HDfree(mdata);
-
-    H5FP_send_reply(req.proc_rank, req.req_id, req.file_id, exit_state);
-    FUNC_LEAVE_NOAPI(ret_value);
-}
-
-/*
  * Function:    H5FP_sap_handle_read_request
  * Purpose:     Handle a read request from a client. The bit of metadata
  *              that the client wants may or may not be in here. It's not
@@ -1171,7 +1172,7 @@ H5FP_sap_handle_write_request(H5FP_request req, char *mdata, int md_size)
 static herr_t
 H5FP_sap_handle_read_request(H5FP_request req)
 {
-    H5FP_file_info *fs;
+    H5FP_file_info *info;
     H5FP_read r;
     herr_t ret_value = SUCCEED;
     char *metadata = NULL;
@@ -1186,13 +1187,25 @@ H5FP_sap_handle_read_request(H5FP_request req)
     r.addr = 0;
     r.status = H5FP_STATUS_MDATA_NOT_CACHED;
 
-    if ((fs = H5FP_find_file_info(req.file_id)) != NULL) {
+    if ((info = H5FP_find_file_info(req.file_id)) != NULL) {
         H5FP_mdata_mod mod;     /* Used to find the correct modification */
         H5TB_NODE *node;
 
+        if (info->num_mods >= H5FP_MDATA_CACHE_HIGHWATER_MARK) {
+            if (H5FP_dump(info, req.proc_rank, req.req_id, req.file_id) == FAIL)
+                HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL,
+                            "can't dump metadata to client");
+
+            /*
+             * We aren't going to find the information we need since it
+             * was just dumped.
+             */
+            HGOTO_DONE(SUCCEED);
+        }
+
         mod.addr = req.addr;    /* This is the key field for the TBBT */
 
-        if ((node = H5TB_dfind(fs->mod_tree, (void *)&mod, NULL)) != NULL) {
+        if ((node = H5TB_dfind(info->mod_tree, (void *)&mod, NULL)) != NULL) {
             H5FP_mdata_mod *fm = (H5FP_mdata_mod *)node->data;
 
             r.md_size = fm->md_size;
@@ -1217,6 +1230,78 @@ done:
 }
 
 /*
+ * Function:    H5FP_sap_handle_write_request
+ * Purpose:     Handle a request to write a piece of metadata in the
+ *              file.
+ * Return:      Success:    SUCCEED
+ *              Failure:    FAIL
+ * Programmer:  Bill Wendling, 06. August, 2002
+ * Modifications:
+ */
+static herr_t
+H5FP_sap_handle_write_request(H5FP_request req, char *mdata, unsigned md_size)
+{
+    H5FP_file_info *info;
+    H5FP_status_t exit_state = H5FP_STATUS_OK;
+    herr_t ret_value = SUCCEED;
+
+    FUNC_ENTER_NOINIT(H5FP_sap_handle_write_request);
+
+    if ((info = H5FP_find_file_info(req.file_id)) != NULL) {
+        if (info->num_mods >= H5FP_MDATA_CACHE_HIGHWATER_MARK) {
+            /*
+             * If there are any modifications not written out yet, dump
+             * them to this process
+             */
+            if (H5FP_send_reply(req.proc_rank, req.req_id, req.file_id,
+                                H5FP_STATUS_DUMPING) == FAIL) {
+                exit_state = H5FP_STATUS_DUMPING_FAILED;
+                HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL,
+                            "can't send message to client");
+            }
+
+            if (H5FP_dump(info, req.proc_rank, req.req_id, req.file_id) == FAIL) {
+                exit_state = H5FP_STATUS_DUMPING_FAILED;
+                HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL,
+                            "metadata dump failed");
+            }
+        }
+
+        if (info->closing) {
+            /* we're closing the file - don't accept anymore changes */
+            exit_state = H5FP_STATUS_FILE_CLOSING;
+            ret_value = FAIL;
+        } else {
+            /* handle the change request */
+            H5FP_object_lock *lock = H5FP_find_object_lock(info, req.oid);
+
+            if (!lock || lock->owned_rank != req.proc_rank
+                    || lock->rw_lock != H5FP_LOCK_WRITE) {
+                /*
+                 * There isn't a write lock or we don't own the write
+                 * lock on this OID
+                 */
+                exit_state = H5FP_STATUS_NO_LOCK;
+                ret_value = FAIL;
+            } else if (H5FP_add_file_mod_to_list(info, req.mem_type, req.type_id,
+                                                 req.addr, req.proc_rank, md_size,
+                                                 mdata) != SUCCEED) {
+                exit_state = H5FP_STATUS_OOM;
+                ret_value = FAIL;
+            }
+        }
+    } else {
+        /* error: there isn't a file opened to change */
+        exit_state = H5FP_STATUS_BAD_FILE_ID;
+        ret_value = FAIL;
+    }
+
+done:
+    H5FP_send_reply(req.proc_rank, req.req_id, req.file_id, exit_state);
+    FUNC_LEAVE_NOAPI(ret_value);
+}
+
+/*
  * Function:    H5FP_sap_handle_close_request
  * Purpose:     Handle a request to close a file.
  * Return:      Success:    SUCCEED
@@ -1227,19 +1312,43 @@ done:
 static herr_t
 H5FP_sap_handle_close_request(H5FP_request req)
 {
-    H5FP_file_info *fs;
+    H5FP_file_info *info;
+    H5FP_status_t exit_state = H5FP_STATUS_OK;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOINIT(H5FP_sap_handle_close_request);
 
-    if ((fs = H5FP_find_file_info(req.file_id)) != NULL)
-        if (++fs->closing == H5FP_comm_size - 1)
+    if ((info = H5FP_find_file_info(req.file_id)) != NULL) {
+        if (info->num_mods) {
+            /*
+             * If there are any modifications not written out yet, dump
+             * them to this process
+             */
+            if (H5FP_send_reply(req.proc_rank, req.req_id, req.file_id,
+                                H5FP_STATUS_DUMPING) == FAIL) {
+                exit_state = H5FP_STATUS_DUMPING_FAILED;
+                HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL,
+                            "can't send message to client");
+            }
+
+            if (H5FP_dump(info, req.proc_rank, req.req_id, req.file_id) == FAIL) {
+                exit_state = H5FP_STATUS_DUMPING_FAILED;
+                HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL,
+                            "can't dump metadata to client");
+            }
+        }
+
+        if (++info->closing == H5FP_comm_size - 1)
             /* all processes have closed the file - remove it from list */
-            if (H5FP_remove_file_id_from_list(req.file_id) != SUCCEED)
+            if (H5FP_remove_file_id_from_list(req.file_id) != SUCCEED) {
+                exit_state = H5FP_STATUS_BAD_FILE_ID;
                 HGOTO_ERROR(H5E_FPHDF5, H5E_NOTFOUND, FAIL,
                             "cannot remove file ID from list");
+            }
+    }
 
 done:
+    H5FP_send_reply(req.proc_rank, req.req_id, req.file_id, exit_state);
     FUNC_LEAVE_NOAPI(ret_value);
 }