[svn-r6336] Purpose:

    Update

Description:
    This is the first conversion of the FPHDF5 code to be a metadata
    cache. There's an extra error message. I rewrote the sync/change code
    to be read metadata/write metadata instead.

    I still need to hook these changes into the HDF5 code so that it
    looks at the SAP first before checking the file for metadata.

Platforms tested:
    Linux

6 files changed, 596 insertions, 816 deletions

diff --git a/src/H5E.c b/src/H5E.c
index 5d664e6..46791f8 100644
--- a/src/H5E.c
+++ b/src/H5E.c
@@ -175,7 +175,8 @@ static const H5E_minor_mesg_t H5E_minor_mesg_g[] = {
     /* FPHDF5 errors */
     {H5E_CANTMAKETREE,  "Can't create a binary tree node"},
     {H5E_CANTRECV,      "Can't receive messages from processes"},
-    {H5E_CANTSENDMDATA, "Can't send metadata message"}
+    {H5E_CANTSENDMDATA, "Can't send metadata message"},
+    {H5E_CANTCHANGE,    "Can't register change with server"}
 };
 
 /* Interface initialization? */
diff --git a/src/H5Epublic.h b/src/H5Epublic.h
index c6ab38a..f9e6e4a 100644
--- a/src/H5Epublic.h
+++ b/src/H5Epublic.h
@@ -206,7 +206,8 @@ typedef enum H5E_minor_t {
     /* FPHDF5 errors */
     H5E_CANTMAKETREE,           /*can't make a TBBT tree                     */
     H5E_CANTRECV,               /*can't receive messages from processes      */
-    H5E_CANTSENDMDATA           /*can't send metadata message                */
+    H5E_CANTSENDMDATA,          /*can't send metadata message                */
+    H5E_CANTCHANGE              /*can't register change on server            */
 } H5E_minor_t;
 
 /* Information about an error */
diff --git a/src/H5FP.c b/src/H5FP.c
index 76b44a3..f45c70e 100644
--- a/src/H5FP.c
+++ b/src/H5FP.c
@@ -29,9 +29,9 @@
 #define INTERFACE_INIT      NULL
 static int interface_initialize_g = 0;
 
-MPI_Datatype SAP_request_t;     /* MPI datatype for the SAP_request obj */
-MPI_Datatype SAP_reply_t;       /* MPI datatype for the SAP_reply obj   */
-MPI_Datatype SAP_sync_t;        /* MPI datatype for the SAP_sync obj    */
+MPI_Datatype H5FP_request_t;    /* MPI datatype for the H5FP_request obj*/
+MPI_Datatype H5FP_reply_t;      /* MPI datatype for the H5FP_reply obj  */
+MPI_Datatype H5FP_read_t;       /* MPI datatype for the H5FP_read obj   */
 
 /* SAP specific variables */
 MPI_Comm H5FP_SAP_COMM;         /* Comm we use: Supplied by user        */
@@ -46,7 +46,11 @@ int H5FP_comm_size;             /* Size of the COMM                     */
 static herr_t H5FP_commit_sap_datatypes(void);
 static herr_t H5FP_request_sap_stop(void);
 
-/** API Functions **/
+/*
+ *===----------------------------------------------------------------------===
+ *                          Public (API) Functions
+ *===----------------------------------------------------------------------===
+ */
 
 /*
  * Function:    H5FPinit
@@ -67,16 +71,17 @@ H5FPinit(MPI_Comm comm, int sap_rank)
     H5TRACE2("e","McIs",comm,sap_rank);
 
     /* initialize to NULL so that we can release if an error occurs */
-    SAP_request_t = MPI_DATATYPE_NULL;
-    SAP_reply_t = MPI_DATATYPE_NULL;
-    SAP_sync_t = MPI_DATATYPE_NULL;
+    H5FP_request_t = MPI_DATATYPE_NULL;
+    H5FP_reply_t = MPI_DATATYPE_NULL;
+    H5FP_read_t = MPI_DATATYPE_NULL;
+
     H5FP_SAP_COMM = MPI_COMM_NULL;
     H5FP_SAP_BARRIER_COMM = MPI_COMM_NULL;
 
     /* Set the global variable to track the SAP's rank */
     H5FP_sap_rank = sap_rank;
 
-    /* Make a private copy of the communicator we were passed */
+    /* Make a private copy of the communicator passed to us */
     if (MPI_Comm_dup(comm, &H5FP_SAP_COMM) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_dup failed");
 
@@ -84,22 +89,24 @@ H5FPinit(MPI_Comm comm, int sap_rank)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_group failed");
 
     /* Exclude the SAP from the barrier group group */
-    if (MPI_Group_excl(sap_group, 1, (int *)&H5FP_sap_rank, &sap_barrier_group)
-            != MPI_SUCCESS)
+    if (MPI_Group_excl(sap_group, 1, (int *)&H5FP_sap_rank,
+                       &sap_barrier_group) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Group_excl failed");
 
     /* Create communicator for barrier group (all processes except the SAP) */
-    if (MPI_Comm_create(H5FP_SAP_COMM, sap_barrier_group, &H5FP_SAP_BARRIER_COMM)
-            != MPI_SUCCESS)
+    if (MPI_Comm_create(H5FP_SAP_COMM, sap_barrier_group,
+                        &H5FP_SAP_BARRIER_COMM) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_create failed");
 
     /* Get the size of all the processes (including the SAP) */
     if (MPI_Comm_size(H5FP_SAP_COMM, &H5FP_comm_size) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Comm_size failed");
 
-    /* we assign the process right after the sap_rank as the one which
-     * will tell the SAP that files have been opened or closed.
-     * we mod it so that we don't go over the size of the communicator. */
+    /*
+     * We assign the process right after the sap_rank as the one which
+     * will tell the SAP that files have been opened or closed. We mod
+     * it so that we don't go over the size of the communicator.
+     */
     H5FP_capt_rank = (H5FP_sap_rank + 1) % H5FP_comm_size;
 
     /* Get this processes rank */
@@ -119,16 +126,16 @@ H5FPinit(MPI_Comm comm, int sap_rank)
 done:
     if (ret_value == FAIL) {
         /* we've encountered an error...clean up */
-        if (SAP_request_t != MPI_DATATYPE_NULL)
-            if (MPI_Type_free(&SAP_request_t) != MPI_SUCCESS)
+        if (H5FP_request_t != MPI_DATATYPE_NULL)
+            if (MPI_Type_free(&H5FP_request_t) != MPI_SUCCESS)
                 HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");
 
-        if (SAP_reply_t != MPI_DATATYPE_NULL)
-            if (MPI_Type_free(&SAP_reply_t) != MPI_SUCCESS)
+        if (H5FP_reply_t != MPI_DATATYPE_NULL)
+            if (MPI_Type_free(&H5FP_reply_t) != MPI_SUCCESS)
                 HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");
 
-        if (SAP_sync_t != MPI_DATATYPE_NULL)
-            if (MPI_Type_free(&SAP_sync_t) != MPI_SUCCESS)
+        if (H5FP_read_t != MPI_DATATYPE_NULL)
+            if (MPI_Type_free(&H5FP_read_t) != MPI_SUCCESS)
                 HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");
 
         if (H5FP_SAP_BARRIER_COMM != MPI_COMM_NULL)
@@ -174,13 +181,13 @@ H5FPfinalize(void)
             HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Error stopping the SAP");
 
     /* Release the MPI types we created */
-    if (MPI_Type_free(&SAP_request_t) != MPI_SUCCESS)
+    if (MPI_Type_free(&H5FP_request_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");
 
-    if (MPI_Type_free(&SAP_reply_t) != MPI_SUCCESS)
+    if (MPI_Type_free(&H5FP_reply_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");
 
-    if (MPI_Type_free(&SAP_sync_t) != MPI_SUCCESS)
+    if (MPI_Type_free(&H5FP_read_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_free failed");
 
     /* Release the barrier communicator */
@@ -197,7 +204,11 @@ done:
     FUNC_LEAVE_API(ret_value);
 }
 
-/** Public Libarary (non-API) Functions **/
+/*
+ *===----------------------------------------------------------------------===
+ *                    Public Library (non-API) Functions
+ *===----------------------------------------------------------------------===
+ */
 
 /*
  * Function:    H5FP_send_metadata
@@ -219,6 +230,7 @@ H5FP_send_metadata(const char *mdata, int len, int rank)
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI(H5FP_send_metadata, FAIL);
+
     assert(mdata);
     assert(len);
 
@@ -231,11 +243,15 @@ done:
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
-/** Private Functions **/
+/*
+ *===----------------------------------------------------------------------===
+ *                            Private Functions
+ *===----------------------------------------------------------------------===
+ */
 
 /*
  * Function:    H5FP_commit_sap_datatypes
- * Purpose:     Commit the SAP_request_t, SAP_reply_t, and SAP_sync_t
+ * Purpose:     Commit the H5FP_request_t, H5FP_reply_t, and H5FP_read_t
  *              structure datatypes to MPI.
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
@@ -245,56 +261,67 @@ done:
 static herr_t
 H5FP_commit_sap_datatypes(void)
 {
-    int block_length[2];
-    MPI_Aint displs[2];
-    MPI_Datatype old_types[2];
-    struct SAP_request req;
-    struct SAP_sync sap_sync;
+    int block_length[3];
+    MPI_Aint displs[3];
+    MPI_Datatype old_types[3];
+    H5FP_request req;
+    H5FP_read sap_read;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI(H5FP_commit_sap_datatypes, FAIL);
 
-    /* Commit the SAP_request_t datatype */
-    block_length[0] = 10;
-    block_length[1] = sizeof(req.oid);
+    /* Commit the H5FP_request_t datatype */
+    block_length[0] = 8;
+    block_length[1] = 1;
+    block_length[2] = sizeof(req.oid);
     old_types[0] = MPI_INT;
-    old_types[1] = MPI_UNSIGNED_CHAR;
+    old_types[1] = HADDR_AS_MPI_TYPE;
+    old_types[2] = MPI_UNSIGNED_CHAR;
     MPI_Address(&req.req_type, &displs[0]);
-    MPI_Address(&req.oid, &displs[1]);
+    MPI_Address(&req.addr, &displs[1]);
+    MPI_Address(&req.oid, &displs[2]);
+    displs[2] -= displs[1];
     displs[1] -= displs[0];
     displs[0] -= displs[0];
 
-    if (MPI_Type_struct(2, block_length, displs, old_types, &SAP_request_t) != MPI_SUCCESS)
+    if (MPI_Type_struct(3, block_length, displs, old_types,
+                        &H5FP_request_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed");
 
-    if (MPI_Type_commit(&SAP_request_t) != MPI_SUCCESS)
+    if (MPI_Type_commit(&H5FP_request_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_commit failed");
 
-    /* Commit the SAP_reply_t datatype */
-    block_length[0] = 3;
+    /* Commit the H5FP_reply_t datatype */
+    block_length[0] = 4;
     displs[0] = 0;
     old_types[0] = MPI_INT;
 
-    if (MPI_Type_struct(1, block_length, displs, old_types, &SAP_reply_t) != MPI_SUCCESS)
+    if (MPI_Type_struct(1, block_length, displs, old_types,
+                        &H5FP_reply_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed");
 
-    if (MPI_Type_commit(&SAP_reply_t) != MPI_SUCCESS)
+    if (MPI_Type_commit(&H5FP_reply_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_commit failed");
 
-    /* Commit the SAP_sync_t datatype */
-    block_length[0] = 9;
+    /* Commit the H5FP_read_t datatype */
+    block_length[0] = 6;
     block_length[1] = 1;
+    block_length[2] = 1;
     old_types[0] = MPI_INT;
-    old_types[1] = MPI_LONG_LONG;
-    MPI_Address(&sap_sync.req_id, &displs[0]);
-    MPI_Address(&sap_sync.size, &displs[1]);
+    old_types[1] = HADDR_AS_MPI_TYPE;
+    old_types[2] = MPI_LONG_LONG;
+    MPI_Address(&sap_read.req_id, &displs[0]);
+    MPI_Address(&sap_read.addr, &displs[1]);
+    MPI_Address(&sap_read.size, &displs[2]);
+    displs[2] -= displs[1];
     displs[1] -= displs[0];
     displs[0] -= displs[0];
 
-    if (MPI_Type_struct(1, block_length, displs, old_types, &SAP_sync_t) != MPI_SUCCESS)
+    if (MPI_Type_struct(3, block_length, displs, old_types,
+                        &H5FP_read_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed");
 
-    if (MPI_Type_commit(&SAP_sync_t) != MPI_SUCCESS)
+    if (MPI_Type_commit(&H5FP_read_t) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_commit failed");
 
 done:
@@ -313,7 +340,7 @@ done:
 static herr_t
 H5FP_request_sap_stop(void)
 {
-    struct SAP_request req;
+    H5FP_request req;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI(H5FP_request_sap_stop, FAIL);
@@ -323,7 +350,7 @@ H5FP_request_sap_stop(void)
     req.req_id = 0;
     req.proc_rank = H5FP_my_rank;
 
-    if (MPI_Send(&req, 1, SAP_request_t, (int)H5FP_sap_rank,
+    if (MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                  H5FP_TAG_REQUEST, H5FP_SAP_COMM) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Send failed");
 
diff --git a/src/H5FPclient.c b/src/H5FPclient.c
index e3b7d09..d1a029c 100644
--- a/src/H5FPclient.c
+++ b/src/H5FPclient.c
@@ -16,6 +16,7 @@
 #define H5G_PACKAGE             /*suppress error about including H5Gpkg */
 
 #include "H5private.h"          /* Generic Functions                    */
+#include "H5ACprivate.h"        /* Metadata Cache                       */
 #include "H5Dprivate.h"         /* Dataset Functions                    */
 #include "H5Eprivate.h"         /* Error Handling                       */
 #include "H5Gpkg.h"             /* Group functions                      */
@@ -38,44 +39,10 @@ static int interface_initialize_g = 0;
 
 /* local functions */
 static unsigned H5FP_gen_request_id(void);
-static herr_t   H5FP_update_file_cache(hid_t file_id, struct SAP_sync *sap_sync, uint8_t *msg);
 
 /** Public Library (non-API) Functions **/
 
 /*
- * Function:    H5FP_fill_fphdf5_struct
- * Purpose:     Nice function to fill in the H5O_fphdf5_t structure
- *              before transmission to the SAP.
- * Return:      Nothing:    Doesn't fail after asserts pass.
- * Programmer:  Bill Wendling, 13. November, 2002
- * Modifications:
- */
-void
-H5FP_fill_fphdf5_struct(H5O_fphdf5_t *fphdf5, uint8_t *oid, haddr_t header,
-                        struct H5S_simple_t *sdim, H5T_t *dtype, time_t *mtime,
-                        H5O_layout_t *layout, H5O_name_t *group, H5O_name_t *dataset,
-                        struct H5P_genplist_t *plist)
-{
-    /* check args */
-    assert(fphdf5);
-    assert(oid);
-    assert(sdim);
-    assert(dtype);
-    assert(layout);
-    assert(plist);
-
-    HDmemcpy(fphdf5->oid, oid, sizeof(fphdf5->oid));
-    fphdf5->header = header;
-    fphdf5->layout = layout;
-    fphdf5->sdim = sdim;
-    fphdf5->mtime = mtime;
-    fphdf5->group = group;
-    fphdf5->dset = dataset;
-    fphdf5->dtype = dtype;
-    fphdf5->plist = plist;
-}
-
-/*
  * Function:    H5FP_request_open
  * Purpose:     Request an open of a file from the SAP. You pass in the
  *              metadata string (MDATA) (the filename), it's length in
@@ -88,20 +55,16 @@ H5FP_fill_fphdf5_struct(H5O_fphdf5_t *fphdf5, uint8_t *oid, haddr_t header,
  *              opening the file, though, should call this function so
  *              that they can get the file ID that the SAP assigns to it.
  *
- *              N.B. This could be expanded to handle the opening of more
- *              than just a file, if that becomes necessary. Right now,
- *              we are only keeping track of file opens.
- *
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 28. August, 2002
  * Modifications:
  */
 herr_t
-H5FP_request_open(const char *mdata, int md_len, enum sap_obj_type obj_type,
+H5FP_request_open(const char *mdata, int md_size, H5FP_obj_t obj_type,
                   unsigned *file_id, unsigned *req_id)
 {
-    struct SAP_request req;
+    H5FP_request req;
     MPI_Status mpi_status;
     int ret_value = SUCCEED, mrc;
 
@@ -119,14 +82,14 @@ H5FP_request_open(const char *mdata, int md_len, enum sap_obj_type obj_type,
         req.req_type = H5FP_REQ_OPEN;
         req.req_id = H5FP_gen_request_id();
         req.proc_rank = H5FP_my_rank;
-        req.md_len = md_len;
+        req.md_size = md_size;
         req.obj_type = obj_type;
 
-        if ((mrc = MPI_Send(&req, 1, SAP_request_t, (int)H5FP_sap_rank,
+        if ((mrc = MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                             H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
             HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
-        if (H5FP_send_metadata(mdata, md_len, (int)H5FP_sap_rank))
+        if (H5FP_send_metadata(mdata, md_size, (int)H5FP_sap_rank))
             HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL,
                         "can't send metadata to server");
     }
@@ -155,11 +118,11 @@ done:
  * Modifications:
  */
 herr_t
-H5FP_request_lock(unsigned int sap_file_id, unsigned char *obj_oid,
-                  enum sap_lock_type rw_lock, int last, unsigned *req_id,
-                  enum sap_status *status)
+H5FP_request_lock(unsigned file_id, unsigned char *obj_oid,
+                  H5FP_lock_t rw_lock, int last, unsigned *req_id,
+                  H5FP_status_t *status)
 {
-    struct SAP_request req;
+    H5FP_request req;
     int ret_value = SUCCEED, mrc;
 
     FUNC_ENTER_NOAPI(H5FP_request_lock, FAIL);
@@ -172,13 +135,13 @@ H5FP_request_lock(unsigned int sap_file_id, unsigned char *obj_oid,
     *status = H5FP_STATUS_OK;
     req.req_type = last ? H5FP_REQ_LOCK_END : H5FP_REQ_LOCK;
     req.req_id = H5FP_gen_request_id();
-    req.sap_file_id = sap_file_id;
+    req.file_id = file_id;
     req.rw_lock = rw_lock;
-    req.md_len = 0;
+    req.md_size = 0;
     req.proc_rank = H5FP_my_rank;
     H5FP_COPY_OID(req.oid, obj_oid);
 
-    if ((mrc = MPI_Send(&req, 1, SAP_request_t, (int)H5FP_sap_rank,
+    if ((mrc = MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
@@ -187,12 +150,12 @@ H5FP_request_lock(unsigned int sap_file_id, unsigned char *obj_oid,
          * On the last lock in the lock-group to be acquired, we expect a
          * reply from the SAP
          */
-        struct SAP_reply sap_reply;
+        H5FP_reply sap_reply;
         MPI_Status mpi_status;
 
         HDmemset(&mpi_status, 0, sizeof(mpi_status));
 
-        if ((mrc = MPI_Recv(&sap_reply, 1, SAP_reply_t, (int)H5FP_sap_rank,
+        if ((mrc = MPI_Recv(&sap_reply, 1, H5FP_reply_t, (int)H5FP_sap_rank,
                             H5FP_TAG_REPLY, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
             HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
 
@@ -220,33 +183,30 @@ done:
  * Modifications:
  */
 herr_t
-H5FP_request_release_lock(unsigned int sap_file_id, unsigned char *obj_oid,
-                          int last, unsigned *req_id, enum sap_status *status)
+H5FP_request_release_lock(unsigned file_id, unsigned char *obj_oid,
+                          int last, unsigned *req_id, H5FP_status_t *status)
 {
-    struct SAP_request req;
+    H5FP_request req;
     herr_t ret_value = SUCCEED;
     int mrc;
 
     FUNC_ENTER_NOAPI(H5FP_request_release_lock, FAIL);
 
+    assert(obj_oid);
     assert(req_id);
     assert(status);
 
     HDmemset(&req, 0, sizeof(req));
 
     *status = H5FP_STATUS_OK;
+    H5FP_COPY_OID(req.oid, obj_oid);
     req.req_type = last ? H5FP_REQ_RELEASE_END : H5FP_REQ_RELEASE;
     req.req_id = H5FP_gen_request_id();
-    req.sap_file_id = sap_file_id;
-    req.md_len = 0;
+    req.file_id = file_id;
+    req.md_size = 0;
     req.proc_rank = H5FP_my_rank;
 
-    if (obj_oid)
-        H5FP_COPY_OID(req.oid, obj_oid);
-    else
-        HDmemset(req.oid, '\0', sizeof(req.oid));
-
-    if ((mrc = MPI_Send(&req, 1, SAP_request_t, (int)H5FP_sap_rank,
+    if ((mrc = MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
@@ -255,12 +215,12 @@ H5FP_request_release_lock(unsigned int sap_file_id, unsigned char *obj_oid,
          * On the last lock released in this lock-group, we expect a
          * reply from the SAP
          */
-        struct SAP_reply sap_reply;
+        H5FP_reply sap_reply;
         MPI_Status mpi_status;
 
         HDmemset(&mpi_status, 0, sizeof(mpi_status));
 
-        if ((mrc = MPI_Recv(&sap_reply, 1, SAP_reply_t, (int)H5FP_sap_rank,
+        if ((mrc = MPI_Recv(&sap_reply, 1, H5FP_reply_t, (int)H5FP_sap_rank,
                             H5FP_TAG_REPLY, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
             HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
 
@@ -276,151 +236,93 @@ done:
 }
 
 /*
- * Function:    H5FP_request_change
- * Purpose:     Tell the SAP that we want to change the structure of the file.
- *              Include the information the SAP will need to send to the
- *              other processes so that they can be synced with what you
- *              are doing. The request ID is returned in a pointer
- *              supplied by the user.
+ * Function:    H5FP_request_write_metadata
+ * Purpose:     Tell the SAP that we want to change a piece of metadata
+ *              associated with the file. The request ID is returned in a
+ *              pointer supplied by the user.
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
 herr_t
-H5FP_request_change(unsigned int sap_file_id, unsigned char *obj_oid,
-                    enum sap_obj_type obj_type, enum sap_action action,
-                    int mdata_len, const char *mdata, unsigned *req_id,
-                    enum sap_status *status)
+H5FP_request_write_metadata(unsigned file_id, unsigned char *obj_oid,
+                            H5FP_obj_t obj_type, H5AC_subid_t type_id,
+                            haddr_t addr, int mdata_size, const char *mdata,
+                            unsigned *req_id, H5FP_status_t *status)
 {
-    struct SAP_reply sap_reply;
+    H5FP_reply sap_reply;
     MPI_Status mpi_status;
-    struct SAP_request req;
+    H5FP_request req;
     herr_t ret_value = SUCCEED;
     int mrc;
 
     FUNC_ENTER_NOAPI(H5FP_request_change, FAIL);
 
     assert(mdata);
+    assert(obj_oid);
+    assert(req_id);
+    assert(status);
+
     HDmemset(&req, 0, sizeof(req));
 
-    req.req_type = H5FP_REQ_CHANGE;
+    H5FP_COPY_OID(req.oid, obj_oid);
+    req.req_type = H5FP_REQ_WRITE;
     req.req_id = H5FP_gen_request_id();
-    req.sap_file_id = sap_file_id;
-    req.obj_type = obj_type;
-    req.action = action;
-    req.md_len = mdata_len;
     req.proc_rank = H5FP_my_rank;
+    req.file_id = file_id;
+    req.obj_type = obj_type;
+    req.type_id = type_id;
+    req.addr = addr;
+    req.md_size = mdata_size;
 
-    if (obj_oid)
-        H5FP_COPY_OID(req.oid, obj_oid);
-    else
-        HDmemset(req.oid, '\0', sizeof(req.oid));
-
-    if ((mrc = MPI_Send(&req, 1, SAP_request_t, (int)H5FP_sap_rank,
+    if ((mrc = MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
     /* The first MPI_Send will have been sent before this one will be read. */
-    if (H5FP_send_metadata(mdata, mdata_len, (int)H5FP_sap_rank) != SUCCEED)
+    if (H5FP_send_metadata(mdata, mdata_size, (int)H5FP_sap_rank) != SUCCEED)
         HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL, "can't send metadata to server");
 
     HDmemset(&mpi_status, 0, sizeof(mpi_status));
 
-    if ((mrc = MPI_Recv(&sap_reply, 1, SAP_reply_t, (int)H5FP_sap_rank,
+    if ((mrc = MPI_Recv(&sap_reply, 1, H5FP_reply_t, (int)H5FP_sap_rank,
                         H5FP_TAG_REPLY, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
 
     *status = sap_reply.status;
 
     if (sap_reply.status != H5FP_STATUS_OK)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTUNLOCK, FAIL, "can't register change with server");
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCHANGE, FAIL, "can't write metadata to server");
 
 done:
     *req_id = req.req_id;
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
-#if 0
-herr_t
-H5FP_generate_message(fid_t fid, const char *dataset, const char *group,
-                      H5D_t *dset, H5S_t *space, H5O_fphdf5_t *fphdf5 /* out */)
-{
-    hobj_ref_t gr_ref, ds_ref;
-    H5O_efl_t efl;
-    H5F_t *file = NULL;
-    H5S_t *space = NULL;
-    H5G_entry_t *ent = NULL;
-    int i;
-    herr_t ret_value = SUCCEED;
-
-    FUNC_ENTER_NOAPI(H5FP_generate_message, FAIL);
-
-    if ((file = H5I_object(file_id)) == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file identifier");
-
-    if ((grp = H5G_open(file, group)) == NULL)
-        HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, FAIL, "unable to open group");
-
-    if ((ent = H5G_entof(grp)) == NULL)
-        /* this will never be executed */
-        HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, FAIL, "unable to get entry info of group");
-
-    memcpy(fphdf5->oid, (void *)&ent->header, sizeof(ent->header));
-
-    if ((loc = H5G_loc(did)) == NULL) {
-        exit(1);
-    }
-
-    memcpy(ds_ref.oid, (void *)&loc->header, sizeof(loc->header));
-
-    memcpy(fphdf5.oid, ds_ref.oid, sizeof(ds_ref.oid));
-    fphdf5->header = dset->ent.header;
-    fphdf5->layout = &dset->layout;
-    fphdf5->sdim = &(space->extent.u.simple);
-#if 0
-    fphdf5->mtime = ;
-#endif
-    fphdf5->group = group;
-    fphdf5->dset = dataset;
-    fphdf5->dtype = dset->type;
-
-    if ((fphdf5->plist = H5P_object_verify(dset->dcpl_id,
-                                           H5P_DATASET_CREATE)) == NULL) {
-        exit(1);
-    }
-
-    if (H5P_get(fphdf5->plist, H5D_CRT_EXT_FILE_LIST_NAME, &efl) < 0)
-        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get external file list");
-
-    fphdf5->efl = &efl;
-
-done:
-    FUNC_LEAVE_NOAPI(ret_value);
-}
-#endif
-
 /*
- * Function:    H5FP_request_sync
- * Purpose:     Tell the SAP that we want all of the structural changes
- *              made on the file that we don't know about. The request ID
- *              is returned in a pointer supplied by the user. The status
- *              of the SAP is returned to the user in the supplied STATUS
- *              pointer.
+ * Function:    H5FP_request_read_metadata
+ * Purpose:     Read a piece of metadata from the SAP. That is, if the
+ *              SAP has access to that metadata. If not, then we'll need
+ *              to read it from disk.
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
 herr_t
-H5FP_request_sync(unsigned int sap_file_id, hid_t hdf_file_id,
-                  unsigned *req_id, enum sap_status *status)
+H5FP_request_read_metadata(unsigned file_id, H5FP_obj_t obj_type,
+                           H5AC_subid_t type_id, haddr_t addr, size_t size,
+                           uint8_t **buf, unsigned *req_id,
+                           H5FP_status_t *status)
 {
-    struct SAP_request req;
+    H5FP_request req;
+    H5FP_read sap_read;     /* metadata info read from the SAP's cache */
     herr_t ret_value = SUCCEED;
+    MPI_Status mpi_status;
     int mrc;
 
-    FUNC_ENTER_NOAPI(H5FP_request_sync, FAIL);
+    FUNC_ENTER_NOAPI(H5FP_request_read_metadata, FAIL);
 
     assert(req_id);
     assert(status);
@@ -428,64 +330,45 @@ H5FP_request_sync(unsigned int sap_file_id, hid_t hdf_file_id,
     HDmemset(&req, 0, sizeof(req));
 
     *status = H5FP_STATUS_OK;
-    req.req_type = H5FP_REQ_SYNC;
+    req.req_type = H5FP_REQ_READ;
     req.req_id = H5FP_gen_request_id();
-    req.sap_file_id = sap_file_id;
+    req.file_id = file_id;
     req.proc_rank = H5FP_my_rank;
+    req.obj_type = obj_type;
+    req.type_id = type_id;
+    req.addr = addr;
 
-    if ((mrc = MPI_Send(&req, 1, SAP_request_t, (int)H5FP_sap_rank,
+    if ((mrc = MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
-    for (;;) {
-        MPI_Status mpi_status;
-        struct SAP_sync sap_sync;
+    HDmemset(&mpi_status, 0, sizeof(mpi_status));
 
+    if ((mrc = MPI_Recv(&sap_read, 1, H5FP_read_t, (int)H5FP_sap_rank, H5FP_TAG_READ,
+                        H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
+
+    if (sap_read.status == H5FP_STATUS_OK) {
+        /* use the info in the H5FP_read_t structure to update the metadata */
+        memset(*buf, '\0', size);
         HDmemset(&mpi_status, 0, sizeof(mpi_status));
 
-        if ((mrc = MPI_Recv(&sap_sync, 1, SAP_sync_t, (int)H5FP_sap_rank, H5FP_TAG_SYNC,
-                            H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
+        if ((mrc = MPI_Recv(*buf, sap_read.md_size, MPI_BYTE, (int)H5FP_sap_rank,
+                            H5FP_TAG_METADATA, H5FP_SAP_COMM,
+                            &mpi_status)) != MPI_SUCCESS)
             HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
-
-        if (sap_sync.status != H5FP_STATUS_OK) {
-            *status = sap_sync.status;
-            HGOTO_DONE(FAIL);
-        }
-
+    } else {
         /*
-         * use the info in the SAP_sync_t structure to update the
-         * metadata
+         * FIXME!!!
+         *
+         * The metadata wasn't in the SAP's cache. Should read from disk
+         * now.
          */
-        if (sap_sync.md_len) {
-            uint8_t *buf;
-
-            if ((buf = (uint8_t *)HDcalloc((size_t)sap_sync.md_len + 1, 1)) == NULL)
-                HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "out of memory");
-
-            HDmemset(&mpi_status, 0, sizeof(mpi_status));
-
-            if ((mrc = MPI_Recv(buf, sap_sync.md_len, MPI_BYTE, (int)H5FP_sap_rank,
-                                H5FP_TAG_METADATA, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS) {
-                HDfree(buf);
-                HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
-            }
-
-            /*
-             * FIXME: perform whatever metadata updates we can with the
-             * metadata returned from the SAP
-             */
-            if (H5FP_update_file_cache(hdf_file_id, &sap_sync, buf) != SUCCEED)
-                HGOTO_DONE(FAIL);
-
-            HDfree(buf);
-        }
-
-        if (sap_sync.last_msg)
-            break;
     }
 
 done:
     *req_id = req.req_id;
+    *status = sap_read.status;
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
@@ -501,20 +384,23 @@ done:
  * Modifications:
  */
 herr_t
-H5FP_request_close(unsigned sap_file_id, unsigned *req_id)
+H5FP_request_close(unsigned file_id, unsigned *req_id)
 {
-    struct SAP_request req;
+    H5FP_request req;
     int ret_value = SUCCEED, mrc;
 
     FUNC_ENTER_NOAPI(H5FP_request_close, FAIL);
 
+    assert(req_id);
+
     HDmemset(&req, 0, sizeof(req));
+
     req.req_type = H5FP_REQ_CLOSE;
     req.req_id = H5FP_gen_request_id();
-    req.sap_file_id = sap_file_id;
+    req.file_id = file_id;
     req.proc_rank = H5FP_my_rank;
 
-    if ((mrc = MPI_Send(&req, 1, SAP_request_t, (int)H5FP_sap_rank,
+    if ((mrc = MPI_Send(&req, 1, H5FP_request_t, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
@@ -524,7 +410,11 @@ done:
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
-/** Private Functions **/
+/*
+ *===----------------------------------------------------------------------===
+ *                    Functions Private to Client Module
+ *===----------------------------------------------------------------------===
+ */
 
 /*
  * Function:    H5FP_gen_request_id
@@ -543,113 +433,4 @@ H5FP_gen_request_id()
     FUNC_LEAVE_NOAPI(i++);
 }
 
-/*
- * Function:    H5FP_update_file_cache
- * Purpose:     Take the information returned by a "sync" call and update
- *              the file cache. The MSG parameter should be an encoded
- *              version of the H5O_fphdf5_t object.
- * Return:      Success:    SUCCEED
- *              Failure:    FAIL
- * Programmer:  Bill Wendling, 16. December, 2002
- * Modifications:
- */
-static herr_t
-H5FP_update_file_cache(hid_t file_id, struct SAP_sync *sap_sync, uint8_t *msg)
-{
-    herr_t ret_value = SUCCEED;
-    H5G_entry_t *ent = NULL, *file_ent = NULL;
-    H5O_fphdf5_t *fmeta = NULL;
-    H5S_t *space = NULL;
-    H5F_t *file = NULL;
-    H5G_t *grp = NULL;
-    H5D_t *dset = NULL;
-
-    FUNC_ENTER_NOINIT(H5FP_update_file_cache);
-
-    /* check args */
-    assert(sap_sync);
-    assert(msg);
-
-    if ((file = H5I_object(file_id)) == NULL)
-        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file identifier");
-
-    if ((fmeta = H5O_FPHDF5[0].decode(file, msg, NULL)) == NULL)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "out of memory");
-
-    if ((file_ent = H5G_loc(file_id)) == NULL)
-        HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, FAIL, "unable to open group");
-
-    if ((grp = H5G_open(file_ent, fmeta->group->s)) == NULL)
-        HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, FAIL, "unable to open group");
-
-    switch (sap_sync->action) {
-#ifdef OLD_WAY
-    case H5FP_ACT_CREATE:
-        if (sap_sync->obj_type == H5FP_OBJ_DATASET) {
-            H5O_efl_t efl;
-
-            if ((ent = H5MM_malloc(sizeof(H5G_entry_t))) == NULL)
-                HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "out of memory");
-
-            if ((space = H5S_create(H5S_SIMPLE)) == NULL)
-                HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL,
-                            "can't create simple dataspace");
-
-            if (H5S_set_extent_simple(space, fmeta->sdim->rank,
-                                      fmeta->sdim->size, fmeta->sdim->max) < 0)
-                HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "can't set dimensions");
-
-            if (H5D_update_entry_info(file, ent, H5G_entof(grp),
-                                       fmeta->dset->s, space,
-                                       fmeta->plist, fmeta->layout, fmeta->dtype,
-                                       FALSE, fmeta->header) == FAIL)
-                HGOTO_ERROR(H5E_FPHDF5, H5E_CANTINIT, FAIL, "can't update metadata cache");
-
-            if (H5P_get(fmeta->plist, H5D_CRT_EXT_FILE_LIST_NAME, &efl) < 0)
-                HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get external file list");
-
-            if (H5D_update_external_storage_info(file, ent, &efl, fmeta->layout) == FAIL)
-                HGOTO_ERROR(H5E_FPHDF5, H5E_CANTINIT, FAIL,
-                            "can't update external file layout metadata cache");
-        }
-
-        break;
-
-    case H5FP_ACT_EXTEND:
-        if (sap_sync->obj_type == H5FP_OBJ_DATASET) {
-            if ((dset = H5D_open(H5G_entof(grp), fmeta->dset->s)) == NULL)
-                HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, FAIL, "unable to open dataset");
-
-            if (H5D_extend(dset, fmeta->sdim->size) != SUCCEED)
-                HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to extend dataset");
-        }
-
-        break;
-#endif /* OLD_WAY */
-
-    case H5FP_ACT_DELETE:
-    default:
-        break;
-    }
-
-done:
-    /* cleanup */
-    if (fmeta)
-        H5O_FPHDF5[0].free(fmeta);
-
-    if (dset)
-        H5D_close(dset);
-
-    if (grp)
-        H5G_close(grp);
-
-    if (space)
-        H5S_close(space);
-
-    if (ret_value == FAIL)
-        H5MM_xfree(ent);
-
-    FUNC_LEAVE_NOAPI(ret_value);
-}
-
 #endif  /* H5_HAVE_FPHDF5 */
diff --git a/src/H5FPprivate.h b/src/H5FPprivate.h
index 9c7caba..1e5c3aa 100644
--- a/src/H5FPprivate.h
+++ b/src/H5FPprivate.h
@@ -11,86 +11,113 @@
  * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html.  If you do not have     *
  * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. *
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
 #ifndef H5FPPRIVATE_H__
 #define H5FPPRIVATE_H__ 0
 
+#include "H5ACpublic.h"         /* Metadata Cache                       */
 #include "H5FPpublic.h"         /* Flexible Parallel HDF5               */
 #include "H5Oprivate.h"         /* Object Headers                       */
 #include "H5Rprivate.h"         /* References                           */
 
-#define H5FP_BYTE_BITS   8
-
-/* Different types of requests */
-
-/*
+/*===----------------------------------------------------------------------===
+ *                             Request Types
+ *===----------------------------------------------------------------------===
+ *
  * The H5FP_REQ_LOCK_END and H5FP_REQ_RELEASE_END are used to lock and
  * release a collection of objects at the same time. The last object you
  * need to lock is sent with the H5FP_REQ_LOCK_END request type (this can
  * also be a null message - where you're just sending a message that has
  * H5FP_REQ_LOCK_END as the request type with no data associated with it.
- * In that case, the md_len MUST be set to 0). The SAP will then try to
- * lock all of the objects you've requested to lock at once. If it cannot
- * do so, then it will fail and you have to try again at a later time.
+ * In that case, the md_size *MUST* be set to 0). The SAP will then try
+ * to lock all of the objects you've requested to lock at once. If it
+ * cannot do so, then it will fail and you have to try again at a later
+ * time.
  *
  * Releasing locks is done in the exact same way, except that the action
  * will always release locks (i.e., not fail) if there is a vaild lock
  * for that object.
  */
-enum sap_request {
+typedef enum {
     H5FP_REQ_OPEN,              /* Open a file (or eventually an object)    */
     H5FP_REQ_LOCK,              /* Lock an object (in a sequence)           */
     H5FP_REQ_LOCK_END,          /* Last lock request in lock sequence       */
     H5FP_REQ_RELEASE,           /* Unlock an object (in a sequence)         */
     H5FP_REQ_RELEASE_END,       /* Last unlock request in unlock sequence   */
-    H5FP_REQ_CHANGE,            /* Change an object                         */
-    H5FP_REQ_SYNC,              /* Syncronize changes in file               */
+    H5FP_REQ_WRITE,             /* Writing a piece of metadata              */
+    H5FP_REQ_READ,              /* Reading a piece of metadata              */
     H5FP_REQ_CLOSE,             /* Close a file (or eventually an object)   */
     H5FP_REQ_STOP               /* Stop SAP                                 */
-};
+} H5FP_req_t;
 
-/* Actions to take when performing a change */
-enum sap_action {
-    H5FP_ACT_CREATE,
-    H5FP_ACT_EXTEND,
-    H5FP_ACT_DELETE,
-    H5FP_ACT_UPDATE
-};
+/*===----------------------------------------------------------------------===
+ *                              Lock Types
+ *===----------------------------------------------------------------------===
+ *
+ * A ``read'' lock indicates that the process is busy reading the
+ * metadata of that object. It's non-exclusive, so any number of
+ * processes can have any number of locks on a given object. However, you
+ * cannot have a write and a read lock on an object.
+ *
+ * A ``write'' lock indicates that the process is busy writing to the
+ * metadata of that object. It's exclusive, so only one process can have
+ * a write lock on an object at any one time. However, that object can
+ * have any number of write locks on that object.
+ *
+ * It's up to the program to release all of the locks it has on a given
+ * object.
+ */
+typedef enum {
+    H5FP_LOCK_READ,
+    H5FP_LOCK_WRITE
+} H5FP_lock_t;
 
-/* Types of objects we can change */
-enum sap_obj_type {
+/*===----------------------------------------------------------------------===
+ *                             Object Types
+ *===----------------------------------------------------------------------===
+ *
+ * The various types of objects we're able to get a lock on or which we
+ * want to modify/read.
+ */
+typedef enum {
     H5FP_OBJ_FILE,
     H5FP_OBJ_GROUP,
     H5FP_OBJ_DATASET,
     H5FP_OBJ_DATATYPE,
     H5FP_OBJ_ATTRIBUTE,
     H5FP_OBJ_MEMORY
-};
+} H5FP_obj_t;
 
-/* Types of locks we can get */
-enum sap_lock_type {
-    H5FP_LOCK_READ,
-    H5FP_LOCK_WRITE
-};
-
-/* The structure sent to the SAP which holds all of the requested action */
-struct SAP_request {
-    enum sap_request req_type;  /* Request type                             */
-    unsigned int req_id;        /* ID for request set by sending process    */
-    unsigned int proc_rank;     /* Rank of sending process                  */
-    unsigned int sap_file_id;   /* SAP's file ID for the specific file      */
-    enum sap_obj_type obj_type; /* Type of object                           */
-    enum sap_action action;     /* Action to do to object (H5FP_REQ_CHANGE only) */
-    enum sap_lock_type rw_lock; /* Indicates read or write lock             */
-    H5FD_mem_t mem_type;        /* Type of memory updated, if req'd         */
-    unsigned long size;         /* Size of memory updated, if req'd         */
-    int md_len;                 /* Length of the metadata sent in next msg  */
-    unsigned char oid[H5R_OBJ_REF_BUF_SIZE]; /* Buffer to store OID of object referenced */
+/*===----------------------------------------------------------------------===
+ *                            MPI Message Tags
+ *===----------------------------------------------------------------------===
+ *
+ * Special tag numbers for requests, replies, and string passing
+ * messages.
+ *
+ * Certain actions (Open, Change, and Close) require a pathname to the
+ * object. This pathname is sent in a separate message and the SAP will
+ * search for it after getting the appropriate request.
+ */
+enum {
+    H5FP_TAG_REQUEST,
+    H5FP_TAG_REPLY,
+    H5FP_TAG_READ,
+    H5FP_TAG_METADATA,
+    H5FP_TAG_FILE_ID
 };
 
-extern MPI_Datatype SAP_request_t;  /* MPI datatype for the SAP_request obj */
-
-/* The status of the SAP */
-enum sap_status {
+/*===----------------------------------------------------------------------===
+ *                                Status
+ *===----------------------------------------------------------------------===
+ *
+ * The status returned by the SAP. If the process receives an
+ * H5FP_STATUS_CATASTROPHIC status, then something *REALLY* bad happened
+ * on the set-aside process. The state of the program is then
+ * indeterminant and the only real course of action is for the program to
+ * abort operation.
+ */
+typedef enum sap_status {
     H5FP_STATUS_OK,
 
     /* For locking */
@@ -106,6 +133,9 @@ enum sap_status {
     H5FP_STATUS_FILE_CLOSING,
     H5FP_STATUS_NO_LOCK,
 
+    /* For read requests */
+    H5FP_STATUS_MDATA_NOT_CACHED,
+
     /* Out of memory error */
     H5FP_STATUS_OOM,
 
@@ -114,47 +144,53 @@ enum sap_status {
 
     /* Reserved for completely disasterous failures which require an abort */
     H5FP_STATUS_CATASTROPHIC
-};
+} H5FP_status_t;
 
-/* Reply from the SAP on an SAP_request send */
-struct SAP_reply {
-    unsigned int req_id;        /* Request ID copied from the SAP_request   */
-    unsigned int sap_file_id;   /* File ID assigned to an open file         */
-    enum sap_status status;     /* Status of the request                    */
-};
+/*
+ * The structure sent to the SAP which holds all of the requested action
+ */
+typedef struct {
+    H5FP_req_t      req_type;   /* Request type                             */
+    unsigned        req_id;     /* ID for request set by sending process    */
+    unsigned        proc_rank;  /* Rank of sending process                  */
+    unsigned        file_id;    /* SAP's file ID for the specific file      */
+    H5FP_obj_t      obj_type;   /* Type of the object                       */
+    H5FP_lock_t     rw_lock;    /* Indicates read or write lock             */
+    H5AC_subid_t    type_id;    /* Type of metadata                         */
+    int             md_size;    /* Size of the metadata sent in next msg    */
+    haddr_t         addr;       /* Address of the metadata                  */
+    unsigned char   oid[H5R_OBJ_REF_BUF_SIZE]; /* Buffer to store OID of object */
+} H5FP_request;
+
+extern MPI_Datatype H5FP_request_t; /* MPI datatype for the H5FP_request obj*/
 
-extern MPI_Datatype SAP_reply_t;    /* MPI datatype for the SAP_reply obj   */
-
-/* The sync message from the SAP on an SAP_request H5FP_REQ_SYNC send */
-struct SAP_sync {
-    unsigned int req_id;        /* Request ID copied from the SAP_request   */
-    unsigned int sync_id;       /* Sync ID to order the sync messages       */
-    unsigned int sap_file_id;   /* SAP's file ID for the specific file      */
-    unsigned int last_msg;      /* Indicates this is the last sync msg sent */
-    enum sap_obj_type obj_type; /* Type of object                           */
-    enum sap_action action;     /* Action done on the object                */
-    enum sap_status status;     /* Status of the request                    */
-    int md_len;                 /* Length of the metadata sent in next msg  */
-    H5FD_mem_t mem_type;        /* Type of memory updated, if req'd         */
-    hsize_t size;               /* Size of memory updated, if req'd         */
-};
+/*
+ * Reply from the SAP on an H5FP_request send
+ */
+typedef struct {
+    unsigned        req_id;     /* Request ID copied from the SAP_request   */
+    unsigned        file_id;    /* File ID assigned to an open file         */
+    H5FP_status_t   status;     /* Status of the request                    */
+    unsigned        md_size;    /* Size of the metadata sent in next msg    */
+} H5FP_reply;
 
-extern MPI_Datatype SAP_sync_t; /* MPI datatype for the SAP_sync obj    */
+extern MPI_Datatype H5FP_reply_t;   /* MPI datatype for the H5FP_reply obj  */
 
 /*
- * Special tag numbers for requests, replies, and string passing messages.
- *
- * Certain actions (Open, Change, and Close) require a pathname to the
- * object. This pathname is sent in a separate message and the SAP will
- * search for it after getting the appropriate request.
+ * The reply message from the SAP on an H5FP_request H5FP_REQ_READ send
  */
-enum {
-    H5FP_TAG_REQUEST,
-    H5FP_TAG_REPLY,
-    H5FP_TAG_SYNC,
-    H5FP_TAG_METADATA,
-    H5FP_TAG_FILE_ID
-};
+typedef struct {
+    unsigned        req_id;     /* Request ID copied from the SAP_request   */
+    unsigned        file_id;    /* SAP's file ID for the specific file      */
+    H5FP_status_t   status;     /* Status of the request                    */
+    H5FD_mem_t      mem_type;   /* Type of memory updated, if req'd         */
+    H5AC_subid_t    type_id;    /* Type of metadata                         */
+    int             md_size;    /* Size of the metadata sent in next msg    */
+    haddr_t         addr;       /* Address of the metadata                  */
+    hsize_t         size;       /* Size of memory updated, if req'd         */
+} H5FP_read;
+
+extern MPI_Datatype H5FP_read_t; /* MPI datatype for the H5FP_read obj      */
 
 /* Handy #define for copying OIDs */
 #define H5FP_COPY_OID(dst, src)     HDmemcpy((dst), (src), H5R_OBJ_REF_BUF_SIZE)
@@ -179,29 +215,24 @@ extern herr_t H5FP_send_metadata(const char *mdata, int len, int rank);
 extern herr_t H5FP_sap_receive_loop(void);
 
 /* Use these functions to communicate with the SAP */
-extern herr_t H5FP_request_open(const char *mdata, int md_len, enum sap_obj_type obj_type,
+extern herr_t H5FP_request_open(const char *mdata, int md_len, H5FP_obj_t obj_type,
                                 unsigned *file_id, unsigned *req_id);
-extern herr_t H5FP_request_lock(unsigned int sap_file_id, unsigned char *mdata,
-                                enum sap_lock_type rw_lock, int last, unsigned *req_id,
-                                enum sap_status *status);
-extern herr_t H5FP_request_release_lock(unsigned int sap_file_id, unsigned char *mdata,
+extern herr_t H5FP_request_lock(unsigned sap_file_id, unsigned char *mdata,
+                                H5FP_lock_t rw_lock, int last, unsigned *req_id,
+                                H5FP_status_t *status);
+extern herr_t H5FP_request_release_lock(unsigned sap_file_id, unsigned char *mdata,
                                         int last, unsigned *req_id,
-                                        enum sap_status *status);
-extern herr_t H5FP_request_change(unsigned int sap_file_id, unsigned char *obj_oid,
-                                  enum sap_obj_type obj_type, enum sap_action action,
-                                  int mdata_len, const char *mdata, unsigned *req_id,
-                                  enum sap_status *status);
-extern herr_t H5FP_request_sync(unsigned int sap_file_id, hid_t hdf_file_id,
-                                unsigned *req_id, enum sap_status *status);
+                                        H5FP_status_t *status);
+extern herr_t H5FP_request_write_metadata(unsigned sap_file_id, unsigned char *obj_oid,
+                                          H5FP_obj_t obj_type, H5AC_subid_t type_id,
+                                          haddr_t addr, int mdata_len, const char *mdata,
+                                          unsigned *req_id, H5FP_status_t *status);
+extern herr_t H5FP_request_read_metadata(unsigned sap_file_id, H5FP_obj_t obj_type,
+                                         H5AC_subid_t type_id, haddr_t addr,
+                                         size_t size, uint8_t **buf, unsigned *req_id,
+                                         H5FP_status_t *status);
 extern herr_t H5FP_request_close(unsigned sap_file_id, unsigned *req_id);
 
-/* Helper functions */
-extern void H5FP_fill_fphdf5_struct(H5O_fphdf5_t *fphdf5, uint8_t *oid, haddr_t header,
-                                    struct H5S_simple_t *sdim, H5T_t *dtype,
-                                    time_t *mtime, H5O_layout_t *layout,
-                                    H5O_name_t *group, H5O_name_t *dataset,
-                                    struct H5P_genplist_t *plist);
-
 #ifdef __cplusplus
 }
 #endif  /* __cplusplus */
diff --git a/src/H5FPserver.c b/src/H5FPserver.c
index 24b7a6f..e073feb 100644
--- a/src/H5FPserver.c
+++ b/src/H5FPserver.c
@@ -26,6 +26,7 @@
  */
 
 #include "H5private.h"          /* Generic Functions                    */
+#include "H5ACprivate.h"        /* Metadata Cache                       */
 #include "H5Eprivate.h"         /* Error Handling                       */
 #include "H5Oprivate.h"         /* Object Headers                       */
 #include "H5TBprivate.h"        /* Threaded, Balanced, Binary Trees     */
@@ -43,120 +44,77 @@ static int interface_initialize_g = 0;
 
 /* Internal SAP structures */
 
-struct sap_obj_lock {
-    uint8_t oid[H5R_OBJ_REF_BUF_SIZE]; /* Buffer to store OID of object referenced */
-    unsigned int owned_rank;    /* rank which has the lock                  */
-    enum sap_obj_type obj_type; /* type of object being locked              */
-    unsigned int ref_count;     /* number of times lock was aquired by proc */
-    enum sap_lock_type rw_lock; /* indicates if it's a read or write lock   */
-};
-
-struct sap_file_mod {
-    unsigned long key;          /* key field for insertion into the TBBT    */
-    unsigned char *procs_notified; /* bitfield of size MPI_Comm_size()      */
-                                /* indicates process was notified of this
-                                 * modification - 0 means no, 1 means yes   */
-    unsigned num_notified;      /* counts number of processes notified      */
-    enum sap_obj_type obj_type; /* type of object                           */
-    enum sap_action action;     /* action done to object (H5FP_REQ_CHANGE only) */
-    int md_len;                 /* the length of the metadata array         */
-    char *metadata;             /* encoded metadata about the object        */
-};
-
-struct sap_file_struct {
-    unsigned int sap_file_id;   /* the file id the SAP keeps per file       */
-    char *filename;             /* the filename - of dubious use            */
-    int closing;                /* we're closing the file - no more changes */
-    H5TB_TREE *mod_tree;        /* a tree of the modifications done         */
-    H5TB_TREE *locks;           /* a tree of locks on objects in the file   */
-};
+typedef struct {
+    uint8_t         oid[H5R_OBJ_REF_BUF_SIZE]; /* Buffer to store OID of object */
+    unsigned        owned_rank; /* rank which has the lock                  */
+    H5FP_obj_t      obj_type;   /* type of object being locked              */
+    unsigned        ref_count;  /* number of times lock was aquired by proc */
+    H5FP_lock_t     rw_lock;    /* indicates if it's a read or write lock   */
+} H5FP_object_lock;
+
+typedef struct {
+    H5AC_subid_t    type_id;    /* id of the type of this metadata          */
+    H5FP_obj_t      obj_type;   /* type of object modified                  */
+    haddr_t         addr;       /* address of the metadata                  */
+    unsigned        md_size;    /* size of the metadata                     */
+    char           *metadata;   /* encoded metadata about the object        */
+} H5FP_mdata_mod;
+
+typedef struct {
+    unsigned        file_id;    /* the file id the SAP keeps per file       */
+    char           *filename;   /* the filename - of dubious use            */
+    int             closing;    /* we're closing the file - no more changes */
+    H5TB_TREE      *mod_tree;   /* a tree of metadata updates done          */
+    H5TB_TREE      *locks;      /* a tree of locks on objects in the file   */
+} H5FP_file_info;
 
 static H5TB_TREE *fs_tree;
 
-/*
- *  Note on procs_notified:
- *
- *  We needed a way of indicating that a process has been notified of a
- *  change. However, we didn't want to waste space, so I implemented an array
- *  of unsigned characters which would emulate such a thing. The code:
- *
- *          rank >> 3
- *
- *  is an integer divide by 8. While the code:
- *
- *          rank & 7
- *
- *  is a mod by 8. (Note that rank should be unsigned at this point). So, the
- *  code
- *
- *          fm->procs_notified[rank >> 3] |= 1 << (rank & 7);
- *
- *  is equivalent to
- *
- *          fm->procs_notified[rank / 8] |= 1 << (rank % 8);
- *
- *  Testing whether a given bit is set or not involves a test like this:
- *
- *          if (mod->procs_notified[rank >> 3] & (1 << (rank & 7)))
- *
- *  Of course, all bytes are 8 bits wide, right? :-)
- *
- *  The following two macros help this be more readable.
- */
-
-#define SET_PROC_NOTIFIED(arr, rank)    \
-    (arr->procs_notified[(rank) >> 3] |= (1 << ((rank) & 7)), ++arr->num_notified)
-#define IS_PROC_NOTIFIED(arr, rank)     \
-    (arr->procs_notified[(rank) >> 3] & (1 << ((rank) & 7)))
-
 /* local functions */
-static herr_t H5FP_sap_receive(struct SAP_request *req, int source,
-                               int tag, char **buf);
+static herr_t H5FP_sap_receive(H5FP_request *req, int source, int tag, char **buf);
 
     /* local functions to generate unique ids for messages */
-static unsigned int H5FP_gen_sap_file_id(void);
+static unsigned H5FP_gen_sap_file_id(void);
 
     /* local functions for handling object locks */
-static int H5FP_object_lock_cmp(struct sap_obj_lock *o1,
-                                struct sap_obj_lock *o2,
-                                int cmparg);
-static struct sap_obj_lock *H5FP_new_object_lock(const unsigned char *oid,
-                                                 unsigned int rank,
-                                                 enum sap_obj_type obj_type,
-                                                 enum sap_lock_type rw_lock);
-static herr_t H5FP_free_object_lock(struct sap_obj_lock *ol);
-static struct sap_obj_lock *H5FP_find_object_lock(struct sap_file_struct *fs,
-                                                  unsigned char *oid);
-static herr_t H5FP_remove_object_lock_from_list(struct sap_file_struct *fs,
-                                                struct sap_obj_lock *ol);
-
-    /* local file structure handling functions */
-static herr_t H5FP_add_new_file_struct_to_list(unsigned int sap_file_id, char *filename);
-static int H5FP_file_struct_cmp(struct sap_file_struct *k1,
-                                struct sap_file_struct *k2, int cmparg);
-static struct sap_file_struct *H5FP_new_file_struct_node(unsigned int sap_file_id,
-                                                         char *filename);
-static struct sap_file_struct *H5FP_find_file_struct(unsigned int sap_file_id);
-static herr_t H5FP_remove_file_id_from_list(unsigned int sap_file_id);
-static herr_t H5FP_free_file_struct_node(struct sap_file_struct *fs);
+static int H5FP_object_lock_cmp(H5FP_object_lock *o1, H5FP_object_lock *o2, int cmparg);
+static H5FP_object_lock *H5FP_new_object_lock(const unsigned char *oid,
+                                              unsigned rank,
+                                              H5FP_obj_t obj_type,
+                                              H5FP_lock_t rw_lock);
+static herr_t H5FP_free_object_lock(H5FP_object_lock *ol);
+static H5FP_object_lock *H5FP_find_object_lock(H5FP_file_info *fs, unsigned char *oid);
+static herr_t H5FP_remove_object_lock_from_list(H5FP_file_info *fs, H5FP_object_lock *ol);
+
+    /* local file information handling functions */
+static herr_t H5FP_add_new_file_info_to_list(unsigned file_id, char *filename);
+static int H5FP_file_info_cmp(H5FP_file_info *k1, H5FP_file_info *k2, int cmparg);
+static H5FP_file_info *H5FP_new_file_info_node(unsigned file_id, char *filename);
+static H5FP_file_info *H5FP_find_file_info(unsigned file_id);
+static herr_t H5FP_remove_file_id_from_list(unsigned file_id);
+static herr_t H5FP_free_file_info_node(H5FP_file_info *fs);
 
     /* local file modification structure handling functions */
-static struct sap_file_mod *H5FP_new_file_mod_node(unsigned rank,
-                                                   enum sap_obj_type obj_type,
-                                                   enum sap_action action,
-                                                   int md_len,
-                                                   char *metadata);
-static herr_t H5FP_free_mod_node(struct sap_file_mod *fs);
+static H5FP_mdata_mod *H5FP_new_file_mod_node(unsigned rank,
+                                              H5AC_subid_t type_id,
+                                              haddr_t addr,
+                                              int md_size,
+                                              char *metadata);
+static herr_t H5FP_free_mod_node(H5FP_mdata_mod *fs);
 
     /* local request handling functions */
-static herr_t H5FP_sap_handle_open_request(struct SAP_request req, char *mdata, int md_len);
-static herr_t H5FP_sap_handle_lock_request(struct SAP_request req);
-static herr_t H5FP_sap_handle_release_lock_request(struct SAP_request req);
-static herr_t H5FP_sap_handle_change_request(struct SAP_request req, char *mdata, int md_len);
-static herr_t H5FP_sap_handle_sync_request(struct SAP_request req);
-static herr_t H5FP_sap_handle_close_request(struct SAP_request req);
+static herr_t H5FP_sap_handle_open_request(H5FP_request req, char *mdata, int md_size);
+static herr_t H5FP_sap_handle_lock_request(H5FP_request req);
+static herr_t H5FP_sap_handle_release_lock_request(H5FP_request req);
+static herr_t H5FP_sap_handle_write_request(H5FP_request req, char *mdata, int md_size);
+static herr_t H5FP_sap_handle_read_request(H5FP_request req);
+static herr_t H5FP_sap_handle_close_request(H5FP_request req);
 
-/** Public Library (non-API) Functions **/
+/*
+ *===----------------------------------------------------------------------===
+ *                    Public Library (non-API) Functions
+ *===----------------------------------------------------------------------===
+ */
 
 /*
  * Function:    H5FP_sap_receive_loop
@@ -177,12 +135,13 @@ H5FP_sap_receive_loop(void)
 
     FUNC_ENTER_NOAPI(H5FP_sap_receive_loop, FAIL);
 
-    if ((fs_tree = H5TB_dmake((H5TB_cmp_t)H5FP_file_struct_cmp,
-                              sizeof(struct sap_file_struct), FALSE)) == NULL)
+    /* Create the file structure tree. */
+    if ((fs_tree = H5TB_dmake((H5TB_cmp_t)H5FP_file_info_cmp,
+                              sizeof(H5FP_file_info), FALSE)) == NULL)
         HGOTO_ERROR(H5E_FPHDF5, H5E_CANTMAKETREE, FAIL, "cannot make TBBT tree");
 
     for (;;) {
-        struct SAP_request req; 
+        H5FP_request req; 
         char *buf = NULL;
         herr_t hrc;
 
@@ -191,9 +150,8 @@ H5FP_sap_receive_loop(void)
 
         switch (req.req_type) {
         case H5FP_REQ_OPEN:
-            if ((hrc = H5FP_sap_handle_open_request(req, buf, req.md_len)) != SUCCEED)
+            if ((hrc = H5FP_sap_handle_open_request(req, buf, req.md_size)) != SUCCEED)
                 HGOTO_ERROR(H5E_FPHDF5, H5E_CANTOPENOBJ, FAIL, "cannot open file");
-
             break;
         case H5FP_REQ_LOCK:
         case H5FP_REQ_LOCK_END:
@@ -203,11 +161,11 @@ H5FP_sap_receive_loop(void)
         case H5FP_REQ_RELEASE_END:
             hrc = H5FP_sap_handle_release_lock_request(req);
             break;
-        case H5FP_REQ_CHANGE:
-            hrc = H5FP_sap_handle_change_request(req, buf, req.md_len);
+        case H5FP_REQ_WRITE:
+            hrc = H5FP_sap_handle_write_request(req, buf, req.md_size);
             break;
-        case H5FP_REQ_SYNC:
-            hrc = H5FP_sap_handle_sync_request(req);
+        case H5FP_REQ_READ:
+            hrc = H5FP_sap_handle_read_request(req);
             break;
         case H5FP_REQ_CLOSE:
             hrc = H5FP_sap_handle_close_request(req);
@@ -215,7 +173,6 @@ H5FP_sap_receive_loop(void)
         case H5FP_REQ_STOP:
             if (++stop == H5FP_comm_size - 1)
                 goto done;
-
             break;
         default:
             HGOTO_ERROR(H5E_FPHDF5, H5E_ARGS, FAIL, "invalid request type");
@@ -240,7 +197,7 @@ done:
  * Modifications:
  */
 static herr_t
-H5FP_sap_receive(struct SAP_request *req, int source, int tag, char **buf)
+H5FP_sap_receive(H5FP_request *req, int source, int tag, char **buf)
 {
     MPI_Status status;
     herr_t ret_value = SUCCEED;
@@ -250,17 +207,21 @@ H5FP_sap_receive(struct SAP_request *req, int source, int tag, char **buf)
 
     HDmemset(&status, 0, sizeof(status));
 
-    if ((mrc = MPI_Recv(req, 1, SAP_request_t, source, tag,
+    if ((mrc = MPI_Recv(req, 1, H5FP_request_t, source, tag,
                         H5FP_SAP_COMM, &status)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
 
-    if (buf && req->md_len) {
-        if ((*buf = (char *)HDmalloc((size_t)req->md_len + 1)) == NULL)
+    if (buf && req->md_size) {
+        /*
+         * There is metadata associated with this request. Get it as a
+         * string (requires another read).
+         */
+        if ((*buf = (char *)HDmalloc((size_t)req->md_size + 1)) == NULL)
             HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "out of memory");
 
-        HDmemset(*buf, 0, (size_t)req->md_len + 1);
+        HDmemset(*buf, 0, (size_t)req->md_size + 1);
 
-        if ((mrc = MPI_Recv(*buf, req->md_len, MPI_BYTE, (int)req->proc_rank,
+        if ((mrc = MPI_Recv(*buf, req->md_size, MPI_BYTE, (int)req->proc_rank,
                             H5FP_TAG_METADATA, H5FP_SAP_COMM, &status)) != MPI_SUCCESS) {
             HDfree(*buf);
             *buf = NULL;
@@ -279,8 +240,8 @@ done:
  * Programmer:  Bill Wendling, 09. September 2002
  * Modifications:
  */
-static int H5FP_object_lock_cmp(struct sap_obj_lock *o1,
-                                struct sap_obj_lock *o2,
+static int H5FP_object_lock_cmp(H5FP_object_lock *o1,
+                                H5FP_object_lock *o2,
                                 int UNUSED cmparg)
 {
     FUNC_ENTER_NOINIT(H5FP_object_lock_cmp);
@@ -303,16 +264,16 @@ static int H5FP_object_lock_cmp(struct sap_obj_lock *o1,
  * Programmer:  Bill Wendling, 09. September 2002
  * Modifications:
  */
-static struct sap_obj_lock *
-H5FP_new_object_lock(const unsigned char *oid, unsigned int rank,
-                     enum sap_obj_type obj_type, enum sap_lock_type rw_lock)
+static H5FP_object_lock *
+H5FP_new_object_lock(const unsigned char *oid, unsigned rank,
+                     H5FP_obj_t obj_type, H5FP_lock_t rw_lock)
 {
-    struct sap_obj_lock *ret_value = NULL;
+    H5FP_object_lock *ret_value = NULL;
     
     FUNC_ENTER_NOINIT(H5FP_new_object_lock);
     assert(oid);
 
-    if ((ret_value = (struct sap_obj_lock *)HDmalloc(sizeof(struct sap_obj_lock))) == NULL)
+    if ((ret_value = (H5FP_object_lock *)HDmalloc(sizeof(H5FP_object_lock))) == NULL)
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "out of memory");
 
     H5FP_COPY_OID(ret_value->oid, oid);
@@ -333,7 +294,7 @@ done:
  * Modifications:
  */
 static herr_t
-H5FP_free_object_lock(struct sap_obj_lock *ol)
+H5FP_free_object_lock(H5FP_object_lock *ol)
 {
     FUNC_ENTER_NOINIT(H5FP_free_object_lock);
     HDfree(ol);
@@ -347,10 +308,10 @@ H5FP_free_object_lock(struct sap_obj_lock *ol)
  * Programmer:  Bill Wendling, 09. September 2002
  * Modifications:
  */
-static struct sap_obj_lock *
-H5FP_find_object_lock(struct sap_file_struct *fs, unsigned char *oid)
+static H5FP_object_lock *
+H5FP_find_object_lock(H5FP_file_info *fs, unsigned char *oid)
 {
-    struct sap_obj_lock *ret_value = NULL;
+    H5FP_object_lock *ret_value = NULL;
 
     FUNC_ENTER_NOINIT(H5FP_find_object_lock);
 
@@ -359,14 +320,14 @@ H5FP_find_object_lock(struct sap_file_struct *fs, unsigned char *oid)
 
     if (fs->locks && fs->locks->root) {
         H5TB_NODE *node;
-        struct sap_obj_lock ol;
+        H5FP_object_lock ol;
 
-        H5FP_COPY_OID(ol.oid, oid);
+        H5FP_COPY_OID(ol.oid, oid); /* This is the key field for the TBBT */
 
         if ((node = H5TB_dfind(fs->locks, (void *)&ol, NULL)) == NULL)
             HGOTO_ERROR(H5E_FPHDF5, H5E_NOTFOUND, NULL, "lock not found");
 
-        ret_value = (struct sap_obj_lock *)node->data;
+        ret_value = (H5FP_object_lock *)node->data;
     }
 
 done:
@@ -383,8 +344,8 @@ done:
  * Modifications:
  */
 static herr_t
-H5FP_remove_object_lock_from_list(struct sap_file_struct *fs,
-                                  struct sap_obj_lock *ol)
+H5FP_remove_object_lock_from_list(H5FP_file_info *fs,
+                                  H5FP_object_lock *ol)
 {
     H5TB_NODE *node;
     herr_t ret_value = FAIL;
@@ -407,29 +368,14 @@ H5FP_remove_object_lock_from_list(struct sap_file_struct *fs,
  * Modifications:
  */
 static int
-H5FP_file_mod_cmp(struct sap_file_mod *k1, struct sap_file_mod *k2, int UNUSED cmparg)
+H5FP_file_mod_cmp(H5FP_mdata_mod *k1,
+                  H5FP_mdata_mod *k2,
+                  int UNUSED cmparg)
 {
     FUNC_ENTER_NOINIT(H5FP_file_mod_cmp);
     assert(k1);
     assert(k2);
-    FUNC_LEAVE_NOAPI(k1->key - k2->key);
-}
-
-/*
- * Function:    H5FP_new_file_mod_key
- * Purpose:     Generate a new key for the sap_file_mode structure.
- * Return:      Success:    New key value.
- *              Failure:    Doesn't.
- * Programmer:  Bill Wendling, 27. August, 2002
- * Modifications:
- */
-static unsigned long
-H5FP_new_file_mod_key(void)
-{
-    static unsigned long key = 0;
-
-    FUNC_ENTER_NOINIT(H5FP_new_file_mod_key);
-    FUNC_LEAVE_NOAPI(key++);
+    FUNC_LEAVE_NOAPI(k1->addr - k2->addr);
 }
 
 /*
@@ -442,12 +388,11 @@ H5FP_new_file_mod_key(void)
  * Modifications:
  */
 static herr_t
-H5FP_free_mod_node(struct sap_file_mod *fs)
+H5FP_free_mod_node(H5FP_mdata_mod *fs)
 {
     FUNC_ENTER_NOINIT(H5FP_free_mod_node);
 
     if (fs) {
-        HDfree(fs->procs_notified);
         HDfree(fs->metadata);
         HDfree(fs);
     }
@@ -457,41 +402,31 @@ H5FP_free_mod_node(struct sap_file_mod *fs)
 
 /*
  * Function:    H5FP_new_file_mod_node
- * Purpose:     Create a new sap_file_mod node and initialize it.
+ * Purpose:     Create a new sap_file_mod node and initialize it. This
+ *              object now has responsibility for freeing the metadata
+ *              information.
  * Return:      Success: Pointer to new sap_file_mod structure.
  *              Failure: NULL
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
-static struct sap_file_mod *
-H5FP_new_file_mod_node(unsigned rank, enum sap_obj_type obj_type,
-                       enum sap_action action, int md_len, char *metadata)
+static H5FP_mdata_mod *
+H5FP_new_file_mod_node(unsigned UNUSED rank, H5AC_subid_t type_id,
+                       haddr_t addr, int md_size, char *metadata)
 {
-    struct sap_file_mod *ret_value = NULL;
+    H5FP_mdata_mod *ret_value = NULL;
 
     FUNC_ENTER_NOINIT(H5FP_new_file_mod_node);
 
-    ret_value = (struct sap_file_mod *)HDmalloc(sizeof(struct sap_file_mod));
+    ret_value = (H5FP_mdata_mod *)HDmalloc(sizeof(H5FP_mdata_mod));
 
     if (!ret_value)
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "out of memory");
 
-    ret_value->procs_notified = (unsigned char *)HDcalloc((size_t)1 +
-                                                   (H5FP_comm_size - 1) / H5FP_BYTE_BITS, 1);
-
-    if (!ret_value->procs_notified) {
-        HDfree(ret_value);
-        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "out of memory");
-    }
-
-    /* we'll just assume the SAP has been notified of this change :) */
-    SET_PROC_NOTIFIED(ret_value, H5FP_sap_rank);
-    SET_PROC_NOTIFIED(ret_value, rank);
-    ret_value->obj_type = obj_type;
-    ret_value->action = action;
-    ret_value->md_len = md_len;
+    ret_value->addr = addr;
+    ret_value->type_id = type_id;
+    ret_value->md_size = md_size;
     ret_value->metadata = metadata;
-    ret_value->key = H5FP_new_file_mod_key();
 
 done:
     FUNC_LEAVE_NOAPI(ret_value);
@@ -499,27 +434,46 @@ done:
 
 /*
  * Function:    H5FP_add_file_mod_to_list
- * Purpose:     Add a modification to a file ID.
+ * Purpose:     Add a metadata write to a file ID. If the metadata is
+ *              already in the cache, then we just replace it with the
+ *              updated bits. (Only the metadata info and size should
+ *              change in this case.)
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
 static herr_t
-H5FP_add_file_mod_to_list(struct sap_file_struct *fs, enum sap_obj_type obj_type,
-                          enum sap_action action, unsigned rank, int md_len,
+H5FP_add_file_mod_to_list(H5FP_file_info *fs, H5AC_subid_t type_id,
+                          haddr_t addr, unsigned rank, int md_size,
                           char *metadata)
 {
-    struct sap_file_mod *fm;
+    H5FP_mdata_mod *fm, mod;
+    H5TB_NODE *node;
     herr_t ret_value = FAIL;
 
     FUNC_ENTER_NOINIT(H5FP_add_file_mod_to_list);
 
     assert(fs);
 
-    if ((fm = H5FP_new_file_mod_node(rank, obj_type, action, md_len, metadata)) != NULL) {
+    mod.addr = addr;    /* This is the key field for the TBBT */
+
+    if ((node = H5TB_dfind(fs->mod_tree, (void *)&mod, NULL)) != NULL) {
+        /*
+         * The metadata is in the cache already. All we have to do is
+         * replace what's there. The addr and type should be the same.
+         * The only things to change is the metadata and its size.
+         */
+        fm = (H5FP_mdata_mod *)node->data;
+        HDfree(fm->metadata);
+        fm->metadata = metadata;
+        fm->md_size = md_size;
+        ret_value = SUCCEED;
+    } else if ((fm = H5FP_new_file_mod_node(rank, type_id, addr,
+                                            md_size, metadata)) != NULL) {
         if (!H5TB_dins(fs->mod_tree, (void *)fm, NULL))
-            HGOTO_ERROR(H5E_FPHDF5, H5E_CANTINSERT, FAIL, "can't insert modification into tree");
+            HGOTO_ERROR(H5E_FPHDF5, H5E_CANTINSERT, FAIL,
+                        "can't insert modification into tree");
 
         ret_value = SUCCEED;
     }
@@ -532,18 +486,13 @@ done:
  * Function:    H5FP_remove_mod_from_list
  * Purpose:     Remove a modification from the list of modifications on
  *              this file.
- *
- *              NOTE: Don't call this function directly. This should only be
- *              called after all processes for a particular modification have
- *              been synced.
- *
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 31. July, 2002
  * Modifications:
  */
 static herr_t
-H5FP_remove_mod_from_list(struct sap_file_struct *fs, struct sap_file_mod *mod)
+H5FP_remove_mod_from_list(H5FP_file_info *fs, H5FP_mdata_mod *mod)
 {
     H5TB_NODE *node;
     herr_t ret_value = FAIL;
@@ -559,7 +508,7 @@ H5FP_remove_mod_from_list(struct sap_file_struct *fs, struct sap_file_mod *mod)
 }
 
 /*
- * Function:    H5FP_free_file_struct_node
+ * Function:    H5FP_free_file_info_node
  * Purpose:     Helper function to free up an SAP_FILE_STRUCT node and all
  *              of the malloced space it has.
  *
@@ -571,9 +520,9 @@ H5FP_remove_mod_from_list(struct sap_file_struct *fs, struct sap_file_mod *mod)
  * Modifications:
  */
 static herr_t
-H5FP_free_file_struct_node(struct sap_file_struct *fs)
+H5FP_free_file_info_node(H5FP_file_info *fs)
 {
-    FUNC_ENTER_NOINIT(H5FP_free_file_struct_node);
+    FUNC_ENTER_NOINIT(H5FP_free_file_info_node);
 
     if (fs) {
         H5TB_dfree(fs->mod_tree, (void (*)(void*))H5FP_free_mod_node, NULL);
@@ -586,54 +535,53 @@ H5FP_free_file_struct_node(struct sap_file_struct *fs)
 }
 
 /*
- * Function:    H5FP_file_struct_cmp
- * Purpose:     Compare two sap_file_struct structs for the TBBT.
+ * Function:    H5FP_file_info_cmp
+ * Purpose:     Compare two sap_file_info structs for the TBBT.
  * Return:      <0, 0, >0
  * Programmer:  Bill Wendling, 27. August, 2002
  * Modifications:
  */
 static int
-H5FP_file_struct_cmp(struct sap_file_struct *k1, struct sap_file_struct *k2, int UNUSED cmparg)
+H5FP_file_info_cmp(H5FP_file_info *k1, H5FP_file_info *k2, int UNUSED cmparg)
 {
-    FUNC_ENTER_NOINIT(H5FP_file_struct_cmp);
+    FUNC_ENTER_NOINIT(H5FP_file_info_cmp);
     assert(k1);
     assert(k2);
-    FUNC_LEAVE_NOAPI(k1->sap_file_id - k2->sap_file_id);
+    FUNC_LEAVE_NOAPI(k1->file_id - k2->file_id);
 }
 
 /*
- * Function:    H5FP_new_file_struct_node
- * Purpose:     Create and initialize an sap_file_struct node.
+ * Function:    H5FP_new_file_info_node
+ * Purpose:     Create and initialize an sap_file_info node.
  * Return:      Success: Pointer to new node
  *              Failure: NULL
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
-static struct sap_file_struct *
-H5FP_new_file_struct_node(unsigned int sap_file_id, char *filename)
+static H5FP_file_info *
+H5FP_new_file_info_node(unsigned file_id, char *filename)
 {
-    struct sap_file_struct *ret_value;
+    H5FP_file_info *ret_value;
 
-    FUNC_ENTER_NOINIT(H5FP_new_file_struct_node);
+    FUNC_ENTER_NOINIT(H5FP_new_file_info_node);
 
-    if ((ret_value = (struct sap_file_struct *)HDmalloc(sizeof(struct sap_file_struct))) == NULL)
+    if ((ret_value = (H5FP_file_info *)HDmalloc(sizeof(H5FP_file_info))) == NULL)
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "out of memory");
 
-    ret_value->sap_file_id = sap_file_id;
+    ret_value->file_id = file_id;
     ret_value->filename = filename;
     ret_value->closing = FALSE;
-    ret_value->mod_tree = NULL;
-    ret_value->locks = NULL;
 
     if ((ret_value->mod_tree = H5TB_dmake((H5TB_cmp_t)H5FP_file_mod_cmp,
-                                          sizeof(struct sap_file_mod), FALSE)) == NULL) {
-        H5FP_free_file_struct_node(ret_value);
+                                          sizeof(H5FP_mdata_mod), FALSE)) == NULL) {
+        HDfree(ret_value);
         HGOTO_ERROR(H5E_FPHDF5, H5E_CANTMAKETREE, NULL, "cannot make TBBT tree");
     }
 
     if ((ret_value->locks = H5TB_dmake((H5TB_cmp_t)H5FP_object_lock_cmp,
-                                       sizeof(struct sap_obj_lock), FALSE)) == NULL) {
-        H5FP_free_file_struct_node(ret_value);
+                                       sizeof(H5FP_object_lock), FALSE)) == NULL) {
+        H5TB_dfree(ret_value->mod_tree, (void (*)(void*))H5FP_free_mod_node, NULL);
+        HDfree(ret_value);
         HGOTO_ERROR(H5E_FPHDF5, H5E_CANTMAKETREE, NULL, "cannot make TBBT tree");
     }
 
@@ -642,50 +590,50 @@ done:
 }
 
 /*
- * Function:    H5FP_find_file_struct
- * Purpose:     Find the file structure for the requested sap_file_id.
+ * Function:    H5FP_find_file_info
+ * Purpose:     Find the file structure for the requested file_id.
  * Return:      Success:    Pointer to the file structure
  *              Failure:    NULL
  * Programmer:  Bill Wendling, 31. July, 2002
  * Modifications:
  */
-static struct sap_file_struct *
-H5FP_find_file_struct(unsigned int sap_file_id)
+static H5FP_file_info *
+H5FP_find_file_info(unsigned file_id)
 {
     H5TB_NODE *node = NULL;
-    struct sap_file_struct *ret_value = NULL;
+    H5FP_file_info *ret_value = NULL;
 
-    FUNC_ENTER_NOINIT(H5FP_find_file_struct);
+    FUNC_ENTER_NOINIT(H5FP_find_file_info);
 
     if (fs_tree && fs_tree->root) {
-        struct sap_file_struct s;
+        H5FP_file_info s;
 
-        s.sap_file_id = sap_file_id;
+        s.file_id = file_id;    /* This is the key field for the TBBT */
 
         if ((node = H5TB_dfind(fs_tree, (void *)&s, NULL)) != NULL)
-            ret_value = (struct sap_file_struct *)node->data;
+            ret_value = (H5FP_file_info *)node->data;
     }
 
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
 /*
- * Function:    H5FP_add_new_file_struct_to_list
- * Purpose:     Add an SAP_FILE_ID to the list of file IDS.
+ * Function:    H5FP_add_new_file_info_to_list
+ * Purpose:     Add a FILE_ID to the list of file IDS.
  * Return:      SUCCEED if the node was added
  *              FAIL otherwise
  * Programmer:  Bill Wendling, 02. August, 2002
  * Modifications:
  */
 static herr_t
-H5FP_add_new_file_struct_to_list(unsigned int sap_file_id, char *filename)
+H5FP_add_new_file_info_to_list(unsigned file_id, char *filename)
 {
-    struct sap_file_struct *fs;
+    H5FP_file_info *fs;
     herr_t ret_value = FAIL;
 
-    FUNC_ENTER_NOINIT(H5FP_add_new_file_struct_to_list);
+    FUNC_ENTER_NOINIT(H5FP_add_new_file_info_to_list);
 
-    if ((fs = H5FP_new_file_struct_node(sap_file_id, filename)) != NULL) {
+    if ((fs = H5FP_new_file_info_node(file_id, filename)) != NULL) {
         if (!H5TB_dins(fs_tree, (void *)fs, NULL))
             HGOTO_ERROR(H5E_FPHDF5, H5E_CANTINSERT, FAIL,
                         "can't insert file structure into tree");
@@ -699,7 +647,7 @@ done:
 
 /*
  * Function:    H5FP_remove_file_id_from_list
- * Purpose:     Remove an SAP_FILE_ID from the list of file IDS.
+ * Purpose:     Remove an FILE_ID from the list of file IDS.
  *
  *              NOTE: This should only be called after all modifications to
  *              the file descriptor have been synced to all processes and the
@@ -711,41 +659,44 @@ done:
  * Modifications:
  */
 static herr_t
-H5FP_remove_file_id_from_list(unsigned int sap_file_id)
+H5FP_remove_file_id_from_list(unsigned file_id)
 {
-    struct sap_file_struct s;
+    H5FP_file_info s;
     H5TB_NODE *node;
     herr_t ret_value = FAIL;
 
     FUNC_ENTER_NOINIT(H5FP_remove_file_id_from_list);
 
-    s.sap_file_id = sap_file_id;
+    s.file_id = file_id;    /* This is the key field for the TBBT */
 
     if ((node = H5TB_dfind(fs_tree, (void *)&s, NULL)) != NULL) {
-        H5FP_free_file_struct_node(H5TB_rem(&fs_tree->root, node, NULL));
+        H5FP_free_file_info_node(H5TB_rem(&fs_tree->root, node, NULL));
         ret_value = SUCCEED;
     }
 
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
-/** Functions to reply to requests **/
+/*
+ *===----------------------------------------------------------------------===
+ *                     Functions to reply to requests
+ *===----------------------------------------------------------------------===
+ */
 
 static herr_t
-H5FP_send_reply(unsigned int to, unsigned int req_id, unsigned int file_id,
-                enum sap_status status)
+H5FP_send_reply(unsigned to, unsigned req_id, unsigned file_id, H5FP_status_t status)
 {
-    struct SAP_reply reply;
+    H5FP_reply reply;
     herr_t ret_value = SUCCEED;
     int mrc;
 
     FUNC_ENTER_NOINIT(H5FP_send_reply);
 
     reply.req_id = req_id;
-    reply.sap_file_id = file_id;
+    reply.file_id = file_id;
     reply.status = status;
 
-    if ((mrc = MPI_Send(&reply, 1, SAP_reply_t, (int)to, H5FP_TAG_REPLY,
+    if ((mrc = MPI_Send(&reply, 1, H5FP_reply_t, (int)to, H5FP_TAG_REPLY,
                         H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
@@ -753,7 +704,11 @@ done:
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
-/** Functions to handle SAP requests **/
+/*
+ *===----------------------------------------------------------------------===
+ *                    Functions to handle SAP requests
+ *===----------------------------------------------------------------------===
+ */
 
 /*
  * Function:    H5FP_gen_request_id
@@ -763,10 +718,10 @@ done:
  * Programmer:  Bill Wendling, 30. July, 2002
  * Modifications:
  */
-static unsigned int
+static unsigned
 H5FP_gen_sap_file_id()
 {
-    static unsigned int i = 0;
+    static unsigned i = 0;
     FUNC_ENTER_NOINIT(H5FP_gen_sap_file_id);
     FUNC_LEAVE_NOAPI(i++);
 }
@@ -780,7 +735,7 @@ H5FP_gen_sap_file_id()
  * Modifications:
  */
 static herr_t
-H5FP_sap_handle_open_request(struct SAP_request req, char *mdata, int UNUSED md_len)
+H5FP_sap_handle_open_request(H5FP_request req, char *mdata, int UNUSED md_size)
 {
     herr_t ret_value = SUCCEED;
     int mrc;
@@ -791,12 +746,17 @@ H5FP_sap_handle_open_request(struct SAP_request req, char *mdata, int UNUSED md_
         unsigned new_file_id = H5FP_gen_sap_file_id();
         int i;
 
-        if (H5FP_add_new_file_struct_to_list(new_file_id, mdata) != SUCCEED)
+        if (H5FP_add_new_file_info_to_list(new_file_id, mdata) != SUCCEED) {
+            HDfree(mdata);
             HGOTO_ERROR(H5E_FPHDF5, H5E_CANTINSERT, FAIL,
                         "can't insert new file structure to list");
+        }
 
         /* broadcast the file id to all processes */
-        /* FIXME: Isn't there some way to broadcast this result to the barrier group? -QAK */
+        /*
+         * FIXME: Isn't there some way to broadcast this result to the
+         * barrier group? -QAK
+         */
         /*
          * XXX:
          *      MPI_Bcast doesn't work in this way and I don't know how
@@ -838,24 +798,25 @@ done:
  * Modifications:
  */
 static herr_t
-H5FP_sap_handle_lock_request(struct SAP_request req)
+H5FP_sap_handle_lock_request(H5FP_request req)
 {
     struct lock_group {
         unsigned char oid[sizeof(req.oid)];
-        unsigned int file_id;
-        unsigned int locked;
-        enum sap_lock_type rw_lock;
-        struct sap_file_struct *fs;
-        struct sap_obj_lock *lock;
+        unsigned file_id;
+        unsigned locked;
+        H5FP_lock_t rw_lock;
+        H5FP_file_info *fs;
+        H5FP_object_lock *lock;
     } *oids;
-    int list_size = 2;  /* the size of the "oids" list */
-    enum sap_status exit_state = H5FP_STATUS_LOCK_ACQUIRED;
+    unsigned list_size = 2; /* the size of the "oids" list */
+    H5FP_status_t exit_state = H5FP_STATUS_LOCK_ACQUIRED;
     herr_t ret_value = SUCCEED;
-    int i, j;
+    unsigned i, j;
 
     FUNC_ENTER_NOINIT(H5FP_sap_handle_lock_request);
 
-    if ((oids = (struct lock_group *)HDmalloc(list_size * sizeof(struct lock_group))) == NULL) {
+    if ((oids = (struct lock_group *)HDmalloc(list_size *
+                                              sizeof(struct lock_group))) == NULL) {
         exit_state = H5FP_STATUS_OOM;
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "out of memory");
     }
@@ -867,7 +828,7 @@ H5FP_sap_handle_lock_request(struct SAP_request req)
     for (i = 0;; ++i) {
         if (req.oid[0]) {
             if (i == list_size) {
-                list_size *= 2;
+                list_size <<= 1;    /* equiv to list_size *= 2; */
                 oids = HDrealloc(oids, list_size * sizeof(struct lock_group));
 
                 if (!oids) {
@@ -877,7 +838,7 @@ H5FP_sap_handle_lock_request(struct SAP_request req)
             }
 
             H5FP_COPY_OID(oids[i].oid, req.oid);
-            oids[i].file_id = req.sap_file_id;
+            oids[i].file_id = req.file_id;
             oids[i].rw_lock = req.rw_lock;
             oids[i].locked = FALSE;
         }
@@ -898,7 +859,7 @@ H5FP_sap_handle_lock_request(struct SAP_request req)
      * not, the it's an error and we need to return.
      */
     for (j = 0; j <= i; ++j) {
-        if ((oids[j].fs = H5FP_find_file_struct(oids[j].file_id)) == NULL) {
+        if ((oids[j].fs = H5FP_find_file_info(oids[j].file_id)) == NULL) {
             /* FIXME: the file ID is invalid */
             exit_state = H5FP_STATUS_BAD_FILE_ID;
             HGOTO_DONE(FAIL);
@@ -969,8 +930,8 @@ H5FP_sap_handle_lock_request(struct SAP_request req)
                 goto rollback;
             }
         } else {
-            struct sap_obj_lock *lock = H5FP_new_object_lock(oids[j].oid, req.proc_rank,
-                                                             req.obj_type, req.rw_lock);
+            H5FP_object_lock *lock = H5FP_new_object_lock(oids[j].oid, req.proc_rank,
+                                                          req.obj_type, req.rw_lock);
 
             if (lock) {
                 if (!H5TB_dins(oids[j].fs->locks, (void *)lock, NULL)) {
@@ -1031,7 +992,7 @@ HDfprintf(stderr, "%s: locking failure (%d)!!\n", FUNC, ret_value);
     }
 
     HDfree(oids);
-    H5FP_send_reply(req.proc_rank, req.req_id, req.sap_file_id, exit_state);
+    H5FP_send_reply(req.proc_rank, req.req_id, req.file_id, exit_state);
     FUNC_LEAVE_NOAPI(ret_value);
 } 
 
@@ -1043,18 +1004,18 @@ HDfprintf(stderr, "%s: locking failure (%d)!!\n", FUNC, ret_value);
  * Modifications:
  */
 static herr_t
-H5FP_sap_handle_release_lock_request(struct SAP_request req)
+H5FP_sap_handle_release_lock_request(H5FP_request req)
 {
     struct release_group {
         unsigned char oid[sizeof(req.oid)];
-        unsigned int file_id;
-        struct sap_file_struct *fs;
-        struct sap_obj_lock *lock;
+        unsigned file_id;
+        H5FP_file_info *fs;
+        H5FP_object_lock *lock;
     } *oids;
-    int list_size = 2;  /* the size of the "oids" list */
-    enum sap_status exit_state = H5FP_STATUS_LOCK_RELEASED;
+    unsigned list_size = 2; /* the size of the "oids" list */
+    H5FP_status_t exit_state = H5FP_STATUS_LOCK_RELEASED;
     herr_t ret_value;
-    int i, j;
+    unsigned i, j;
 
     FUNC_ENTER_NOINIT(H5FP_sap_handle_release_lock_request);
 
@@ -1071,7 +1032,7 @@ H5FP_sap_handle_release_lock_request(struct SAP_request req)
     for (i = 0;; ++i) {
         if (req.oid[0]) {
             if (i == list_size) {
-                list_size *= 2;
+                list_size <<= 1;    /* equiv to list_size *= 2; */
                 oids = HDrealloc(oids, list_size * sizeof(struct release_group));
 
                 if (!oids) {
@@ -1081,7 +1042,7 @@ H5FP_sap_handle_release_lock_request(struct SAP_request req)
             }
 
             H5FP_COPY_OID(oids[i].oid, req.oid);
-            oids[i].file_id = req.sap_file_id;
+            oids[i].file_id = req.file_id;
         }
 
         if (req.req_type == H5FP_REQ_RELEASE_END)
@@ -1099,7 +1060,7 @@ H5FP_sap_handle_release_lock_request(struct SAP_request req)
      * will help keep us from being in a catastrophic state.
      */
     for (j = 0; j <= i; ++j) {
-        if ((oids[j].fs = H5FP_find_file_struct(oids[j].file_id)) == NULL) {
+        if ((oids[j].fs = H5FP_find_file_info(oids[j].file_id)) == NULL) {
             /* FIXME: the file ID is invalid */
             exit_state = H5FP_STATUS_BAD_FILE_ID;
             HGOTO_DONE(FAIL);
@@ -1136,37 +1097,37 @@ H5FP_sap_handle_release_lock_request(struct SAP_request req)
     }
 
 done:
-HDfprintf(stderr, "exit_state == %d\n", exit_state);
     HDfree(oids);
-    H5FP_send_reply(req.proc_rank, req.req_id, req.sap_file_id, exit_state);
+    H5FP_send_reply(req.proc_rank, req.req_id, req.file_id, exit_state);
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
 /*
- * Function:    H5FP_sap_handle_change_request
- * Purpose:     Handle a request to change the structure of an object.
+ * Function:    H5FP_sap_handle_write_request
+ * Purpose:     Handle a request to write a piece of metadata in the
+ *              file.
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 06. August, 2002
  * Modifications:
  */
 static herr_t
-H5FP_sap_handle_change_request(struct SAP_request req, char *mdata, int md_len)
+H5FP_sap_handle_write_request(H5FP_request req, char *mdata, int md_size)
 {
-    struct sap_file_struct *fs;
-    enum sap_status exit_state = H5FP_STATUS_OK;
+    H5FP_file_info *fs;
+    H5FP_status_t exit_state = H5FP_STATUS_OK;
     herr_t ret_value = SUCCEED;
 
-    FUNC_ENTER_NOINIT(H5FP_sap_handle_change_request);
+    FUNC_ENTER_NOINIT(H5FP_sap_handle_write_request);
 
-    if ((fs = H5FP_find_file_struct(req.sap_file_id)) != NULL) {
+    if ((fs = H5FP_find_file_info(req.file_id)) != NULL) {
         if (fs->closing) {
             /* we're closing the file - don't accept anymore changes */
             exit_state = H5FP_STATUS_FILE_CLOSING;
             ret_value = FAIL;
         } else {
             /* handle the change request */
-            struct sap_obj_lock *lock = H5FP_find_object_lock(fs, req.oid);
+            H5FP_object_lock *lock = H5FP_find_object_lock(fs, req.oid);
 
             if (!lock || lock->owned_rank != req.proc_rank
                     || lock->rw_lock != H5FP_LOCK_WRITE) {
@@ -1176,8 +1137,9 @@ H5FP_sap_handle_change_request(struct SAP_request req, char *mdata, int md_len)
                  */
                 exit_state = H5FP_STATUS_NO_LOCK;
                 ret_value = FAIL;
-            } else if (H5FP_add_file_mod_to_list(fs, req.obj_type, req.action,
-                                                 req.proc_rank, md_len, mdata) != SUCCEED) {
+            } else if (H5FP_add_file_mod_to_list(fs, req.type_id, req.addr,
+                                                 req.proc_rank, md_size,
+                                                 mdata) != SUCCEED) {
                 exit_state = H5FP_STATUS_OOM;
                 ret_value = FAIL;
             }
@@ -1188,87 +1150,68 @@ H5FP_sap_handle_change_request(struct SAP_request req, char *mdata, int md_len)
         ret_value = FAIL;
     }
 
-    H5FP_send_reply(req.proc_rank, req.req_id, req.sap_file_id, exit_state);
+    if (ret_value == FAIL)
+        HDfree(mdata);
+
+    H5FP_send_reply(req.proc_rank, req.req_id, req.file_id, exit_state);
     FUNC_LEAVE_NOAPI(ret_value);
 }
 
 /*
- * Function:    H5FP_sap_handle_sync_request
- * Purpose:     Handle a request to sync changes to a file with the requesting
- *              process.
+ * Function:    H5FP_sap_handle_read_request
+ * Purpose:     Handle a read request from a client. The bit of metadata
+ *              that the client wants may or may not be in here. It's not
+ *              an error if it isn't here. When that's the case, the
+ *              client just goes and reads it from the file.
  * Return:      Success:    SUCCEED
  *              Failure:    FAIL
  * Programmer:  Bill Wendling, 06. August, 2002
  * Modifications:
  */
 static herr_t
-H5FP_sap_handle_sync_request(struct SAP_request req)
+H5FP_sap_handle_read_request(H5FP_request req)
 {
-    struct sap_file_struct *fs;
-    struct SAP_sync s;
+    H5FP_file_info *fs;
+    H5FP_read r;
     herr_t ret_value = SUCCEED;
-    int mrc, sync_id = 0;
-
-    FUNC_ENTER_NOINIT(H5FP_sap_handle_sync_request);
-
-HDfprintf(stderr, "%s: Trying to Synchronize!\n", FUNC);
-
-    s.req_id = req.req_id;
-    s.sap_file_id = req.sap_file_id;
-    s.last_msg = FALSE;
-    s.md_len = 0;
-    s.obj_type = 0;
-    s.action = 0;
-    s.status = H5FP_STATUS_OK;
-
-    if ((fs = H5FP_find_file_struct(req.sap_file_id)) != NULL) {
-        if (fs->mod_tree->root) {
-            H5TB_NODE *mod_node = H5TB_first(fs->mod_tree->root);
-
-            while (mod_node) {
-                struct sap_file_mod *mod = (struct sap_file_mod *)mod_node->data;
+    char *metadata = NULL;
+    int mrc;
 
-                if (mod && !IS_PROC_NOTIFIED(mod, req.proc_rank)) {
-                    s.sync_id = sync_id++;
-                    s.md_len = mod->md_len;
-                    s.obj_type = mod->obj_type;
-                    s.action = mod->action;
+    FUNC_ENTER_NOINIT(H5FP_sap_handle_read_request);
 
-                    if ((mrc = MPI_Send(&s, 1, SAP_sync_t, (int)req.proc_rank,
-                                        H5FP_TAG_SYNC, H5FP_SAP_COMM)) != MPI_SUCCESS)
-                        HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
+    r.req_id = req.req_id;
+    r.file_id = req.file_id;
+    r.md_size = 0;
+    r.type_id = 0;
+    r.addr = 0;
+    r.status = H5FP_STATUS_MDATA_NOT_CACHED;
 
+    if ((fs = H5FP_find_file_info(req.file_id)) != NULL) {
+        H5FP_mdata_mod mod;     /* Used to find the correct modification */
+        H5TB_NODE *node;
 
-                    if (H5FP_send_metadata(mod->metadata, mod->md_len, (int)req.proc_rank)
-                            != SUCCEED)
-                        HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL,
-                                    "can't send metadata to client");
+        mod.addr = req.addr;    /* This is the key field for the TBBT */
 
-                    SET_PROC_NOTIFIED(mod, req.proc_rank);
-                }
+        if ((node = H5TB_dfind(fs->mod_tree, (void *)&mod, NULL)) != NULL) {
+            H5FP_mdata_mod *fm = (H5FP_mdata_mod *)node->data;
 
-                /*
-                 * check to see if this modification has been sent to all
-                 * clients
-                 */
-                if (mod->num_notified == (unsigned)H5FP_comm_size)
-                    if (H5FP_remove_mod_from_list(fs, mod) == FAIL)
-                        HGOTO_ERROR(H5E_FPHDF5, H5E_NOTFOUND, FAIL, "cannot remove modification");
-
-                mod_node = H5TB_next(mod_node);
-            }
+            r.md_size = fm->md_size;
+            r.type_id = fm->type_id;
+            r.addr = fm->addr;
+            r.status = H5FP_STATUS_OK;
+            metadata = fm->metadata;    /* Sent out in a separate message */
         }
-    } else {
-        s.status = H5FP_STATUS_BAD_FILE_ID;
     }
 
-    s.sync_id = sync_id;
-    s.last_msg = TRUE;
-
-    if ((mrc = MPI_Send(&s, 1, SAP_sync_t, (int)req.proc_rank,
-                        H5FP_TAG_SYNC, H5FP_SAP_COMM)) != MPI_SUCCESS)
+    if ((mrc = MPI_Send(&r, 1, H5FP_read_t, (int)req.proc_rank,
+                        H5FP_TAG_READ, H5FP_SAP_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
 
+    if (r.md_size)
+        if (H5FP_send_metadata(metadata, (int)r.md_size, (int)req.proc_rank) != SUCCEED)
+            HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL,
+                        "can't send metadata to client");
+
 done:
     FUNC_LEAVE_NOAPI(ret_value);
 }
@@ -1282,23 +1225,19 @@ done:
  * Modifications:
  */
 static herr_t
-H5FP_sap_handle_close_request(struct SAP_request req)
+H5FP_sap_handle_close_request(H5FP_request req)
 {
-    struct sap_file_struct *fs;
+    H5FP_file_info *fs;
     herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOINIT(H5FP_sap_handle_close_request);
 
-    if ((fs = H5FP_find_file_struct(req.sap_file_id)) != NULL) {
-        /* do a sync with the closing process */
-        if (H5FP_sap_handle_sync_request(req) != SUCCEED)
-            HGOTO_ERROR(H5E_FPHDF5, H5E_NOTFOUND, FAIL, "couldn't sync with process");
-
+    if ((fs = H5FP_find_file_info(req.file_id)) != NULL)
         if (++fs->closing == H5FP_comm_size - 1)
             /* all processes have closed the file - remove it from list */
-            if (H5FP_remove_file_id_from_list(req.sap_file_id) != SUCCEED)
-                HGOTO_ERROR(H5E_FPHDF5, H5E_NOTFOUND, FAIL, "cannot remove file ID from list");
-    }
+            if (H5FP_remove_file_id_from_list(req.file_id) != SUCCEED)
+                HGOTO_ERROR(H5E_FPHDF5, H5E_NOTFOUND, FAIL,
+                            "cannot remove file ID from list");
 
 done:
     FUNC_LEAVE_NOAPI(ret_value);