[svn-r7367] Purpose:

    Update

Description:
    Added an extra flag to the clear functions that triggers a "destroy"
    of the object being cleared if necessary. This is a fix for the
    FPHDF5 stuff which had an object sticking around after it was
    cleared. (In FPHDF5, some processes are in charge of destroying the
    object, but all processes might actually allocate the object.)

Platforms tested:
    AIX (Copper: Fortran & C++)
    Linux (Verbena: Fortran & C++)
    IRIX (Modi4: Parallel & Fortran)
    (Sol is down)

Misc. update:

18 files changed, 416 insertions, 196 deletions

diff --git a/src/H5AC.c b/src/H5AC.c
index 849bf2a..c6f1f43 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -807,7 +807,7 @@ H5AC_flush(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, unsi
                 HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve xfer mode")
 
             /* Sanity check transfer mode */
-            assert(xfer_mode==H5FD_MPIO_COLLECTIVE);
+            assert(xfer_mode == H5FD_MPIO_COLLECTIVE || IS_H5FD_FPHDF5(f));
 #endif /* NDEBUG */
 
             /* Create the mapping */
@@ -879,8 +879,8 @@ H5AC_flush(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, unsi
                 /* Clear the dirty flag only, if requested */
                 if(clear_only) {
                     /* Call the callback routine to clear all dirty flags for object */
-                    if(((*info)->type->clear)(*info)<0)
-                        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear cache")
+                    if(((*info)->type->clear)(*info, destroy)<0)
+                        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear cache");
                 } /* end if */
                 else {
                     flush = (*info)->type->flush;
@@ -982,8 +982,8 @@ H5AC_flush(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, unsi
             /* Clear the dirty flag only, if requested */
             if(clear_only) {
                 /* Call the callback routine to clear all dirty flags for object */
-                if(((*info)->type->clear)(*info)<0)
-                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear cache")
+                if(((*info)->type->clear)(*info, destroy)<0)
+                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear cache");
             } /* end if */
             else {
                 flush = (*info)->type->flush;
@@ -1771,8 +1771,8 @@ H5AC_unprotect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr,
     } /* end if */
     else {
         /* Mark the thing as clean (prerequite for destroy routine) */
-        if((type->clear)(thing)<0)
-            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear object")
+        if((type->clear)(thing, FALSE)<0)
+            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear object");
 
         /* Destroy previously cached thing */
         if ((type->dest)(f, thing)<0)
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index 58942b3..616ed91 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -77,7 +77,7 @@ typedef enum H5AC_subid_t {
 typedef void *(*H5AC_load_func_t)(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *udata1, void *udata2);
 typedef herr_t (*H5AC_flush_func_t)(H5F_t *f, hid_t dxpl_id, hbool_t dest, haddr_t addr, void *thing);
 typedef herr_t (*H5AC_dest_func_t)(H5F_t *f, void *thing);
-typedef herr_t (*H5AC_clear_func_t)(void *thing);
+typedef herr_t (*H5AC_clear_func_t)(void *thing, hbool_t dest);
 
 typedef struct H5AC_class_t {
     H5AC_subid_t	id;
diff --git a/src/H5B.c b/src/H5B.c
index 965c2e4..0f0f94a 100644
--- a/src/H5B.c
+++ b/src/H5B.c
@@ -154,7 +154,7 @@ static herr_t H5B_assert(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_
 static H5B_t *H5B_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *_type, void *udata);
 static herr_t H5B_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5B_t *b);
 static herr_t H5B_dest(H5F_t *f, H5B_t *b);
-static herr_t H5B_clear(H5B_t *b);
+static herr_t H5B_clear(H5B_t *b, hbool_t destroy);
 
 /* H5B inherits cache-like properties from H5AC */
 static const H5AC_class_t H5AC_BT[1] = {{
@@ -560,9 +560,10 @@ H5B_dest(H5F_t UNUSED *f, H5B_t *bt)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5B_clear(H5B_t *bt)
+H5B_clear(H5B_t *bt, hbool_t destroy)
 {
     unsigned	u;      /* Local index variable */
+    herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOINIT(H5B_clear)
 
@@ -575,8 +576,13 @@ H5B_clear(H5B_t *bt)
     for (u=0; u<=bt->nchildren; u++)
         bt->key[u].dirty = FALSE;
     bt->cache_info.dirty = FALSE;
+ 
+    if (destroy)
+        if (H5B_dest(NULL, bt) < 0)
+	    HGOTO_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to destroy B-tree node");
 
-    FUNC_LEAVE_NOAPI(SUCCEED)
+done:
+    FUNC_LEAVE_NOAPI(ret_value);
 } /* end H5B_clear() */
 
 
diff --git a/src/H5D.c b/src/H5D.c
index 06b8d53..00bd280 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -13,6 +13,7 @@
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define H5D_PACKAGE		/*suppress error about including H5Dpkg	  */
+#define H5F_PACKAGE		/*suppress error about including H5Fpkg	  */
 
 /* Pablo information */
 /* (Put before include files to avoid problems with inline functions) */
@@ -21,6 +22,8 @@
 #include "H5private.h"		/* Generic Functions			*/
 #include "H5Dpkg.h"		/* Datasets 				*/
 #include "H5Eprivate.h"		/* Error handling		  	*/
+#include "H5Fprivate.h"         /* Files                                */
+#include "H5Fpkg.h"             /* Files access                         */
 #include "H5FDprivate.h"	/* File drivers				*/
 #include "H5FLprivate.h"	/* Free Lists                           */
 #include "H5FOprivate.h"        /* File objects                         */
@@ -30,6 +33,10 @@
 #include "H5Sprivate.h"		/* Dataspaces 				*/
 #include "H5Vprivate.h"		/* Vectors and arrays 			*/
 
+#ifdef H5_HAVE_FPHDF5
+#include "H5FPprivate.h"        /* Flexible PHDF5                       */
+#endif  /* H5_HAVE_FPHDF5 */
+
 /*#define H5D_DEBUG*/
 
 /* Interface initialization */
@@ -1602,6 +1609,10 @@ H5D_create(H5G_entry_t *loc, const char *name, hid_t type_id, const H5S_t *space
     H5P_genplist_t 	*dc_plist=NULL;         /* New Property list */
     hbool_t             has_vl_type=FALSE;      /* Flag to indicate a VL-type for dataset */
     H5D_t		*ret_value;             /* Return value */
+#ifdef H5_HAVE_FPHDF5
+    hbool_t             locked_grp = FALSE;     /* The parent group is locked */
+    hobj_ref_t          grp_oid;
+#endif  /* H5_HAVE_FPHDF5 */
 
     FUNC_ENTER_NOAPI(H5D_create, NULL)
 
@@ -1641,6 +1652,55 @@ H5D_create(H5G_entry_t *loc, const char *name, hid_t type_id, const H5S_t *space
     if (NULL==(file=H5G_insertion_file(loc, name, dxpl_id)))
         HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to locate insertion point")
 
+#ifdef H5_HAVE_FPHDF5
+    if (H5FD_is_fphdf5_driver(file->shared->lf)) {
+        unsigned file_id = H5FD_fphdf5_file_id(file->shared->lf);
+        char *tmp_name = H5MM_xstrdup(name);
+        char *last = HDstrrchr(tmp_name, '/');
+        hid_t gid;
+        H5G_t *grp = NULL;
+        H5G_entry_t *grp_loc = NULL;
+        H5FP_status_t status;
+        unsigned req_id;
+
+        /* Chop off the dataset name (we only want to lock the group) */
+        if (last)
+            last[1] = '\0';
+
+        /* Open the group */
+        if ((grp = H5G_open(loc, tmp_name, dxpl_id)) == NULL)
+            HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, NULL, "unable to open group");
+
+        /* Register the group so we can grab information about it */
+        if ((gid = H5I_register(H5I_GROUP, grp)) < 0) {
+            H5G_close(grp);
+            HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, NULL, "unable to register group");
+        }
+
+        /* Grab the location of the group */
+        if ((grp_loc = H5G_loc(gid)) == NULL) {
+            H5I_dec_ref(gid);
+            HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a location ID")
+        }
+
+        grp_oid = grp_loc->header;
+
+        if (H5FP_request_lock(file_id, grp_oid, H5FP_LOCK_WRITE,
+                              TRUE, &req_id, &status) != SUCCEED) {
+            /* FIXME: This is bad. We should check the "status" variable */
+            H5I_dec_ref(gid);
+            H5MM_xfree(tmp_name);
+            HGOTO_ERROR(H5E_FPHDF5, H5E_CANTLOCK, NULL, "unable to lock group");
+        }
+
+        if (H5I_dec_ref(gid) != SUCCEED)
+            HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, NULL, "unable to close group");
+
+        locked_grp = TRUE;
+        H5MM_xfree(tmp_name);
+    }
+#endif  /* H5_HAVE_FPHDF5 */
+
     /* Copy datatype for dataset */
     if((new_dset->type = H5T_copy(type, H5T_COPY_ALL))==NULL)
         HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, NULL, "can't copy datatype")
@@ -1863,6 +1923,20 @@ done:
         H5FL_FREE(H5D_t,new_dset);
     }
 
+#ifdef H5_HAVE_FPHDF5
+    if (locked_grp) {
+        unsigned file_id = H5FD_fphdf5_file_id(file->shared->lf);
+        H5FP_status_t status;
+        unsigned req_id;
+
+        /* Request a release of the lock */
+        if (H5FP_request_release_lock(file_id, grp_oid, TRUE, &req_id, &status)) {
+            /* FIXME: This is bad. We should check the "status" variable */
+            HGOTO_ERROR(H5E_FPHDF5, H5E_CANTLOCK, NULL, "unable to lock group");
+        }
+    }
+#endif  /* H5_HAVE_FPHDF5 */
+
     FUNC_LEAVE_NOAPI(ret_value)
 }
 
diff --git a/src/H5F.c b/src/H5F.c
index 097093d..f312934 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -1573,6 +1573,7 @@ H5F_new(H5F_file_t *shared, hid_t fcpl_id, hid_t fapl_id)
 	 */
 	if ((n=H5AC_create(f, f->shared->mdc_nelmts))<0)
 	    HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create meta data cache");
+
 	f->shared->mdc_nelmts = n;
 
 	/* Create the chunk cache */
@@ -1917,6 +1918,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t d
 	 */
 	if (NULL==(file = H5F_new(NULL, fcpl_id, fapl_id)))
 	    HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create new file object");
+
 	file->shared->flags = flags;
 	file->shared->lf = lf;
     }
@@ -1984,9 +1986,9 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t d
 	if (H5F_flush(file, dxpl_id, H5F_SCOPE_LOCAL, H5F_FLUSH_ALLOC_ONLY) < 0)
 	    HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to write file superblock");
 
-	/* Create and open the root group */
-	if (H5G_mkroot(file, dxpl_id, NULL)<0)
-	    HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create/open root group");
+        /* Create and open the root group */
+        if (H5G_mkroot(file, dxpl_id, NULL)<0)
+            HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create/open root group");
 
 #ifdef H5_HAVE_FPHDF5
         /*
@@ -1996,6 +1998,9 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t d
         if (H5FD_is_fphdf5_driver(lf)) {
             H5P_genplist_t *d_plist;
 
+            /* Wait for all processes to catch up */
+            MPI_Barrier(H5FP_SAP_BARRIER_COMM);
+
             /* Get the data xfer property list */
             if ((d_plist = H5I_object(dxpl_id)) == NULL)
                 HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, NULL, "not a dataset transfer list");
@@ -2302,8 +2307,7 @@ done:
  *-------------------------------------------------------------------------
  */
 hid_t
-H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id,
-	  hid_t fapl_id)
+H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
 {
     
     H5F_t	*new_file = NULL;	/*file struct for new file	*/
@@ -2348,7 +2352,7 @@ H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id,
      */
     if (NULL==(new_file=H5F_open(filename, flags, fcpl_id, fapl_id, H5AC_dxpl_id)))
 	HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create file");
-    
+
     /* Get an atom for the file */
     if ((ret_value = H5I_register(H5I_FILE, new_file))<0)
 	HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file");
@@ -2647,11 +2651,12 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, H5F_scope_t scope, unsigned flags)
              * the clients...
              */
             if (H5FD_is_fphdf5_driver(f->shared->lf) && !H5FD_fphdf5_is_sap(f->shared->lf)) {
-                unsigned        req_id;
-                H5FP_status_t   status;
+                unsigned        req_id = 0;
+                H5FP_status_t   status = H5FP_STATUS_OK;
 
                 /* Send the request to the SAP */
-                if (H5FP_request_free(f->shared->lf, &req_id, &status) != SUCCEED)
+                if (H5FP_request_update_eoma_eosda(f->shared->lf,
+                                                   &req_id, &status) != SUCCEED)
                     /* FIXME: Should we check the "status" variable here? */
                     HGOTO_ERROR(H5E_FPHDF5, H5E_CANTFREE, FAIL,
                                 "server couldn't free from file");
@@ -2704,11 +2709,17 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, H5F_scope_t scope, unsigned flags)
         } /* end if */
 
         /* flush the entire raw data cache */
-        if (H5F_istore_flush(f, dxpl_id, flags & (H5F_FLUSH_INVALIDATE|H5F_FLUSH_CLEAR_ONLY)) < 0)
+        if (H5F_istore_flush(f, dxpl_id, flags & (H5F_FLUSH_INVALIDATE | H5F_FLUSH_CLEAR_ONLY)) < 0)
             HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush raw data cache");
         
         /* flush (and invalidate) the entire meta data cache */
-        if (H5AC_flush(f, dxpl_id, NULL, HADDR_UNDEF, flags & (H5F_FLUSH_INVALIDATE|H5F_FLUSH_CLEAR_ONLY)) < 0)
+        /*
+         * FIXME: This should be CLEAR_ONLY for non-captain processes.
+         * Need to fix the H5G_mkroot() call so that only the captain
+         * allocates object headers (calls the H5O_init function...via a
+         * lot of other functions first)....
+         */
+        if (H5AC_flush(f, dxpl_id, NULL, HADDR_UNDEF, flags & (H5F_FLUSH_INVALIDATE | H5F_FLUSH_CLEAR_ONLY)) < 0)
             HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush meta data cache");
     } /* end if */
 
@@ -3122,7 +3133,6 @@ H5F_close(H5F_t *f)
             if (H5F_flush(f, H5AC_dxpl_id, H5F_SCOPE_LOCAL,
                           H5F_FLUSH_INVALIDATE | H5F_FLUSH_CLOSING | H5F_FLUSH_CLEAR_ONLY) < 0)
                 HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache");
-
         }
 
         /* Let's all meet up now... */
diff --git a/src/H5FD.c b/src/H5FD.c
index 37e14e4..aa1935c 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -1313,9 +1313,9 @@ H5FD_alloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size)
      * try to do the actual allocations.
      */
     if (H5FD_is_fphdf5_driver(file) && !H5FD_fphdf5_is_sap(file)) {
-        unsigned        req_id;
+        unsigned        req_id = 0;
         unsigned        capt_only = 0;
-        H5FP_status_t   status;
+        H5FP_status_t   status = H5FP_STATUS_OK;
         H5P_genplist_t *plist;
         H5FP_alloc_t    fp_alloc;
 
@@ -1325,8 +1325,9 @@ H5FD_alloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size)
 
         if (H5P_exist_plist(plist, H5FD_FPHDF5_CAPTN_ALLOC_ONLY) > 0)
             if (H5P_get(plist, H5FD_FPHDF5_CAPTN_ALLOC_ONLY, &capt_only) < 0)
-                HGOTO_ERROR(H5E_PLIST, H5E_CANTDELETE, HADDR_UNDEF,
-                            "can't remove FPHDF5 property");
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTDELETE, HADDR_UNDEF, "can't retrieve FPHDF5 property");
+
+        HDmemset(&fp_alloc, 0, sizeof(fp_alloc));
 
         /*
          * If the captain is the only one who should allocate resources,
@@ -2081,6 +2082,27 @@ H5FD_free(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t si
     assert(file->cls);
     assert(type >= 0 && type < H5FD_MEM_NTYPES);
 
+#ifdef H5_HAVE_FPHDF5
+    /*
+     * When we're using the FPHDF5 driver, allocate from the SAP. If this
+     * is the SAP executing this code, then skip the send to the SAP and
+     * try to do the actual allocations.
+     */
+    if (H5FD_is_fphdf5_driver(file) && !H5FD_fphdf5_is_sap(file)) {
+        unsigned        req_id;
+        H5FP_status_t   status;
+
+        /* Send the request to the SAP */
+        if (H5FP_request_free(file, type, addr, size, &req_id, &status) != SUCCEED)
+            /* FIXME: Should we check the "status" variable here? */
+            HGOTO_ERROR(H5E_FPHDF5, H5E_CANTFREE, FAIL,
+                        "server couldn't free from file");
+
+        /* We've succeeded -- return the value */
+        HGOTO_DONE(ret_value);
+    }
+#endif  /* H5_HAVE_FPHDF5 */
+
     if (!H5F_addr_defined(addr) || addr>file->maxaddr ||
             H5F_addr_overflow(addr, size) || addr+size>file->maxaddr)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid region");
@@ -2284,6 +2306,7 @@ H5FD_free(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t si
             HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver free request failed");
     } else {
         /* leak memory */
+HDfprintf(stderr, "%s: LEAKED MEMORY!!!!!!\n", FUNC);
     }
 
 done:
diff --git a/src/H5FDfphdf5.c b/src/H5FDfphdf5.c
index 3cbe426..c825950 100644
--- a/src/H5FDfphdf5.c
+++ b/src/H5FDfphdf5.c
@@ -652,7 +652,7 @@ H5Pset_dxpl_fphdf5(hid_t dxpl_id, H5FD_mpio_xfer_t xfer_mode)
                     "can't set values in default property list");
 
     /* Check arguments */
-    if ((plist = H5P_object_verify(dxpl_id,H5P_DATASET_XFER)) == NULL)
+    if ((plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER)) == NULL)
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     if (xfer_mode != H5FD_MPIO_INDEPENDENT && xfer_mode != H5FD_MPIO_COLLECTIVE)
@@ -663,7 +663,7 @@ H5Pset_dxpl_fphdf5(hid_t dxpl_id, H5FD_mpio_xfer_t xfer_mode)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value");
 
     /* Initialize driver-specific properties */
-    ret_value = H5P_set_driver(plist, H5FD_MPIO, NULL);
+    ret_value = H5P_set_driver(plist, H5FD_FPHDF5, NULL);
 
 done:
     FUNC_LEAVE_API(ret_value);
@@ -692,7 +692,7 @@ H5Pget_dxpl_fphdf5(hid_t dxpl_id, H5FD_mpio_xfer_t *xfer_mode)
     FUNC_ENTER_API(H5Pget_dxpl_fphdf5, FAIL);
     H5TRACE2("e","i*Dt",dxpl_id,xfer_mode);
 
-    if ((plist = H5P_object_verify(dxpl_id,H5P_DATASET_XFER)) == NULL)
+    if ((plist = H5P_object_verify(dxpl_id, H5P_DATASET_XFER)) == NULL)
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dxpl");
 
     if (H5P_get_driver(plist) != H5FD_FPHDF5)
@@ -783,8 +783,7 @@ H5FD_fphdf5_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd
     hsize_t                     threshold;
     hsize_t                     alignment;
     unsigned                    file_id;
-    unsigned                    req_id;
-    MPI_Status                  status;
+    unsigned                    req_id = 0;
     H5FD_t                     *ret_value = NULL;
 
     /* Flag to indicate that the file was successfully opened */
@@ -847,18 +846,16 @@ H5FD_fphdf5_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd
         HGOTO_ERROR(H5E_FPHDF5, H5E_CANTOPENFILE, NULL,
                     "can't inform SAP of file open");
 
-    /* Grab the rank of this process */
-    if ((mrc = MPI_Comm_rank(H5FP_SAP_COMM, &mpi_rank)) != MPI_SUCCESS)
-        HMPI_GOTO_ERROR(NULL, "MPI_Comm_rank failed", mrc);
-
-    HDmemset(&status, 0, sizeof(status));
-
     /* Broadcast the file ID */
     if ((mrc = MPI_Bcast(&file_id, 1, MPI_UNSIGNED,
                          (int)H5FP_capt_barrier_rank,
                          H5FP_SAP_BARRIER_COMM)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mrc);
 
+    /* Grab the rank of this process */
+    if ((mrc = MPI_Comm_rank(H5FP_SAP_COMM, &mpi_rank)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(NULL, "MPI_Comm_rank failed", mrc);
+
     /* The captain rank will get the filesize and broadcast it. */
     if ((unsigned)mpi_rank == H5FP_capt_rank)
         /* Get current file size */
@@ -924,8 +921,8 @@ static herr_t
 H5FD_fphdf5_close(H5FD_t *_file)
 {
     H5FD_fphdf5_t  *file = (H5FD_fphdf5_t *)_file;
-    H5FP_status_t   status;
-    unsigned        req_id;
+    unsigned        req_id = 0;
+    H5FP_status_t   status = H5FP_STATUS_OK;
     int             mrc;
     herr_t          ret_value = SUCCEED;
 
@@ -935,14 +932,19 @@ H5FD_fphdf5_close(H5FD_t *_file)
     assert(file);
     assert(file->pub.driver_id == H5FD_FPHDF5);
 
-    /* MPI_File_close sets argument to MPI_FILE_NULL */
-    if ((mrc = MPI_File_close(&file->f)) != MPI_SUCCESS)
-        HMPI_GOTO_ERROR(FAIL, "MPI_File_close failed", mrc);
-
+    /* Tell the SAP that we're closing the file... */
     if (H5FP_request_close(_file, file->file_id, &req_id, &status) == FAIL)
         HGOTO_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL,
                     "can't inform SAP of file close");
 
+    /* Let all of the processes catch up to this point. */
+    if ((mrc = MPI_Barrier(H5FP_SAP_BARRIER_COMM)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mrc);
+
+    /* MPI_File_close sets argument to MPI_FILE_NULL */
+    if ((mrc = MPI_File_close(&file->f)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(FAIL, "MPI_File_close failed", mrc);
+
     /* Clean up other stuff */
     H5MM_xfree(file);
 
@@ -969,9 +971,6 @@ H5FD_fphdf5_query(const H5FD_t UNUSED *_file, unsigned long *flags /* out */)
 
     FUNC_ENTER_NOAPI(H5FD_fphdf5_query, FAIL);
 
-    /* check args */
-    assert(flags);
-
     /* Set the VFL feature flags that this driver supports */
     if (flags) {
         *flags = 0;
@@ -1189,7 +1188,7 @@ H5FD_fphdf5_read(H5FD_t *_file, H5FD_mem_t mem_type, hid_t dxpl_id,
     assert(file->pub.driver_id == H5FD_FPHDF5);
     assert(buf);
 
-    /* make certain we have the correct type of property list */
+    /* Make certain we have the correct type of property list */
     assert(H5I_get_type(dxpl_id) == H5I_GENPROP_LST);
     assert(H5P_isa_class(dxpl_id, H5P_DATASET_XFER) == TRUE);
 
@@ -1260,8 +1259,8 @@ H5FD_fphdf5_read(H5FD_t *_file, H5FD_mem_t mem_type, hid_t dxpl_id,
          * This is metadata - we want to try to read it from the SAP
          * first.
          */
-        H5FP_status_t   sap_status;
-        unsigned        req_id;
+        unsigned        req_id = 0;
+        H5FP_status_t   sap_status = H5FP_STATUS_OK;
 
         if (H5FP_request_read_metadata(_file, file->file_id, dxpl_id, mem_type,
                                        mpi_off, size, (uint8_t**)&buf,
@@ -1288,49 +1287,11 @@ HDfprintf(stderr, "%s:%d: Metadata cache read failed!\n", FUNC, __LINE__);
             HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mrc);
     } else {
         if ((mrc = MPI_File_read_at_all(file->f, mpi_off, buf, size_i,
-                                        buf_type, &status )) != MPI_SUCCESS)
+                                        buf_type, &status)) != MPI_SUCCESS)
             HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at_all failed", mrc);
     }
 
     /*
-     * KLUDGE, Robb Matzke, 2000-12-29
-     * The LAM implementation of MPI_Get_count() says
-     *
-     *    MPI_Get_count: invalid argument (rank 0, MPI_COMM_WORLD)
-     *
-     * So I'm commenting this out until it can be investigated. The
-     * returned `bytes_written' isn't used anyway because of Kim's kludge
-     * to avoid bytes_written < 0. Likewise in H5FD_fphdf5_write().
-     */
-
-#ifdef H5_HAVE_MPI_GET_COUNT /* Bill and Albert's kludge*/
-    /*
-     * Yet Another KLUDGE, Albert Cheng & Bill Wendling, 2001-05-11.
-     * Many systems don't support MPI_Get_count so we need to do a
-     * configure thingy to fix this.
-     */
-
-    /*
-     * Calling MPI_Get_count with "MPI_BYTE" is only valid when we
-     * actually had the 'buf_type' set to MPI_BYTE -QAK
-     */
-    if (use_view_this_time) {
-        /*
-         * Figure out the mapping from the MPI 'buf_type' to bytes,
-         * someday... If this gets fixed (and MPI_Get_count() is
-         * reliable), the kludge below where the 'bytes_read' value from
-         * MPI_Get_count() is overwritten with the 'size_i' parameter can
-         * be removed. -QAK
-         */
-    } else {
-        /* How many bytes were actually read? */
-        if ((mrc = MPI_Get_count(&status, MPI_BYTE, &bytes_read)) != MPI_SUCCESS)
-            HMPI_GOTO_ERROR(FAIL, "MPI_Get_count failed", mrc);
-    }
-#endif /* H5_HAVE_MPI_GET_COUNT */
-
-    /*
-     * KLUGE rky 1998-02-02
      * MPI_Get_count incorrectly returns negative count; fake a complete
      * read.
      */
@@ -1459,8 +1420,8 @@ H5FD_fphdf5_write(H5FD_t *_file, H5FD_mem_t mem_type, hid_t dxpl_id,
      * dumping the data from the SAP...
      */
     if (mem_type != H5FD_MEM_DRAW && !dumping) {
-        unsigned req_id;
-        H5FP_status_t sap_status;
+        unsigned        req_id = 0;
+        H5FP_status_t   sap_status = H5FP_STATUS_OK;
 
         if (H5FP_request_write_metadata(_file, file->file_id, dxpl_id, mem_type,
                                         mpi_off, size_i, buf, &req_id,
@@ -1517,7 +1478,6 @@ H5FD_fphdf5_write_real(H5FD_t *_file, hid_t dxpl_id, MPI_Offset mpi_off, int siz
     MPI_Status          status;
     MPI_Datatype        buf_type;
     MPI_Datatype        file_type;
-    MPI_Offset          mpi_disp;
     int                 mrc;
     int                 bytes_written;
     unsigned            use_view_this_time = 0;
@@ -1553,7 +1513,10 @@ H5FD_fphdf5_write_real(H5FD_t *_file, hid_t dxpl_id, MPI_Offset mpi_off, int siz
         if (H5P_get(plist, H5FD_FPHDF5_XFER_USE_VIEW_NAME, &use_view_this_time) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get MPI-I/O type property");
 
+
     if (use_view_this_time) {
+        MPI_Offset          mpi_disp;
+
         /* Prepare for a full-blown xfer using btype, ftype, and disp */
         if (H5P_get(plist, H5FD_FPHDF5_XFER_MEM_MPI_TYPE_NAME, &buf_type) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get MPI-I/O type property");
@@ -1576,11 +1539,10 @@ H5FD_fphdf5_write_real(H5FD_t *_file, hid_t dxpl_id, MPI_Offset mpi_off, int siz
     } else {
         /*
          * Prepare for a simple xfer of a contiguous block of bytes. The
-         * btype, ftype, and disp fields are not used.
+         * btype and ftype.
          */
         buf_type = MPI_BYTE;
         file_type = MPI_BYTE;
-        mpi_disp = 0;   /* mpi_off is already set */
     }
 
 
@@ -1603,43 +1565,6 @@ H5FD_fphdf5_write_real(H5FD_t *_file, hid_t dxpl_id, MPI_Offset mpi_off, int siz
             HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at_all failed", mrc);
     }
 
-    /*
-     * KLUDGE, Robb Matzke, 2000-12-29
-     * The LAM implementation of MPI_Get_count() says
-     *
-     *    MPI_Get_count: invalid argument (rank 0, MPI_COMM_WORLD)
-     *
-     * So I'm commenting this out until it can be investigated. The
-     * returned `bytes_written' isn't used anyway because of Kim's kludge
-     * to avoid bytes_written<0. Likewise in H5FD_fphdf5_read().
-     */
-
-#ifdef H5_HAVE_MPI_GET_COUNT /* Bill and Albert's kludge*/
-    /*
-     * Yet Another KLUDGE, Albert Cheng & Bill Wendling, 2001-05-11.
-     * Many systems don't support MPI_Get_count so we need to do a
-     * configure thingy to fix this.
-     */
-
-    /*
-     * Calling MPI_Get_count with "MPI_BYTE" is only valid when we
-     * actually had the 'buf_type' set to MPI_BYTE -QAK
-     */
-    if (use_view_this_time) {
-        /*
-         * Figure out the mapping from the MPI 'buf_type' to bytes,
-         * someday... If this gets fixed (and MPI_Get_count() is
-         * reliable), the kludge below where the 'bytes_written' value
-         * from MPI_Get_count() is overwritten with the 'size'
-         * parameter can be removed. -QAK
-         */
-    } else {
-        /* How many bytes were actually written? */
-        if ((mrc = MPI_Get_count(&status, MPI_BYTE, &bytes_written)) != MPI_SUCCESS)
-            HMPI_GOTO_ERROR(FAIL, "MPI_Get_count failed", mrc);
-    }
-#endif /* H5_HAVE_MPI_GET_COUNT */
-
     /* Reset the file view when we used MPI derived types */
     if (use_view_this_time)
         if ((mrc = MPI_File_set_view(file->f, (MPI_Offset)0, MPI_BYTE, MPI_BYTE,
@@ -1680,8 +1605,8 @@ H5FD_fphdf5_flush(H5FD_t *_file, hid_t dxpl_id, unsigned closing)
     H5FD_fphdf5_t  *file = (H5FD_fphdf5_t*)_file;
     MPI_Offset      mpi_off;
     int	            mrc;
-    unsigned        req_id;
-    H5FP_status_t   status;
+    unsigned        req_id = 0;
+    H5FP_status_t   status = H5FP_STATUS_OK;
     herr_t          ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI(H5FD_fphdf5_flush, FAIL);
diff --git a/src/H5FDfphdf5.h b/src/H5FDfphdf5.h
index 20a5c83..8cf27fd 100644
--- a/src/H5FDfphdf5.h
+++ b/src/H5FDfphdf5.h
@@ -107,9 +107,9 @@ H5_DLL MPI_Comm H5FD_fphdf5_barrier_communicator(H5FD_t *_file);
 H5_DLL herr_t   H5FD_fphdf5_setup(hid_t dxpl_id, MPI_Datatype btype,
                                   MPI_Datatype ftype, unsigned use_view);
 H5_DLL herr_t   H5FD_fphdf5_teardown(hid_t dxpl_id);
+H5_DLL unsigned H5FD_fphdf5_file_id(H5FD_t *_file);
 H5_DLL int      H5FD_fphdf5_mpi_rank(H5FD_t *_file);
 H5_DLL int      H5FD_fphdf5_mpi_size(H5FD_t *_file);
-H5_DLL unsigned H5FD_fphdf5_file_id(H5FD_t *_file);
 H5_DLL hbool_t  H5FD_fphdf5_is_sap(H5FD_t *_file);
 H5_DLL hbool_t  H5FD_fphdf5_is_captain(H5FD_t *_file);
 H5_DLL hbool_t  H5FD_is_fphdf5_driver(H5FD_t *_file);
diff --git a/src/H5FP.c b/src/H5FP.c
index 60c73e3..5f38f5e 100644
--- a/src/H5FP.c
+++ b/src/H5FP.c
@@ -119,6 +119,8 @@ H5FPinit(MPI_Comm comm, int sap_rank, MPI_Comm *sap_comm, MPI_Comm *sap_barrier_
     if ((mrc = MPI_Comm_rank(H5FP_SAP_COMM, (int *)&my_rank)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Comm_rank failed", mrc);
 
+    H5FP_capt_barrier_rank = 0;
+
     /* Get the rank of the captain in the barrier Comm */
     if (H5FP_capt_rank == (unsigned)my_rank)
         if ((mrc = MPI_Comm_rank(H5FP_SAP_BARRIER_COMM,
@@ -341,24 +343,21 @@ H5FP_commit_sap_datatypes(void)
     block_length[0] = 8;
     block_length[1] = 1;
     block_length[2] = 4;
-    block_length[3] = 1;
-    block_length[4] = sizeof(sap_req.oid);
+    block_length[3] = 2;
     old_types[0] = MPI_UNSIGNED;
     old_types[1] = MPI_UNSIGNED_LONG;
     old_types[2] = MPI_LONG_LONG_INT;
     old_types[3] = HADDR_AS_MPI_TYPE;
-    old_types[4] = MPI_UNSIGNED_CHAR;
     MPI_Address(&sap_req.req_id, &displs[0]);
     MPI_Address(&sap_req.feature_flags, &displs[1]);
     MPI_Address(&sap_req.meta_block_size, &displs[2]);
     MPI_Address(&sap_req.addr, &displs[3]);
-    MPI_Address(&sap_req.oid, &displs[4]);
 
     /* Calculate the displacements */
     for (i = 4; i >= 0; --i)
         displs[i] -= displs[0];
 
-    if (MPI_Type_struct(5, block_length, displs, old_types, &H5FP_request) != MPI_SUCCESS)
+    if (MPI_Type_struct(4, block_length, displs, old_types, &H5FP_request) != MPI_SUCCESS)
         HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "MPI_Type_struct failed");
 
     if (MPI_Type_commit(&H5FP_request) != MPI_SUCCESS)
@@ -366,7 +365,7 @@ H5FP_commit_sap_datatypes(void)
 
     /* Commit the H5FP_reply datatype */
     block_length[0] = 4;
-    old_types[0] = MPI_INT;
+    old_types[0] = MPI_UNSIGNED;
     MPI_Address(&sap_reply.req_id, &displs[0]);
 
     /* Calculate the displacements */
diff --git a/src/H5FPclient.c b/src/H5FPclient.c
index 75149e8..d5bcfdb 100644
--- a/src/H5FPclient.c
+++ b/src/H5FPclient.c
@@ -25,6 +25,7 @@
 #include "H5Iprivate.h"         /* ID Functions                         */
 #include "H5MMprivate.h"        /* Memory Allocation                    */
 #include "H5Oprivate.h"         /* Object Headers                       */
+#include "H5Rprivate.h"         /* References                           */
 #include "H5Spkg.h"             /* Dataspace Functions                  */
 #include "H5TBprivate.h"        /* Threaded, Balanced, Binary Trees     */
 
@@ -135,7 +136,7 @@ done:
  * Modifications:
  */
 herr_t
-H5FP_request_lock(unsigned file_id, unsigned char *obj_oid,
+H5FP_request_lock(unsigned file_id, hobj_ref_t obj_oid,
                   H5FP_lock_t rw_lock, int last, unsigned *req_id,
                   H5FP_status_t *status)
 {
@@ -162,7 +163,7 @@ H5FP_request_lock(unsigned file_id, unsigned char *obj_oid,
     req.rw_lock = rw_lock;
     req.md_size = 0;
     req.proc_rank = my_rank;
-    H5FP_COPY_OID(req.oid, obj_oid);
+    req.oid = obj_oid;
 
     if ((mrc = MPI_Send(&req, 1, H5FP_request, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS) {
@@ -208,7 +209,7 @@ done:
  * Modifications:
  */
 herr_t
-H5FP_request_release_lock(unsigned file_id, unsigned char *obj_oid,
+H5FP_request_release_lock(unsigned file_id, hobj_ref_t obj_oid,
                           int last, unsigned *req_id, H5FP_status_t *status)
 {
     H5FP_request_t req;
@@ -228,7 +229,7 @@ H5FP_request_release_lock(unsigned file_id, unsigned char *obj_oid,
         HMPI_GOTO_ERROR(FAIL, "MPI_Comm_rank failed", mrc);
 
     *status = H5FP_STATUS_OK;
-    H5FP_COPY_OID(req.oid, obj_oid);
+    req.oid = obj_oid;
     req.req_type = last ? H5FP_REQ_RELEASE_END : H5FP_REQ_RELEASE;
     req.req_id = H5FP_gen_request_id();
     req.file_id = file_id;
@@ -424,6 +425,7 @@ H5FP_request_write_metadata(H5FD_t *file, unsigned file_id, hid_t dxpl_id,
         HGOTO_ERROR(H5E_FPHDF5, H5E_CANTSENDMDATA, FAIL, "can't send metadata to server");
 
     HDmemset(&mpi_status, 0, sizeof(mpi_status));
+    HDmemset(&sap_reply, 0, sizeof(sap_reply));
 
     if ((mrc = MPI_Recv(&sap_reply, 1, H5FP_reply, (int)H5FP_sap_rank,
                         H5FP_TAG_REPLY, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
@@ -484,8 +486,7 @@ H5FP_request_flush_metadata(H5FD_t *file, unsigned file_id, hid_t dxpl_id,
     H5FP_reply_t sap_reply;
     H5FP_request_t req;
     MPI_Status mpi_status;
-    int mrc, my_rank;
-    int ret_value = SUCCEED;
+    int mrc, ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI(H5FP_request_flush_metadata, FAIL);
 
@@ -496,13 +497,12 @@ H5FP_request_flush_metadata(H5FD_t *file, unsigned file_id, hid_t dxpl_id,
 
     HDmemset(&req, 0, sizeof(req));
 
-    if ((mrc = MPI_Comm_rank(H5FP_SAP_COMM, &my_rank)) != MPI_SUCCESS)
+    if ((mrc = MPI_Comm_rank(H5FP_SAP_COMM, &req.proc_rank)) != MPI_SUCCESS)
         HMPI_GOTO_ERROR(FAIL, "MPI_Comm_rank failed", mrc);
 
     req.req_type = H5FP_REQ_FLUSH;
     req.req_id = H5FP_gen_request_id();
     req.file_id = file_id;
-    req.proc_rank = my_rank;
 
     if ((mrc = MPI_Send(&req, 1, H5FP_request, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
@@ -669,7 +669,8 @@ done:
  * Modifications:
  */
 herr_t
-H5FP_request_free(H5FD_t *file, unsigned *req_id, H5FP_status_t *status)
+H5FP_request_free(H5FD_t *file, H5FD_mem_t mem_type, haddr_t addr, hsize_t size,
+                  unsigned *req_id, H5FP_status_t *status)
 {
     H5FP_alloc_t    sap_alloc;
     H5FP_request_t  req;
@@ -684,12 +685,78 @@ H5FP_request_free(H5FD_t *file, unsigned *req_id, H5FP_status_t *status)
     assert(req_id);
     assert(status);
 
+    /* Allow zero-sized frees to occur without penalty */
+    if (size == 0)
+        HGOTO_DONE(SUCCEED);
+
     HDmemset(&req, 0, sizeof(req));
 
     req.req_type = H5FP_REQ_FREE;
     req.req_id = H5FP_gen_request_id();
     req.file_id = H5FD_fphdf5_file_id(file);
     req.proc_rank = H5FD_fphdf5_mpi_rank(file);
+    req.mem_type = mem_type;
+    req.addr = addr;
+    req.meta_block_size = size;
+
+    if ((mrc = MPI_Send(&req, 1, H5FP_request, (int)H5FP_sap_rank,
+                        H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(FAIL, "MPI_Send failed", mrc);
+
+    HDmemset(&mpi_status, 0, sizeof(mpi_status));
+
+    if ((mrc = MPI_Recv(&sap_alloc, 1, H5FP_alloc, (int)H5FP_sap_rank,
+                        H5FP_TAG_ALLOC, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(FAIL, "MPI_Recv failed", mrc);
+
+    if (sap_alloc.status != H5FP_STATUS_OK)
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCHANGE, FAIL, "can't free space on server");
+
+    if ((mrc = MPI_Bcast(&sap_alloc, 1, H5FP_alloc, (int)H5FP_capt_barrier_rank,
+                         H5FP_SAP_BARRIER_COMM)) != MPI_SUCCESS)
+        HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed!", mrc);
+
+    /* Set the EOA for all processes. This call doesn't fail. */
+    file->cls->set_eoa(file, sap_alloc.eoa);
+    *status = H5FP_STATUS_OK;
+
+done:
+    *req_id = req.req_id;
+    FUNC_LEAVE_NOAPI(ret_value);
+}
+
+/*
+ * Function:    H5FP_request_update_eoma_eosda
+ * Purpose:     Request the SAP updates the EOMA and EOSDA information
+ *              for the file.
+ * Return:      Success:    SUCCEED
+ *              Failure:    FAIL
+ * Programmer:  Bill Wendling, 02. April 2003
+ * Modifications:
+ */
+herr_t
+H5FP_request_update_eoma_eosda(H5FD_t *file, unsigned *req_id,
+                               H5FP_status_t *status)
+{
+    H5FP_alloc_t    sap_alloc;
+    H5FP_request_t  req;
+    MPI_Status      mpi_status;
+    int             mrc;
+    herr_t          ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(H5FP_request_update_eoma_eosda, FAIL);
+
+    /* check args */
+    assert(file);
+    assert(req_id);
+    assert(status);
+
+    HDmemset(&req, 0, sizeof(req));
+
+    req.req_type = H5FP_REQ_UPDATE_EOMA_EOSDA;
+    req.req_id = H5FP_gen_request_id();
+    req.file_id = H5FD_fphdf5_file_id(file);
+    req.proc_rank = H5FD_fphdf5_mpi_rank(file);
 
     if ((mrc = MPI_Send(&req, 1, H5FP_request, (int)H5FP_sap_rank,
                         H5FP_TAG_REQUEST, H5FP_SAP_COMM)) != MPI_SUCCESS)
@@ -750,11 +817,11 @@ H5FP_gen_request_id()
 static herr_t
 H5FP_dump_to_file(H5FD_t *file, hid_t dxpl_id)
 {
-    H5FP_read_t sap_read;
-    hid_t new_dxpl_id = FAIL;
+    H5FP_read_t     sap_read;
+    hid_t           new_dxpl_id = FAIL;
     H5P_genplist_t *plist = NULL, *old_plist;
-    unsigned dumping = 1;
-    herr_t ret_value = SUCCEED;
+    unsigned        dumping = 1;
+    herr_t          ret_value = SUCCEED;
 
     FUNC_ENTER_NOINIT(H5FP_dump_to_file);
 
@@ -768,9 +835,6 @@ H5FP_dump_to_file(H5FD_t *file, hid_t dxpl_id)
     if ((new_dxpl_id = H5P_copy_plist(old_plist)) < 0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "can't copy property list");
 
-    if (new_dxpl_id == FAIL)
-        HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "can't copy property list");
-
     if ((plist = H5P_object_verify(new_dxpl_id, H5P_DATASET_XFER)) == NULL)
         HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dataset transfer list");
 
@@ -791,6 +855,7 @@ H5FP_dump_to_file(H5FD_t *file, hid_t dxpl_id)
         char *mdata;
 
         HDmemset(&mpi_status, 0, sizeof(mpi_status));
+        HDmemset(&sap_read, 0, sizeof(sap_read));
 
         if ((mrc = MPI_Recv(&sap_read, 1, H5FP_read, (int)H5FP_sap_rank,
                             H5FP_TAG_DUMP, H5FP_SAP_COMM, &mpi_status)) != MPI_SUCCESS)
@@ -818,9 +883,8 @@ H5FP_dump_to_file(H5FD_t *file, hid_t dxpl_id)
     }
 
 done:
-    if (plist)
-        /* FIXME: What can I do if this fails?? */
-        H5P_close(plist);
+    if (new_dxpl_id > 0)
+        H5Pclose(new_dxpl_id);
 
     FUNC_LEAVE_NOAPI(ret_value);
 }
diff --git a/src/H5FPprivate.h b/src/H5FPprivate.h
index 5ae04ca..0bc03c3 100644
--- a/src/H5FPprivate.h
+++ b/src/H5FPprivate.h
@@ -55,7 +55,8 @@ typedef enum {
 
     /* Allocation Requests */
     H5FP_REQ_ALLOC,             /* Allocate a region of metadata            */
-    H5FP_REQ_FREE               /* Free a region of metadata                */
+    H5FP_REQ_FREE,              /* Free a region of metadata                */
+    H5FP_REQ_UPDATE_EOMA_EOSDA  /* Update the EOMA and EOSDA information    */
 } H5FP_req_t;
 
 /*===----------------------------------------------------------------------===
@@ -186,7 +187,7 @@ typedef struct {
     hsize_t         threshold;  /* Alignment threshold                      */
     hsize_t         alignment;  /* Alignment (really!)                      */
     haddr_t         addr;       /* Address of the metadata                  */
-    unsigned char   oid[H5R_OBJ_REF_BUF_SIZE]; /* Buffer to store OID of object */
+    haddr_t         oid;        /* Buffer to store OID of object            */
 } H5FP_request_t;
 
 extern MPI_Datatype H5FP_request; /* MPI datatype for the H5FP_request obj  */
@@ -240,9 +241,6 @@ typedef struct {
 
 extern MPI_Datatype H5FP_alloc; /* MPI datatype for the H5FP_alloc obj      */
 
-/* Handy #define for copying OIDs */
-#define H5FP_COPY_OID(dst, src)     HDmemcpy((dst), (src), H5R_OBJ_REF_BUF_SIZE)
-
 /* SAP specific variables */
 extern MPI_Comm H5FP_SAP_COMM;  /* Comm we use: Supplied by user            */
 extern MPI_Comm H5FP_SAP_BARRIER_COMM; /* Comm if you want to do a barrier  */
@@ -268,10 +266,10 @@ extern herr_t H5FP_request_open(H5FP_obj_t obj_type,
                                 hsize_t alignment,
                                 unsigned *file_id,
                                 unsigned *req_id);
-extern herr_t H5FP_request_lock(unsigned sap_file_id, unsigned char *mdata,
+extern herr_t H5FP_request_lock(unsigned sap_file_id, hobj_ref_t oid,
                                 H5FP_lock_t rw_lock, int last, unsigned *req_id,
                                 H5FP_status_t *status);
-extern herr_t H5FP_request_release_lock(unsigned sap_file_id, unsigned char *mdata,
+extern herr_t H5FP_request_release_lock(unsigned sap_file_id, hobj_ref_t oid,
                                         int last, unsigned *req_id,
                                         H5FP_status_t *status);
 extern herr_t H5FP_request_read_metadata(H5FD_t *file, unsigned sap_file_id, hid_t dxpl_id,
@@ -291,7 +289,11 @@ extern herr_t H5FP_request_allocate(H5FD_t *file, H5FD_mem_t mem_type,
                                     hsize_t size, haddr_t *addr,
                                     haddr_t *eoa, unsigned *req_id,
                                     H5FP_status_t *status);
-extern herr_t H5FP_request_free(H5FD_t *file, unsigned *req_id, H5FP_status_t *status);
+extern herr_t H5FP_request_free(H5FD_t *file, H5FD_mem_t mem_type,
+                                haddr_t addr, hsize_t size,
+                                unsigned *req_id, H5FP_status_t *status);
+extern herr_t H5FP_request_update_eoma_eosda(H5FD_t *file, unsigned *req_id,
+                                             H5FP_status_t *status);
 
 /* NOTE: Don't use these functions outside of the H5FP* modules! */
 extern herr_t H5FP_send_metadata(const char *mdata, int len, int to);
diff --git a/src/H5FPserver.c b/src/H5FPserver.c
index 36a6fa8..d9f5d7c 100644
--- a/src/H5FPserver.c
+++ b/src/H5FPserver.c
@@ -28,8 +28,10 @@
 #include "H5private.h"          /* Generic Functions                    */
 #include "H5ACprivate.h"        /* Metadata Cache                       */
 #include "H5Eprivate.h"         /* Error Handling                       */
-#include "H5FDprivate.h"        /* File driver                          */
+#include "H5FDprivate.h"        /* File Driver                          */
 #include "H5Oprivate.h"         /* Object Headers                       */
+#include "H5Pprivate.h"         /* Property List                        */
+#include "H5Rprivate.h"         /* References                           */
 #include "H5TBprivate.h"        /* Threaded, Balanced, Binary Trees     */
 
 #ifdef H5_HAVE_FPHDF5
@@ -54,7 +56,7 @@ static int interface_initialize_g = 0;
  * appropriate H5FP_file_info structure.
  */
 typedef struct {
-    uint8_t         oid[H5R_OBJ_REF_BUF_SIZE]; /* Buffer to store OID of object */
+    hobj_ref_t      oid;        /* Buffer to store OID of object            */
     unsigned        owned_rank; /* rank which has the lock                  */
     H5FP_obj_t      obj_type;   /* type of object being locked              */
     unsigned        ref_count;  /* number of times lock was aquired by proc */
@@ -116,13 +118,13 @@ static unsigned H5FP_gen_sap_file_id(void);
 static int H5FP_object_lock_cmp(H5FP_object_lock *o1,
                                 H5FP_object_lock *o2,
                                 int cmparg);
-static H5FP_object_lock *H5FP_new_object_lock(const unsigned char *oid,
+static H5FP_object_lock *H5FP_new_object_lock(hobj_ref_t oid,
                                               unsigned rank,
                                               H5FP_obj_t obj_type,
                                               H5FP_lock_t rw_lock);
 static herr_t H5FP_free_object_lock(H5FP_object_lock *ol);
 static H5FP_object_lock *H5FP_find_object_lock(H5FP_file_info *info,
-                                               unsigned char *oid);
+                                               hobj_ref_t oid);
 static herr_t H5FP_remove_object_lock_from_list(H5FP_file_info *info,
                                                 H5FP_object_lock *ol);
 
@@ -162,6 +164,7 @@ static herr_t   H5FP_sap_handle_flush_request(H5FP_request_t *req);
 static herr_t   H5FP_sap_handle_close_request(H5FP_request_t *req);
 static herr_t   H5FP_sap_handle_alloc_request(H5FP_request_t *req);
 static herr_t   H5FP_sap_handle_free_request(H5FP_request_t *req);
+static herr_t   H5FP_sap_handle_update_eoma_eosda_request(H5FP_request_t *req);
 
 /*
  *===----------------------------------------------------------------------===
@@ -237,6 +240,9 @@ H5FP_sap_receive_loop(void)
         case H5FP_REQ_FREE:
             hrc = H5FP_sap_handle_free_request(&req);
             break;
+        case H5FP_REQ_UPDATE_EOMA_EOSDA:
+            hrc = H5FP_sap_handle_update_eoma_eosda_request(&req);
+            break;
         case H5FP_REQ_STOP:
             hrc = SUCCEED;
             if (++stop == comm_size - 1)
@@ -307,7 +313,7 @@ H5FP_object_lock_cmp(H5FP_object_lock *o1,
     FUNC_ENTER_NOINIT(H5FP_object_lock_cmp);
     assert(o1);
     assert(o2);
-    FUNC_LEAVE_NOAPI(HDmemcmp(o1->oid, o2->oid, sizeof(o1->oid)));
+    FUNC_LEAVE_NOAPI(o1->oid == o2->oid);
 }
 
 /*
@@ -325,18 +331,17 @@ H5FP_object_lock_cmp(H5FP_object_lock *o1,
  * Modifications:
  */
 static H5FP_object_lock *
-H5FP_new_object_lock(const unsigned char *oid, unsigned rank,
-                     H5FP_obj_t obj_type, H5FP_lock_t rw_lock)
+H5FP_new_object_lock(hobj_ref_t oid, unsigned rank, H5FP_obj_t obj_type,
+                     H5FP_lock_t rw_lock)
 {
     H5FP_object_lock *ret_value = NULL;
     
     FUNC_ENTER_NOINIT(H5FP_new_object_lock);
-    assert(oid);
 
     if ((ret_value = (H5FP_object_lock *)HDmalloc(sizeof(H5FP_object_lock))) == NULL)
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "out of memory");
 
-    H5FP_COPY_OID(ret_value->oid, oid);
+    ret_value->oid = oid;
     ret_value->owned_rank = rank;
     ret_value->obj_type = obj_type;
     ret_value->ref_count = 1;
@@ -370,7 +375,7 @@ H5FP_free_object_lock(H5FP_object_lock *ol)
  * Modifications:
  */
 static H5FP_object_lock *
-H5FP_find_object_lock(H5FP_file_info *info, unsigned char *oid)
+H5FP_find_object_lock(H5FP_file_info *info, hobj_ref_t oid)
 {
     H5FP_object_lock *ret_value = NULL;
 
@@ -383,7 +388,7 @@ H5FP_find_object_lock(H5FP_file_info *info, unsigned char *oid)
         H5TB_NODE *node;
         H5FP_object_lock ol;
 
-        H5FP_COPY_OID(ol.oid, oid); /* This is the key field for the TBBT */
+        ol.oid = oid;   /* This is the key field for the TBBT */
 
         if ((node = H5TB_dfind(info->locks, (void *)&ol, NULL)) == NULL)
             HGOTO_ERROR(H5E_FPHDF5, H5E_NOTFOUND, NULL, "lock not found");
@@ -924,7 +929,7 @@ static herr_t
 H5FP_sap_handle_lock_request(H5FP_request_t *req)
 {
     struct lock_group {
-        unsigned char       oid[sizeof(req->oid)];
+        hobj_ref_t          oid;
         unsigned            file_id;
         unsigned            locked;
         H5FP_lock_t         rw_lock;
@@ -949,7 +954,7 @@ H5FP_sap_handle_lock_request(H5FP_request_t *req)
      * at one time.
      */
     for (i = 0;; ++i) {
-        if (req->oid[0]) {
+        if (req->oid) {
             if (i == list_size) {
                 list_size <<= 1;    /* equiv to list_size *= 2; */
                 oids = HDrealloc(oids, list_size * sizeof(struct lock_group));
@@ -960,7 +965,7 @@ H5FP_sap_handle_lock_request(H5FP_request_t *req)
                 }
             }
 
-            H5FP_COPY_OID(oids[i].oid, req->oid);
+            oids[i].oid = req->oid;
             oids[i].file_id = req->file_id;
             oids[i].rw_lock = req->rw_lock;
             oids[i].locked = FALSE;
@@ -1127,7 +1132,7 @@ static herr_t
 H5FP_sap_handle_release_lock_request(H5FP_request_t *req)
 {
     struct release_group {
-        unsigned char       oid[sizeof(req->oid)];
+        hobj_ref_t          oid;
         unsigned            file_id;
         H5FP_file_info     *info;
         H5FP_object_lock   *lock;
@@ -1150,7 +1155,7 @@ H5FP_sap_handle_release_lock_request(H5FP_request_t *req)
      * release locks at one time.
      */
     for (i = 0;; ++i) {
-        if (req->oid[0]) {
+        if (req->oid) {
             if (i == list_size) {
                 list_size <<= 1;    /* equiv to list_size *= 2; */
                 oids = HDrealloc(oids, list_size * sizeof(struct release_group));
@@ -1161,7 +1166,7 @@ H5FP_sap_handle_release_lock_request(H5FP_request_t *req)
                 }
             }
 
-            H5FP_COPY_OID(oids[i].oid, req->oid);
+            oids[i].oid = req->oid;
             oids[i].file_id = req->file_id;
         }
 
@@ -1548,6 +1553,54 @@ H5FP_sap_handle_free_request(H5FP_request_t *req)
     sap_alloc.req_id = req->req_id;
     sap_alloc.file_id = req->file_id;
     sap_alloc.status = H5FP_STATUS_OK;
+    sap_alloc.mem_type = req->mem_type;
+
+    if ((info = H5FP_find_file_info(req->file_id)) != NULL) {
+        if (H5FD_free((H5FD_t*)&info->file, req->mem_type, H5P_DEFAULT,
+                      req->addr, req->meta_block_size) != SUCCEED) {
+            exit_state = H5FP_STATUS_CANT_FREE;
+            sap_alloc.status = H5FP_STATUS_CANT_FREE;
+            HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL,
+                        "SAP unable to free metadata block");
+        }
+
+        /* Get the EOA - This call doesn't fail */
+        sap_alloc.eoa = ((H5FD_t*)&info->file)->cls->get_eoa((H5FD_t*)&info->file);
+        sap_alloc.status = H5FP_STATUS_OK;
+    } else {
+        sap_alloc.status = H5FP_STATUS_CANT_FREE;
+    }
+
+done:
+    if ((mrc = MPI_Send(&sap_alloc, 1, H5FP_alloc, (int)req->proc_rank,
+                        H5FP_TAG_ALLOC, H5FP_SAP_COMM)) != MPI_SUCCESS)
+        HMPI_DONE_ERROR(FAIL, "MPI_Send failed", mrc);
+
+    FUNC_LEAVE_NOAPI(ret_value);
+}
+
+/*
+ * Function:    H5FP_sap_handle_update_eoma_eosda_request
+ * Purpose:     Handle a request to update the EOMA and EOSDA for a file.
+ * Return:      Success:    SUCCEED
+ *              Failure:    FAIL
+ * Programmer:  Bill Wendling, 02. April 2003
+ * Modifications:
+ */
+static herr_t
+H5FP_sap_handle_update_eoma_eosda_request(H5FP_request_t *req)
+{
+    H5FP_alloc_t    sap_alloc;
+    H5FP_file_info *info;
+    H5FP_status_t   exit_state = H5FP_STATUS_OK;
+    herr_t          ret_value = SUCCEED;
+    int             mrc;
+
+    FUNC_ENTER_NOINIT(H5FP_sap_handle_update_eoma_eosda_request);
+
+    sap_alloc.req_id = req->req_id;
+    sap_alloc.file_id = req->file_id;
+    sap_alloc.status = H5FP_STATUS_OK;
     sap_alloc.addr = HADDR_UNDEF;
 
     if ((info = H5FP_find_file_info(req->file_id)) != NULL) {
diff --git a/src/H5Gnode.c b/src/H5Gnode.c
index 2f20a2c..a7880b4 100644
--- a/src/H5Gnode.c
+++ b/src/H5Gnode.c
@@ -70,7 +70,7 @@ static H5G_node_t *H5G_node_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const vo
 static herr_t H5G_node_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr,
 			     H5G_node_t *sym);
 static herr_t H5G_node_dest(H5F_t *f, H5G_node_t *sym);
-static herr_t H5G_node_clear(H5G_node_t *sym);
+static herr_t H5G_node_clear(H5G_node_t *sym, hbool_t destroy);
 
 /* B-tree callbacks */
 static size_t H5G_node_sizeof_rkey(H5F_t *f, const void *_udata);
@@ -550,9 +550,10 @@ H5G_node_dest(H5F_t UNUSED *f, H5G_node_t *sym)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5G_node_clear(H5G_node_t *sym)
+H5G_node_clear(H5G_node_t *sym, hbool_t destroy)
 {
     int i;              /* Local index variable */
+    herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOINIT(H5G_node_clear);
 
@@ -567,7 +568,16 @@ H5G_node_clear(H5G_node_t *sym)
         sym->entry[i].dirty=FALSE;
     sym->cache_info.dirty = FALSE;
 
-    FUNC_LEAVE_NOAPI(SUCCEED);
+    /*
+     * Destroy the symbol node?	 This might happen if the node is being
+     * preempted from the cache.
+     */
+    if (destroy)
+        if (H5G_node_dest(NULL, sym) < 0)
+	    HGOTO_ERROR(H5E_SYM, H5E_CANTFREE, FAIL, "unable to destroy symbol table node");
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value);
 } /* end H5G_node_clear() */
 
 
diff --git a/src/H5HG.c b/src/H5HG.c
index 57a3320..679b301 100644
--- a/src/H5HG.c
+++ b/src/H5HG.c
@@ -119,7 +119,7 @@ static H5HG_heap_t *H5HG_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void
 static herr_t H5HG_flush(H5F_t *f, hid_t dxpl_id, hbool_t dest, haddr_t addr,
 			 H5HG_heap_t *heap);
 static herr_t H5HG_dest(H5F_t *f, H5HG_heap_t *heap);
-static herr_t H5HG_clear(H5HG_heap_t *heap);
+static herr_t H5HG_clear(H5HG_heap_t *heap, hbool_t destroy);
 
 /*
  * H5HG inherits cache-like properties from H5AC
@@ -519,7 +519,7 @@ H5HG_dest (H5F_t *f, H5HG_heap_t *heap)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5HG_clear(H5HG_heap_t *heap)
+H5HG_clear(H5HG_heap_t *heap, hbool_t destroy)
 {
     FUNC_ENTER_NOINIT(H5HG_clear);
 
@@ -529,6 +529,12 @@ H5HG_clear(H5HG_heap_t *heap)
     /* Mark heap as clean */
     heap->cache_info.dirty = 0;
 
+    if (destroy) {
+        heap->chunk = H5FL_BLK_FREE(heap_chunk,heap->chunk);
+        heap->obj = H5FL_ARR_FREE(H5HG_obj_t,heap->obj);
+        H5FL_FREE(H5HG_heap_t, heap);
+    }
+
     FUNC_LEAVE_NOAPI(SUCCEED);
 } /* H5HG_clear() */
 
diff --git a/src/H5HL.c b/src/H5HL.c
index 550d5f6..3d72889 100644
--- a/src/H5HL.c
+++ b/src/H5HL.c
@@ -68,7 +68,7 @@ static H5HL_t *H5HL_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *udat
 			 void *udata2);
 static herr_t H5HL_flush(H5F_t *f, hid_t dxpl_id, hbool_t dest, haddr_t addr, H5HL_t *heap);
 static herr_t H5HL_dest(H5F_t *f, H5HL_t *heap);
-static herr_t H5HL_clear(H5HL_t *heap);
+static herr_t H5HL_clear(H5HL_t *heap, hbool_t destroy);
 
 /*
  * H5HL inherits cache-like properties from H5AC
@@ -568,8 +568,10 @@ H5HL_dest(H5F_t UNUSED *f, H5HL_t *heap)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5HL_clear(H5HL_t *heap)
+H5HL_clear(H5HL_t *heap, hbool_t destroy)
 {
+    herr_t ret_value = SUCCEED;
+
     FUNC_ENTER_NOINIT(H5HL_clear);
 
     /* check arguments */
@@ -578,7 +580,12 @@ H5HL_clear(H5HL_t *heap)
     /* Mark heap as clean */
     heap->cache_info.dirty = 0;
 
-    FUNC_LEAVE_NOAPI(SUCCEED);
+    if (destroy)
+        if (H5HL_dest(NULL, heap) < 0)
+	    HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to destroy local heap collection");
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value);
 } /* end H5HL_clear() */
 
 
diff --git a/src/H5O.c b/src/H5O.c
index 82d7cc3..45ca8b7 100644
--- a/src/H5O.c
+++ b/src/H5O.c
@@ -33,12 +33,17 @@
 #include "H5Eprivate.h"
 #include "H5Fpkg.h"
 #include "H5FLprivate.h"	/*Free Lists	  */
+#include "H5FSprivate.h"        /* Function Stack                           */
 #include "H5Iprivate.h"
 #include "H5MFprivate.h"
 #include "H5MMprivate.h"
 #include "H5Opkg.h"             /* Object header functions                 */
 #include "H5Pprivate.h"
 
+#ifdef H5_HAVE_FPHDF5
+#include "H5FDfphdf5.h"         /* FPHDF5 File Descriptor                  */
+#endif  /* H5_HAVE_FPHDF5 */
+
 #ifdef H5_HAVE_GETTIMEOFDAY
 #include <sys/time.h>
 #endif /* H5_HAVE_GETTIMEOFDAY */
@@ -82,7 +87,7 @@ static H5O_t *H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *_udata
 		       void *_udata2);
 static herr_t H5O_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5O_t *oh);
 static herr_t H5O_dest(H5F_t *f, H5O_t *oh);
-static herr_t H5O_clear(H5O_t *oh);
+static herr_t H5O_clear(H5O_t *oh, hbool_t destroy);
 
 /* H5O inherits cache-like properties from H5AC */
 static const H5AC_class_t H5AC_OHDR[1] = {{
@@ -228,7 +233,7 @@ H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, H5G_entry_t *ent/*out*/)
 
     /* initialize the object header */
     if (H5O_init(f, dxpl_id, size_hint, ent, header) != SUCCEED)
-	HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to initialize object header");
+        HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to initialize object header");
 
 done:
     FUNC_LEAVE_NOAPI(ret_value);
@@ -261,6 +266,9 @@ H5O_init(H5F_t *f, hid_t dxpl_id, size_t size_hint, H5G_entry_t *ent/*out*/, had
     H5O_t      *oh = NULL;
     haddr_t     tmp_addr;
     herr_t      ret_value = SUCCEED;    /* return value */
+#ifdef H5_HAVE_FPHDF5
+    unsigned    capt_only = 0;
+#endif  /* H5_HAVE_FPHDF5 */
 
     FUNC_ENTER_NOINIT(H5O_init);
 
@@ -309,6 +317,29 @@ H5O_init(H5F_t *f, hid_t dxpl_id, size_t size_hint, H5G_entry_t *ent/*out*/, had
     oh->mesg[0].raw_size = size_hint - H5O_SIZEOF_MSGHDR(f);
     oh->mesg[0].chunkno = 0;
 
+#ifdef H5_HAVE_FPHDF5
+    if (H5FD_is_fphdf5_driver(f->shared->lf)) {
+        H5P_genplist_t *plist;
+
+        /* Get the data xfer property list */
+        if ((plist = H5I_object(dxpl_id)) == NULL)
+            HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dataset transfer list");
+
+        /* Check if the "Captain Only" flag's been set */
+        if (H5P_exist_plist(plist, H5FD_FPHDF5_CAPTN_ALLOC_ONLY) > 0)
+            if (H5P_get(plist, H5FD_FPHDF5_CAPTN_ALLOC_ONLY, &capt_only) < 0)
+                HGOTO_ERROR(H5E_PLIST, H5E_CANTDELETE, FAIL, "can't retrieve FPHDF5 property");
+    }
+
+    /*
+     * We only want to initialize the object header if this isn't an
+     * FPHDF5 driver or it is, but the captain only flag is set or if the
+     * captain only flag just isn't set.
+     */
+    if (!H5FD_is_fphdf5_driver(f->shared->lf) || !capt_only || (H5FD_fphdf5_is_captain(f->shared->lf) && capt_only))
+        ;
+#endif  /* H5_HAVE_FPHDF5 */
+
     /* cache it */
     if (H5AC_set(f, dxpl_id, H5AC_OHDR, ent->header, oh) < 0)
         HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to cache object header");
@@ -864,9 +895,10 @@ H5O_dest(H5F_t UNUSED *f, H5O_t *oh)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5O_clear(H5O_t *oh)
+H5O_clear(H5O_t *oh, hbool_t destroy)
 {
     unsigned	u;      /* Local index variable */
+    herr_t ret_value = SUCCEED;
 
     FUNC_ENTER_NOINIT(H5O_clear);
 
@@ -884,7 +916,12 @@ H5O_clear(H5O_t *oh)
     /* Mark whole header as clean */
     oh->cache_info.dirty=FALSE;
 
-    FUNC_LEAVE_NOAPI(SUCCEED);
+    if (destroy)
+        if (H5O_dest(NULL, oh) < 0)
+            HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to destroy object header data");
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value);
 } /* end H5O_clear() */
 
 
diff --git a/src/H5config.h.in b/src/H5config.h.in
index 45d725d..083f2ee 100644
--- a/src/H5config.h.in
+++ b/src/H5config.h.in
@@ -36,6 +36,9 @@
 /* Define to 1 if you have the `fork' function. */
 #undef HAVE_FORK
 
+/* Define if we want flexible parallel HDF5 support */
+#undef HAVE_FPHDF5
+
 /* Define to 1 if you have the `fseek64' function. */
 #undef HAVE_FSEEK64
 
diff --git a/src/hdf5.h b/src/hdf5.h
index 4bb9d10..3ed7203 100644
--- a/src/hdf5.h
+++ b/src/hdf5.h
@@ -29,6 +29,7 @@
 #include "H5Epublic.h"		/* Errors				*/
 #include "H5Fpublic.h"		/* Files				*/
 #include "H5FDpublic.h"		/* File drivers				*/
+#include "H5FPpublic.h"		/* Flexible Parallel HDF5               */
 #include "H5Gpublic.h"		/* Groups				*/
 #include "H5HGpublic.h"		/* Global heaps				*/
 #include "H5HLpublic.h"		/* Local heaps				*/