Merge pull request #236 in HDFFV/hdf5 from develop_merge_cache_image_04 to develop

Merged file shutdown changes from cache_image.

* commit '49fa0e8dbabe5019b1491845a963d8cfe41b59c1':
  Bring file shutdown code from cache image branch
  Merge code from cache image branch to split FSM ring into two types: raw data and metadata.  Also, some more ring reset safeties and minor code cleanups.
  Align w/incoming cache image changes, and improve recovery on errors.
  Revert accidental changes to freespace open and close routines.
  Remove 'const' from cache client pre_serialize callback, to reduce warnings, and correspondingly remove 'const' from some internal routines.  Also rename some H5MF* routines to reflect their static/package usage.
  Align with incoming cache_image branch changes: use the index list (instead of the hash buckets) for scanning the entries during a flush, and also add in counters for tracking operations during cache flushes.

31 files changed, 2259 insertions, 529 deletions

diff --git a/src/H5AC.c b/src/H5AC.c
index 86e5ed7..6449584 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -1287,6 +1287,44 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ *
+ * Function:    H5AC_prep_for_file_close
+ *
+ * Purpose:     This function should be called just prior to the cache
+ *              flushes at file close.  
+ *
+ *              The objective of the call is to allow the metadata cache
+ *              to do any preparatory work prior to generation of a
+ *              cache image.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  John Mainzer
+ *              7/3/15
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5AC_prep_for_file_close(H5F_t *f, hid_t dxpl_id)
+{
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Sanity checks */
+    HDassert(f);
+    HDassert(f->shared);
+    HDassert(f->shared->cache);
+
+    if(H5C_prep_for_file_close(f, dxpl_id) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "cache prep for file close failed")
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_prep_for_file_close() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5AC_create_flush_dependency()
  *
  * Purpose:	Create a flush dependency between two entries in the metadata
@@ -2777,6 +2815,54 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5AC_unsettle_entry_ring()
+ *
+ * Purpose:     Advise the metadata cache that the specified entry's metadata
+ *              cache manager ring is no longer settled (if it was on entry).
+ *
+ *              If the target metadata cache manager ring is already
+ *              unsettled, do nothing, and return SUCCEED.
+ *
+ *              If the target metadata cache manager ring is settled, and
+ *              we are not in the process of a file shutdown, mark
+ *              the ring as unsettled, and return SUCCEED.
+ *
+ *              If the target metadata cache  manager is settled, and we
+ *              are in the process of a file shutdown, post an error
+ *              message, and return FAIL.
+ *
+ *		Note that this function simply passes the call on to
+ *		the metadata cache proper, and returns the result.
+ *
+ * Return:	Success:	Non-negative
+ *		Failure:	Negative
+ *
+ * Programmer:  Quincey Koziol
+ *              September 17, 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5AC_unsettle_entry_ring(void *_entry)
+{
+    H5AC_info_t *entry = (H5AC_info_t *)_entry; /* Entry to remove */
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Sanity checks */
+    HDassert(entry);
+
+    /* Unsettle the entry's ring */
+    if(H5C_unsettle_entry_ring(entry) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry")
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_unsettle_entry_ring() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5AC_remove_entry()
  *
  * Purpose:     Remove an entry from the cache.  Must be not protected, pinned,
diff --git a/src/H5ACdbg.c b/src/H5ACdbg.c
index 6120242..8d99c6f 100644
--- a/src/H5ACdbg.c
+++ b/src/H5ACdbg.c
@@ -249,3 +249,43 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* H5AC__open_trace_file() */
 
+
+/*-------------------------------------------------------------------------
+ *
+ * Function:    H5AC_cache_is_clean()
+ *
+ * Purpose:     Debugging function that verifies that all rings in the
+ *              metadata cache are clean from the outermost ring, inwards
+ *              to the inner ring specified.
+ *
+ *              Returns TRUE if all specified rings are clean, and FALSE
+ *              if not.  Throws an assertion failure on error.
+ *
+ * Return:      TRUE if the indicated ring(s) are clean, and FALSE otherwise.
+ *
+ * Programmer:  John Mainzer, 6/18/16
+ *
+ * Changes:     None.
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef NDEBUG
+hbool_t
+H5AC_cache_is_clean(const H5F_t *f, H5AC_ring_t inner_ring)
+{
+    H5C_t *cache_ptr;
+    hbool_t ret_value;          /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+    /* Sanity checks */
+    HDassert(f);
+    HDassert(f->shared);
+    cache_ptr = f->shared->cache;
+
+    ret_value = H5C_cache_is_clean(cache_ptr, inner_ring);
+
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_cache_is_clean() */
+#endif /* NDEBUG */
+
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index 07bb8fa..96f23cb 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -154,7 +154,8 @@ typedef enum {
 typedef H5C_ring_t       H5AC_ring_t;
 #define H5AC_RING_INV    H5C_RING_UNDEFINED
 #define H5AC_RING_USER   H5C_RING_USER
-#define H5AC_RING_FSM    H5C_RING_FSM
+#define H5AC_RING_RDFSM  H5C_RING_RDFSM
+#define H5AC_RING_MDFSM  H5C_RING_MDFSM
 #define H5AC_RING_SBE    H5C_RING_SBE
 #define H5AC_RING_SB     H5C_RING_SB
 #define H5AC_RING_NTYPES H5C_RING_NTYPES
@@ -364,6 +365,7 @@ H5_DLL herr_t H5AC_get_entry_status(const H5F_t *f, haddr_t addr,
 H5_DLL herr_t H5AC_insert_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type,
     haddr_t addr, void *thing, unsigned int flags);
 H5_DLL herr_t H5AC_pin_protected_entry(void *thing);
+H5_DLL herr_t H5AC_prep_for_file_close(H5F_t *f, hid_t dxpl_id);
 H5_DLL herr_t H5AC_create_flush_dependency(void *parent_thing, void *child_thing);
 H5_DLL void * H5AC_protect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type,
     haddr_t addr, void *udata, unsigned flags);
@@ -403,6 +405,7 @@ H5_DLL herr_t H5AC_get_entry_ring(const H5F_t *f, haddr_t addr, H5AC_ring_t *rin
 H5_DLL herr_t H5AC_set_ring(hid_t dxpl_id, H5AC_ring_t ring, H5P_genplist_t **dxpl,
     H5AC_ring_t *orig_ring);
 H5_DLL herr_t H5AC_reset_ring(H5P_genplist_t *dxpl, H5AC_ring_t orig_ring);
+H5_DLL herr_t H5AC_unsettle_entry_ring(void *entry);
 H5_DLL herr_t H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags);
 H5_DLL herr_t H5AC_get_tag(const void *thing, /*OUT*/ haddr_t *tag);
 
@@ -422,6 +425,9 @@ H5_DLL herr_t H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr);
 /* Debugging functions */
 H5_DLL herr_t H5AC_stats(const H5F_t *f);
 H5_DLL herr_t H5AC_dump_cache(const H5F_t *f);
+#ifndef NDEBUG
+H5_DLL hbool_t H5AC_cache_is_clean(const H5F_t *f, H5AC_ring_t inner_ring);
+#endif /* NDEBUG */ /* end debugging functions */
 
 #endif /* !_H5ACprivate_H */
 
diff --git a/src/H5C.c b/src/H5C.c
index e753da4..37c833e 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -138,12 +138,10 @@ static herr_t H5C__autoadjust__ageout__remove_excess_markers(H5C_t * cache_ptr);
 static herr_t H5C__flash_increase_cache_size(H5C_t * cache_ptr,
     size_t old_entry_size, size_t new_entry_size);
 
-static herr_t H5C_flush_invalidate_cache(const H5F_t *  f,
-                                          hid_t    dxpl_id,
-			                  unsigned flags);
+static herr_t H5C_flush_invalidate_cache(H5F_t *f, hid_t dxpl_id, unsigned flags);
 
-static herr_t H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id,
-    H5C_ring_t ring, unsigned flags);
+static herr_t H5C_flush_invalidate_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring,
+    unsigned flags);
 
 static herr_t H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring,
     unsigned flags);
@@ -169,7 +167,7 @@ static herr_t H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry);
 static herr_t H5C__verify_len_eoa(H5F_t *f, const H5C_class_t * type,
     haddr_t addr, size_t *len, hbool_t actual);
 
-static herr_t H5C__generate_image(const H5F_t *f, H5C_t * cache_ptr, H5C_cache_entry_t *entry_ptr, 
+static herr_t H5C__generate_image(H5F_t *f, H5C_t * cache_ptr, H5C_cache_entry_t *entry_ptr, 
                                   hid_t dxpl_id);
 
 #if H5C_DO_SLIST_SANITY_CHECKS
@@ -438,6 +436,14 @@ H5C_create(size_t		      max_cache_size,
         ((cache_ptr->epoch_markers)[i]).type		 = &H5C__epoch_marker_class;
     }
 
+    cache_ptr->entries_loaded_counter		= 0;
+    cache_ptr->entries_inserted_counter		= 0;
+    cache_ptr->entries_relocated_counter	= 0;
+
+    /* initialize free space manager related fields: */
+    cache_ptr->rdfsm_settled		= FALSE;
+    cache_ptr->mdfsm_settled		= FALSE;
+
     if ( H5C_reset_cache_hit_rate_stats(cache_ptr) != SUCCEED ) {
 
         /* this should be impossible... */
@@ -685,6 +691,53 @@ H5C_free_tag_list_cb(void *_item, void H5_ATTR_UNUSED *key, void H5_ATTR_UNUSED
 
 
 /*-------------------------------------------------------------------------
+ *
+ * Function:    H5C_prep_for_file_close
+ *
+ * Purpose:     This function should be called just prior to the cache 
+ *		flushes at file close.  There should be no protected 
+ *		entries in the cache at this point.
+ *		
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  John Mainzer
+ *              7/3/15
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id)
+{
+    H5C_t *     cache_ptr;
+    herr_t	ret_value = SUCCEED;      /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Sanity checks */
+    HDassert(f);
+    HDassert(f->shared);
+    HDassert(f->shared->cache);
+    cache_ptr = f->shared->cache;
+    HDassert(cache_ptr);
+    HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+
+    /* For now at least, it is possible to receive the 
+     * close warning more than once -- the following 
+     * if statement handles this.
+     */
+    if(cache_ptr->close_warning_received)
+        HGOTO_DONE(SUCCEED)
+    cache_ptr->close_warning_received = TRUE;
+
+    /* Make certain there aren't any protected entries */
+    HDassert(cache_ptr->pl_len == 0);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_prep_for_file_close() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5C_dest
  *
  * Purpose:     Flush all data to disk and destroy the cache.
@@ -719,6 +772,7 @@ H5C_dest(H5F_t * f, hid_t dxpl_id)
     /* Sanity check */
     HDassert(cache_ptr);
     HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+    HDassert(cache_ptr->close_warning_received);
 
     /* Flush and invalidate all cache entries */
     if(H5C_flush_invalidate_cache(f, dxpl_id, H5C__NO_FLAGS_SET) < 0 )
@@ -902,6 +956,10 @@ done:
  *
  *						JRM -- 8/30/15
  *
+ *		Modified function to call the free space manager 
+ *		settling functions.
+ *						JRM -- 6/9/16
+ *
  *-------------------------------------------------------------------------
  */
 herr_t
@@ -981,6 +1039,55 @@ H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags)
          */
         ring = H5C_RING_USER;
 	while(ring < H5C_RING_NTYPES) {
+	    /* only call the free space manager settle routines when close
+             * warning has been received, and then only when the index is 
+             * non-empty for that ring.
+             */
+	    if(cache_ptr->close_warning_received) {
+		switch(ring) {
+		    case H5C_RING_USER:
+			break;
+
+		    case H5C_RING_RDFSM:
+			if(!cache_ptr->rdfsm_settled) {
+                            hbool_t fsm_settled = FALSE;        /* Whether the FSM was actually settled */
+
+                            /* Settle raw data FSM */
+			    if(H5MF_settle_raw_data_fsm(f, dxpl_id, &fsm_settled) < 0)
+                                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "RD FSM settle failed")
+
+                            /* Only set the flag if the FSM was actually settled */
+                            if(fsm_settled)
+                                cache_ptr->rdfsm_settled = TRUE;
+                        } /* end if */
+			break;
+
+		    case H5C_RING_MDFSM:
+			if(!cache_ptr->mdfsm_settled) {
+                            hbool_t fsm_settled = FALSE;        /* Whether the FSM was actually settled */
+
+                            /* Settle metadata FSM */
+			    if(H5MF_settle_meta_data_fsm(f, dxpl_id, &fsm_settled) < 0)
+                                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "MD FSM settle failed")
+
+                            /* Only set the flag if the FSM was actually settled */
+                            if(fsm_settled)
+                                cache_ptr->mdfsm_settled = TRUE;
+                        } /* end if */
+			break;
+
+		    case H5C_RING_SBE:
+			break;
+
+		    case H5C_RING_SB:
+			break;
+
+		    default:
+                        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown ring?!?!")
+			break;
+		} /* end switch */
+            } /* end if */
+
 	    if(H5C_flush_ring(f, dxpl_id, ring, flags) < 0)
                 HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush ring failed.")
             ring++;
@@ -2417,6 +2524,9 @@ H5C_protect(H5F_t *		f,
          */
         H5C__UPDATE_RP_FOR_INSERTION(cache_ptr, entry_ptr, NULL)
 
+        /* Update entries loaded in cache counter */
+        cache_ptr->entries_loaded_counter++;
+
         /* Record that the entry was loaded, to trigger a notify callback later */
         /* (After the entry is fully added to the cache) */
         was_loaded = TRUE;
@@ -3224,10 +3334,10 @@ H5C_unprotect(H5F_t *		  f,
             /* Delete the entry from the skip list on destroy */
             flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG;
 
+            HDassert(((!was_clean) || dirtied) == entry_ptr->in_slist);
             if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flush_flags) < 0)
                 HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "Can't flush entry")
-
-        }
+        } /* end if */
 #ifdef H5_HAVE_PARALLEL
         else if(clear_entry) {
 
@@ -3264,6 +3374,82 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ *
+ * Function:    H5C_unsettle_entry_ring
+ *
+ * Purpose:     Advise the metadata cache that the specified entry's metadata
+ *              cache manager ring is no longer settled (if it was on entry).
+ *
+ *              If the target metadata cache manager ring is already
+ *              unsettled, do nothing, and return SUCCEED.
+ *
+ *              If the target metadata cache manager ring is settled, and
+ *              we are not in the process of a file shutdown, mark
+ *              the ring as unsettled, and return SUCCEED.
+ *
+ *              If the target metadata cache  manager is settled, and we
+ *              are in the process of a file shutdown, post an error
+ *              message, and return FAIL.
+ *
+ *		Note that this function simply passes the call on to
+ *		the metadata cache proper, and returns the result.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Quincey Koziol
+ *              January 3, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5C_unsettle_entry_ring(void *_entry)
+{
+    H5C_cache_entry_t *entry = (H5C_cache_entry_t *)_entry;     /* Entry whose ring to unsettle */
+    H5C_t *cache;               /* Cache for file */
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Sanity checks */
+    HDassert(entry);
+    HDassert(entry->ring != H5C_RING_UNDEFINED);
+    HDassert((H5C_RING_USER == entry->ring) || (H5C_RING_RDFSM == entry->ring) || (H5C_RING_MDFSM == entry->ring));
+    cache = entry->cache_ptr;
+    HDassert(cache);
+    HDassert(cache->magic == H5C__H5C_T_MAGIC);
+
+    switch(entry->ring) {
+	case H5C_RING_USER:
+            /* Do nothing */
+	    break;
+
+	case H5C_RING_RDFSM:
+	    if(cache->rdfsm_settled) {
+		if(cache->flush_in_progress || cache->close_warning_received)
+		    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected rdfsm ring unsettle")
+		cache->rdfsm_settled = FALSE;
+	    } /* end if */
+	    break;
+
+	case H5C_RING_MDFSM:
+	    if(cache->mdfsm_settled) {
+		if(cache->flush_in_progress || cache->close_warning_received)
+		    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected mdfsm ring unsettle")
+		cache->mdfsm_settled = FALSE;
+	    } /* end if */
+	    break;
+
+	default:
+	    HDassert(FALSE); /* this should be un-reachable */
+	    break;
+    } /* end switch */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_unsettle_entry_ring() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5C_validate_resize_config()
  *
  * Purpose:	Run a sanity check on the specified sections of the
@@ -4998,7 +5184,7 @@ done:
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5C_flush_invalidate_cache(const H5F_t * f, hid_t dxpl_id, unsigned flags)
+H5C_flush_invalidate_cache(H5F_t *f, hid_t dxpl_id, unsigned flags)
 {
     H5C_t *		cache_ptr;
     H5C_ring_t		ring;
@@ -5143,25 +5329,25 @@ done:
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring,
+H5C_flush_invalidate_ring(H5F_t * f, hid_t dxpl_id, H5C_ring_t ring,
     unsigned flags)
 {
-    H5C_t *		cache_ptr;
+    H5C_t              *cache_ptr;
     hbool_t             restart_slist_scan;
-    int32_t		protected_entries = 0;
-    int32_t		i;
-    int32_t		cur_ring_pel_len;
-    int32_t		old_ring_pel_len;
-    unsigned		cooked_flags;
-    unsigned		evict_flags;
-    H5SL_node_t * 	node_ptr = NULL;
-    H5C_cache_entry_t *	entry_ptr = NULL;
-    H5C_cache_entry_t *	next_entry_ptr = NULL;
+    int32_t             protected_entries = 0;
+    int32_t             i;
+    int32_t             cur_ring_pel_len;
+    int32_t             old_ring_pel_len;
+    unsigned            cooked_flags;
+    unsigned            evict_flags;
+    H5SL_node_t        *node_ptr = NULL;
+    H5C_cache_entry_t  *entry_ptr = NULL;
+    H5C_cache_entry_t  *next_entry_ptr = NULL;
 #if H5C_DO_SANITY_CHECKS
     int64_t             initial_slist_len = 0;
     size_t              initial_slist_size = 0;
 #endif /* H5C_DO_SANITY_CHECKS */
-    herr_t		ret_value = SUCCEED;
+    herr_t              ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI(FAIL)
 
@@ -5407,77 +5593,92 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring,
          *
          * Writes to disk are possible here.
          */
-        for(i = 0; i < H5C__HASH_TABLE_LEN; i++) {
-            next_entry_ptr = cache_ptr->index[i];
 
-            while(next_entry_ptr != NULL) {
-                entry_ptr = next_entry_ptr;
-                HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
-                HDassert(entry_ptr->ring >= ring);
+        /* reset the counters so that we can detect insertions, loads,
+         * and moves caused by the pre_serialize and serialize calls.
+         */
+        cache_ptr->entries_loaded_counter         = 0;
+        cache_ptr->entries_inserted_counter       = 0;
+        cache_ptr->entries_relocated_counter      = 0;
+
+        next_entry_ptr = cache_ptr->il_head;
+        while(next_entry_ptr != NULL) {
+            entry_ptr = next_entry_ptr;
+            HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
+            HDassert(entry_ptr->ring >= ring);
 
-                next_entry_ptr = entry_ptr->ht_next;
-                HDassert((next_entry_ptr == NULL) ||
-                        (next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC));
+            next_entry_ptr = entry_ptr->il_next;
+            HDassert((next_entry_ptr == NULL) ||
+                     (next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC));
 
-                if(((!entry_ptr->flush_me_last) ||
-                       ((entry_ptr->flush_me_last) &&
-                            (cache_ptr->num_last_entries >= cache_ptr->slist_len))) &&
-                       (entry_ptr->flush_dep_nchildren == 0) &&
-                       (entry_ptr->ring == ring)) {
+            if((!entry_ptr->flush_me_last || (entry_ptr->flush_me_last && cache_ptr->num_last_entries >= cache_ptr->slist_len))
+                    && entry_ptr->flush_dep_nchildren == 0 && entry_ptr->ring == ring) {
+                if(entry_ptr->is_protected) {
+                    /* we have major problems -- but lets flush and 
+                     * destroy everything we can before we flag an 
+                     * error.
+                     */
+                    protected_entries++;
+                    if(!entry_ptr->in_slist)
+                        HDassert(!(entry_ptr->is_dirty));
+                } /* end if */
+                else if(!(entry_ptr->is_pinned)) {
+                    /* if *entry_ptr is dirty, it is possible 
+                     * that one or more other entries may be 
+                     * either removed from the cache, loaded 
+                     * into the cache, or moved to a new location
+                     * in the file as a side effect of the flush.
+                     *
+                     * It's also possible that removing a clean
+                     * entry will remove the last child of a proxy
+                     * entry, allowing it to be removed also and
+                     * invalidating the next_entry_ptr.
+                     *
+                     * If either of these happen, and one of the target 
+                     * or proxy entries happens to be the next entry in 
+                     * the hash bucket, we could either find ourselves
+                     * either scanning a non-existant entry, scanning
+                     * through a different bucket, or skipping an entry.
+                     *
+                     * Neither of these are good, so restart the 
+                     * the scan at the head of the hash bucket 
+                     * after the flush if we detect that the next_entry_ptr
+                     * becomes invalid.
+                     *
+                     * This is not as inefficient at it might seem,
+                     * as hash buckets typically have at most two
+                     * or three entries.
+                     */
+                    cache_ptr->entry_watched_for_removal = next_entry_ptr;
 
-                    if(entry_ptr->is_protected) {
-                        /* we have major problems -- but lets flush and 
-                         * destroy everything we can before we flag an 
-                         * error.
-                         */
-                        protected_entries++;
-                        if(!entry_ptr->in_slist)
-                            HDassert(!(entry_ptr->is_dirty));
-                    } else if(!(entry_ptr->is_pinned)) {
-
-                        /* if *entry_ptr is dirty, it is possible 
-                         * that one or more other entries may be 
-                         * either removed from the cache, loaded 
-                         * into the cache, or moved to a new location
-                         * in the file as a side effect of the flush.
-                         *
-                         * It's also possible that removing a clean
-                         * entry will remove the last child of a proxy
-                         * entry, allowing it to be removed also and
-                         * invalidating the next_entry_ptr.
-                         *
-                         * If either of these happen, and one of the target 
-                         * or proxy entries happens to be the next entry in 
-                         * the hash bucket, we could either find ourselves
-                         * either scanning a non-existant entry, scanning
-                         * through a different bucket, or skipping an entry.
-                         *
-                         * Neither of these are good, so restart the 
-                         * the scan at the head of the hash bucket 
-                         * after the flush if we detect that the next_entry_ptr
-                         * becomes invalid.
-                         *
-                         * This is not as inefficient at it might seem,
-                         * as hash buckets typically have at most two
-                         * or three entries.
-                         */
-                        cache_ptr->entry_watched_for_removal = next_entry_ptr;
-                        if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, (cooked_flags | H5C__DURING_FLUSH_FLAG | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG)) < 0)
-                            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Entry flush destroy failed.")
-
-                        /* Check for the next entry getting removed */
-                        if(NULL != next_entry_ptr && NULL == cache_ptr->entry_watched_for_removal) {
-                            /* update stats for hash bucket scan restart here.
-                             *                   -- JRM 
-                             */
-                            next_entry_ptr = cache_ptr->index[i];
-                            H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr)
-                        } /* end if */
-                        else
-                            cache_ptr->entry_watched_for_removal = NULL;
+                    if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, (cooked_flags | H5C__DURING_FLUSH_FLAG | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG)) < 0)
+                        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Entry flush destroy failed.")
+
+                    /* Restart the index list scan if necessary.  Must
+                     * do this if the next entry is evicted, and also if 
+                     * one or more entries are inserted, loaded, or moved
+                     * as these operations can result in part of the scan
+                     * being skipped -- which can cause a spurious failure
+                     * if this results in the size of the pinned entry 
+                     * failing to decline during the pass.
+                     */
+                    if((NULL != next_entry_ptr && NULL == cache_ptr->entry_watched_for_removal)
+                            || (cache_ptr->entries_loaded_counter > 0)
+                            || (cache_ptr->entries_inserted_counter > 0)
+                            || (cache_ptr->entries_relocated_counter > 0)) {
+
+                        next_entry_ptr = cache_ptr->il_head;
+
+                        cache_ptr->entries_loaded_counter         = 0;
+                        cache_ptr->entries_inserted_counter       = 0;
+                        cache_ptr->entries_relocated_counter      = 0;
+
+                        H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr)
                     } /* end if */
+                    else
+                       cache_ptr->entry_watched_for_removal = NULL;
                 } /* end if */
-            } /* end while loop scanning hash table bin */
+            } /* end if */
         } /* end for loop scanning hash table */
 
         /* We can't do anything if entries are pinned.  The
@@ -5488,32 +5689,33 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring,
          * of pinned entries from pass to pass.  If it stops
          * shrinking before it hits zero, we scream and die.
          */
-	old_ring_pel_len = cur_ring_pel_len;
+        old_ring_pel_len = cur_ring_pel_len;
         entry_ptr = cache_ptr->pel_head_ptr;
         cur_ring_pel_len = 0;
         while(entry_ptr != NULL) {
             HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
             HDassert(entry_ptr->ring >= ring);
 
-	    if(entry_ptr->ring == ring)
+            if(entry_ptr->ring == ring)
                 cur_ring_pel_len++;
 
             entry_ptr = entry_ptr->next;
         } /* end while */
 
-	if((cur_ring_pel_len > 0) && (cur_ring_pel_len >= old_ring_pel_len)) {
+       /* Check if the number of pinned entries in the ring is positive, and 
+        * it is not declining.  Scream and die if so.
+        */
+        if(cur_ring_pel_len > 0 && cur_ring_pel_len >= old_ring_pel_len) {
             /* Don't error if allowed to have pinned entries remaining */
-	    if(evict_flags)
+            if(evict_flags)
                 HGOTO_DONE(TRUE)
 
-	   /* The number of pinned entries in the ring is positive, and 
-            * it is not declining.  Scream and die.
-	    */
             HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Pinned entry count not decreasing, cur_ring_pel_len = %d, old_ring_pel_len = %d, ring = %d", (int)cur_ring_pel_len, (int)old_ring_pel_len, (int)ring)
         } /* end if */
 
         HDassert(protected_entries == cache_ptr->pl_len);
-        if((protected_entries > 0) && (protected_entries == cache_ptr->index_len))
+
+        if(protected_entries > 0 && protected_entries == cache_ptr->index_len)
             HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Only protected entries left in cache, protected_entries = %d", (int)protected_entries)
     } /* main while loop */
 
@@ -5719,7 +5921,7 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring,  unsigned flags)
             if(!flush_marked_entries || entry_ptr->flush_marker)
                 HDassert(entry_ptr->ring >= ring);
 
-            /* advance node pointer now, before we delete its target
+            /* Advance node pointer now, before we delete its target
              * from the slist.
              */
             node_ptr = H5SL_next(node_ptr);
@@ -5728,19 +5930,26 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring,  unsigned flags)
                 if(NULL == next_entry_ptr)
                     HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!")
 
+                HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
+                HDassert(next_entry_ptr->is_dirty);
+                HDassert(next_entry_ptr->in_slist);
+
+                if(!flush_marked_entries || next_entry_ptr->flush_marker)
+                    HDassert(next_entry_ptr->ring >= ring);
+
                 HDassert(entry_ptr != next_entry_ptr);
             } /* end if */
             else
                 next_entry_ptr = NULL;
 
-            if(((!flush_marked_entries) || (entry_ptr->flush_marker)) &&
-                    ((!entry_ptr->flush_me_last) ||
-                       (entry_ptr->flush_me_last &&
-                         ((cache_ptr->num_last_entries >= cache_ptr->slist_len) ||
-                           (flush_marked_entries && entry_ptr->flush_marker)))) &&
-                       ( ( entry_ptr->flush_dep_nchildren == 0 ) ||
-                         ( entry_ptr->flush_dep_ndirty_children == 0 ) ) &&
-                     (entry_ptr->ring == ring)) {
+            if((!flush_marked_entries || entry_ptr->flush_marker) 
+                    && (!entry_ptr->flush_me_last ||
+                        (entry_ptr->flush_me_last 
+                            && (cache_ptr->num_last_entries >= cache_ptr->slist_len
+                               || (flush_marked_entries && entry_ptr->flush_marker)))) 
+                    && (entry_ptr->flush_dep_nchildren == 0
+                        || entry_ptr->flush_dep_ndirty_children == 0) 
+                    && entry_ptr->ring == ring) {
                 if(entry_ptr->is_protected) {
                     /* we probably have major problems -- but lets 
                      * flush everything we can before we decide 
@@ -5857,7 +6066,7 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
+H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
     unsigned flags)
 {
     H5C_t *	     	cache_ptr;              /* Cache for file */
@@ -5930,7 +6139,7 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_
 #endif /* H5C_DO_SANITY_CHECKS */
 
     if(entry_ptr->is_protected) {
-	HDassert(!entry_ptr->is_protected);
+        HDassert(!entry_ptr->is_protected);
 
         /* Attempt to flush a protected entry -- scream and die. */
         HGOTO_ERROR(H5E_CACHE, H5E_PROTECT, FAIL, "Attempt to flush a protected entry.")
@@ -6117,11 +6326,11 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_
                 HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared")
 
             /* Propagate the clean flag up the flush dependency chain if appropriate */
-	    HDassert(entry_ptr->flush_dep_ndirty_children == 0);
-	    if(entry_ptr->flush_dep_nparents > 0)
-		if(H5C__mark_flush_dep_clean(entry_ptr) < 0)
-		    HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep clean flag")
-	} /* end if */
+            HDassert(entry_ptr->flush_dep_ndirty_children == 0);
+            if(entry_ptr->flush_dep_nparents > 0)
+                if(H5C__mark_flush_dep_clean(entry_ptr) < 0)
+                    HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep clean flag")
+        } /* end if */
     } /* end else */
 
     /* reset the flush_in progress flag */
@@ -7736,7 +7945,7 @@ H5C__assert_flush_dep_nocycle(const H5C_cache_entry_t * entry,
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr, 
+H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr, 
     hid_t dxpl_id)
 {
     haddr_t		new_addr = HADDR_UNDEF;
@@ -7836,7 +8045,8 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p
          * for a move 
          */
         if(serialize_flags & H5C__SERIALIZE_MOVED_FLAG) {
-            H5C__UPDATE_STATS_FOR_MOVE(cache_ptr, entry_ptr);
+            /* Update stats and entries relocated counter */
+            H5C__UPDATE_STATS_FOR_MOVE(cache_ptr, entry_ptr)
 
             /* We must update cache data structures for the change in address */
             if(entry_ptr->addr == old_addr) {
diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c
index 9266acd..5697bff 100644
--- a/src/H5Cdbg.c
+++ b/src/H5Cdbg.c
@@ -952,3 +952,46 @@ H5C__dump_entry(H5C_t *cache_ptr, const H5C_cache_entry_t *entry_ptr,
     if(entry_ptr->flush_dep_nchildren)
         H5C__dump_children(cache_ptr, entry_ptr, FALSE, "Child", indent);
 } /* end H5C__dump_entry() */
+/*-------------------------------------------------------------------------
+ *
+ * Function:    H5C_cache_is_clean()
+ *
+ * Purpose:     Debugging function that verifies that all rings in the 
+ *		metadata cache are clean from the outermost ring, inwards
+ *		to the inner ring specified.
+ *
+ *		Returns TRUE if all specified rings are clean, and FALSE
+ *		if not.  Throws an assertion failure on error.
+ *
+ * Return:      TRUE if the indicated ring(s) are clean, and FALSE otherwise.
+ *
+ * Programmer:  John Mainzer, 6/18/16
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef NDEBUG
+hbool_t
+H5C_cache_is_clean(const H5C_t *cache_ptr, H5C_ring_t inner_ring)
+{
+    H5C_ring_t		ring = H5C_RING_USER;
+    hbool_t             ret_value = TRUE;      /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+    /* Sanity checks */
+    HDassert(cache_ptr);
+    HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+    HDassert(inner_ring >= H5C_RING_USER);
+    HDassert(inner_ring <= H5C_RING_SB);
+
+    while(ring <= inner_ring) {
+	if(cache_ptr->dirty_index_ring_size[ring] > 0)
+            ret_value = FALSE;
+
+	ring++;
+    } /* end while */
+
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_cache_is_clean() */
+#endif /* NDEBUG */
+
diff --git a/src/H5Cepoch.c b/src/H5Cepoch.c
index f9c809b..e576028 100644
--- a/src/H5Cepoch.c
+++ b/src/H5Cepoch.c
@@ -66,7 +66,7 @@ static void * H5C__epoch_marker_deserialize(const void * image_ptr,
     size_t len, void * udata, hbool_t * dirty_ptr);
 static herr_t H5C__epoch_marker_image_len(const void * thing,
     size_t *image_len_ptr);
-static herr_t H5C__epoch_marker_pre_serialize(const H5F_t *f,
+static herr_t H5C__epoch_marker_pre_serialize(H5F_t *f,
     hid_t dxpl_id, void * thing, haddr_t addr, size_t len,
     haddr_t * new_addr_ptr, size_t * new_len_ptr, unsigned * flags_ptr);
 static herr_t H5C__epoch_marker_serialize(const H5F_t *f,
@@ -183,7 +183,7 @@ H5C__epoch_marker_image_len(const void H5_ATTR_UNUSED *thing,
 
 
 static herr_t
-H5C__epoch_marker_pre_serialize(const H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id,
+H5C__epoch_marker_pre_serialize(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id,
     void H5_ATTR_UNUSED *thing, haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED len,
     haddr_t H5_ATTR_UNUSED *new_addr_ptr, size_t H5_ATTR_UNUSED *new_len_ptr,
     unsigned H5_ATTR_UNUSED *flags_ptr)
diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h
index e9ff6ef..306f5f9 100644
--- a/src/H5Cpkg.h
+++ b/src/H5Cpkg.h
@@ -656,7 +656,8 @@ if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) &&             \
             ((cache_ptr)->cache_flush_moves[(entry_ptr)->type->id])++; \
         if ( entry_ptr->flush_in_progress )                            \
             ((cache_ptr)->entry_flush_moves[(entry_ptr)->type->id])++; \
-	(((cache_ptr)->moves)[(entry_ptr)->type->id])++;
+	(((cache_ptr)->moves)[(entry_ptr)->type->id])++;               \
+        (cache_ptr)->entries_relocated_counter++;
 
 #define H5C__UPDATE_STATS_FOR_ENTRY_SIZE_CHANGE(cache_ptr, entry_ptr, new_size)\
 	if ( cache_ptr->flush_in_progress )                                    \
@@ -782,6 +783,7 @@ if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) &&             \
             ((cache_ptr)->max_size)[(entry_ptr)->type->id] )         \
         ((cache_ptr)->max_size)[(entry_ptr)->type->id]               \
              = (entry_ptr)->size;                                    \
+    cache_ptr->entries_inserted_counter++;                           \
 }
 
 #define H5C__UPDATE_STATS_FOR_PROTECT(cache_ptr, entry_ptr, hit)            \
@@ -866,6 +868,7 @@ if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) &&             \
         (cache_ptr)->max_slist_len = (cache_ptr)->slist_len;         \
     if ( (cache_ptr)->slist_size > (cache_ptr)->max_slist_size )     \
         (cache_ptr)->max_slist_size = (cache_ptr)->slist_size;       \
+    cache_ptr->entries_inserted_counter++;                           \
 }
 
 #define H5C__UPDATE_STATS_FOR_PROTECT(cache_ptr, entry_ptr, hit)            \
@@ -3465,6 +3468,9 @@ typedef struct H5C_tag_info_t {
  *              clean data so as to avoid case b) above.  Again, this is
  *              a soft limit.
  *
+ * close_warning_received: Boolean flag indicating that a file closing 
+ *		warning has been received.
+ *
  *
  * In addition to the call back functions required for each entry, the
  * cache requires the following call back functions for this instance of
@@ -4057,6 +4063,52 @@ typedef struct H5C_tag_info_t {
  *	this field will be reset every automatic resize epoch.
  *
  *
+ * entries_loaded_counter: Number of entries loaded into the cache 
+ *		since the last time this field was reset.
+ *
+ * entries_inserted_counter: Number of entries inserted into the cache 
+ *		since the last time this field was reset.
+ *
+ * entries relocated_counter: Number of entries whose base address has
+ *		been changed since the last time this field was reset.
+ *
+ * Free Space Manager Related fields:
+ *
+ * The free space managers must be informed when we are about to close 
+ * or flush the file so that they order themselves accordingly.  This used
+ * to be done much later in the close process, but with cache image and 
+ * page buffering, this is no longer viable, as we must finalize the on 
+ * disk image of all metadata much sooner.
+ *
+ * This is handled by the H5FS_settle_raw_data_fsm() and 
+ * H5FS_settle_meta_data_fsm() routines.  As these calls are expensive,
+ * the following fields are used to track whether the target free space
+ * managers are clean.  
+ *
+ * They are also used in sanity checking, as once a free space manager is
+ * settled, it should not become unsettled (i.e. be asked to allocate or
+ * free file space) either ever (in the case of a file close) or until the
+ * flush is complete.
+ *
+ * rdfsm_settled:  Boolean flag indicating whether the raw data free space
+ *		manager is settled -- i.e. whether the correct space has 
+ *		been allocated for it in the file.
+ *
+ *		Note that the name of this field is deceptive.  In the 
+ *		multi file case, the flag applies to all free space 
+ *		managers that are not involved in allocating space for
+ *		free space manager metadata.
+ *
+ * mdfsm_settled:  Boolean flag indicating whether the meta data free space
+ *              manager is settled -- i.e. whether the correct space has 
+ *              been allocated for it in the file.
+ *
+ *              Note that the name of this field is deceptive.  In the 
+ *              multi file case, the flag applies only to free space 
+ *		managers that are involved in allocating space for free 
+ *		space managers.
+ *
+ *
  * Statistics collection fields:
  *
  * When enabled, these fields are used to collect statistics as described
@@ -4337,6 +4389,7 @@ struct H5C_t {
     hbool_t			write_permitted;
     H5C_log_flush_func_t	log_flush;
     hbool_t			evictions_enabled;
+    hbool_t			close_warning_received;
 
     /* Fields for maintaining [hash table] index of entries */
     int32_t                     index_len;
@@ -4439,6 +4492,13 @@ struct H5C_t {
     int64_t			cache_hits;
     int64_t			cache_accesses;
 
+    int64_t			entries_loaded_counter;
+    int64_t			entries_inserted_counter;
+    int64_t			entries_relocated_counter;
+    /* Free Space Manager Related fields */
+    hbool_t 			rdfsm_settled;
+    hbool_t			mdfsm_settled;
+
 #if H5C_COLLECT_CACHE_STATS
     /* stats fields */
     int64_t                     hits[H5C__MAX_NUM_TYPE_IDS + 1];
@@ -4532,7 +4592,7 @@ H5_DLLVAR const H5C_class_t H5C__epoch_marker_class;
 /******************************/
 
 /* General routines */
-H5_DLL herr_t H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id,
+H5_DLL herr_t H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id,
     H5C_cache_entry_t *entry_ptr, unsigned flags);
 H5_DLL herr_t H5C__flush_marked_entries(H5F_t * f, hid_t dxpl_id);
 H5_DLL herr_t H5C__iter_tagged_entries(H5C_t *cache, haddr_t tag, hbool_t match_global,
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index 365fd6c..3b9634f 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -559,7 +559,7 @@ typedef struct H5C_t H5C_t;
  *
  *	The typedef for the pre-serialize callback is as follows:
  *
- *	typedef herr_t (*H5C_pre_serialize_func_t)(const H5F_t *f,
+ *	typedef herr_t (*H5C_pre_serialize_func_t)(H5F_t *f,
  *                                             hid_t dxpl_id,
  *                                             void * thing,
  *                                             haddr_t addr,
@@ -878,7 +878,7 @@ typedef htri_t (*H5C_verify_chksum_func_t)(const void *image_ptr, size_t len, vo
 typedef void *(*H5C_deserialize_func_t)(const void *image_ptr,
     size_t len, void *udata_ptr, hbool_t *dirty_ptr);
 typedef herr_t (*H5C_image_len_func_t)(const void *thing, size_t *image_len_ptr);
-typedef herr_t (*H5C_pre_serialize_func_t)(const H5F_t *f, hid_t dxpl_id,
+typedef herr_t (*H5C_pre_serialize_func_t)(H5F_t *f, hid_t dxpl_id,
     void *thing, haddr_t addr, size_t len, haddr_t *new_addr_ptr,
     size_t *new_len_ptr, unsigned *flags_ptr);
 typedef herr_t (*H5C_serialize_func_t)(const H5F_t *f, void *image_ptr,
@@ -937,11 +937,13 @@ typedef herr_t (*H5C_log_flush_func_t)(H5C_t *cache_ptr, haddr_t addr,
  * ring.  
  *
  * Free space managers managing file space must be flushed next,
- * and are assigned to the second outermost ring.
+ * and are assigned to the second and third outermost rings.  Two rings
+ * are used here as the raw data free space manager must be flushed before
+ * the metadata free space manager.
  *
  * The object header and associated chunks used to implement superblock 
  * extension messages must be flushed next, and are thus assigned to 
- * the third outermost ring.
+ * the fourth outermost ring.
  *
  * The superblock proper must be flushed last, and is thus assigned to 
  * the innermost ring.
@@ -957,10 +959,11 @@ typedef herr_t (*H5C_log_flush_func_t)(H5C_t *cache_ptr, haddr_t addr,
 
 #define H5C_RING_UNDEFINED  0 /* shouldn't appear in the cache */
 #define H5C_RING_USER       1 /* outermost ring */
-#define H5C_RING_FSM	    2
-#define H5C_RING_SBE        4 /* temporarily merged with H5C_RING_SB */
-#define H5C_RING_SB         4 /* innermost ring */
-#define H5C_RING_NTYPES     5
+#define H5C_RING_RDFSM      2
+#define H5C_RING_MDFSM      3
+#define H5C_RING_SBE        4
+#define H5C_RING_SB         5 /* innermost ring */
+#define H5C_RING_NTYPES     6
 
 typedef int H5C_ring_t;
 
@@ -1770,6 +1773,7 @@ H5_DLL herr_t H5C_mark_entry_clean(void *thing);
 H5_DLL herr_t H5C_move_entry(H5C_t *cache_ptr, const H5C_class_t *type,
     haddr_t old_addr, haddr_t new_addr);
 H5_DLL herr_t H5C_pin_protected_entry(void *thing);
+H5_DLL herr_t H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id);
 H5_DLL herr_t H5C_create_flush_dependency(void *parent_thing, void *child_thing);
 H5_DLL void * H5C_protect(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type,
     haddr_t addr, void *udata, unsigned flags);
@@ -1794,7 +1798,8 @@ H5_DLL hbool_t H5C_get_ignore_tags(const H5C_t *cache_ptr);
 H5_DLL herr_t H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest_tag);
 H5_DLL herr_t H5C_cork(H5C_t *cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked);
 H5_DLL herr_t H5C_get_entry_ring(const H5F_t *f, haddr_t addr, H5C_ring_t *ring);
-H5_DLL herr_t H5C_remove_entry(void * thing);
+H5_DLL herr_t H5C_unsettle_entry_ring(void *thing);
+H5_DLL herr_t H5C_remove_entry(void *thing);
 
 #ifdef H5_HAVE_PARALLEL
 H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id,
@@ -1807,5 +1812,9 @@ H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t *f, hid_t dxpl_id, int32_t ce_arra
     haddr_t *ce_array_ptr);
 #endif /* H5_HAVE_PARALLEL */
 
+#ifndef NDEBUG	/* debugging functions */
+H5_DLL hbool_t H5C_cache_is_clean(const H5C_t *cache_ptr, H5C_ring_t inner_ring);
+#endif /* NDEBUG */
+
 #endif /* !_H5Cprivate_H */
 
diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c
index f0d4dba..4c4cad0 100644
--- a/src/H5FDfamily.c
+++ b/src/H5FDfamily.c
@@ -1357,7 +1357,7 @@ H5FD_family_lock(H5FD_t *_file, hbool_t rw)
     if(u < file->nmembs) {
         unsigned v;                         /* Local index variable */
 
-        for(v = 0; v < v; v++) {
+        for(v = 0; v < u; v++) {
             if(H5FD_unlock(file->memb[v]) < 0)
                 /* Push error, but keep going */
                 HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files")
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index 427b42c..45f0187 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -151,7 +151,7 @@ H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H
 H5_DLL herr_t H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f,
     haddr_t addr, hsize_t size);
 H5_DLL htri_t H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f,
-    haddr_t blk_end, hsize_t extra_requested);
+    hid_t dxpl_id, haddr_t blk_end, hsize_t extra_requested);
 H5_DLL haddr_t H5FD_get_eoa(const H5FD_t *file, H5FD_mem_t type);
 H5_DLL herr_t H5FD_set_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t addr);
 H5_DLL haddr_t H5FD_get_eof(const H5FD_t *file, H5FD_mem_t type);
diff --git a/src/H5FDspace.c b/src/H5FDspace.c
index a7899ac..fcddecf 100644
--- a/src/H5FDspace.c
+++ b/src/H5FDspace.c
@@ -243,9 +243,9 @@ H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, H5F_t *f, hsize_t size,
     if(!H5F_addr_defined(ret_value))
         HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, HADDR_UNDEF, "real 'alloc' request failed")
 
-    /* Mark superblock dirty in cache, so change to EOA will get encoded */
-    if(H5F_super_dirty(f) < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, HADDR_UNDEF, "unable to mark superblock as dirty")
+    /* Mark EOA info dirty in cache, so change will get encoded */
+    if(H5F_eoa_dirty(f, dxpl_id) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, HADDR_UNDEF, "unable to mark EOA info as dirty")
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -350,8 +350,8 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, H5F_t *f, haddr_t addr,
-    hsize_t size)
+H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, H5F_t *f,
+    haddr_t addr, hsize_t size)
 {
     herr_t      ret_value = SUCCEED;       /* Return value */
 
@@ -367,9 +367,9 @@ H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, H5F_t *f, haddr_t addr,
     if(H5FD_free_real(file, dxpl_id, type, addr, size) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "real 'free' request failed")
 
-    /* Mark superblock dirty in cache, so change to EOA will get encoded */
-    if(H5F_super_dirty(f) < 0)
-        HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
+    /* Mark EOA info dirty in cache, so change will get encoded */
+    if(H5F_eoa_dirty(f, dxpl_id) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, FAIL, "unable to mark EOA info as dirty")
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -395,8 +395,8 @@ done:
  *-------------------------------------------------------------------------
  */
 htri_t
-H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, haddr_t blk_end,
-    hsize_t extra_requested)
+H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, hid_t dxpl_id,
+    haddr_t blk_end, hsize_t extra_requested)
 {
     haddr_t eoa;                /* End of allocated space in file */
     htri_t ret_value = FALSE;   /* Return value */
@@ -423,9 +423,9 @@ H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, haddr_t blk_end,
         if(HADDR_UNDEF == H5FD_extend(file, type, FALSE, extra_requested, NULL, NULL))
             HGOTO_ERROR(H5E_VFL, H5E_CANTEXTEND, FAIL, "driver extend request failed")
 
-        /* Mark superblock dirty in cache, so change to EOA will get encoded */
-        if(H5F_super_dirty(f) < 0)
-            HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
+        /* Mark EOA info dirty in cache, so change will get encoded */
+        if(H5F_eoa_dirty(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, FAIL, "unable to mark EOA info as dirty")
 
         /* Indicate success */
         HGOTO_DONE(TRUE)
diff --git a/src/H5FS.c b/src/H5FS.c
index af02e0f..8644a6f 100644
--- a/src/H5FS.c
+++ b/src/H5FS.c
@@ -888,6 +888,10 @@ H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id)
         if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, fspace->sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0)
             HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sections to cache")
 
+        /* Since space has been allocated for the section info and the sinfo
+         * has been inserted into the cache, relinquish owership (i.e. float)
+         * the section info.
+         */
         fspace->sinfo = NULL;
     } /* end if */
 
@@ -897,6 +901,169 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ * Function:	H5FS_alloc_vfd_alloc_hdr_and_section_info
+ *
+ * Purpose:	This function is part of a hack to patch over a design
+ *              flaw in the free space managers for file space allocation.  
+ *		Specifically, if a free space manager allocates space for 
+ *		its own section info, it is possible for it to 
+ *		go into an infinite loop as it:
+ *
+ *                      1) computes the size of the section info
+ *
+ *                      2) allocates file space for the section info
+ *
+ *                      3) notices that the size of the section info
+ *                         has changed
+ *
+ *                      4) deallocates the section info file space and
+ *                         returns to 1) above.
+ *
+ *		Similarly, if it allocates space for its own header, it 
+ *		can go into an infinte loop as it:
+ *
+ *			1) allocates space for the header
+ *
+ *			2) notices that the free space manager is empty
+ *			   and thus should not have file space allocated 
+ *			   to its header
+ *
+ *			3) frees the space allocated to the header
+ *
+ *			4) notices that the free space manager is not 
+ *			   empty and thus must have file space allocated
+ *			   to it, and thus returns to 1) above.
+ *
+ *              In a nutshell, the solution applied in this hack is to
+ *		deallocate file space for the free space manager(s) that
+ *		handle FSM header and/or section info file space allocations,
+ *              wait until all other allocation/deallocation requests have
+ *              been handled, and then test to see if the free space manager(s)
+ *		in question are empty.  If they are, do nothing.  If they 
+ *		are not, allocate space for them at end of file bypassing the
+ *		usual file space allocation calls, and thus avoiding the 
+ *		potential infinite loops.
+ *
+ *		The purpose of this function is to allocate file space for 
+ *		the header and section info of the target free space manager 
+ *		directly from the VFD if needed.  In this case the function 
+ *		also re-inserts the header and section info in the metadata 
+ *		cache with this allocation.
+ *
+ *		Note that if f->shared->alignment > 1, and EOA is not a 
+ *		multiple of the alignment, it is possible that performing 
+ *		this allocation may generate a fragment of file space in 
+ *		addition to the space allocated for the section info.
+ *
+ *		At present we deal with this by screaming and dying.  
+ *		Obviously, this is not acceptatable, but it should do 
+ *		for now.
+ *
+ *		
+ * Return:	Success:	non-negative
+ *		Failure:	negative
+ *
+ * Programmer:	John Mainzer
+ *		6/6/16
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id, 
+    H5FS_t *fspace, haddr_t *fs_addr_ptr)
+{
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+
+    /* Check arguments */
+    HDassert(f);
+    HDassert(fspace);
+    HDassert(fs_addr_ptr);
+
+    /* The section info should be unlocked */
+    HDassert(fspace->sinfo_lock_count == 0);
+
+    /* No space should be allocated */
+    HDassert(*fs_addr_ptr == HADDR_UNDEF);
+    HDassert(fspace->addr == HADDR_UNDEF);
+    HDassert(fspace->sect_addr == HADDR_UNDEF);
+    HDassert(fspace->alloc_sect_size == 0);
+
+    /* Check if any space will be needed */
+    if(fspace->serial_sect_count > 0) {
+        haddr_t sect_addr;      /* Address of sinfo */
+
+        /* The section info is floating, so fspace->sinfo should be defined */
+        HDassert(fspace->sinfo);
+
+        /* Start by allocating file space for the header */
+        if(HADDR_UNDEF == (fspace->addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_HDR,
+                (hsize_t)H5FS_HEADER_SIZE(f), FALSE)))
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr")
+
+        /* Cache the new free space header (pinned) */
+        if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space header to cache")
+
+
+        /* Now allocate file space for the section info */
+        if(HADDR_UNDEF == (sect_addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_SINFO,
+                fspace->sect_size, FALSE)))
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space")
+
+        /* Update fspace->alloc_sect_size and fspace->sect_addr to reflect 
+         * the allocation 
+         */
+        fspace->alloc_sect_size = fspace->sect_size;
+        fspace->sect_addr = sect_addr;
+
+        /* We have changed the sinfo address -- Mark free space header dirty */
+        if(H5AC_mark_entry_dirty(fspace) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
+
+        /* Insert the new section info into the metadata cache.  */
+
+        /* Question: Do we need to worry about this insertion causing an 
+         * eviction from the metadata cache?  Think on this.  If so, add a 
+         * flag to H5AC_insert() to force it to skip the call to make space in
+         * cache.
+         *
+         * On reflection, no.
+         *
+         * On a regular file close, any eviction will not change the 
+         * the contents of the free space manger(s), as all entries 
+         * should have correct file space allocated by the time this 
+         * function is called.
+         *
+         * In the cache image case, the selection of entries for inclusion
+         * in the cache image will not take place until after this call.
+         * (Recall that this call is made during the metadata fsm settle
+         * routine, which is called during the serialization routine in 
+         * the cache image case.  Entries are not selected for inclusion
+         * in the image until after the cache is serialized.)  
+         *
+         *                                        JRM -- 11/4/16
+         */
+        if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space sinfo to cache")
+
+        /* Since space has been allocated for the section info and the sinfo
+         * has been inserted into the cache, relinquish owership (i.e. float)
+         * the section info.
+         */
+        fspace->sinfo = NULL;
+
+        /* Set the address of the free space header, on success */
+        *fs_addr_ptr = fspace->addr;
+    } /* end if */
+
+done:
+    FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* H5FS_alloc_vfd_alloc_hdr_and_section_info() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5FS_free()
  *
  * Purpose:     Free space for free-space manager header and section info header
@@ -923,6 +1090,7 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id)
 
     cache_flags = H5AC__DELETED_FLAG | H5AC__TAKE_OWNERSHIP_FLAG;;
 
+    /* Free space for section info */
     if(H5F_addr_defined(fspace->sect_addr)) {
         hsize_t	saved_size;     /* Size of previous section info */
         unsigned sinfo_status = 0;      /* Section info cache status */
@@ -964,6 +1132,7 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id)
             HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
     } /* end if */
 
+    /* Free space for header */
     if(H5F_addr_defined(fspace->addr)) {
         unsigned hdr_status = 0;        /* Header entry status */
 
diff --git a/src/H5FScache.c b/src/H5FScache.c
index 400f07c..6235509 100644
--- a/src/H5FScache.c
+++ b/src/H5FScache.c
@@ -83,11 +83,12 @@ static htri_t H5FS__cache_hdr_verify_chksum(const void *image_ptr, size_t len, v
 static void *H5FS__cache_hdr_deserialize(const void *image, size_t len,
     void *udata, hbool_t *dirty);
 static herr_t H5FS__cache_hdr_image_len(const void *thing, size_t *image_len);
-static herr_t H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id,
+static herr_t H5FS__cache_hdr_pre_serialize(H5F_t *f, hid_t dxpl_id,
     void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len,
     unsigned *flags);
 static herr_t H5FS__cache_hdr_serialize(const H5F_t *f, void *image, 
     size_t len, void *thing);
+static herr_t H5FS__cache_hdr_notify(H5AC_notify_action_t action, void *thing);
 static herr_t H5FS__cache_hdr_free_icr(void *thing);
 
 static herr_t H5FS__cache_sinfo_get_initial_load_size(void *udata, size_t *image_len);
@@ -95,7 +96,7 @@ static htri_t H5FS__cache_sinfo_verify_chksum(const void *image_ptr, size_t len,
 static void *H5FS__cache_sinfo_deserialize(const void *image, size_t len,
     void *udata, hbool_t *dirty);
 static herr_t H5FS__cache_sinfo_image_len(const void *thing, size_t *image_len);
-static herr_t H5FS__cache_sinfo_pre_serialize(const H5F_t *f, hid_t dxpl_id,
+static herr_t H5FS__cache_sinfo_pre_serialize(H5F_t *f, hid_t dxpl_id,
     void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len,
     unsigned *flags);
 static herr_t H5FS__cache_sinfo_serialize(const H5F_t *f, void *image,
@@ -121,7 +122,7 @@ const H5AC_class_t H5AC_FSPACE_HDR[1] = {{
     H5FS__cache_hdr_image_len,          /* 'image_len' callback */
     H5FS__cache_hdr_pre_serialize,      /* 'pre_serialize' callback */
     H5FS__cache_hdr_serialize,          /* 'serialize' callback */
-    NULL,                               /* 'notify' callback */
+    H5FS__cache_hdr_notify, 		/* 'notify' callback */
     H5FS__cache_hdr_free_icr,           /* 'free_icr' callback */
     NULL,                               /* 'fsf_size' callback */
 }};
@@ -404,7 +405,7 @@ H5FS__cache_hdr_image_len(const void *_thing, size_t *image_len)
  *-------------------------------------------------------------------------
  */
 static herr_t 
-H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing,
+H5FS__cache_hdr_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing,
     haddr_t addr, size_t H5_ATTR_UNUSED len, haddr_t *new_addr, size_t *new_len,
     unsigned *flags)
 {
@@ -768,6 +769,63 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len,
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5FS__cache_hdr_notify
+ *
+ * Purpose:     Handle cache action notifications
+ *
+ * Return:      SUCCEED/FAIL
+ *
+ * Programmer:  Quincey Koziol
+ *              koziol@lbl.gov
+ *              January 3, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5FS__cache_hdr_notify(H5AC_notify_action_t action, void *_thing)
+{
+    H5FS_t *fspace = (H5FS_t *)_thing;  /* Pointer to the object */
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT
+    
+    /* Sanity check */
+    HDassert(fspace);
+
+    /* Determine which action to take */
+    switch(action) {
+        case H5AC_NOTIFY_ACTION_AFTER_INSERT:
+        case H5AC_NOTIFY_ACTION_AFTER_LOAD:
+        case H5AC_NOTIFY_ACTION_AFTER_FLUSH:
+            /* do nothing */
+            break;
+
+        case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED:
+            if(H5AC_unsettle_entry_ring(fspace) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTFLUSH, FAIL, "unable to mark FSM ring as unsettled")
+            break;
+
+        case H5AC_NOTIFY_ACTION_ENTRY_CLEANED:
+        case H5AC_NOTIFY_ACTION_CHILD_DIRTIED:
+        case H5AC_NOTIFY_ACTION_CHILD_CLEANED:
+        case H5AC_NOTIFY_ACTION_BEFORE_EVICT:
+            /* do nothing */
+            break;
+
+        default:
+#ifdef NDEBUG
+            HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, FAIL, "unknown action from metadata cache")
+#else /* NDEBUG */
+            HDassert(0 && "Unknown action?!?");
+#endif /* NDEBUG */
+    } /* end switch */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+}   /* end H5FS__cache_hdr_notify() */
+
+
+/*-------------------------------------------------------------------------
  * Function:	H5FS__cache_hdr_free_icr
  *
  * Purpose:	Destroys a free space header in memory.
@@ -1091,7 +1149,7 @@ H5FS__cache_sinfo_image_len(const void *_thing, size_t *image_len)
  *-------------------------------------------------------------------------
  */
 static herr_t 
-H5FS__cache_sinfo_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing,
+H5FS__cache_sinfo_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing,
     haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags)
 {
     H5FS_sinfo_t 	*sinfo = (H5FS_sinfo_t *)_thing;        /* Pointer to the object */
diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h
index 20fdff1..1816a6e 100644
--- a/src/H5FSprivate.h
+++ b/src/H5FSprivate.h
@@ -182,6 +182,8 @@ H5_DLL herr_t H5FS_delete(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr);
 H5_DLL herr_t H5FS_close(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace);
 H5_DLL herr_t H5FS_alloc_hdr(H5F_t *f, H5FS_t *fspace, haddr_t *fs_addr, hid_t dxpl_id);
 H5_DLL herr_t H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id);
+H5_DLL herr_t H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id, 
+    H5FS_t *fspace, haddr_t *fs_addr_ptr);
 H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id);
 
 /* Free space section routines */
diff --git a/src/H5Fint.c b/src/H5Fint.c
index f1e9fb5..8ad97a8 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -76,6 +76,8 @@ typedef struct H5F_olist_t {
 static int H5F_get_objects_cb(void *obj_ptr, hid_t obj_id, void *key);
 static herr_t H5F_build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl,
     const char *name, char ** /*out*/ actual_name);/* Declare a free list to manage the H5F_t struct */
+static herr_t H5F__flush_phase1(H5F_t *f, hid_t dxpl_id);
+static herr_t H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing);
 
 
 /*********************/
@@ -820,15 +822,40 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
         int actype;                         /* metadata cache type (enum value) */
         H5F_io_info_t fio_info;             /* I/O info for operation */
 
-        /* Flush at this point since the file will be closed.
+        /* Flush at this point since the file will be closed (phase 1).
          * Only try to flush the file if it was opened with write access, and if
          * the caller requested a flush.
          */
         if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush)
-            if(H5F_flush(f, dxpl_id, TRUE) < 0)
+            if(H5F__flush_phase1(f, dxpl_id) < 0)
                 /* Push error, but keep going*/
                 HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
 
+        /* Notify the metadata cache that the file is about to be closed.
+         * This allows the cache to set up for creating a metadata cache 
+         * image if this has been requested.
+         */
+        if(H5AC_prep_for_file_close(f, dxpl_id) < 0)
+            /* Push error, but keep going */
+            HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "metadata cache prep for close failed")
+
+        /* Flush at this point since the file will be closed (phase 2).
+         * Only try to flush the file if it was opened with write access, and if
+         * the caller requested a flush.
+         */
+        if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush)
+            if(H5F__flush_phase2(f, dxpl_id, TRUE) < 0)
+                /* Push error, but keep going*/
+                HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
+
+        /* With the shutdown modifications, the contents of the metadata cache
+         * should be clean at this point, with the possible exception of the 
+         * the superblock and superblock extension.
+         *
+         * Verify this.
+         */
+        HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
+
         /* Release the external file cache */
         if(f->shared->efc) {
             if(H5F_efc_destroy(f->shared->efc) < 0)
@@ -837,36 +864,72 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
             f->shared->efc = NULL;
         } /* end if */
 
+        /* With the shutdown modifications, the contents of the metadata cache
+         * should be clean at this point, with the possible exception of the 
+         * the superblock and superblock extension.
+         *
+         * Verify this.
+         */
+        HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
+
         /* Release objects that depend on the superblock being initialized */
         if(f->shared->sblock) {
             /* Shutdown file free space manager(s) */
-            /* (We should release the free space information now (before truncating
-             *      the file and before the metadata cache is shut down) since the
-             *      free space manager is holding some data structures in memory
-             *      and also because releasing free space can shrink the file's
-             *      'eoa' value)
+            /* (We should release the free space information now (before 
+             *      truncating the file and before the metadata cache is shut 
+             *      down) since the free space manager is holding some data 
+             *      structures in memory and also because releasing free space 
+             *      can shrink the file's 'eoa' value)
+             *
+             * Update 11/1/16:
+             *
+             *      With recent library shutdown modifications, the free space
+             *      managers should be settled and written to file at this point
+             *      (assuming they are persistent).  In this case, closing the
+             *      free space managers should have no effect on EOA.
+             *
+             *                                          -- JRM 
              */
             if(H5F_ACC_RDWR & H5F_INTENT(f)) {
                 if(H5MF_close(f, dxpl_id) < 0)
                     /* Push error, but keep going*/
                     HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file free space info")
 
-                /* Flush the file again (if requested), as shutting down the
-                 * free space manager may dirty some data structures again.
+                /* at this point, only the superblock and superblock 
+                 * extension should be dirty.
                  */
+                HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
+
                 if(flush) {
 		    /* Clear status_flags */
                     f->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_WRITE_ACCESS);
                     f->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_SWMR_WRITE_ACCESS);
 
-                    /* Mark superblock dirty in cache, so change will get encoded */
-                    if(H5F_super_dirty(f) < 0)
+                    /* Mark EOA info dirty in cache, so change will get encoded */
+                    if(H5F_eoa_dirty(f, dxpl_id) < 0)
                         /* Push error, but keep going*/
                         HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
 
-                    if(H5F_flush(f, dxpl_id, TRUE) < 0)
+                    /* Release any space allocated to space aggregators, 
+                     * so that the eoa value corresponds to the end of the 
+                     * space written to in the file.
+                     *
+                     * At most, this should change the superblock or the 
+                     * superblock extension messages.
+                     */
+                    if(H5MF_free_aggrs(f, dxpl_id) < 0)
+                        /* Push error, but keep going*/
+                        HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space")
+
+                    /* Truncate the file to the current allocated size */
+                    if(H5FD_truncate(f->shared->lf, dxpl_id, TRUE) < 0)
                         /* Push error, but keep going*/
-                        HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
+                        HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed")
+
+                    /* at this point, only the superblock and superblock 
+                     * extension should be dirty.
+                     */
+                    HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
 		} /* end if */
             } /* end if */
 
@@ -884,6 +947,13 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
                 HDONE_ERROR(H5E_FSPACE, H5E_CANTUNPIN, FAIL, "unable to unpin superblock")
             f->shared->sblock = NULL;
         } /* end if */
+
+        /* with the possible exception of the superblock and superblock 
+         * extension, the metadata cache should be clean at this point.
+         *
+         * Verify this.
+         */
+        HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
  
         /* Remove shared file struct from list of open files */
         if(H5F_sfile_remove(f->shared) < 0)
@@ -1358,25 +1428,24 @@ done:
 
 
 /*-------------------------------------------------------------------------
- * Function:	H5F_flush
+ * Function:	H5F_flush_phase1
  *
- * Purpose:	Flushes cached data.
+ * Purpose:	First phase of flushing cached data.
  *
  * Return:	Non-negative on success/Negative on failure
  *
- * Programmer:	Robb Matzke
- *		matzke@llnl.gov
- *		Aug 29 1997
+ * Programmer:	Quincey Koziol
+ *		koziol@lbl.gov
+ *		Jan  1 2017
  *
  *-------------------------------------------------------------------------
  */
-herr_t
-H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
+static herr_t
+H5F__flush_phase1(H5F_t *f, hid_t dxpl_id)
 {
-    H5F_io_info_t fio_info;             /* I/O info for operation */
     herr_t   ret_value = SUCCEED;       /* Return value */
 
-    FUNC_ENTER_NOAPI(FAIL)
+    FUNC_ENTER_STATIC
 
     /* Sanity check arguments */
     HDassert(f);
@@ -1396,6 +1465,34 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
         /* Push error, but keep going*/
         HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space")
 
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F__flush_phase1() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5F__flush_phase2
+ *
+ * Purpose:	Second phase of flushing cached data.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ * Programmer:	Quincey Koziol
+ *		koziol@lbl.gov
+ *		Jan  1 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing)
+{
+    H5F_io_info_t fio_info;             /* I/O info for operation */
+    herr_t   ret_value = SUCCEED;       /* Return value */
+
+    FUNC_ENTER_STATIC
+
+    /* Sanity check arguments */
+    HDassert(f);
+
     /* Flush the entire metadata cache */
     if(H5AC_flush(f, dxpl_id) < 0)
         /* Push error, but keep going*/
@@ -1403,6 +1500,7 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
 
     /* Truncate the file to the current allocated size */
     if(H5FD_truncate(f->shared->lf, dxpl_id, closing) < 0)
+        /* Push error, but keep going*/
         HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed")
 
     /* Flush the entire metadata cache again since the EOA could have changed in the truncate call. */
@@ -1413,6 +1511,7 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
     /* Set up I/O info for operation */
     fio_info.f = f;
     if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+        /* Push error, but keep going*/
         HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
 
     /* Flush out the metadata accumulator */
@@ -1425,6 +1524,43 @@ H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
         /* Push error, but keep going*/
         HDONE_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "low level flush failed")
 
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F__flush_phase2() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5F_flush
+ *
+ * Purpose:	Flushes cached data.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ * Programmer:	Robb Matzke
+ *		matzke@llnl.gov
+ *		Aug 29 1997
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
+{
+    herr_t   ret_value = SUCCEED;       /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Sanity check arguments */
+    HDassert(f);
+
+    /* First phase of flushing data */
+    if(H5F__flush_phase1(f, dxpl_id) < 0)
+        /* Push error, but keep going*/
+        HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data")
+
+    /* Second phase of flushing data */
+    if(H5F__flush_phase2(f, dxpl_id, closing) < 0)
+        /* Push error, but keep going*/
+        HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data")
+
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5F_flush() */
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index 1adf74b..11665f4 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -259,6 +259,14 @@ struct H5F_file_t {
                                  * block is present.  At all other times
                                  * it should be NULL.
                                  */
+    hbool_t drvinfo_sb_msg_exists;  /* Convenience field used to track 
+                                     * whether the driver info superblock 
+                                     * extension message has been created 
+                                     * yet. This field should be TRUE iff the
+                                     * superblock extension exists and contains
+                                     * a driver info message.  Under all other
+                                     * circumstances, it must be set to FALSE.
+                                     */
     unsigned	nrefs;		/* Ref count for times file is opened	*/
     unsigned	flags;		/* Access Permissions for file          */
     H5F_mtab_t	mtab;		/* File mount table                     */
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index c099d32..bcc56c6 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -756,6 +756,7 @@ H5_DLL herr_t H5F_fake_free(H5F_t *f);
 
 /* Superblock related routines */
 H5_DLL herr_t H5F_super_dirty(H5F_t *f);
+H5_DLL herr_t H5F_eoa_dirty(H5F_t *f, hid_t dxpl_id);
 
 /* Parallel I/O (i.e. MPI) related routines */
 #ifdef H5_HAVE_PARALLEL
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index 9e93394..893ce26 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -53,6 +53,7 @@
 /* Local Prototypes */
 /********************/
 static herr_t H5F_super_ext_create(H5F_t *f, hid_t dxpl_id, H5O_loc_t *ext_ptr);
+static herr_t H5F__update_super_ext_driver_msg(H5F_t *f, hid_t dxpl_id);
 
 
 /*********************/
@@ -222,6 +223,86 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5F__update_super_ext_driver_msg
+ *
+ * Purpose:	Update the superblock extension file driver info message if 
+ *		we are using a V 2 superblock.  Observe that the function
+ *		is a NO-OP if the file driver info message does not exist.
+ *              This is necessary, as the function is called whenever the 
+ *		EOA is updated, and were it to create the file driver info
+ *		message, it would find itself in an infinite recursion.
+ *
+ * Return:      Success:        SUCCEED
+ *              Failure:        FAIL
+ *
+ * Programmer:  John Mainzer
+ *              11/10/15
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F__update_super_ext_driver_msg(H5F_t *f, hid_t dxpl_id)
+{
+    H5F_super_t *sblock;        /* Pointer to the super block */
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_STATIC
+
+    /* Sanity check */
+    HDassert(f);
+    HDassert(f->shared);
+    sblock = f->shared->sblock;
+    HDassert(sblock);
+    HDassert(sblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
+    HDassert(sblock->cache_info.type == H5AC_SUPERBLOCK);
+
+    /* Update the driver information message in the superblock extension
+     * if appropriate.
+     */
+    if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2) {
+        if(H5F_addr_defined(sblock->ext_addr)) {
+            /* Check for ignoring the driver info for this file */
+            if(!H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) {
+                size_t     driver_size;    /* Size of driver info block (bytes)*/
+
+                /* Check for driver info */
+                H5_CHECKED_ASSIGN(driver_size, size_t, H5FD_sb_size(f->shared->lf), hsize_t);
+
+		/* Nothing to do unless there is both driver info and 
+                 * the driver info superblock extension message has 
+                 * already been created.
+                 */
+                if(driver_size > 0) {
+                    H5O_drvinfo_t drvinfo;      /* Driver info */
+                    uint8_t dbuf[H5F_MAX_DRVINFOBLOCK_SIZE];  /* Driver info block encoding buffer */
+
+                    /* Sanity check */
+                    HDassert(driver_size <= H5F_MAX_DRVINFOBLOCK_SIZE);
+
+                    /* Encode driver-specific data */
+                    if(H5FD_sb_encode(f->shared->lf, drvinfo.name, dbuf) < 0)
+                        HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to encode driver information")
+
+                    /* Write the message to the superblock extension.
+                     *
+                     * Note that the superblock extension and the 
+                     * file driver info message must already exist.
+                     */
+                    drvinfo.len = driver_size;
+                    drvinfo.buf = dbuf;
+                    if(H5F_super_ext_write_msg(f, dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE) < 0)
+                        HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message")
+                } /* end if driver_size > 0 */
+            } /* end if !H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO) */
+        } /* end if superblock extension exists */
+    } /* end if sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F__update_super_ext_driver_msg() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5F__super_read
  *
  * Purpose:     Reads the superblock from the file or from the BUF. If
@@ -549,6 +630,9 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
 
                 /* Reset driver info message */
                 H5O_msg_reset(H5O_DRVINFO_ID, &drvinfo);
+
+		HDassert(FALSE == f->shared->drvinfo_sb_msg_exists);
+		f->shared->drvinfo_sb_msg_exists = TRUE;
             } /* end else */
         } /* end if */
 
@@ -844,8 +928,18 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
 
     /* Compute the size of the driver information block */
     H5_CHECKED_ASSIGN(driver_size, size_t, H5FD_sb_size(f->shared->lf), hsize_t);
+
+    /* The following code sets driver_size to the valued needed 
+     * for the driver info block, and sets the driver info block
+     * address regardless of the version of the superblock.
+     */
     if(driver_size > 0) {
-        driver_size += H5F_DRVINFOBLOCK_HDR_SIZE;
+        /* Add in the driver info header, for older superblocks */
+        /* Superblock versions >= 2 will put the driver info in a message
+         *      and don't need the header -QAK, 1/4/2017
+         */
+        if(super_vers < HDF5_SUPERBLOCK_VERSION_2)
+            driver_size += H5F_DRVINFOBLOCK_HDR_SIZE;
 
         /*
          * The file driver information block begins immediately after the
@@ -976,6 +1070,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
             info.buf = dbuf;
             if(H5O_msg_create(&ext_loc, H5O_DRVINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &info, dxpl_id) < 0)
                 HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update driver info header message")
+
+            HDassert(FALSE == f->shared->drvinfo_sb_msg_exists);
+            f->shared->drvinfo_sb_msg_exists = TRUE;
         } /* end if */
 
         /* Check for non-default free space settings */
@@ -1077,20 +1174,20 @@ done:
 
 
 /*-------------------------------------------------------------------------
- * Function:    H5F_super_dirty
+ * Function:    H5F_eoa_dirty
  *
- * Purpose:     Mark the file's superblock dirty
+ * Purpose:     Mark the file's EOA info dirty
  *
  * Return:      Success:        non-negative on success
  *              Failure:        Negative
  *
  * Programmer:  Quincey Koziol
- *              August 14, 2009
+ *              January 4, 2017
  *
  *-------------------------------------------------------------------------
  */
 herr_t
-H5F_super_dirty(H5F_t *f)
+H5F_eoa_dirty(H5F_t *f, hid_t dxpl_id)
 {
     herr_t ret_value = SUCCEED;         /* Return value */
 
@@ -1102,16 +1199,56 @@ H5F_super_dirty(H5F_t *f)
     HDassert(f->shared->sblock);
 
     /* Mark superblock dirty in cache, so change to EOA will get encoded */
-    if(H5AC_mark_entry_dirty(f->shared->sblock) < 0)
+    if(H5F_super_dirty(f) < 0)
         HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
 
-    /* if the driver information block exists, mark it dirty as well 
+    /* If the driver information block exists, mark it dirty as well 
      * so that the change in eoa will be reflected there as well if 
      * appropriate.
      */
-    if ( f->shared->drvinfo )
+    if(f->shared->drvinfo) {
         if(H5AC_mark_entry_dirty(f->shared->drvinfo) < 0)
             HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark drvinfo as dirty")
+    } /* end if */
+    /* If the driver info is stored as a message, update that instead */
+    else if(f->shared->drvinfo_sb_msg_exists) {
+        if(H5F__update_super_ext_driver_msg(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark drvinfo message as dirty")
+    } /* end if */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5F_eoa_dirty() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5F_super_dirty
+ *
+ * Purpose:     Mark the file's superblock dirty
+ *
+ * Return:      Success:        non-negative on success
+ *              Failure:        Negative
+ *
+ * Programmer:  Quincey Koziol
+ *              August 14, 2009
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F_super_dirty(H5F_t *f)
+{
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Sanity check */
+    HDassert(f);
+    HDassert(f->shared);
+    HDassert(f->shared->sblock);
+
+    /* Mark superblock dirty in cache, so change to EOA will get encoded */
+    if(H5AC_mark_entry_dirty(f->shared->sblock) < 0)
+        HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c
index 8e832d7..fa893e1 100644
--- a/src/H5Fsuper_cache.c
+++ b/src/H5Fsuper_cache.c
@@ -74,9 +74,6 @@ static htri_t H5F__cache_superblock_verify_chksum(const void *image_ptr, size_t
 static void *H5F__cache_superblock_deserialize(const void *image, size_t len,
     void *udata, hbool_t *dirty);
 static herr_t H5F__cache_superblock_image_len(const void *thing, size_t *image_len);
-static herr_t H5F__cache_superblock_pre_serialize(const H5F_t *f, 
-    hid_t dxpl_id, void *thing, haddr_t addr, size_t len, 
-    haddr_t *new_addr, size_t *new_len, unsigned *flags);
 static herr_t H5F__cache_superblock_serialize(const H5F_t *f, void *image, size_t len,
     void *thing);
 static herr_t H5F__cache_superblock_free_icr(void *thing);
@@ -115,7 +112,7 @@ const H5AC_class_t H5AC_SUPERBLOCK[1] = {{
     H5F__cache_superblock_verify_chksum, /* 'verify_chksum' callback */
     H5F__cache_superblock_deserialize,  /* 'deserialize' callback */
     H5F__cache_superblock_image_len,    /* 'image_len' callback */
-    H5F__cache_superblock_pre_serialize,/* 'pre_serialize' callback */
+    NULL,                               /* 'pre_serialize' callback */
     H5F__cache_superblock_serialize,    /* 'serialize' callback */
     NULL,       			/* 'notify' callback */
     H5F__cache_superblock_free_icr,     /* 'free_icr' callback */
@@ -653,120 +650,6 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len)
 
 
 /*-------------------------------------------------------------------------
- * Function:    H5FS__cache_superblock_pre_serialize
- *
- * Purpose:	The current use of this function is a cludge to repair an 
- *		oversight in the conversion of the superblock code to use the 
- *		version 3 cache.  
- *
- *		In the V2 metadata cache callbacks, the superblock dirver info 
- *     		message was updated in the flush routine.  Note that this 
- *		operation only applies to version 2 or later superblocks.
- *
- *     		Somehow, this functionality was lost in the conversion to use 
- *     		the V3 cache, causing failures with the multi file driver 
- *     		(and possibly the family file driver as well).
- *
- *     		Performing this operation is impossible in the current 
- *		serialize routine, as the dxpl_id is not available.  While 
- *		I am pretty sure that this is not the correct place for this 
- *		functionality, as I can see it causing problems with both 
- *		journaling and possibly parallel HDF5 as well, I am placing 
- *		code for the necessary update in the pre_serialize call for 
- *		now for testing purposes.  We will almost certainly want to 
- *		change this.
- *
- * Return:      Success:        SUCCEED
- *              Failure:        FAIL
- *
- * Programmer:  John Mainzer
- *              10/82/14
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5F__cache_superblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, 
-    void *_thing, haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED len, 
-    haddr_t H5_ATTR_UNUSED *new_addr, size_t H5_ATTR_UNUSED *new_len, 
-    unsigned H5_ATTR_UNUSED *flags)
-{
-    H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
-    H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
-    H5F_super_t *sblock = (H5F_super_t *)_thing; /* Pointer to the super block */
-    herr_t ret_value = SUCCEED; /* Return value */
-
-    FUNC_ENTER_NOAPI_NOINIT
-
-    /* Sanity check */
-    HDassert(f);
-    HDassert(sblock);
-    HDassert(sblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
-    HDassert(sblock->cache_info.type == H5AC_SUPERBLOCK);
-    HDassert(flags);
-
-    if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2) {
-        /* WARNING: This code almost certainly doesn't belong here.  Must
-         *          discuss with Quincey where to put it.  Note issues
-         *          for journaling and possibly parallel.
-         *
-         *                                            -- JRM
-         */
-        /* Update the driver information message in the superblock extension
-         * if appropriate.
-         */
-        if(H5F_addr_defined(sblock->ext_addr)) {
-            size_t     driver_size;    /* Size of driver info block (bytes)*/
-            H5O_loc_t  ext_loc;        /* "Object location" for superblock extension */
-
-            HDassert(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2);
-
-            /* Open the superblock extension's object header */
-            if(H5F_super_ext_open((H5F_t *)f, sblock->ext_addr, &ext_loc) < 0)
-                HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension")
-
-            /* Check for ignoring the driver info for this file */
-            if(!H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) {
-                /* Check for driver info message */
-                H5_CHECKED_ASSIGN(driver_size, size_t, H5FD_sb_size(f->shared->lf), hsize_t);
-                if(driver_size > 0) {
-                    H5O_drvinfo_t drvinfo;      /* Driver info */
-                    uint8_t dbuf[H5F_MAX_DRVINFOBLOCK_SIZE];  /* Driver info block encoding buffer */
-
-                    /* Sanity check */
-                    HDassert(driver_size <= H5F_MAX_DRVINFOBLOCK_SIZE);
-
-                    /* Encode driver-specific data */
-                    if(H5FD_sb_encode(f->shared->lf, drvinfo.name, dbuf) < 0)
-                        HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to encode driver information")
-
-                    /* Set the ring type in the DXPL */
-                    if(H5AC_set_ring(dxpl_id, H5AC_RING_SBE, &dxpl, &orig_ring) < 0)
-                        HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set ring value")
-
-                    /* Write driver info information to the superblock extension */
-                    drvinfo.len = driver_size;
-                    drvinfo.buf = dbuf;
-                    if(H5O_msg_write(&ext_loc, H5O_DRVINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &drvinfo, dxpl_id) < 0)
-                        HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message")
-                } /* end if */
-            } /* end if */
-
-            /* Close the superblock extension object header */
-            if(H5F_super_ext_close((H5F_t *)f, &ext_loc, dxpl_id, FALSE) < 0)
-                HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension")
-        } /* end if */
-    } /* end if */
-
-done:
-    /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set property value")
-
-    FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FS_cache_superblock_pre_serialize() */
-
-
-/*-------------------------------------------------------------------------
  * Function:	H5F__cache_superblock_serialize
  *
  * Purpose:	Flushes a dirty object to disk.
diff --git a/src/H5HFcache.c b/src/H5HFcache.c
index 1e2eb8f..e72cc6c 100644
--- a/src/H5HFcache.c
+++ b/src/H5HFcache.c
@@ -81,7 +81,7 @@ static htri_t H5HF__cache_hdr_verify_chksum(const void *image_ptr, size_t len, v
 static void *H5HF__cache_hdr_deserialize(const void *image, size_t len,
     void *udata, hbool_t *dirty); 
 static herr_t H5HF__cache_hdr_image_len(const void *thing, size_t *image_len);
-static herr_t H5HF__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id,
+static herr_t H5HF__cache_hdr_pre_serialize(H5F_t *f, hid_t dxpl_id,
     void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, 
     unsigned *flags); 
 static herr_t H5HF__cache_hdr_serialize(const H5F_t *f, void *image,
@@ -93,7 +93,7 @@ static htri_t H5HF__cache_iblock_verify_chksum(const void *image_ptr, size_t len
 static void *H5HF__cache_iblock_deserialize(const void *image, size_t len,
     void *udata, hbool_t *dirty); 
 static herr_t H5HF__cache_iblock_image_len(const void *thing, size_t *image_len);
-static herr_t H5HF__cache_iblock_pre_serialize(const H5F_t *f, hid_t dxpl_id,
+static herr_t H5HF__cache_iblock_pre_serialize(H5F_t *f, hid_t dxpl_id,
     void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len,
     unsigned *flags); 
 static herr_t H5HF__cache_iblock_serialize(const H5F_t *f, void *image,
@@ -106,7 +106,7 @@ static htri_t H5HF__cache_dblock_verify_chksum(const void *image_ptr, size_t len
 static void *H5HF__cache_dblock_deserialize(const void *image, size_t len,
     void *udata, hbool_t *dirty); 
 static herr_t H5HF__cache_dblock_image_len(const void *thing, size_t *image_len);
-static herr_t H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id,
+static herr_t H5HF__cache_dblock_pre_serialize(H5F_t *f, hid_t dxpl_id,
     void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len,
     unsigned *flags); 
 static herr_t H5HF__cache_dblock_serialize(const H5F_t *f, void *image,
@@ -664,7 +664,7 @@ H5HF__cache_hdr_image_len(const void *_thing, size_t *image_len)
  *-------------------------------------------------------------------------
  */
 static herr_t 
-H5HF__cache_hdr_pre_serialize(const H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id,
+H5HF__cache_hdr_pre_serialize(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id,
     void *_thing, haddr_t addr, size_t len, haddr_t H5_ATTR_UNUSED *new_addr,
     size_t H5_ATTR_UNUSED *new_len, unsigned *flags)
 {
@@ -1188,7 +1188,7 @@ H5HF__cache_iblock_image_len(const void *_thing, size_t *image_len)
  *-------------------------------------------------------------------------
  */
 static herr_t 
-H5HF__cache_iblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing,
+H5HF__cache_iblock_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing,
     haddr_t addr, size_t H5_ATTR_UNUSED len, haddr_t *new_addr,
     size_t H5_ATTR_UNUSED *new_len, unsigned *flags)
 {
@@ -2053,7 +2053,7 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len)
  *-------------------------------------------------------------------------
  */
 static herr_t 
-H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing,
+H5HF__cache_dblock_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing,
     haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags)
 {
     hbool_t 		 at_tmp_addr;  /* Flag to indicate direct block is */
diff --git a/src/H5MF.c b/src/H5MF.c
index 09341e6..3ed6d28 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -86,7 +86,8 @@ typedef struct {
 /* Allocator routines */
 static herr_t H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
 static herr_t H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
-static herr_t H5MF__close_delete(H5F_t *f, hid_t dxpl_id);
+static herr_t H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl);
+static herr_t H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id);
 
 
 /*********************/
@@ -222,7 +223,7 @@ done:
 
 
 /*-------------------------------------------------------------------------
- * Function:	H5MF_alloc_open
+ * Function:	H5MF__alloc_open
  *
  * Purpose:	Open an existing free space manager of TYPE for file by
  *		creating a free-space structure
@@ -237,12 +238,14 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5MF_alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
+H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
 {
     const H5FS_section_class_t *classes[] = { /* Free space section classes implemented for file */
         H5MF_FSPACE_SECT_CLS_SIMPLE};
     H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t fsm_ring;               /* Free space manager ring */
     H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
+    hbool_t reset_ring = FALSE;         /* Whether the ring was set */
     herr_t ret_value = SUCCEED;         /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
@@ -255,10 +258,17 @@ H5MF_alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
     HDassert(type != H5FD_MEM_NOLIST);
     HDassert(H5F_addr_defined(f->shared->fs_addr[type]));
     HDassert(f->shared->fs_state[type] == H5F_FS_STATE_CLOSED);
+    HDassert(type == H5MF_ALLOC_TO_FS_TYPE(f, type));
 
     /* Set the ring type in the DXPL */
-    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
+    if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+            || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+        fsm_ring = H5AC_RING_MDFSM;
+    else
+        fsm_ring = H5AC_RING_RDFSM;
+    if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+    reset_ring = TRUE;
 
     /* Open an existing free space structure for the file */
     if(NULL == (f->shared->fs_man[type] = H5FS_open(f, dxpl_id, f->shared->fs_addr[type],
@@ -271,11 +281,12 @@ H5MF_alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
 
 done:
     /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
 
     FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* end H5MF_alloc_open() */
+} /* end H5MF__alloc_open() */
 
 
 /*-------------------------------------------------------------------------
@@ -334,7 +345,7 @@ done:
 
 
 /*-------------------------------------------------------------------------
- * Function:	H5MF_alloc_start
+ * Function:	H5MF__alloc_start
  *
  * Purpose:	Open or create a free space manager of a given type
  *
@@ -348,7 +359,7 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5MF_alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
+H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
 {
     herr_t ret_value = SUCCEED;         /* Return value */
 
@@ -364,7 +375,7 @@ H5MF_alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
     /* Check if the free space manager exists already */
     if(H5F_addr_defined(f->shared->fs_addr[type])) {
         /* Open existing free space manager */
-        if(H5MF_alloc_open(f, dxpl_id, type) < 0)
+        if(H5MF__alloc_open(f, dxpl_id, type) < 0)
             HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space")
     } /* end if */
     else {
@@ -375,7 +386,7 @@ H5MF_alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5MF_alloc_start() */
+} /* end H5MF__alloc_start() */
 
 
 /*-------------------------------------------------------------------------
@@ -438,8 +449,10 @@ haddr_t
 H5MF_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size)
 {
     H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t fsm_ring = H5AC_RING_INV;       /* free space manager ring */
     H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
     H5FD_mem_t  fs_type;                /* Free space type (mapped from allocation type) */
+    hbool_t reset_ring = FALSE;         /* Whether the ring was set */
     haddr_t ret_value = HADDR_UNDEF;    /* Return value */
 
     FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, HADDR_UNDEF)
@@ -457,14 +470,20 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ
     fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type);
 
     /* Set the ring type in the DXPL */
-    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
+    if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+            || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+        fsm_ring = H5AC_RING_MDFSM;
+    else
+        fsm_ring = H5AC_RING_RDFSM;
+    if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, HADDR_UNDEF, "unable to set ring value")
+    reset_ring = TRUE;
 
     /* Check if we are using the free space manager for this file */
     if(H5F_HAVE_FREE_SPACE_MANAGER(f)) {
         /* Check if the free space manager for the file has been initialized */
         if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type]))
-            if(H5MF_alloc_open(f, dxpl_id, fs_type) < 0)
+            if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0)
                 HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, HADDR_UNDEF, "can't initialize file free space")
 
         /* Search for large enough space in the free space manager */
@@ -533,8 +552,9 @@ HDfprintf(stderr, "%s: Check 2.0\n", FUNC);
 
 done:
     /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, HADDR_UNDEF, "unable to set property value")
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, HADDR_UNDEF, "unable to set property value")
 
 #ifdef H5MF_ALLOC_DEBUG
 HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size);
@@ -622,14 +642,16 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5MF_xfree(const H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr,
+H5MF_xfree(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr,
     hsize_t size)
 {
     H5F_io_info_t fio_info;             /* I/O info for operation */
     H5MF_free_section_t *node = NULL;   /* Free space section pointer */
     H5MF_sect_ud_t udata;               /* User data for callback */
     H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t fsm_ring;               /* Free space manager ring */
     H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
+    hbool_t reset_ring = FALSE;         /* Whether the ring was set */
     H5FD_mem_t fs_type;                 /* Free space type (mapped from allocation type) */
     herr_t ret_value = SUCCEED;         /* Return value */
 
@@ -648,9 +670,21 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN
     if(H5F_addr_le(f->shared->tmp_addr, addr))
         HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "attempting to free temporary file space")
 
+    /* Get free space type from allocation type */
+    fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type);
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
     /* Set the ring type in the DXPL */
-    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
+    if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+            || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+        fsm_ring = H5AC_RING_MDFSM;
+    else
+        fsm_ring = H5AC_RING_RDFSM;
+    if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+    reset_ring = TRUE;
 
     /* Set up I/O info for operation */
     fio_info.f = f;
@@ -661,12 +695,6 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN
     if(H5F__accum_free(&fio_info, alloc_type, addr, size) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't check free space intersection w/metadata accumulator")
 
-    /* Get free space type from allocation type */
-    fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type);
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
-
     /* Check if the free space manager for the file has been initialized */
     if(!f->shared->fs_man[fs_type]) {
         /* If there's no free space manager for objects of this type,
@@ -717,7 +745,7 @@ HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, a
          *  space isn't at the end of the file, so start up (or create)
          *  the file space manager
          */
-        if(H5MF_alloc_start(f, dxpl_id, fs_type) < 0)
+        if(H5MF__alloc_start(f, dxpl_id, fs_type) < 0)
             HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
     } /* end if */
 
@@ -760,8 +788,9 @@ HDfprintf(stderr, "%s: After H5FS_sect_add()\n", FUNC);
 
 done:
     /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
 
     /* Release section node, if allocated and not added to section list or merged */
     if(node)
@@ -797,9 +826,12 @@ H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, haddr_t addr,
     hsize_t size, hsize_t extra_requested)
 {
     H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t fsm_ring;               /* Free space manager ring */
     H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
-    haddr_t     end;            /* End of block to extend */
-    H5FD_mem_t  map_type;       /* Mapped type */
+    haddr_t     end;                    /* End of block to extend */
+    H5FD_mem_t fs_type;			/* Memory type of the free space manager */
+    H5FD_mem_t  map_type;               /* Mapped type */
+    hbool_t reset_ring = FALSE;         /* Whether the ring was set */
     htri_t	ret_value = FAIL;       /* Return value */
 
     FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
@@ -817,29 +849,34 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r
     /* Compute end of block to extend */
     end = addr + size;
 
+    /* Get free space type from allocation type */
+    fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type);
+
     /* Set the ring type in the DXPL */
-    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
+    if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+            || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+        fsm_ring = H5AC_RING_MDFSM;
+    else
+        fsm_ring = H5AC_RING_RDFSM;
+    if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+    reset_ring = TRUE;
 
     /* Check if the block is exactly at the end of the file */
-    if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, end, extra_requested)) < 0)
+    if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, dxpl_id, end, extra_requested)) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file")
     else if(ret_value == FALSE) {
         H5F_blk_aggr_t *aggr;   /* Aggregator to use */
 
         /* Check for test block able to extend aggregation block */
         aggr = (map_type == H5FD_MEM_DRAW) ?  &(f->shared->sdata_aggr) : &(f->shared->meta_aggr);
-        if((ret_value = H5MF_aggr_try_extend(f, aggr, map_type, end, extra_requested)) < 0)
+        if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0)
             HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block")
         else if(ret_value == FALSE) {
-            H5FD_mem_t  fs_type;                /* Free space type (mapped from allocation type) */
-
-            /* Get free space type from allocation type */
-            fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type);
 
             /* Check if the free space for the file has been initialized */
             if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type]))
-                if(H5MF_alloc_open(f, dxpl_id, fs_type) < 0)
+                if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0)
                     HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
 
             /* Check for test block able to block in free space manager */
@@ -851,8 +888,9 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r
 
 done:
     /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
 
 #ifdef H5MF_ALLOC_DEBUG
 HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value);
@@ -886,7 +924,9 @@ H5MF_sects_dump(f, dxpl_id, stderr);
 herr_t
 H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size)
 {
-    H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+    H5AC_ring_t curr_ring;      /* Current ring value */
+    H5AC_ring_t needed_ring;    /* Ring value needed for loop iteration  */
     H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
     haddr_t eoa;                /* End of allocated space in the file */
     haddr_t ma_addr = HADDR_UNDEF;    /* Base "metadata aggregator" address */
@@ -898,6 +938,7 @@ H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_si
     H5FD_mem_t type;            /* Memory type for iteration */
     hbool_t fs_started[H5FD_MEM_NTYPES]; /* Indicate whether the free-space manager has been started */
     hbool_t eoa_shrank;		/* Whether an EOA shrink occurs */
+    hbool_t reset_ring = FALSE; /* Whether the ring was set */
     herr_t ret_value = SUCCEED; /* Return value */
 
     FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
@@ -912,8 +953,10 @@ H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_si
 	HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed")
 
     /* Set the ring type in the DXPL */
-    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
+    if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+    reset_ring = TRUE;
+    curr_ring = H5AC_RING_RDFSM;
 
     /* Retrieve metadata aggregator info, if available */
     if(H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size) < 0)
@@ -928,9 +971,22 @@ H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_si
 
 	fs_started[type] = FALSE;
 
+        /* test to see if we need to switch rings -- do so if required */
+        if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+            needed_ring = H5AC_RING_MDFSM;
+        else
+            needed_ring = H5AC_RING_RDFSM;
+
+        if(needed_ring != curr_ring) {
+            if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+                HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)")
+            curr_ring = needed_ring;
+        } /* end if */
+
 	/* Check if the free space for the file has been initialized */
         if(!f->shared->fs_man[type] && H5F_addr_defined(f->shared->fs_addr[type])) {
-            if(H5MF_alloc_open(f, dxpl_id, type) < 0)
+            if(H5MF__alloc_open(f, dxpl_id, type) < 0)
                 HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
             HDassert(f->shared->fs_man[type]);
             fs_started[type] = TRUE;
@@ -990,9 +1046,24 @@ H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_si
 
     /* Close the free-space managers if they were opened earlier in this routine */
     for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
-	if(fs_started[type])
+        if(fs_started[type]) {
+
+            /* test to see if we need to switch rings -- do so if required */
+            if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                    || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+                needed_ring = H5AC_RING_MDFSM;
+            else
+                needed_ring = H5AC_RING_RDFSM;
+
+            if(needed_ring != curr_ring) {
+                if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+                    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)")
+                curr_ring = needed_ring;
+            } /* end if */
+
             if(H5MF__alloc_close(f, dxpl_id, type) < 0)
                 HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space")
+        } /* end if */
     } /* end for */
 
     /* Set the value(s) to return */
@@ -1004,8 +1075,9 @@ H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_si
 
 done:
     /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
 
     FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
 } /* end H5MF_get_freespace() */
@@ -1031,8 +1103,6 @@ H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr,
 {
     H5MF_free_section_t *node = NULL;   /* Free space section pointer */
     H5MF_sect_ud_t udata;               /* User data for callback */
-    H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
-    H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
     htri_t ret_value = FAIL;                   /* Return value */
 
     FUNC_ENTER_NOAPI(FAIL)
@@ -1047,10 +1117,6 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN
     HDassert(H5F_addr_defined(addr));
     HDassert(size > 0);
 
-    /* Set the ring type in the DXPL */
-    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
-
     /* Create free space section for block */
     if(NULL == (node = H5MF_sect_simple_new(addr, size)))
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section")
@@ -1072,9 +1138,6 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN
     } /* end if */
 
 done:
-    /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
 
     /* Free section node allocated */
     if(node && H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0)
@@ -1088,7 +1151,7 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value);
 
 
 /*-------------------------------------------------------------------------
- * Function:    H5MF_close_shrink_eoa
+ * Function:    H5MF__close_shrink_eoa
  *
  * Purpose:     Shrink the EOA while closing
  *
@@ -1100,15 +1163,22 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value);
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5MF_close_shrink_eoa(H5F_t *f, hid_t dxpl_id)
+H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id)
 {
     H5FD_mem_t type;            /* Memory type for iteration */
+    H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
+    H5AC_ring_t curr_ring = H5AC_RING_INV;      /* current ring value */
+    H5AC_ring_t needed_ring = H5AC_RING_INV;    /* ring value needed for this
+                                                 * iteration.
+                                                 */
     hbool_t eoa_shrank;		/* Whether an EOA shrink occurs */
+    hbool_t reset_ring = FALSE; /* Whether the ring was set */
     htri_t status;		/* Status value */
     H5MF_sect_ud_t udata;	/* User data for callback */
     herr_t ret_value = SUCCEED;	/* Return value */
 
-    FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+    FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
 
     /* check args */
     HDassert(f);
@@ -1120,12 +1190,35 @@ H5MF_close_shrink_eoa(H5F_t *f, hid_t dxpl_id)
     udata.allow_sect_absorb = FALSE;    
     udata.allow_eoa_shrink_only = TRUE; 
 
+    /* Set the ring type in the DXPL.  In most cases, we will 
+     * need H5AC_RING_RDFSM, so initialy set the ring type in 
+     * the DXPL to that value.  We will alter this later if needed.
+     */
+    if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value(1)")
+    reset_ring = TRUE;
+    curr_ring = H5AC_RING_RDFSM;
+
     /* Iterate until no more EOA shrinking occurs */
     do {
 	eoa_shrank = FALSE;
 
 	/* Check the last section of each free-space manager */
 	for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+
+            /* test to see if we need to switch rings -- do so if required */
+            if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                    || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+                needed_ring = H5AC_RING_MDFSM;
+            else
+                needed_ring = H5AC_RING_RDFSM;
+
+            if(needed_ring != curr_ring) {
+                if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+                    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)")
+                curr_ring = needed_ring;
+            } /* end if */
+
 	    if(f->shared->fs_man[type]) {
 		udata.alloc_type = type;
 		if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[type], &udata)) < 0)
@@ -1143,8 +1236,639 @@ H5MF_close_shrink_eoa(H5F_t *f, hid_t dxpl_id)
     } while(eoa_shrank);
 
 done:
+    /* Reset the ring in the DXPL */
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+
+    FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF__close_shrink_eoa() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5MF_settle_raw_data_fsm()
+ *
+ * Purpose: 	Handle any tasks required before the metadata cache 
+ *		can serialize or flush the raw data free space manager
+ *		and any metadata free space managers that reside in the 
+ *		raw data free space manager ring. 
+ *
+ *		Specifically, any metadata managers that DON'T handle 
+ *		space allocation for free space manager header or section 
+ *		info will reside in the raw data free space manager ring.  
+ *		As of this writing, the plan is to move to only two free space 
+ *		managers, one for raw data and one for metadata -- which
+ *		means that only the raw data free space manager will reside
+ *		in the free space manager ring.  However, this has not been
+ *		fully implemented yet, so this code must support the 
+ *		possibilty of multiple metadata free space managers, at most
+ *		two of which handle free space manager header or section info, 
+ *		and thus reside in the metadata free space manager ring.
+ *
+ *		At present, the task list is:
+ *
+ *		1) Reduce the EOA to the extent possible.  To do this:
+ *
+ *		    a) Free both aggregators.  Space not at EOA will be
+ *		       added to the appropriate free space manager.
+ *		    
+ *		       The raw data aggregator should not be restarted 
+ *		       after this point.  It is possible that the metadata
+ *		       aggregator will be.
+ *		    
+ *		    b) Free all file space currently allocated to free
+ *		       space managers.  
+ *
+ *		    c) Delete the free space manager superblock 
+ *		       extension message if allocated.
+ *
+ *		   This done, reduce the EOA by moving it to just before
+ *		   the last piece of free memory in the file.
+ *		   
+ *		2) Ensure that space is allocated for the free space
+ *                 manager superblock extension message.  Must do this
+ *                 now, before reallocating file space for free space 
+ *		   managers, as it is possible that this allocation may
+ *		   grab the last section in a FSM -- making it unnecessary
+ *		   to re-allocate file space for it.
+ *
+ *		3) Scan all free space managers not involved in allocating
+ *		   space for free space managers.  For each such free space
+ *		   manager, test to see if it contains free space.  If 
+ *		   it does, allocate file space for its header and section
+ *		   data.  If it contains no free space, leave it without 
+ *		   allocated file space as there is no need to save it to 
+ *		   file.
+ *
+ *		   Note that all free space managers in this class should
+ *		   see no further space allocations / deallocations as 
+ *		   at this point, all raw data allocations should be 
+ *		   finalized, as should all metadata allocations not 
+ *		   involving free space managers.
+ *
+ *		   We will allocate space for free space managers involved
+ *		   in the allocation of file space for free space managers
+ *		   in H5MF_settle_meta_data_fsm()
+ * Return:	SUCCEED/FAIL
+ *
+ * Programmer:  John Mainzer
+ *	        5/25/16
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
+{
+    H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
+    hbool_t reset_ring = FALSE;         /* Whether the ring was set */
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+
+    /* check args */
+    HDassert(f);
+    HDassert(f->shared);
+    HDassert(fsm_settled);
+
+    /* Only need to settle things if we are persisting the free space info */
+    if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
+        H5O_fsinfo_t fsinfo;                    /* Free space manager info message */
+        H5FD_mem_t	type;			/* Memory type for iteration */
+        H5AC_ring_t curr_ring = H5AC_RING_INV;  /* Current ring value */
+        H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration  */
+        hbool_t fsm_opened[H5FD_MEM_NTYPES];    /* State of FSM */
+        hbool_t fsm_visited[H5FD_MEM_NTYPES];   /* State of FSM */
+
+        /* Initialize fsm_opened and fsm_visited */
+        HDmemset(fsm_opened, 0, sizeof(fsm_opened));
+        HDmemset(fsm_visited, 0, sizeof(fsm_visited));
+
+        /* 1) Reduce the EOA to the extent possible. */
+
+        /* a) Free the space in aggregators:
+         *
+         * (for space not at EOF, it may be put into free space managers) 
+         *
+         * Do this now so that the raw data FSM (and any other FSM that isn't 
+         * involved in space allocation for FSMs) will have no further activity.
+         *
+         * Note that while the raw data aggregator should not be restarted during
+         * the close process, this need not be the case for the metadata aggregator.
+         */
+        if(H5MF_free_aggrs(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't free aggregators")
+
+        /* Set the ring type in the DXPL.  In most cases, we will
+         * need H5AC_RING_MDFSM first, so initialy set the ring in
+         * the DXPL to that value.  We will alter this later if
+         * needed.
+         */
+        if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value(0)")
+        reset_ring = TRUE;
+        curr_ring = H5AC_RING_MDFSM;
+
+        /* b) Free the file space (if any) allocated to each free space manager.
+         *
+         * Do this to facilitate reduction of the size of the file to the 
+         * extent possible.  We will re-allocate space to free space managers 
+         * that have free space to save after this reduction.
+         *
+         * In the case of the raw data free space manager, and any other free
+         * space manager that does not allocate space for free space managers,
+         * allocations should be complete at this point, as all raw data should
+         * have space allocated and be flushed to file at this point.  Thus we
+         * can examine such free space managers and only re-allocate space for 
+         * them if they contain free space.  Do this later in this function after
+         * the EOA has been reduced to the extent possible.
+         *
+         * For free space managers that allocate file space for free space 
+         * managers (usually just a single metadata free space manager, but for 
+         * now at least, free space managers for different types of metadata 
+         * are possible), the matter is more ticklish due to the self-
+         * referential nature of the problem.  These FSMs are dealt with in 
+         * H5MF_settle_meta_data_fsm().
+         */
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+            H5FS_stat_t fs_stat;        /* Information for free-space manager */
+            H5FD_mem_t  fsm_type;       /* File memory type for FSM */
+
+            /* There is potentially a many-to-one mapping from memory types to 
+             * free space managers.  Use the fsm_visited[] array to avoid visiting 
+             * a given FSM more than once.  Use fsm_opened[] to track which FSMs
+             * must be closed at the end of this function.
+             */
+            fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type);
+            if(!fsm_visited[fsm_type]) {
+                fsm_visited[fsm_type] = TRUE;
+
+                /* If there is no active FSM for this type, but such a FSM has 
+                 * space allocated in file, open it so that we can free its file 
+                 * space.
+                 */
+                if(NULL == f->shared->fs_man[fsm_type]) {
+                    if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) {
+                        /* Sanity check */
+                        HDassert(fsm_opened[fsm_type] == FALSE);
+
+                        /* Start up FSM for the file memory type */
+                        if(H5MF__alloc_open(f, dxpl_id, fsm_type) < 0)
+                            HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
+                        fsm_opened[fsm_type] = TRUE;
+                    } /* end if */
+                } /* end if */
+
+                /* Check for an actual FSM for this type now */
+                /* (Possibly opened in previous step) */
+                if(f->shared->fs_man[fsm_type]) {
+                    /* Test to see if we need to switch rings -- do so if required */
+                    if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                           || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+                        needed_ring = H5AC_RING_MDFSM;
+                    else
+                        needed_ring = H5AC_RING_RDFSM;
+                    if(needed_ring != curr_ring) {
+                        if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0)
+                            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value.")
+                        curr_ring = needed_ring;
+                    } /* end if */
+
+                    /* Query free space manager info for this type */
+                    if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
+                        HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+
+                    /* Check if the free space manager has space in the file */
+                    if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) {
+                        /* Delete the free space manager in the file.  Will 
+                         * reallocate later if the free space manager contains
+                         * any free space.
+                         */
+                        if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id) < 0)
+                            HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
+                        f->shared->fs_addr[fsm_type] = HADDR_UNDEF;
+                    } /* end if */
+                } /* end if */
+
+                /* note that we are tracking opened FSM -- we will close them
+                 * at the end of the function.
+                 */
+            } /* end if */
+        } /* end for */
+
+        /* c) Delete the free space manager superblock extension message 
+         *    if allocated.
+         *
+         *    Must do this since the routine that writes / creates superblock
+         *    extension messages will choke if the target message is 
+         *    unexpectedly either absent or present.
+         */
+        if(H5F_addr_defined(f->shared->sblock->ext_addr))
+            if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension")
+
+        /* As the final element in 1), shrink the EOA for the file */
+        if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+
+     
+        /* 2) Ensure that space is allocated for the free space manager superblock 
+         *    extension message.  Must do this now, before reallocating file space 
+         *    for free space managers, as it is possible that this allocation may
+         *    grab the last section in a FSM -- making it unnecessary to 
+         *    re-allocate file space for it.
+         *
+         * Do this by writing a free space manager superblock extension message.
+         * 
+         * Since no free space manager has file space allocated for it, this 
+         * message must be invalid since we can't save addresses of FSMs when
+         * those addresses are unknown.  This is OK -- we will write the correct
+         * values to the message at free space manager shutdown.
+         */
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) 
+            fsinfo.fs_addr[type-1] = HADDR_UNDEF;
+        fsinfo.strategy = f->shared->fs_strategy;
+        fsinfo.threshold = f->shared->fs_threshold;
+        if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
+
+
+        /* 3) Scan all free space managers not involved in allocating
+         *    space for free space managers.  For each such free space
+         *    manager, test to see if it contains free space.  If 
+         *    it does, allocate file space for its header and section
+         *    data.  If it contains no free space, leave it without 
+         *    allocated file space as there is no need to save it to 
+         *    file.
+         *
+         *    Note that all free space managers in this class should
+         *    see no further space allocations / deallocations as 
+         *    at this point, all raw data allocations should be 
+         *    finalized, as should all metadata allocations not involving
+         *    free space managers.
+         *
+         *    We will allocate space for free space managers involved
+         *    in the allocation of file space for free space managers
+         *    in H5MF_settle_meta_data_fsm()
+         */
+
+        /* Reinitialize fsm_visited */
+        HDmemset(fsm_visited, 0, sizeof(fsm_visited));
+
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+            H5FD_mem_t  fsm_type;       /* File memory type for FSM */
+
+            fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type);
+
+            /* test to see if we need to switch rings -- do so if required */
+            if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                    || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+                needed_ring = H5AC_RING_MDFSM;
+            else
+                needed_ring = H5AC_RING_RDFSM;
+
+            if(needed_ring != curr_ring) {
+                if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0)
+                    HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value.")
+                curr_ring = needed_ring;
+            } /* end if */
+
+            /* Since there can be a many-to-one mapping from memory types 
+             * to free space managers, ensure that we don't visit any FSM
+             * more than once.
+             */
+            if(!fsm_visited[fsm_type]) {
+                fsm_visited[fsm_type] = TRUE;
+
+                if(f->shared->fs_man[fsm_type]) {
+                    /* Only allocate file space if the target free space manager 
+                     * doesn't allocate file space for free space managers.  Note 
+                     * that this is also the deciding factor as to whether a FSM 
+                     * in in the raw data FSM ring.
+                     */
+                    if((fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) 
+                            && (fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) {
+                        H5FS_stat_t fs_stat;        /* Information for free-space manager */
+
+                        /* The current ring should be H5AC_RING_RDFSM */
+                        HDassert(curr_ring == H5AC_RING_RDFSM);
+
+                        /* Query free space manager info for this type */
+                        if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 )
+                            HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+
+                        /* If the free space manager contains section info, 
+                         * allocate space for the header and sinfo (note that 
+                         * space must not be allocated at present -- verify
+                         * verify this with assertions).
+                         */
+                        if(fs_stat.serial_sect_count > 0) {
+                            /* Sanity check */
+                            HDassert(!H5F_addr_defined(fs_stat.addr));
+
+                            /* Allocate FSM header */
+                            if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0)
+                                HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocated free-space header")
+
+                            /* Allocate FSM section info */
+                            HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+                            HDassert(fs_stat.alloc_sect_size == 0);
+                            if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 )
+                                HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info")
+
+#ifndef NDEBUG
+                            /* Re-Query free space manager info for this type */
+                            if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
+                                HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+
+                            HDassert(H5F_addr_defined(fs_stat.addr));
+                            HDassert(H5F_addr_defined(fs_stat.sect_addr));
+                            HDassert(fs_stat.serial_sect_count > 0);
+                            HDassert(fs_stat.alloc_sect_size > 0);
+                            HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size);
+#endif /* NDEBUG */
+                        } /* end if */
+                        else {
+                            HDassert(!H5F_addr_defined(fs_stat.addr));
+                            HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+                            HDassert(fs_stat.serial_sect_count == 0);
+                            HDassert(fs_stat.alloc_sect_size == 0);
+                        } /* end else */
+                    } /* end if */
+                } /* end if */
+
+                /* Close any opened FSMs */
+                if(fsm_opened[fsm_type]) {
+                    if(H5MF__alloc_close(f, dxpl_id, fsm_type) < 0)
+                       HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't close file free space manager")
+                    fsm_opened[fsm_type] = FALSE;
+                } /* end if */
+            } /* end if */
+        } /* end for */
+
+        /* verify that all opened FSMs were closed */
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
+            HDassert(!fsm_opened[type]);
+
+        /* Indicate that the FSM was settled successfully */
+        *fsm_settled = TRUE;
+    } /* end if */
+
+done:
+    /* Reset the ring in the DXPL */
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set property value")
+
     FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* end H5MF_close_shrink_eoa() */
+} /* H5MF_settle_raw_data_fsm() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5MF_settle_meta_data_fsm()
+ *
+ * Purpose: 	If the free space manager is persistent, handle any tasks 
+ *		required before the metadata cache can serialize or flush 
+ *		the metadata free space manager(sI) that handle file space 
+ *		allocation for free space managers.
+ *
+ *		In most cases, there will be only one manager assigned 
+ *		to this role.  However, since for reason or reason unknown,
+ *		free space manager headers and section info blocks are 
+ *		different classes of memory, it is possible that two free 
+ *		space managers will be involved.
+ *
+ *		On entry to this function, the raw data settle routine
+ *		(H5MF_settle_raw_data_fsm()) should have:
+ *
+ *		1) Freed the accumulators.
+ *
+ *		2) Freed all file space allocated to the free space managers.
+ *
+ *		3) Deleted the free space manager superblock extension message
+ *
+ *		4) Reduced the EOA to the extent possible.
+ *
+ *		5) Re-created the free space manager superblock extension
+ *		   message.
+ *
+ *		6) Reallocated file space for all non-empty free space 
+ *		   managers NOT involved in allocation of space for free 
+ *		   space managers.
+ *
+ *		   Note that these free space managers (if not empty) should
+ *		   have been written to file by this point, and that no
+ *		   further space allocations involving them should take 
+ *		   place during file close.
+ *
+ *		On entry to this routine. the free space manager(s) involved
+ *		in allocation of file space for free space managers should
+ *		still be floating. (i.e. should not have any file space 
+ *		allocated to them.)
+ *
+ *		Similarly, the raw data aggregator should not have been 
+ *		restarted.  Note that it is probable that reallocation of 
+ *		space in 5) and 6) above will have re-started the metadata 
+ *		aggregator.
+ *
+ *
+ *		In this routine, we proceed as follows:
+ *
+ *		1) Verify that the free space manager(s) involved in file
+ *		   space allocation for free space managers are still floating.
+ *
+ *		2) Free the accumulators.
+ *
+ *		3) Reduce the EOA to the extent possible.
+ *
+ *		4) Re-allocate space for any free space manager(s) that:
+ *
+ *		   a) are involved in allocation of space for free space 
+ *		      managers, and 
+ *
+ *		   b) contain free space.
+ *
+ *		   It is possible that we could allocate space for one 
+ *		   of these free space manager(s) only to have the allocation 
+ *		   result in the free space manager being empty and thus 
+ *		   obliging us to free the space again.  Thus there is the
+ *		   potential for an infinte loop if we want to avoid saving
+ *		   empty free space managers.
+ *
+ *		   Similarly, it is possible that we could allocate space 
+ *		   for a section info block, only to discover that this 
+ *		   allocation has changed the size of the section info -- 
+ *		   forcing us to deallocate and start the loop over again.
+ *
+ *		   To avoid this, simply allocate file space for these
+ *		   FSM(s) directly from the VFD layer if allocation is 
+ *		   indicated.  This avoids the issue by bypassing the FSMs
+ *		   in this case.  
+ *
+ *		   Note that this may increase the size of the file needlessly. 
+ *		   A better solution would be to modify the FSM code to 
+ *		   save empty FSMs to file, and to allow section info blocks
+ *		   to be oversized.  However, given that the FSM code is 
+ *		   also used by the fractal heaps, and that we are under 
+ *		   severe time pressure at the moment, the above brute 
+ *		   force solution is attractive. 
+ *
+ * Return:	SUCCEED/FAIL
+ *
+ * Programmer:  John Mainzer
+ *	        5/25/16
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
+{
+    H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+    hbool_t	reset_ring = FALSE;     /* Whether we set the ring */
+    herr_t 	ret_value = SUCCEED;        /* Return value */
+
+    FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+
+    /* Check args */
+    HDassert(f);
+    HDassert(f->shared);
+    HDassert(fsm_settled);
+
+    /* Only need to settle things if we are persisting the free space info */
+    if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
+        H5FS_t     *hdr_fspace;             /* Ptr to FSM hdr alloc FSM */
+        H5FS_t     *sinfo_fspace;           /* Ptr to FSM sinfo alloc FSM */
+        H5FS_stat_t fs_stat;		    /* Information for FSM */
+        H5FD_mem_t  hdr_fsm_alloc_type;	    /* FSM hdr alloc type */
+        H5FD_mem_t  sinfo_fsm_alloc_type;   /* FSM info alloc type */
+
+        /* Get the file memory types for the FSM header & section info */
+        hdr_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR);
+        sinfo_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO);
+
+        /* Sanity checks */
+        HDassert(hdr_fsm_alloc_type > H5FD_MEM_NOLIST);
+        HDassert(hdr_fsm_alloc_type < H5FD_MEM_NTYPES);
+        HDassert(sinfo_fsm_alloc_type > H5FD_MEM_NOLIST);
+        HDassert(sinfo_fsm_alloc_type < H5FD_MEM_NTYPES);
+        HDassert(!H5F_addr_defined(f->shared->fs_addr[hdr_fsm_alloc_type]));
+        HDassert(!H5F_addr_defined(f->shared->fs_addr[sinfo_fsm_alloc_type]));
+
+        /* Note that in most cases, hdr_fspace will equal sinfo_fspace. */
+        hdr_fspace = f->shared->fs_man[hdr_fsm_alloc_type];
+        sinfo_fspace = f->shared->fs_man[sinfo_fsm_alloc_type];
+
+        /* Set the ring in the dxpl appropriately for subsequent calls */
+        if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value")
+        reset_ring = TRUE;
+
+#ifndef NDEBUG
+        /* Verify that hdr_fspace is floating if it exists */
+        if(hdr_fspace) {
+            /* Query free space manager info for this type */
+            if(H5FS_stat_info(f, hdr_fspace, &fs_stat) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "can't get free-space info")
+
+            HDassert(!H5F_addr_defined(fs_stat.addr));
+            HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+            HDassert(fs_stat.alloc_sect_size == 0);
+        } /* end if */
+
+        /* Verify that sinfo_fspace is floating if it exists */
+        if((sinfo_fspace) && (hdr_fspace != sinfo_fspace)) {
+            /* Query free space manager info for this type */
+            if(H5FS_stat_info(f, sinfo_fspace, &fs_stat) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "can't get free-space info")
+
+            HDassert(!H5F_addr_defined(fs_stat.addr));
+            HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+            HDassert(fs_stat.alloc_sect_size == 0);
+        } /* end if */
+#endif /* NDEBUG */
+
+        /* Free the space in the metadata aggregator.  Do this via the 
+         * H5MF_free_aggrs() call.  Note that the raw data aggregator must
+         * have already been freed.  Sanity checks for this?
+         */
+        /* (for space not at EOF, it may be put into free space managers) */
+        if(H5MF_free_aggrs(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't free aggregators")
+
+        /* Trying shrinking the EOA for the file */
+        if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+
+        /* ******************* PROBLEM: ********************
+         *
+         * If the file has an alignement other than 1, and if 
+         * the EOA is not a multiple of this alignment, allocating sapce 
+         * for the section via the VFD info has the potential of generating 
+         * a fragment that will be added to the free space manager.  This 
+         * of course undoes everything we have been doing here.
+         *
+         * Need a way around this.  Obvious solution is to force the EOA to 
+         * be a multiple of the alignment.  
+         *
+         * Fortunately, alignment is typically 1, so this is a non-issue in
+         * most cases.  In cases where the alignment is not 1, for now we 
+         * have decided to drop the fragment on the floor.  
+         *
+         * Eventually, we should fix this by modifying the on disk representations
+         * of free space managers to allow for empty space, so as to bypass the
+         * issues created by self-referential free space managers, and make
+         * this issue moot.
+         */
+        /* HDassert(f->shared->alignment == 1); */
+
+
+        /* The free space manager(s) that handle space allocations for free 
+         * space managers should be settled now, albeit without file space 
+         * allocated to them.  To avoid the possibility of changing the sizes
+         * of their section info blocks, allocate space for them now at the 
+         * end of file via H5FD_alloc().
+         *
+         * In the past, this issue of allocating space without touching the 
+         * free space managers has been deal with by calling 
+         * H5MF_aggr_vfd_alloc(), which in turn calls H5MF_aggr_alloc().  
+         * This is problematic since (if I read the code correctly) it will 
+         * re-constitute the metadata aggregator, which will add any left 
+         * over space to one of the free space managers when freed.
+         *
+         * This is a non-starter, since the entire objective is to settle the
+         * free space managers.
+         *
+         * Hence the decision to call H5FD_alloc() directly.  
+         * 
+         * As discussed in PROBLEM above, if f->shared->alignment is not 1, 
+         * this has the possibility of generating a fragment of file space
+         * that would typically be inserted into one of the free space managers.
+         *
+         * This is isn't good, but due to schedule pressure, we will just drop
+         * the fragement on the floor for now.
+         */
+        if(hdr_fspace)
+            if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, hdr_fspace,
+                    &(f->shared->fs_addr[hdr_fsm_alloc_type])) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't vfd allocate hdr FSM file space")
+
+        if(sinfo_fspace && (sinfo_fspace != hdr_fspace))
+            if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, sinfo_fspace,
+                    &(f->shared->fs_addr[sinfo_fsm_alloc_type])) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't vfd allocate sinfo FSM file space")
+
+        /* Indicate that the FSM was settled successfully */
+        *fsm_settled = TRUE;
+    } /* end if */
+
+done:
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set property value")
+
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_settle_meta_data_fsm() */
 
 
 /*-------------------------------------------------------------------------
@@ -1161,9 +1885,10 @@ done:
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5MF__close_delete(H5F_t *f, hid_t dxpl_id)
+H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl)
 {
     H5FD_mem_t type;                    /* Memory type for iteration */
+    H5AC_ring_t curr_ring = H5AC_RING_INV;      /* current ring value */
     herr_t ret_value = SUCCEED;         /* Return value */
 
     FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
@@ -1177,9 +1902,25 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
 
     /* Iterate over all the free space types that have managers and get each free list's space */
     for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+        H5AC_ring_t needed_ring;        /* Ring value needed for this iteration */
+
 #ifdef H5MF_ALLOC_DEBUG_MORE
 HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]);
 #endif /* H5MF_ALLOC_DEBUG_MORE */
+
+        /* test to see if we need to switch rings -- do so if required */
+        if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+            needed_ring = H5AC_RING_MDFSM;
+        else
+            needed_ring = H5AC_RING_RDFSM;
+
+        if(needed_ring != curr_ring) {
+            if(H5AC_set_ring(dxpl_id, needed_ring, dxpl, &curr_ring) < 0)
+                HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (4)")
+            curr_ring = needed_ring;
+        } /* end if */
+
         /* If the free space manager for this type is open, close it */
         if(f->shared->fs_man[type]) {
 #ifdef H5MF_ALLOC_DEBUG_MORE
@@ -1252,6 +1993,7 @@ H5MF_try_close(H5F_t *f, hid_t dxpl_id)
 {
     H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
     H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
+    hbool_t reset_ring = FALSE;         /* Whether the ring was set */
     herr_t ret_value = SUCCEED;         /* Return value */
 
     FUNC_ENTER_NOAPI(FAIL)
@@ -1263,17 +2005,19 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
     HDassert(f);
 
     /* Set the ring type in the DXPL */
-    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
+    if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+    reset_ring = TRUE;
 
     /* Close and delete freespace managers from the file */
-    if(H5MF__close_delete(f, dxpl_id) < 0)
+    if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to close delete free-space managers")
 
 done:
     /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
 
 #ifdef H5MF_ALLOC_DEBUG
 HDfprintf(stderr, "%s: Leaving\n", FUNC);
@@ -1304,8 +2048,10 @@ herr_t
 H5MF_close(H5F_t *f, hid_t dxpl_id)
 {
     H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t curr_ring;              /* Current ring value */
     H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
     H5FD_mem_t type;                    /* Memory type for iteration */
+    hbool_t reset_ring = FALSE;         /* Whether the ring was set */
     herr_t ret_value = SUCCEED;         /* Return value */
 
     FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
@@ -1319,9 +2065,15 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
     HDassert(f->shared->lf);
     HDassert(f->shared->sblock);
 
-    /* Set the ring type in the DXPL */
-    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
+    /* Set the ring type in the DXPL.  In most cases, we will
+     * need H5AC_RING_RDFSM, so initialy set the ring in
+     * the DXPL to that value.  We will alter this later if 
+     * needed.
+     */
+    if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+    reset_ring = TRUE;
+    curr_ring = H5AC_RING_RDFSM;
 
     /* Free the space in aggregators */
     /* (for space not at EOF, it may be put into free space managers) */
@@ -1329,91 +2081,60 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
         HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators")
 
     /* Trying shrinking the EOA for the file */
-    if(H5MF_close_shrink_eoa(f, dxpl_id) < 0)
+    if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
 
     /* Making free-space managers persistent for superblock version >= 2 */
     if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2
             && f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
         H5O_fsinfo_t fsinfo;		/* Free space manager info message */
-        hbool_t update = FALSE;		/* To update info for the message */
 
-        /* Check to remove free-space manager info message from superblock extension */
-        if(H5F_addr_defined(f->shared->sblock->ext_addr))
-            if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0)
-                HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension")
+        /* Superblock extension and free space manager message should 
+         * exist at this point -- verify at least the former.
+         */
+        HDassert(H5F_addr_defined(f->shared->sblock->ext_addr));
 
-	/* Free free-space manager header and/or section info header */
-	for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
-	    H5FS_stat_t	fs_stat;	/* Information for free-space manager */
-
-	    /* Check for free space manager of this type */
-            if(f->shared->fs_man[type]) {
-		/* Switch to "about to be deleted" state */
-                f->shared->fs_state[type] = H5F_FS_STATE_DELETING;
-
-		/* Query the free space manager's information */
-                if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0)
-		    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
-
-		/* Check if the free space manager has space in the file */
-                if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) {
-		    /* Delete the free space manager in the file */
-                     /* (will re-allocate later) */
-                    if(H5FS_free(f, f->shared->fs_man[type], dxpl_id) < 0)
-			HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
-		    f->shared->fs_addr[type] = HADDR_UNDEF;
-		} /* end if */
-	    } /* end iif */
-	    fsinfo.fs_addr[type-1] = HADDR_UNDEF;
-	} /* end for */
+        /* Note that unlike the previous version of this code, we do not 
+         * delete free space managers that have no section to store.
+         *
+         * Can't do this, as that would involve freeing file space, which would 
+         * dirty the free space manager in question. 
+         * 
+         * Fortunately, the code doesn't seem to care about this.
+         */
 
+        /* Gather data for the free space manager superblock extension message.
+         * 
+         * In passing, verify that all the free space managers are closed.
+         */
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
+            fsinfo.fs_addr[type - 1] = f->shared->fs_addr[type];
 	fsinfo.strategy = f->shared->fs_strategy;
 	fsinfo.threshold = f->shared->fs_threshold;
 
-	/* Write free-space manager info message to superblock extension object header */
-	/* Create the superblock extension object header in advance if needed */
-	if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE) < 0)
-	    HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
-
-	/* Re-allocate free-space manager header and/or section info header */
-	for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
-	    H5FS_stat_t	fs_stat;		/* Information for free-space manager */
-
-	    /* Check for active free space manager of this type */
-            if(f->shared->fs_man[type]) {
-		/* Re-query free space manager info for this type */
-                if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0)
-		    HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
-
-		/* Are there sections to persist? */
-                if(fs_stat.serial_sect_count) {
-		    /* Allocate space for free-space manager header */
-		    if(H5FS_alloc_hdr(f, f->shared->fs_man[type], &f->shared->fs_addr[type], dxpl_id) < 0)
-			HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "can't allocated free-space header")
-
-		    /* Allocate space for free-space maanger section info header */
-		    if(H5FS_alloc_sect(f, f->shared->fs_man[type], dxpl_id) < 0)
-			HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate free-space section info")
-
-		    HDassert(f->shared->fs_addr[type]);
-		    fsinfo.fs_addr[type-1] = f->shared->fs_addr[type];
-		    update = TRUE;
-		} /* end if */
-	    } else if(H5F_addr_defined(f->shared->fs_addr[type])) {
-		fsinfo.fs_addr[type-1] = f->shared->fs_addr[type];
-		update = TRUE;
-	    } /* end else-if */
-	} /* end for */
-
-	/* Update the free space manager info message in superblock extension object header */
-	if(update)
-	    if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE) < 0)
-	        HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
+        /* Write the free space manager message -- message must already exist */
+        if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE) < 0)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
 
 	/* Final close of free-space managers */
-	for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+	for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
 	    if(f->shared->fs_man[type]) {
+                H5AC_ring_t needed_ring;    /* Ring value needed for this iteration */
+
+                /* test to see if we need to switch rings -- do so if required */
+                if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                        || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+                    needed_ring = H5AC_RING_MDFSM;
+                else
+                    needed_ring = H5AC_RING_RDFSM;
+
+                if(needed_ring != curr_ring) {
+                    if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+                        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)")
+                    curr_ring = needed_ring;
+                } /* end if */
+
+                HDassert(f->shared->fs_state[type] == H5F_FS_STATE_OPEN);
 		if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0)
 		    HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't close free space manager")
 		f->shared->fs_man[type] = NULL;
@@ -1421,27 +2142,47 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
 	    } /* end if */
 	    f->shared->fs_addr[type] = HADDR_UNDEF;
 	} /* end for */
+
+        /* Verify that we haven't dirtied any metadata cache entries 
+         * from the metadata free space manager ring out.
+         */
+        HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
+
+        /* Verify that the aggregators are still shutdown. */
+        HDassert(f->shared->sdata_aggr.tot_size == 0);
+        HDassert(f->shared->sdata_aggr.addr == 0);
+        HDassert(f->shared->sdata_aggr.size == 0);
+        HDassert(f->shared->meta_aggr.tot_size == 0);
+        HDassert(f->shared->meta_aggr.addr == 0);
+        HDassert(f->shared->meta_aggr.size == 0);
     } /* end if */
     else {  /* super_vers can be 0, 1, 2 */
         /* Close and delete freespace managers from the file */
-	if(H5MF__close_delete(f, dxpl_id) < 0)
+        if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0)
             HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
-    } /* end else */
 
-    /* Free the space in aggregators (again) */
-    /* (in case any free space information re-started them) */
-    if(H5MF_free_aggrs(f, dxpl_id) < 0)
-        HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators")
-
-    /* Trying shrinking the EOA for the file */
-    /* (in case any free space is now at the EOA) */
-    if(H5MF_close_shrink_eoa(f, dxpl_id) < 0)
-        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+          /* moved code that was for both the persistant and non persistant free
+           * space managers to the non-persistant case.  In the persistant
+           * case, the EOA should already be as shrunked as it can be, and the 
+           * aggregators should alread be shut down.
+           */
+
+        /* Free the space in aggregators (again) */
+        /* (in case any free space information re-started them) */
+        if(H5MF_free_aggrs(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators")
+
+        /* Trying shrinking the EOA for the file */
+        /* (in case any free space is now at the EOA) */
+        if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+    } /* end else */
 
 done:
     /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
 
 #ifdef H5MF_ALLOC_DEBUG
 HDfprintf(stderr, "%s: Leaving\n", FUNC);
@@ -1501,9 +2242,12 @@ H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects,
     size_t 	total_sects = 0;	/* total number of sections */
     H5MF_sect_iter_ud_t sect_udata;     /* User data for callback */
     H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t curr_ring;              /* Current ring value */
+    H5AC_ring_t needed_ring;            /* Ring value needed for this iteration */
     H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
     H5FD_mem_t	start_type, end_type;   /* Memory types to iterate over */
     H5FD_mem_t 	ty;     		/* Memory type for iteration */
+    hbool_t reset_ring = FALSE;         /* Whether the ring was set */
     ssize_t 	ret_value = -1;         /* Return value */
 
     FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
@@ -1528,17 +2272,40 @@ H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects,
     sect_udata.sect_count = nsects;
     sect_udata.sect_idx = 0;
 
-    /* Set the ring type in the DXPL */
-    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
+    /* Set the ring type in the DXPL.  Note that if we are 
+     * scanning a number of different free space managers, 
+     * we may have to change the ring
+     */
+    if((H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+            || (H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+        needed_ring = H5AC_RING_MDFSM;
+    else
+        needed_ring = H5AC_RING_RDFSM;
+    curr_ring = needed_ring;
+    if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &orig_ring) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+    reset_ring = TRUE;
 
     /* Iterate over memory types, retrieving the number of sections of each type */
     for(ty = start_type; ty < end_type; H5_INC_ENUM(H5FD_mem_t, ty)) {
 	hbool_t fs_started = FALSE;
 
+        /* test to see if we need to switch rings -- do so if required */
+        if((H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                || (H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+            needed_ring = H5AC_RING_MDFSM;
+        else
+            needed_ring = H5AC_RING_RDFSM;
+
+        if(needed_ring != curr_ring) {
+            if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+                HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)")
+            curr_ring = needed_ring;
+        } /* end if */
+
 	/* Open free space manager of this type, if it isn't already */
         if(!f->shared->fs_man[ty] && H5F_addr_defined(f->shared->fs_addr[ty])) {
-	    if(H5MF_alloc_open(f, dxpl_id, ty) < 0)
+	    if(H5MF__alloc_open(f, dxpl_id, ty) < 0)
 		HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space")
             HDassert(f->shared->fs_man[ty]);
 	    fs_started = TRUE;
@@ -1576,8 +2343,9 @@ H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects,
 
 done:
     /* Reset the ring in the DXPL */
-    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
-        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
 
     FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
 } /* H5MF_get_free_sections() */
diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c
index 8faff0d..7b5a298 100644
--- a/src/H5MFaggr.c
+++ b/src/H5MFaggr.c
@@ -78,6 +78,83 @@ static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type,
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5MF_vfd_alloc
+ *
+ * Purpose:     Allocate SIZE bytes of file memory via H5FD_alloc() and return
+ *              the relative address where that contiguous chunk of file memory
+ *              exists.
+ *		The TYPE argument describes the purpose for which the storage
+ *		is being requested.
+ *
+ * Return:      Success:        The file address of new chunk.
+ *              Failure:        HADDR_UNDEF
+ *
+ * Programmer:  Quincey Koziol
+ *              January 2, 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+haddr_t
+H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, hsize_t size,
+    hbool_t keep_fragment)
+{
+    haddr_t eoa;                        /* Initial EOA for the file */
+    haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */
+    hsize_t eoa_frag_size = 0;          /* Size of fragment at EOA */
+    haddr_t ret_value = HADDR_UNDEF;    /* Return value */
+
+    FUNC_ENTER_NOAPI(HADDR_UNDEF)
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size);
+#endif /* H5MF_ALLOC_DEBUG */
+
+    /* check arguments */
+    HDassert(f);
+    HDassert(f->shared);
+    HDassert(f->shared->lf);
+    HDassert(size > 0);
+
+    /* Get the EOA for the file -- need for sanity check below */
+    if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type)))
+       HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa")
+
+    /* Check for overlap into temporary allocation space */
+    if(H5F_addr_gt((eoa + size), f->shared->tmp_addr))
+        HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, HADDR_UNDEF, "hdr file space alloc will overlap into 'temporary' file space")
+
+    /* Allocate space for the header */
+    if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size)))
+        HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space for hdr")
+
+    /* Sanity check for overlapping into file's temporary allocation space */
+    HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr));
+
+    /* If the file alignment is 1, there should be no eoa fragment */
+    HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1));
+
+    /* Check if fragment was generated and we want to keep it */
+    if(keep_fragment && eoa_frag_size > 0) {
+        /* Sanity check */
+        HDassert(H5F_addr_defined(eoa_frag_addr));
+
+        /* Put fragment on the free list */
+        if(H5MF_xfree(f, alloc_type, dxpl_id, eoa_frag_addr, eoa_frag_size) < 0)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment")
+    } /* end if */
+
+done:
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size);
+#endif /* H5MF_ALLOC_DEBUG */
+#ifdef H5MF_ALLOC_DEBUG_DUMP
+H5MF_sects_dump(f, dxpl_id, stderr);
+#endif /* H5MF_ALLOC_DEBUG_DUMP */
+
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5MF_vfd_alloc() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5MF_aggr_vfd_alloc
  *
  * Purpose:     Allocate SIZE bytes of file memory via H5MF_aggr_alloc()
@@ -221,7 +298,7 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz
                 if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr))
                     HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
 
-		if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, aggr->addr + aggr->size, ext_size)) < 0)
+		if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0)
 		    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space")
 		else if(was_extended) {
 		    /* aggr->size is unchanged */
@@ -264,7 +341,7 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC);
                 if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr))
                     HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
 
-		if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, aggr->addr + aggr->size, ext_size)) < 0)
+		if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0)
 		    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space")
 		else if(was_extended) {
 		    aggr->addr += aggr_frag_size;
@@ -404,8 +481,8 @@ HDfprintf(stderr, "%s: ret_value = %a\n", FUNC, ret_value);
  *-------------------------------------------------------------------------
  */
 htri_t
-H5MF_aggr_try_extend(H5F_t *f, H5F_blk_aggr_t *aggr, H5FD_mem_t type,
-    haddr_t blk_end, hsize_t extra_requested)
+H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, 
+    H5FD_mem_t type, haddr_t blk_end, hsize_t extra_requested)
 {
     htri_t ret_value = FALSE;           /* Return value */
 
@@ -447,7 +524,7 @@ H5MF_aggr_try_extend(H5F_t *f, H5F_blk_aggr_t *aggr, H5FD_mem_t type,
 		else {
 		    hsize_t extra = (extra_requested < aggr->alloc_size) ? aggr->alloc_size : extra_requested;
 
-		    if((ret_value = H5FD_try_extend(f->shared->lf, type, f, (aggr->addr + aggr->size), extra)) < 0)
+		    if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, (aggr->addr + aggr->size), extra)) < 0)
 			HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file")
 		    else if(ret_value == TRUE) {
 			/* Shift the aggregator block by the extra requested */
diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c
index ef4f1bb..57bac9e 100644
--- a/src/H5MFdbg.c
+++ b/src/H5MFdbg.c
@@ -171,7 +171,7 @@ H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int ind
     for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
 	if(H5F_addr_eq(f->shared->fs_addr[type], fs_addr)) {
 	    if(!f->shared->fs_man[type])
-		if(H5MF_alloc_open(f, dxpl_id, type) < 0)
+		if(H5MF__alloc_open(f, dxpl_id, type) < 0)
 		    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
 
 	    if(f->shared->fs_man[type]) {
diff --git a/src/H5MFpkg.h b/src/H5MFpkg.h
index 1a62710..43fc1cd 100644
--- a/src/H5MFpkg.h
+++ b/src/H5MFpkg.h
@@ -143,8 +143,8 @@ H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1];
 /******************************/
 
 /* Allocator routines */
-H5_DLL herr_t H5MF_alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
-H5_DLL herr_t H5MF_alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
+H5_DLL herr_t H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
+H5_DLL herr_t H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
 H5_DLL herr_t H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream);
 
 /* 'simple' section routines */
@@ -159,7 +159,7 @@ H5_DLL herr_t H5MF_sect_simple_free(H5FS_section_info_t *sect);
 /* Block aggregator routines */
 H5_DLL haddr_t H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
     H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size);
-H5_DLL htri_t H5MF_aggr_try_extend(H5F_t *f, H5F_blk_aggr_t *aggr,
+H5_DLL htri_t H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
     H5FD_mem_t type, haddr_t abs_blk_end, hsize_t extra_requested);
 H5_DLL htri_t H5MF_aggr_can_absorb(const H5F_t *f, const H5F_blk_aggr_t *aggr,
     const H5MF_free_section_t *sect, H5MF_shrink_type_t *shrink);
diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h
index cb797b7..330fe80 100644
--- a/src/H5MFprivate.h
+++ b/src/H5MFprivate.h
@@ -58,8 +58,11 @@ H5_DLL herr_t H5MF_try_close(H5F_t *f, hid_t dxpl_id);
 
 /* File space allocation routines */
 H5_DLL haddr_t H5MF_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
-H5_DLL haddr_t H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
-H5_DLL herr_t H5MF_xfree(const H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr,
+H5_DLL haddr_t H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id,
+    hsize_t size);
+H5_DLL haddr_t H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type,
+    hsize_t size, hbool_t keep_fragment);
+H5_DLL herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr,
     hsize_t size);
 H5_DLL herr_t H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type,
     haddr_t addr, hsize_t size, hsize_t extra_requested);
@@ -75,6 +78,10 @@ H5_DLL haddr_t H5MF_alloc_tmp(H5F_t *f, hsize_t size);
 H5_DLL herr_t H5MF_free_aggrs(H5F_t *f, hid_t dxpl_id);
 H5_DLL htri_t H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id);
 
+/* Settling routines */
+H5_DLL herr_t H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled);
+H5_DLL herr_t H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled);
+
 /* Debugging routines */
 #ifdef H5MF_DEBUGGING
 H5_DLL herr_t H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr,
diff --git a/src/H5O.c b/src/H5O.c
index 0a4e4eb..e0c0f0b 100644
--- a/src/H5O.c
+++ b/src/H5O.c
@@ -129,12 +129,12 @@ const H5O_msg_class_t *const H5O_msg_class_g[] = {
     H5O_MSG_DRVINFO,		/*0x0014 Driver info settings		*/
     H5O_MSG_AINFO,		/*0x0015 Attribute information		*/
     H5O_MSG_REFCOUNT,		/*0x0016 Object's ref. count		*/
-    H5O_MSG_FSINFO,		/*0x0017 Free-space manager info message */
+    H5O_MSG_FSINFO,		/*0x0017 Free-space manager info        */
     H5O_MSG_UNKNOWN,		/*0x0018 Placeholder for unknown message */
 #ifdef H5O_ENABLE_BOGUS
     H5O_MSG_BOGUS_INVALID, 	/*0x0019 "Bogus invalid" (for testing) 	*/
 #else /* H5O_ENABLE_BOGUS */
-    NULL,                      /*0x0019 "Bogus invalid" (for testing)  	*/
+    NULL,                       /*0x0019 "Bogus invalid" (for testing) 	*/
 #endif /* H5O_ENABLE_BOGUS */
 };
 
diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h
index 5851fe4..f6df874 100644
--- a/src/H5Oprivate.h
+++ b/src/H5Oprivate.h
@@ -203,7 +203,7 @@ typedef struct H5O_copy_t {
 #define H5O_DRVINFO_ID  0x0014          /* Driver info message.  */
 #define H5O_AINFO_ID    0x0015          /* Attribute info message.  */
 #define H5O_REFCOUNT_ID 0x0016          /* Reference count message.  */
-#define H5O_FSINFO_ID   0x0017          /* Free-space manager info message.  */
+#define H5O_FSINFO_ID   0x0017          /* File space info message.  */
 #define H5O_UNKNOWN_ID  0x0018          /* Placeholder message ID for unknown message.  */
                                         /* (this should never exist in a file) */
 /* 
diff --git a/test/cache.c b/test/cache.c
index ed4fceb..8f4935b 100644
--- a/test/cache.c
+++ b/test/cache.c
@@ -15912,6 +15912,11 @@ check_destroy_pinned_err(void)
         protect_entry(file_ptr, 0, 0);
 	unprotect_entry(file_ptr, 0, 0, H5C__PIN_ENTRY_FLAG);
 
+        if(H5C_prep_for_file_close(file_ptr, H5P_DATASET_XFER_DEFAULT) < 0 ) {
+            pass = FALSE;
+            failure_mssg = "unexpected failure of prep for file close.\n";
+        } /* end if */
+
         if(H5C_dest(file_ptr, H5AC_ind_read_dxpl_id) >= 0) {
             pass = FALSE;
             failure_mssg = "destroy succeeded on cache with pinned entry.\n";
@@ -15987,6 +15992,20 @@ check_destroy_protected_err(void)
         file_ptr = setup_cache((size_t)(2 * 1024),
                                 (size_t)(1 * 1024));
 
+        /* Note: normally this call would go just before the series of 
+         * flushes prior to file close -- in particular, all entries 
+         * should be unprotected when this call is made.
+         *
+         * Thus H5C_prep_for_file_close() contains an assert to verify
+         * this.  Since this assert would be triggered by the condition 
+         * we are trying to test, put the call to H5C_prep_for_file_close()
+         * prior to the final protect call.
+         */
+        if(H5C_prep_for_file_close(file_ptr, H5P_DATASET_XFER_DEFAULT) < 0 ) {
+            pass = FALSE;
+            failure_mssg = "unexpected failure of prep for file close.\n";
+        } /* end if */
+
         protect_entry(file_ptr, 0, 0);
 
         if(H5C_dest(file_ptr, H5AC_ind_read_dxpl_id) >= 0) {
diff --git a/test/cache_common.c b/test/cache_common.c
index f387f05..daf9777 100644
--- a/test/cache_common.c
+++ b/test/cache_common.c
@@ -128,37 +128,37 @@ static herr_t monster_image_len(const void *thing, size_t *image_len_ptr);
 static herr_t variable_image_len(const void *thing, size_t *image_len_ptr);
 static herr_t notify_image_len(const void *thing, size_t *image_len_ptr);
 
-static herr_t pico_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t pico_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t nano_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t nano_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t micro_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t micro_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t tiny_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t tiny_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t small_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t small_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t medium_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t medium_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t large_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t large_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t huge_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t huge_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t monster_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t monster_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t variable_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t variable_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
-static herr_t notify_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t notify_pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, 
     size_t *new_len_ptr, unsigned *flags_ptr);
 
@@ -211,7 +211,7 @@ static herr_t get_final_load_size(const void *image, size_t image_len,
 static void *deserialize(const void *image_ptr, size_t len, void *udata_ptr,
     hbool_t *dirty_ptr, int32_t entry_type);
 static herr_t image_len(const void *thing, size_t *image_len_ptr, int32_t entry_type);
-static herr_t pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing,
+static herr_t pre_serialize(H5F_t *f, hid_t dxpl_id, void *thing,
     haddr_t addr, size_t len, haddr_t *new_addr_ptr, size_t *new_len_ptr, 
     unsigned *flags_ptr);
 static herr_t serialize(const H5F_t *f, void *image_ptr, size_t len, 
@@ -1114,7 +1114,7 @@ notify_image_len(const void *thing, size_t *image_length)
  *-------------------------------------------------------------------------
  */
 herr_t
-pre_serialize(const H5F_t *f,
+pre_serialize(H5F_t *f,
               hid_t H5_ATTR_UNUSED dxpl_id,
               void *thing,
               haddr_t addr,
@@ -1199,7 +1199,7 @@ pre_serialize(const H5F_t *f,
 } /* pre_serialize() */
 
 herr_t
-pico_pre_serialize(const H5F_t *f,
+pico_pre_serialize(H5F_t *f,
                    hid_t dxpl_id,
                    void *thing,
                    haddr_t addr,
@@ -1213,7 +1213,7 @@ pico_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-nano_pre_serialize(const H5F_t *f,
+nano_pre_serialize(H5F_t *f,
                    hid_t dxpl_id,
                    void *thing,
                    haddr_t addr,
@@ -1227,7 +1227,7 @@ nano_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-micro_pre_serialize(const H5F_t *f,
+micro_pre_serialize(H5F_t *f,
                     hid_t dxpl_id,
                     void *thing,
                     haddr_t addr,
@@ -1241,7 +1241,7 @@ micro_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-tiny_pre_serialize(const H5F_t *f,
+tiny_pre_serialize(H5F_t *f,
                    hid_t dxpl_id,
                    void *thing,
                    haddr_t addr,
@@ -1255,7 +1255,7 @@ tiny_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-small_pre_serialize(const H5F_t *f,
+small_pre_serialize(H5F_t *f,
                     hid_t dxpl_id,
                     void *thing,
                     haddr_t addr,
@@ -1269,7 +1269,7 @@ small_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-medium_pre_serialize(const H5F_t *f,
+medium_pre_serialize(H5F_t *f,
                      hid_t dxpl_id,
                      void *thing,
                      haddr_t addr,
@@ -1283,7 +1283,7 @@ medium_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-large_pre_serialize(const H5F_t *f,
+large_pre_serialize(H5F_t *f,
                     hid_t dxpl_id,
                     void *thing,
                     haddr_t addr,
@@ -1297,7 +1297,7 @@ large_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-huge_pre_serialize(const H5F_t *f,
+huge_pre_serialize(H5F_t *f,
                    hid_t dxpl_id,
                    void *thing,
                    haddr_t addr,
@@ -1311,7 +1311,7 @@ huge_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-monster_pre_serialize(const H5F_t *f,
+monster_pre_serialize(H5F_t *f,
                       hid_t dxpl_id,
                       void *thing,
                       haddr_t addr,
@@ -1325,7 +1325,7 @@ monster_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-variable_pre_serialize(const H5F_t *f,
+variable_pre_serialize(H5F_t *f,
                        hid_t dxpl_id,
                        void *thing,
                        haddr_t addr,
@@ -1339,7 +1339,7 @@ variable_pre_serialize(const H5F_t *f,
 }
 
 herr_t
-notify_pre_serialize(const H5F_t *f,
+notify_pre_serialize(H5F_t *f,
                      hid_t dxpl_id,
                      void *thing,
                      haddr_t addr,
@@ -3408,6 +3408,12 @@ takedown_cache(H5F_t * file_ptr,
             H5C_stats(cache_ptr, "test cache", dump_detailed_stats);
         }
 
+	if ( H5C_prep_for_file_close(file_ptr, H5P_DATASET_XFER_DEFAULT) < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "unexpected failure of prep for file close.\n";
+        }
+
         flush_cache(file_ptr, TRUE, FALSE, FALSE);
 
         H5C_dest(file_ptr, H5AC_ind_read_dxpl_id);
diff --git a/test/freespace.c b/test/freespace.c
index af53eba..d963a6e 100644
--- a/test/freespace.c
+++ b/test/freespace.c
@@ -2869,7 +2869,7 @@ test_fs_sect_iterate(hid_t fapl)
 	if(H5FS_sect_add(f, dxpl_id, frsp, (H5FS_section_info_t *)sect_node,
 		H5FS_ADD_RETURNED_SPACE, NULL) < 0)
 	    FAIL_STACK_ERROR
-    }
+    } /* end for */
 
     if(H5FS_sect_iterate(f, dxpl_id, frsp, TEST_sects_cb, &udata) < 0)
 	TEST_ERROR
@@ -2905,7 +2905,7 @@ error:
     H5E_BEGIN_TRY {
         if(frsp)
             H5FS_close(f, dxpl_id, frsp);
-	    H5Fclose(file);
+        H5Fclose(file);
         H5Pclose(dxpl_id);
     } H5E_END_TRY;
     return 1;
@@ -2915,9 +2915,9 @@ error:
 int
 main(void)
 {
-    hid_t       	fapl = -1;	/* File access property list for data files */
-    unsigned    	nerrors = 0;    /* Cumulative error count */
-    const char *env_h5_drvr = NULL;     /* File Driver value from environment */
+    hid_t           fapl = -1;              /* File access property list for data files */
+    unsigned        nerrors = 0;            /* Cumulative error count */
+    const char     *env_h5_drvr = NULL;     /* File Driver value from environment */
 
     /* Get the VFD to use */
     env_h5_drvr = HDgetenv("HDF5_DRIVER");
@@ -2926,11 +2926,16 @@ main(void)
 
     h5_reset();
 
-    fapl = h5_fileaccess();
+    if((fapl = h5_fileaccess()) < 0) {
+        nerrors++;
+        PUTS_ERROR("Can't get VFD-dependent fapl")
+    } /* end if */
 
     /* make sure alignment is not set for tests to succeed */
-    if(H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1) < 0)
-        TEST_ERROR
+    if(H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1) < 0) {
+        nerrors++;
+        PUTS_ERROR("Can't set alignment")
+    } /* end if */
 
     nerrors += test_fs_create(fapl);
     nerrors += test_fs_sect_add(fapl);
@@ -2946,16 +2951,16 @@ main(void)
 
     if(nerrors)
         goto error;
-    puts("All free-space tests passed.");
+    HDputs("All free-space tests passed.");
 
     h5_cleanup(FILENAME, fapl);
-    return 0;
+    HDexit(EXIT_SUCCESS);
 
 error:
-    puts("*** TESTS FAILED ***");
+    HDputs("*** TESTS FAILED ***");
     H5E_BEGIN_TRY {
         H5Pclose(fapl);
     } H5E_END_TRY;
-    return 1;
+    HDexit(EXIT_FAILURE);
 } /* main() */
 
diff --git a/test/mf.c b/test/mf.c
index e3d1845..f56c7ab 100644
--- a/test/mf.c
+++ b/test/mf.c
@@ -980,7 +980,7 @@ test_mf_fs_start(hid_t fapl)
     frspace_state_t 	state;
 
 
-    TESTING("H5MF_alloc_create()/H5MF_alloc_open() of free-space manager");
+    TESTING("H5MF_alloc_create()/H5MF__alloc_open() of free-space manager");
 
     /* Set the filename to use for this test (dependent on fapl) */
     h5_fixname(FILENAME[0], fapl, filename, sizeof(filename));
@@ -1014,7 +1014,7 @@ test_mf_fs_start(hid_t fapl)
     /* Start up free-space manager */
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -1131,7 +1131,7 @@ test_mf_fs_alloc_free(hid_t fapl)
 
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -1218,7 +1218,7 @@ test_mf_fs_alloc_free(hid_t fapl)
 
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -1303,7 +1303,7 @@ test_mf_fs_alloc_free(hid_t fapl)
 
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -1489,7 +1489,7 @@ test_mf_fs_extend(hid_t fapl)
 
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -1607,7 +1607,7 @@ test_mf_fs_extend(hid_t fapl)
 
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -1720,7 +1720,7 @@ test_mf_fs_extend(hid_t fapl)
 
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -1833,7 +1833,7 @@ test_mf_fs_extend(hid_t fapl)
 
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -2025,7 +2025,7 @@ test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl)
 
         type = H5FD_MEM_SUPER;
 
-        if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+        if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
             TEST_ERROR
 
         if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -2095,7 +2095,7 @@ test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl)
 
         type = H5FD_MEM_SUPER;
 
-        if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+        if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
             TEST_ERROR
 
         if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -4083,7 +4083,7 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl)
 
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -4153,7 +4153,7 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl)
 
     type = H5FD_MEM_SUPER;
 
-    if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
         TEST_ERROR
 
     if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -4243,7 +4243,7 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl)
 
         type = H5FD_MEM_SUPER;
 
-        if(H5MF_alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
+        if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0)
             TEST_ERROR
 
         if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN)
@@ -6223,7 +6223,7 @@ test_mf_fs_persist(hid_t fapl_new, hid_t fcpl)
         TEST_ERROR
 
     /* Start up H5FD_MEM_SUPER free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
         FAIL_STACK_ERROR
 
     /* Get info for free-space manager */
@@ -6368,7 +6368,7 @@ test_mf_fs_gone(hid_t fapl_new, hid_t fcpl)
         TEST_ERROR
 
     /* Start up H5FD_MEM_SUPER free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
         FAIL_STACK_ERROR
 
     /* Get info for H5FD_MEM_SUPER free-space manager */
@@ -6495,7 +6495,7 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl)
         TEST_ERROR
 
     /* Start up H5FD_MEM_SUPER free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
         FAIL_STACK_ERROR
 
     /* Get free-space info */
@@ -6519,7 +6519,7 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl)
         TEST_ERROR
 
     /* Start up H5FD_MEM_DRAW free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, stype) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, stype) < 0)
         FAIL_STACK_ERROR
 
     /* Get free-space info */
@@ -6583,7 +6583,7 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl)
         TEST_ERROR
 
     /* Start up H5FD_MEM_SUPER free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
         FAIL_STACK_ERROR
 
     /* Get free-space info */
@@ -6623,7 +6623,7 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl)
 	TEST_ERROR
 
     /* Start up H5FD_MEM_SUPER free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
         FAIL_STACK_ERROR
 
     /* Get free-space info */
@@ -6730,7 +6730,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl)
         TEST_ERROR
 
     /* Start up H5FD_MEM_SUPER free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
         FAIL_STACK_ERROR
 
     /* Get free-space info */
@@ -6754,7 +6754,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl)
         TEST_ERROR
 
     /* Start up H5FD_MEM_DRAW free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, stype) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, stype) < 0)
         FAIL_STACK_ERROR
 
     /* Get free-space info */
@@ -6812,7 +6812,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl)
         TEST_ERROR
 
     /* Start up H5FD_MEM_SUPER free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
         FAIL_STACK_ERROR
 
     /* Get free-space info */
@@ -6838,7 +6838,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl)
         TEST_ERROR
 
     /* Start up H5FD_MEM_BTREE free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, btype) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, btype) < 0)
         FAIL_STACK_ERROR
 
     /* Get free-space info */
@@ -6876,7 +6876,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl)
     /* If H5FD_MEM_SUPER is there, should not find block #1 & #3 */
     if(H5F_addr_defined(f->shared->fs_addr[type])) {
 	/* Start up H5FD_MEM_SUPER free-space manager */
-	if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
+	if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
 	    FAIL_STACK_ERROR
 
 	if((node_found = H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type],
@@ -6895,7 +6895,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl)
 	TEST_ERROR
 
     /* Start up H5FD_MEM_GHEAP free-space manager */
-    if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, gtype) < 0)
+    if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, gtype) < 0)
 	FAIL_STACK_ERROR
 
     /* Get free-space info */
@@ -7189,7 +7189,7 @@ test_filespace_strategy_threshold(hid_t fapl_new)
 			    TEST_ERROR
 
 			/* Open the free-space manager */
-			if(H5MF_alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
+			if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0)
 			    FAIL_STACK_ERROR
 
 			/* Retrieve the total amount of free space and # of free-space sections */