Bring file shutdown code from cache image branch

1 files changed, 654 insertions, 70 deletions

diff --git a/src/H5MF.c b/src/H5MF.c
index 19ca523..3ed6d28 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -863,14 +863,14 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r
     reset_ring = TRUE;
 
     /* Check if the block is exactly at the end of the file */
-    if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, end, extra_requested)) < 0)
+    if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, dxpl_id, end, extra_requested)) < 0)
         HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file")
     else if(ret_value == FALSE) {
         H5F_blk_aggr_t *aggr;   /* Aggregator to use */
 
         /* Check for test block able to extend aggregation block */
         aggr = (map_type == H5FD_MEM_DRAW) ?  &(f->shared->sdata_aggr) : &(f->shared->meta_aggr);
-        if((ret_value = H5MF_aggr_try_extend(f, aggr, map_type, end, extra_requested)) < 0)
+        if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0)
             HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block")
         else if(ret_value == FALSE) {
 
@@ -1246,6 +1246,632 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5MF_settle_raw_data_fsm()
+ *
+ * Purpose: 	Handle any tasks required before the metadata cache 
+ *		can serialize or flush the raw data free space manager
+ *		and any metadata free space managers that reside in the 
+ *		raw data free space manager ring. 
+ *
+ *		Specifically, any metadata managers that DON'T handle 
+ *		space allocation for free space manager header or section 
+ *		info will reside in the raw data free space manager ring.  
+ *		As of this writing, the plan is to move to only two free space 
+ *		managers, one for raw data and one for metadata -- which
+ *		means that only the raw data free space manager will reside
+ *		in the free space manager ring.  However, this has not been
+ *		fully implemented yet, so this code must support the 
+ *		possibilty of multiple metadata free space managers, at most
+ *		two of which handle free space manager header or section info, 
+ *		and thus reside in the metadata free space manager ring.
+ *
+ *		At present, the task list is:
+ *
+ *		1) Reduce the EOA to the extent possible.  To do this:
+ *
+ *		    a) Free both aggregators.  Space not at EOA will be
+ *		       added to the appropriate free space manager.
+ *		    
+ *		       The raw data aggregator should not be restarted 
+ *		       after this point.  It is possible that the metadata
+ *		       aggregator will be.
+ *		    
+ *		    b) Free all file space currently allocated to free
+ *		       space managers.  
+ *
+ *		    c) Delete the free space manager superblock 
+ *		       extension message if allocated.
+ *
+ *		   This done, reduce the EOA by moving it to just before
+ *		   the last piece of free memory in the file.
+ *		   
+ *		2) Ensure that space is allocated for the free space
+ *                 manager superblock extension message.  Must do this
+ *                 now, before reallocating file space for free space 
+ *		   managers, as it is possible that this allocation may
+ *		   grab the last section in a FSM -- making it unnecessary
+ *		   to re-allocate file space for it.
+ *
+ *		3) Scan all free space managers not involved in allocating
+ *		   space for free space managers.  For each such free space
+ *		   manager, test to see if it contains free space.  If 
+ *		   it does, allocate file space for its header and section
+ *		   data.  If it contains no free space, leave it without 
+ *		   allocated file space as there is no need to save it to 
+ *		   file.
+ *
+ *		   Note that all free space managers in this class should
+ *		   see no further space allocations / deallocations as 
+ *		   at this point, all raw data allocations should be 
+ *		   finalized, as should all metadata allocations not 
+ *		   involving free space managers.
+ *
+ *		   We will allocate space for free space managers involved
+ *		   in the allocation of file space for free space managers
+ *		   in H5MF_settle_meta_data_fsm()
+ * Return:	SUCCEED/FAIL
+ *
+ * Programmer:  John Mainzer
+ *	        5/25/16
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
+{
+    H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
+    hbool_t reset_ring = FALSE;         /* Whether the ring was set */
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+
+    /* check args */
+    HDassert(f);
+    HDassert(f->shared);
+    HDassert(fsm_settled);
+
+    /* Only need to settle things if we are persisting the free space info */
+    if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
+        H5O_fsinfo_t fsinfo;                    /* Free space manager info message */
+        H5FD_mem_t	type;			/* Memory type for iteration */
+        H5AC_ring_t curr_ring = H5AC_RING_INV;  /* Current ring value */
+        H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration  */
+        hbool_t fsm_opened[H5FD_MEM_NTYPES];    /* State of FSM */
+        hbool_t fsm_visited[H5FD_MEM_NTYPES];   /* State of FSM */
+
+        /* Initialize fsm_opened and fsm_visited */
+        HDmemset(fsm_opened, 0, sizeof(fsm_opened));
+        HDmemset(fsm_visited, 0, sizeof(fsm_visited));
+
+        /* 1) Reduce the EOA to the extent possible. */
+
+        /* a) Free the space in aggregators:
+         *
+         * (for space not at EOF, it may be put into free space managers) 
+         *
+         * Do this now so that the raw data FSM (and any other FSM that isn't 
+         * involved in space allocation for FSMs) will have no further activity.
+         *
+         * Note that while the raw data aggregator should not be restarted during
+         * the close process, this need not be the case for the metadata aggregator.
+         */
+        if(H5MF_free_aggrs(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't free aggregators")
+
+        /* Set the ring type in the DXPL.  In most cases, we will
+         * need H5AC_RING_MDFSM first, so initialy set the ring in
+         * the DXPL to that value.  We will alter this later if
+         * needed.
+         */
+        if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value(0)")
+        reset_ring = TRUE;
+        curr_ring = H5AC_RING_MDFSM;
+
+        /* b) Free the file space (if any) allocated to each free space manager.
+         *
+         * Do this to facilitate reduction of the size of the file to the 
+         * extent possible.  We will re-allocate space to free space managers 
+         * that have free space to save after this reduction.
+         *
+         * In the case of the raw data free space manager, and any other free
+         * space manager that does not allocate space for free space managers,
+         * allocations should be complete at this point, as all raw data should
+         * have space allocated and be flushed to file at this point.  Thus we
+         * can examine such free space managers and only re-allocate space for 
+         * them if they contain free space.  Do this later in this function after
+         * the EOA has been reduced to the extent possible.
+         *
+         * For free space managers that allocate file space for free space 
+         * managers (usually just a single metadata free space manager, but for 
+         * now at least, free space managers for different types of metadata 
+         * are possible), the matter is more ticklish due to the self-
+         * referential nature of the problem.  These FSMs are dealt with in 
+         * H5MF_settle_meta_data_fsm().
+         */
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+            H5FS_stat_t fs_stat;        /* Information for free-space manager */
+            H5FD_mem_t  fsm_type;       /* File memory type for FSM */
+
+            /* There is potentially a many-to-one mapping from memory types to 
+             * free space managers.  Use the fsm_visited[] array to avoid visiting 
+             * a given FSM more than once.  Use fsm_opened[] to track which FSMs
+             * must be closed at the end of this function.
+             */
+            fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type);
+            if(!fsm_visited[fsm_type]) {
+                fsm_visited[fsm_type] = TRUE;
+
+                /* If there is no active FSM for this type, but such a FSM has 
+                 * space allocated in file, open it so that we can free its file 
+                 * space.
+                 */
+                if(NULL == f->shared->fs_man[fsm_type]) {
+                    if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) {
+                        /* Sanity check */
+                        HDassert(fsm_opened[fsm_type] == FALSE);
+
+                        /* Start up FSM for the file memory type */
+                        if(H5MF__alloc_open(f, dxpl_id, fsm_type) < 0)
+                            HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
+                        fsm_opened[fsm_type] = TRUE;
+                    } /* end if */
+                } /* end if */
+
+                /* Check for an actual FSM for this type now */
+                /* (Possibly opened in previous step) */
+                if(f->shared->fs_man[fsm_type]) {
+                    /* Test to see if we need to switch rings -- do so if required */
+                    if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                           || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+                        needed_ring = H5AC_RING_MDFSM;
+                    else
+                        needed_ring = H5AC_RING_RDFSM;
+                    if(needed_ring != curr_ring) {
+                        if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0)
+                            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value.")
+                        curr_ring = needed_ring;
+                    } /* end if */
+
+                    /* Query free space manager info for this type */
+                    if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
+                        HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+
+                    /* Check if the free space manager has space in the file */
+                    if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) {
+                        /* Delete the free space manager in the file.  Will 
+                         * reallocate later if the free space manager contains
+                         * any free space.
+                         */
+                        if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id) < 0)
+                            HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
+                        f->shared->fs_addr[fsm_type] = HADDR_UNDEF;
+                    } /* end if */
+                } /* end if */
+
+                /* note that we are tracking opened FSM -- we will close them
+                 * at the end of the function.
+                 */
+            } /* end if */
+        } /* end for */
+
+        /* c) Delete the free space manager superblock extension message 
+         *    if allocated.
+         *
+         *    Must do this since the routine that writes / creates superblock
+         *    extension messages will choke if the target message is 
+         *    unexpectedly either absent or present.
+         */
+        if(H5F_addr_defined(f->shared->sblock->ext_addr))
+            if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension")
+
+        /* As the final element in 1), shrink the EOA for the file */
+        if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+
+     
+        /* 2) Ensure that space is allocated for the free space manager superblock 
+         *    extension message.  Must do this now, before reallocating file space 
+         *    for free space managers, as it is possible that this allocation may
+         *    grab the last section in a FSM -- making it unnecessary to 
+         *    re-allocate file space for it.
+         *
+         * Do this by writing a free space manager superblock extension message.
+         * 
+         * Since no free space manager has file space allocated for it, this 
+         * message must be invalid since we can't save addresses of FSMs when
+         * those addresses are unknown.  This is OK -- we will write the correct
+         * values to the message at free space manager shutdown.
+         */
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) 
+            fsinfo.fs_addr[type-1] = HADDR_UNDEF;
+        fsinfo.strategy = f->shared->fs_strategy;
+        fsinfo.threshold = f->shared->fs_threshold;
+        if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
+
+
+        /* 3) Scan all free space managers not involved in allocating
+         *    space for free space managers.  For each such free space
+         *    manager, test to see if it contains free space.  If 
+         *    it does, allocate file space for its header and section
+         *    data.  If it contains no free space, leave it without 
+         *    allocated file space as there is no need to save it to 
+         *    file.
+         *
+         *    Note that all free space managers in this class should
+         *    see no further space allocations / deallocations as 
+         *    at this point, all raw data allocations should be 
+         *    finalized, as should all metadata allocations not involving
+         *    free space managers.
+         *
+         *    We will allocate space for free space managers involved
+         *    in the allocation of file space for free space managers
+         *    in H5MF_settle_meta_data_fsm()
+         */
+
+        /* Reinitialize fsm_visited */
+        HDmemset(fsm_visited, 0, sizeof(fsm_visited));
+
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+            H5FD_mem_t  fsm_type;       /* File memory type for FSM */
+
+            fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type);
+
+            /* test to see if we need to switch rings -- do so if required */
+            if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
+                    || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+                needed_ring = H5AC_RING_MDFSM;
+            else
+                needed_ring = H5AC_RING_RDFSM;
+
+            if(needed_ring != curr_ring) {
+                if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0)
+                    HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value.")
+                curr_ring = needed_ring;
+            } /* end if */
+
+            /* Since there can be a many-to-one mapping from memory types 
+             * to free space managers, ensure that we don't visit any FSM
+             * more than once.
+             */
+            if(!fsm_visited[fsm_type]) {
+                fsm_visited[fsm_type] = TRUE;
+
+                if(f->shared->fs_man[fsm_type]) {
+                    /* Only allocate file space if the target free space manager 
+                     * doesn't allocate file space for free space managers.  Note 
+                     * that this is also the deciding factor as to whether a FSM 
+                     * in in the raw data FSM ring.
+                     */
+                    if((fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) 
+                            && (fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) {
+                        H5FS_stat_t fs_stat;        /* Information for free-space manager */
+
+                        /* The current ring should be H5AC_RING_RDFSM */
+                        HDassert(curr_ring == H5AC_RING_RDFSM);
+
+                        /* Query free space manager info for this type */
+                        if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 )
+                            HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+
+                        /* If the free space manager contains section info, 
+                         * allocate space for the header and sinfo (note that 
+                         * space must not be allocated at present -- verify
+                         * verify this with assertions).
+                         */
+                        if(fs_stat.serial_sect_count > 0) {
+                            /* Sanity check */
+                            HDassert(!H5F_addr_defined(fs_stat.addr));
+
+                            /* Allocate FSM header */
+                            if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0)
+                                HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocated free-space header")
+
+                            /* Allocate FSM section info */
+                            HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+                            HDassert(fs_stat.alloc_sect_size == 0);
+                            if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 )
+                                HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info")
+
+#ifndef NDEBUG
+                            /* Re-Query free space manager info for this type */
+                            if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
+                                HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+
+                            HDassert(H5F_addr_defined(fs_stat.addr));
+                            HDassert(H5F_addr_defined(fs_stat.sect_addr));
+                            HDassert(fs_stat.serial_sect_count > 0);
+                            HDassert(fs_stat.alloc_sect_size > 0);
+                            HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size);
+#endif /* NDEBUG */
+                        } /* end if */
+                        else {
+                            HDassert(!H5F_addr_defined(fs_stat.addr));
+                            HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+                            HDassert(fs_stat.serial_sect_count == 0);
+                            HDassert(fs_stat.alloc_sect_size == 0);
+                        } /* end else */
+                    } /* end if */
+                } /* end if */
+
+                /* Close any opened FSMs */
+                if(fsm_opened[fsm_type]) {
+                    if(H5MF__alloc_close(f, dxpl_id, fsm_type) < 0)
+                       HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't close file free space manager")
+                    fsm_opened[fsm_type] = FALSE;
+                } /* end if */
+            } /* end if */
+        } /* end for */
+
+        /* verify that all opened FSMs were closed */
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
+            HDassert(!fsm_opened[type]);
+
+        /* Indicate that the FSM was settled successfully */
+        *fsm_settled = TRUE;
+    } /* end if */
+
+done:
+    /* Reset the ring in the DXPL */
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set property value")
+
+    FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* H5MF_settle_raw_data_fsm() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5MF_settle_meta_data_fsm()
+ *
+ * Purpose: 	If the free space manager is persistent, handle any tasks 
+ *		required before the metadata cache can serialize or flush 
+ *		the metadata free space manager(sI) that handle file space 
+ *		allocation for free space managers.
+ *
+ *		In most cases, there will be only one manager assigned 
+ *		to this role.  However, since for reason or reason unknown,
+ *		free space manager headers and section info blocks are 
+ *		different classes of memory, it is possible that two free 
+ *		space managers will be involved.
+ *
+ *		On entry to this function, the raw data settle routine
+ *		(H5MF_settle_raw_data_fsm()) should have:
+ *
+ *		1) Freed the accumulators.
+ *
+ *		2) Freed all file space allocated to the free space managers.
+ *
+ *		3) Deleted the free space manager superblock extension message
+ *
+ *		4) Reduced the EOA to the extent possible.
+ *
+ *		5) Re-created the free space manager superblock extension
+ *		   message.
+ *
+ *		6) Reallocated file space for all non-empty free space 
+ *		   managers NOT involved in allocation of space for free 
+ *		   space managers.
+ *
+ *		   Note that these free space managers (if not empty) should
+ *		   have been written to file by this point, and that no
+ *		   further space allocations involving them should take 
+ *		   place during file close.
+ *
+ *		On entry to this routine. the free space manager(s) involved
+ *		in allocation of file space for free space managers should
+ *		still be floating. (i.e. should not have any file space 
+ *		allocated to them.)
+ *
+ *		Similarly, the raw data aggregator should not have been 
+ *		restarted.  Note that it is probable that reallocation of 
+ *		space in 5) and 6) above will have re-started the metadata 
+ *		aggregator.
+ *
+ *
+ *		In this routine, we proceed as follows:
+ *
+ *		1) Verify that the free space manager(s) involved in file
+ *		   space allocation for free space managers are still floating.
+ *
+ *		2) Free the accumulators.
+ *
+ *		3) Reduce the EOA to the extent possible.
+ *
+ *		4) Re-allocate space for any free space manager(s) that:
+ *
+ *		   a) are involved in allocation of space for free space 
+ *		      managers, and 
+ *
+ *		   b) contain free space.
+ *
+ *		   It is possible that we could allocate space for one 
+ *		   of these free space manager(s) only to have the allocation 
+ *		   result in the free space manager being empty and thus 
+ *		   obliging us to free the space again.  Thus there is the
+ *		   potential for an infinte loop if we want to avoid saving
+ *		   empty free space managers.
+ *
+ *		   Similarly, it is possible that we could allocate space 
+ *		   for a section info block, only to discover that this 
+ *		   allocation has changed the size of the section info -- 
+ *		   forcing us to deallocate and start the loop over again.
+ *
+ *		   To avoid this, simply allocate file space for these
+ *		   FSM(s) directly from the VFD layer if allocation is 
+ *		   indicated.  This avoids the issue by bypassing the FSMs
+ *		   in this case.  
+ *
+ *		   Note that this may increase the size of the file needlessly. 
+ *		   A better solution would be to modify the FSM code to 
+ *		   save empty FSMs to file, and to allow section info blocks
+ *		   to be oversized.  However, given that the FSM code is 
+ *		   also used by the fractal heaps, and that we are under 
+ *		   severe time pressure at the moment, the above brute 
+ *		   force solution is attractive. 
+ *
+ * Return:	SUCCEED/FAIL
+ *
+ * Programmer:  John Mainzer
+ *	        5/25/16
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
+{
+    H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+    hbool_t	reset_ring = FALSE;     /* Whether we set the ring */
+    herr_t 	ret_value = SUCCEED;        /* Return value */
+
+    FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+
+    /* Check args */
+    HDassert(f);
+    HDassert(f->shared);
+    HDassert(fsm_settled);
+
+    /* Only need to settle things if we are persisting the free space info */
+    if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
+        H5FS_t     *hdr_fspace;             /* Ptr to FSM hdr alloc FSM */
+        H5FS_t     *sinfo_fspace;           /* Ptr to FSM sinfo alloc FSM */
+        H5FS_stat_t fs_stat;		    /* Information for FSM */
+        H5FD_mem_t  hdr_fsm_alloc_type;	    /* FSM hdr alloc type */
+        H5FD_mem_t  sinfo_fsm_alloc_type;   /* FSM info alloc type */
+
+        /* Get the file memory types for the FSM header & section info */
+        hdr_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR);
+        sinfo_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO);
+
+        /* Sanity checks */
+        HDassert(hdr_fsm_alloc_type > H5FD_MEM_NOLIST);
+        HDassert(hdr_fsm_alloc_type < H5FD_MEM_NTYPES);
+        HDassert(sinfo_fsm_alloc_type > H5FD_MEM_NOLIST);
+        HDassert(sinfo_fsm_alloc_type < H5FD_MEM_NTYPES);
+        HDassert(!H5F_addr_defined(f->shared->fs_addr[hdr_fsm_alloc_type]));
+        HDassert(!H5F_addr_defined(f->shared->fs_addr[sinfo_fsm_alloc_type]));
+
+        /* Note that in most cases, hdr_fspace will equal sinfo_fspace. */
+        hdr_fspace = f->shared->fs_man[hdr_fsm_alloc_type];
+        sinfo_fspace = f->shared->fs_man[sinfo_fsm_alloc_type];
+
+        /* Set the ring in the dxpl appropriately for subsequent calls */
+        if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set ring value")
+        reset_ring = TRUE;
+
+#ifndef NDEBUG
+        /* Verify that hdr_fspace is floating if it exists */
+        if(hdr_fspace) {
+            /* Query free space manager info for this type */
+            if(H5FS_stat_info(f, hdr_fspace, &fs_stat) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "can't get free-space info")
+
+            HDassert(!H5F_addr_defined(fs_stat.addr));
+            HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+            HDassert(fs_stat.alloc_sect_size == 0);
+        } /* end if */
+
+        /* Verify that sinfo_fspace is floating if it exists */
+        if((sinfo_fspace) && (hdr_fspace != sinfo_fspace)) {
+            /* Query free space manager info for this type */
+            if(H5FS_stat_info(f, sinfo_fspace, &fs_stat) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "can't get free-space info")
+
+            HDassert(!H5F_addr_defined(fs_stat.addr));
+            HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+            HDassert(fs_stat.alloc_sect_size == 0);
+        } /* end if */
+#endif /* NDEBUG */
+
+        /* Free the space in the metadata aggregator.  Do this via the 
+         * H5MF_free_aggrs() call.  Note that the raw data aggregator must
+         * have already been freed.  Sanity checks for this?
+         */
+        /* (for space not at EOF, it may be put into free space managers) */
+        if(H5MF_free_aggrs(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't free aggregators")
+
+        /* Trying shrinking the EOA for the file */
+        if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+
+        /* ******************* PROBLEM: ********************
+         *
+         * If the file has an alignement other than 1, and if 
+         * the EOA is not a multiple of this alignment, allocating sapce 
+         * for the section via the VFD info has the potential of generating 
+         * a fragment that will be added to the free space manager.  This 
+         * of course undoes everything we have been doing here.
+         *
+         * Need a way around this.  Obvious solution is to force the EOA to 
+         * be a multiple of the alignment.  
+         *
+         * Fortunately, alignment is typically 1, so this is a non-issue in
+         * most cases.  In cases where the alignment is not 1, for now we 
+         * have decided to drop the fragment on the floor.  
+         *
+         * Eventually, we should fix this by modifying the on disk representations
+         * of free space managers to allow for empty space, so as to bypass the
+         * issues created by self-referential free space managers, and make
+         * this issue moot.
+         */
+        /* HDassert(f->shared->alignment == 1); */
+
+
+        /* The free space manager(s) that handle space allocations for free 
+         * space managers should be settled now, albeit without file space 
+         * allocated to them.  To avoid the possibility of changing the sizes
+         * of their section info blocks, allocate space for them now at the 
+         * end of file via H5FD_alloc().
+         *
+         * In the past, this issue of allocating space without touching the 
+         * free space managers has been deal with by calling 
+         * H5MF_aggr_vfd_alloc(), which in turn calls H5MF_aggr_alloc().  
+         * This is problematic since (if I read the code correctly) it will 
+         * re-constitute the metadata aggregator, which will add any left 
+         * over space to one of the free space managers when freed.
+         *
+         * This is a non-starter, since the entire objective is to settle the
+         * free space managers.
+         *
+         * Hence the decision to call H5FD_alloc() directly.  
+         * 
+         * As discussed in PROBLEM above, if f->shared->alignment is not 1, 
+         * this has the possibility of generating a fragment of file space
+         * that would typically be inserted into one of the free space managers.
+         *
+         * This is isn't good, but due to schedule pressure, we will just drop
+         * the fragement on the floor for now.
+         */
+        if(hdr_fspace)
+            if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, hdr_fspace,
+                    &(f->shared->fs_addr[hdr_fsm_alloc_type])) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't vfd allocate hdr FSM file space")
+
+        if(sinfo_fspace && (sinfo_fspace != hdr_fspace))
+            if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, sinfo_fspace,
+                    &(f->shared->fs_addr[sinfo_fsm_alloc_type])) < 0)
+                HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't vfd allocate sinfo FSM file space")
+
+        /* Indicate that the FSM was settled successfully */
+        *fsm_settled = TRUE;
+    } /* end if */
+
+done:
+    if(reset_ring)
+        if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+            HDONE_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set property value")
+
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_settle_meta_data_fsm() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5MF__close_delete
  *
  * Purpose:     Common code for closing and deleting freespace managers from
@@ -1462,80 +2088,33 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
     if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2
             && f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
         H5O_fsinfo_t fsinfo;		/* Free space manager info message */
-        hbool_t update = FALSE;		/* To update info for the message */
 
-        /* Check to remove free-space manager info message from superblock extension */
-        if(H5F_addr_defined(f->shared->sblock->ext_addr))
-            if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0)
-                HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension")
+        /* Superblock extension and free space manager message should 
+         * exist at this point -- verify at least the former.
+         */
+        HDassert(H5F_addr_defined(f->shared->sblock->ext_addr));
 
-	/* Free free-space manager header and/or section info header */
-	for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
-	    H5FS_stat_t	fs_stat;	/* Information for free-space manager */
-
-	    /* Check for free space manager of this type */
-            if(f->shared->fs_man[type]) {
-		/* Switch to "about to be deleted" state */
-                f->shared->fs_state[type] = H5F_FS_STATE_DELETING;
-
-		/* Query the free space manager's information */
-                if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0)
-		    HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
-
-		/* Check if the free space manager has space in the file */
-                if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) {
-		    /* Delete the free space manager in the file */
-                     /* (will re-allocate later) */
-                    if(H5FS_free(f, f->shared->fs_man[type], dxpl_id) < 0)
-			HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
-		    f->shared->fs_addr[type] = HADDR_UNDEF;
-		} /* end if */
-	    } /* end iif */
-	    fsinfo.fs_addr[type-1] = HADDR_UNDEF;
-	} /* end for */
+        /* Note that unlike the previous version of this code, we do not 
+         * delete free space managers that have no section to store.
+         *
+         * Can't do this, as that would involve freeing file space, which would 
+         * dirty the free space manager in question. 
+         * 
+         * Fortunately, the code doesn't seem to care about this.
+         */
 
+        /* Gather data for the free space manager superblock extension message.
+         * 
+         * In passing, verify that all the free space managers are closed.
+         */
+        for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
+            fsinfo.fs_addr[type - 1] = f->shared->fs_addr[type];
 	fsinfo.strategy = f->shared->fs_strategy;
 	fsinfo.threshold = f->shared->fs_threshold;
 
-	/* Write free-space manager info message to superblock extension object header */
-	/* Create the superblock extension object header in advance if needed */
-	if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE) < 0)
-	    HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
-
-	/* Re-allocate free-space manager header and/or section info header */
-	for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
-	    H5FS_stat_t	fs_stat;		/* Information for free-space manager */
-
-	    /* Check for active free space manager of this type */
-            if(f->shared->fs_man[type]) {
-		/* Re-query free space manager info for this type */
-                if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0)
-		    HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
-
-		/* Are there sections to persist? */
-                if(fs_stat.serial_sect_count) {
-		    /* Allocate space for free-space manager header */
-		    if(H5FS_alloc_hdr(f, f->shared->fs_man[type], &f->shared->fs_addr[type], dxpl_id) < 0)
-			HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "can't allocated free-space header")
-
-		    /* Allocate space for free-space maanger section info header */
-		    if(H5FS_alloc_sect(f, f->shared->fs_man[type], dxpl_id) < 0)
-			HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate free-space section info")
-
-		    HDassert(f->shared->fs_addr[type]);
-		    fsinfo.fs_addr[type-1] = f->shared->fs_addr[type];
-		    update = TRUE;
-		} /* end if */
-	    } else if(H5F_addr_defined(f->shared->fs_addr[type])) {
-		fsinfo.fs_addr[type-1] = f->shared->fs_addr[type];
-		update = TRUE;
-	    } /* end else-if */
-	} /* end for */
-
-	/* Update the free space manager info message in superblock extension object header */
-	if(update)
-	    if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE) < 0)
-	        HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
+        /* Write the free space manager message -- message must already exist */
+        if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE) < 0)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
 
 	/* Final close of free-space managers */
 	for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
@@ -1564,6 +2143,11 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
 	    f->shared->fs_addr[type] = HADDR_UNDEF;
 	} /* end for */
 
+        /* Verify that we haven't dirtied any metadata cache entries 
+         * from the metadata free space manager ring out.
+         */
+        HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
+
         /* Verify that the aggregators are still shutdown. */
         HDassert(f->shared->sdata_aggr.tot_size == 0);
         HDassert(f->shared->sdata_aggr.addr == 0);