[svn-r22666] Description:

    Merge "flush me last" & "flush me collectively" feature from the
avoid_truncate branch back to the trunk.  (To help enable allowing the
free space section info to reside in temporary address space)

Tested on:
    Mac OSX/64 10.7.4 (amazon) w/debug, gcc 4.7.x, C++, FORTRAN & threadsafe
    (h5committest forthcoming)

7 files changed, 401 insertions, 181 deletions

diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index 456eb0d..26fa051 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -321,6 +321,8 @@ H5_DLLVAR hid_t H5AC_ind_dxpl_id;
 #define H5AC__FLUSH_IGNORE_PROTECTED_FLAG H5C__FLUSH_IGNORE_PROTECTED_FLAG
 #define H5AC__FREE_FILE_SPACE_FLAG	  H5C__FREE_FILE_SPACE_FLAG
 #define H5AC__TAKE_OWNERSHIP_FLAG         H5C__TAKE_OWNERSHIP_FLAG
+#define H5AC__FLUSH_LAST_FLAG		  H5C__FLUSH_LAST_FLAG
+#define H5AC__FLUSH_COLLECTIVELY_FLAG	  H5C__FLUSH_COLLECTIVELY_FLAG
 
 
 /* #defines of flags used to report entry status in the
diff --git a/src/H5C.c b/src/H5C.c
index f651fb2..c229a46 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -414,12 +414,6 @@ done:
  * Programmer:  John Mainzer
  *              3/17/10
  *
- * Modifications:
- *
- *		Heavily reworked to have each process flush a group of 
- *		adjacent entries.
- *						JRM -- 4/15/10
- *
  *-------------------------------------------------------------------------
  */
 #ifdef H5_HAVE_PARALLEL
@@ -442,8 +436,13 @@ H5C_apply_candidate_list(H5F_t * f,
     int			last_entry_to_flush;
     int			entries_to_clear = 0;
     int			entries_to_flush = 0;
+    int			entries_to_flush_or_clear_last = 0;
+    int			entries_to_flush_collectively = 0;
     int			entries_cleared = 0;
     int			entries_flushed = 0;
+    int			entries_delayed = 0;
+    int			entries_flushed_or_cleared_last = 0;
+    int			entries_flushed_collectively = 0;
     int			entries_examined = 0;
     int			initial_list_len;
     int               * candidate_assignment_table = NULL;
@@ -451,6 +450,7 @@ H5C_apply_candidate_list(H5F_t * f,
     H5C_cache_entry_t *	clear_ptr = NULL;
     H5C_cache_entry_t *	entry_ptr = NULL;
     H5C_cache_entry_t *	flush_ptr = NULL;
+    H5C_cache_entry_t * delayed_ptr = NULL;
 #if H5C_DO_SANITY_CHECKS
     haddr_t		last_addr;
 #endif /* H5C_DO_SANITY_CHECKS */
@@ -612,12 +612,28 @@ H5C_apply_candidate_list(H5F_t * f,
               (int)(cache_ptr->LRU_list_len));
 #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
 
+    /* ====================================================================== *
+     * Now scan the LRU and PEL lists, flushing or clearing entries as
+     * needed.
+     *
+     * The flush_me_last and flush_me_collectively flags may dictate how or
+     * when some entries can be flushed, and should be addressed here.
+     * However, in their initial implementation, these flags only apply to the
+     * superblock, so there's only a relatively small change to this function
+     * to account for this one case where they come into play. If these flags
+     * are ever expanded upon, this function and the following flushing steps
+     * should be reworked to account for additional cases.
+     * ====================================================================== */
+
     entries_examined = 0;
     initial_list_len = cache_ptr->LRU_list_len;
     entry_ptr = cache_ptr->LRU_tail_ptr;
 
+    /* Examine each entry in the LRU list */
     while((entry_ptr != NULL) && (entries_examined <= initial_list_len) &&
             ((entries_cleared + entries_flushed) < num_candidates)) {
+
+        /* If this process needs to clear this entry. */
         if(entry_ptr->clear_on_unprotect) {
             entry_ptr->clear_on_unprotect = FALSE;
             clear_ptr = entry_ptr;
@@ -625,7 +641,7 @@ H5C_apply_candidate_list(H5F_t * f,
             entries_cleared++;
 
 #if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
-    HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, 
+    HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank,
               (long long)clear_ptr->addr);
 #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
 
@@ -638,14 +654,18 @@ H5C_apply_candidate_list(H5F_t * f,
                                       &first_flush,
                                       TRUE) < 0)
                 HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
-        } else if(entry_ptr->flush_immediately) {
+        } /* end if */
+
+        /* Else, if this process needs to flush this entry. */
+        else if (entry_ptr->flush_immediately) {
+
             entry_ptr->flush_immediately = FALSE;
             flush_ptr = entry_ptr;
             entry_ptr = entry_ptr->prev;
             entries_flushed++;
 
 #if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
-    HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, 
+    HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank,
               (long long)flush_ptr->addr);
 #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
 
@@ -657,17 +677,20 @@ H5C_apply_candidate_list(H5F_t * f,
                                       H5C__NO_FLAGS_SET,
                                       &first_flush,
                                       TRUE) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
-        } else {
+                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry.")
+        } /* end else-if */
+
+        /* Otherwise, no action to be taken on this entry. Grab the next. */
+        else {
             entry_ptr = entry_ptr->prev;
-        }
+        } /* end else */
 
         entries_examined++;
     } /* end while */
 
 #if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %d/%d/%d.\n", 
-              FUNC, mpi_rank, entries_examined, 
+    HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %d/%d/%d.\n",
+              FUNC, mpi_rank, entries_examined,
               entries_cleared, entries_flushed);
 #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
 
@@ -676,69 +699,168 @@ H5C_apply_candidate_list(H5F_t * f,
      */
 
 #if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n", 
+    HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n",
              FUNC, mpi_rank, (int)(cache_ptr->pel_len));
 #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
 
     entry_ptr = cache_ptr->pel_head_ptr;
     while((entry_ptr != NULL) &&
-            ((entries_cleared + entries_flushed) < num_candidates)) {
-        if(entry_ptr->clear_on_unprotect) {
-            entry_ptr->clear_on_unprotect = FALSE;
-            clear_ptr = entry_ptr;
-            entry_ptr = entry_ptr->next;
-            entries_cleared++;
+          ((entries_cleared + entries_flushed + entries_delayed)
+            < num_candidates)) {
+
+        /* If entry is marked for flush or for clear */
+        if((entry_ptr->clear_on_unprotect||entry_ptr->flush_immediately)) {
+
+            /* If this entry needs to be flushed last */
+            if (entry_ptr->flush_me_last) {
+
+                /* At this time, only the superblock supports being
+                   flushed last. Conveniently, it also happens to be the only
+                   entry that supports being flushed collectively, as well. Also
+                   conveniently, it's always pinned, so we only need to check
+                   for it while scanning the PEL here. Finally, it's never
+                   included in a candidate list that excludes other dirty
+                   entries in a cache, so we can handle this relatively simple
+                   case here.
+                   
+                   For now, this function asserts this and saves the entry
+                   to flush it after scanning the rest of the PEL list.
+
+                   If there are ever more entries that either need to be
+                   flushed last and/or flushed collectively, this whole routine
+                   will need to be reworked to handle all additional cases. As
+                   it is the simple case of a single pinned entry needing
+                   flushed last and collectively is just a minor addition to
+                   this routine, but signficantly buffing up the usage of
+                   flush_me_last or flush_me_collectively will require a more
+                   intense rework of this function and potentially the function
+                   of candidate lists as a whole. */
+
+                HDassert(entry_ptr->flush_me_collectively);
+                entries_to_flush_or_clear_last++;
+                entries_to_flush_collectively++;
+                HDassert(entries_to_flush_or_clear_last == 1);
+                HDassert(entries_to_flush_collectively == 1);
+
+                /* Delay the entry. It will be flushed later. */
+                delayed_ptr = entry_ptr;
+                entries_delayed++;
+                HDassert(entries_delayed == 1);
+
+            } /* end if */
+
+            /* Else, this process needs to clear this entry. */
+            else if (entry_ptr->clear_on_unprotect) {
+                HDassert(!entry_ptr->flush_immediately);
+                entry_ptr->clear_on_unprotect = FALSE;
+                clear_ptr = entry_ptr;
+                entry_ptr = entry_ptr->next;
+                entries_cleared++;
 
 #if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
-            HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, 
+            HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank,
                       (long long)clear_ptr->addr);
 #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
 
-            if(H5C_flush_single_entry(f,
-                                      primary_dxpl_id,
-                                      secondary_dxpl_id,
-                                      clear_ptr->type,
-                                      clear_ptr->addr,
-                                      H5C__FLUSH_CLEAR_ONLY_FLAG,
-                                      &first_flush,
-                                      TRUE) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
-        } else if(entry_ptr->flush_immediately) {
-            entry_ptr->flush_immediately = FALSE;
-            flush_ptr = entry_ptr;
-            entry_ptr = entry_ptr->next;
-            entries_flushed++;
+                if(H5C_flush_single_entry(f,
+                                          primary_dxpl_id,
+                                          secondary_dxpl_id,
+                                          clear_ptr->type,
+                                          clear_ptr->addr,
+                                          H5C__FLUSH_CLEAR_ONLY_FLAG,
+                                          &first_flush,
+                                          TRUE) < 0)
+                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
+            } /* end else-if */
+
+            /* Else, if this process needs to independently flush this entry. */
+            else if (entry_ptr->flush_immediately) {
+                entry_ptr->flush_immediately = FALSE;
+                flush_ptr = entry_ptr;
+                entry_ptr = entry_ptr->next;
+                entries_flushed++;
 
 #if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
-            HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, 
+            HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank,
                       (long long)flush_ptr->addr);
 #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
 
-            if(H5C_flush_single_entry(f,
-                                      primary_dxpl_id,
-                                      secondary_dxpl_id,
-                                      flush_ptr->type,
-                                      flush_ptr->addr,
-                                      H5C__NO_FLAGS_SET,
-                                      &first_flush,
-                                      TRUE) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
-        } else {
+                if(H5C_flush_single_entry(f,
+                                          primary_dxpl_id,
+                                          secondary_dxpl_id,
+                                          flush_ptr->type,
+                                          flush_ptr->addr,
+                                          H5C__NO_FLAGS_SET,
+                                          &first_flush,
+                                          TRUE) < 0)
+                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry.")
+            } /* end else-if */
+        } /* end if */
+
+        /* Otherwise, this entry is not marked for flush or clear. Grab the next. */
+        else {
             entry_ptr = entry_ptr->next;
-        }
+        } /* end else */
+
     } /* end while */
 
 #if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    HDfprintf(stdout, 
-              "%s:%d: pel entries examined/cleared/flushed = %d/%d/%d.\n", 
-              FUNC, mpi_rank, entries_examined, 
+    HDfprintf(stdout,
+              "%s:%d: pel entries examined/cleared/flushed = %d/%d/%d.\n",
+              FUNC, mpi_rank, entries_examined,
               entries_cleared, entries_flushed);
     HDfprintf(stdout, "%s:%d: done.\n", FUNC, mpi_rank);
 
     fsync(stdout);
 #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
 
-    if((entries_flushed != entries_to_flush) || (entries_cleared != entries_to_clear))
+    /* ====================================================================== *
+     * Now, handle all delayed entries.                                       *
+     *                                                                        *
+     * This can *only* be the superblock at this time, so it's relatively     *
+     * easy to deal with. We're collectively flushing the entry saved from    *
+     * above. This will need to be handled differently if there are ever more *
+     * than one entry needing this special treatment.)                        *
+     * ====================================================================== */
+
+    if (delayed_ptr) {
+
+        if (delayed_ptr->clear_on_unprotect) {
+            entry_ptr->clear_on_unprotect = FALSE;
+            entries_cleared++;
+        } else if (delayed_ptr->flush_immediately) {
+            entry_ptr->flush_immediately = FALSE;
+            entries_flushed++;
+        } /* end if */
+
+        if(H5C_flush_single_entry(f,
+                                  primary_dxpl_id,
+                                  secondary_dxpl_id,
+                                  delayed_ptr->type,
+                                  delayed_ptr->addr,
+                                  H5C__NO_FLAGS_SET,
+                                  &first_flush,
+                                  TRUE) < 0)
+            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL,
+                        "Can't flush entry collectively.")
+
+        entries_flushed_collectively++;
+        entries_flushed_or_cleared_last++;
+    } /* end if */
+
+    /* ====================================================================== *
+     * Finished flushing everything.                                          *
+     * ====================================================================== */
+
+    HDassert((entries_flushed == entries_to_flush));
+    HDassert((entries_cleared == entries_to_clear));
+    HDassert((entries_flushed_or_cleared_last == entries_to_flush_or_clear_last));
+    HDassert((entries_flushed_collectively == entries_to_flush_collectively));
+    
+    if((entries_flushed != entries_to_flush) || 
+       (entries_cleared != entries_to_clear) ||
+       (entries_flushed_or_cleared_last != entries_to_flush_or_clear_last) ||
+       (entries_flushed_collectively != entries_to_flush_collectively))
         HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry count mismatch.")
 
 done:
@@ -924,7 +1046,8 @@ H5C_construct_candidate_list__min_clean(H5C_t * cache_ptr)
         entry_ptr = cache_ptr->dLRU_tail_ptr;
         while((nominated_entries_size < space_needed) &&
                 (nominated_entries_count < cache_ptr->slist_len) &&
-                (entry_ptr != NULL)) {
+                (entry_ptr != NULL) &&
+                (!entry_ptr->flush_me_last)) {
             haddr_t		nominated_addr;
 
             HDassert( ! (entry_ptr->is_protected) );
@@ -1771,8 +1894,12 @@ H5C_flush_cache(H5F_t *f, hid_t primary_dxpl_id, hid_t secondary_dxpl_id, unsign
                     HDassert( entry_ptr != NULL );
                     HDassert( entry_ptr->in_slist );
 
-                    if ( ( ! flush_marked_entries ) ||
-                         ( entry_ptr->flush_marker ) ) {
+                    if ( ( ( ! flush_marked_entries ) ||
+                           ( entry_ptr->flush_marker ) ) &&
+                         ( ( ! entry_ptr->flush_me_last ) ||
+                           ( ( entry_ptr->flush_me_last ) &&
+                             ( cache_ptr->num_last_entries >=
+                               cache_ptr->slist_len ) ) ) ) {
 
                         if ( entry_ptr->is_protected ) {
 
@@ -2524,6 +2651,10 @@ H5C_insert_entry(H5F_t *             f,
     herr_t		result;
     hbool_t		first_flush = TRUE;
     hbool_t		insert_pinned;
+    hbool_t     flush_last;
+#ifdef H5_HAVE_PARALLEL
+    hbool_t     flush_collectively;
+#endif
     hbool_t             set_flush_marker;
     hbool_t		write_permitted = TRUE;
     size_t		empty_space;
@@ -2562,8 +2693,12 @@ H5C_insert_entry(H5F_t *             f,
     }
 #endif /* H5C_DO_EXTREME_SANITY_CHECKS */
 
-    set_flush_marker = ( (flags & H5C__SET_FLUSH_MARKER_FLAG) != 0 );
-    insert_pinned    = ( (flags & H5C__PIN_ENTRY_FLAG) != 0 );
+    set_flush_marker   = ( (flags & H5C__SET_FLUSH_MARKER_FLAG) != 0 );
+    insert_pinned      = ( (flags & H5C__PIN_ENTRY_FLAG) != 0 );
+    flush_last         = ( (flags & H5C__FLUSH_LAST_FLAG) != 0 );
+#ifdef H5_HAVE_PARALLEL
+    flush_collectively = ( (flags & H5C__FLUSH_COLLECTIVELY_FLAG) != 0 );
+#endif
 
     entry_ptr = (H5C_cache_entry_t *)thing;
 
@@ -2604,6 +2739,10 @@ H5C_insert_entry(H5F_t *             f,
 
     entry_ptr->is_pinned = insert_pinned;
     entry_ptr->pinned_from_client = insert_pinned;
+    entry_ptr->flush_me_last = flush_last;
+#ifdef H5_HAVE_PARALLEL
+    entry_ptr->flush_me_collectively = flush_collectively;
+#endif
 
     /* newly inserted entries are assumed to be dirty */
     entry_ptr->is_dirty = TRUE;
@@ -7710,85 +7849,91 @@ H5C_flush_invalidate_cache(H5F_t * f,
                 HDassert( entry_ptr != NULL );
                 HDassert( entry_ptr->in_slist );
 
-#if H5C_DO_SANITY_CHECKS
-                /* update actual_slist_len & actual_slist_size before
-                 * the flush.  Note that the entry will be removed
-                 * from the slist after the flush, and thus may be
-                 * resized by the flush callback.  This is OK, as
-                 * we will catch the size delta in
-                 * cache_ptr->slist_size_increase.
-                 *
-                 * Note that we include pinned entries in this count, even
-                 * though we will not actually flush them.
-                 */
-                actual_slist_len++;
-                actual_slist_size += entry_ptr->size;
-#endif /* H5C_DO_SANITY_CHECKS */
+                if ( ( ! entry_ptr->flush_me_last ) ||
+                     ( ( entry_ptr->flush_me_last ) &&
+                       ( cache_ptr->num_last_entries >=
+                         cache_ptr->slist_len ) ) ) {
+
+    #if H5C_DO_SANITY_CHECKS
+                    /* update actual_slist_len & actual_slist_size before
+                     * the flush.  Note that the entry will be removed
+                     * from the slist after the flush, and thus may be
+                     * resized by the flush callback.  This is OK, as
+                     * we will catch the size delta in
+                     * cache_ptr->slist_size_increase.
+                     *
+                     * Note that we include pinned entries in this count, even
+                     * though we will not actually flush them.
+                     */
+                    actual_slist_len++;
+                    actual_slist_size += entry_ptr->size;
+    #endif /* H5C_DO_SANITY_CHECKS */
 
-                if ( entry_ptr->is_protected ) {
+                    if ( entry_ptr->is_protected ) {
 
-                    /* we have major problems -- but lets flush
-                     * everything we can before we flag an error.
-                     */
-                    protected_entries++;
+                        /* we have major problems -- but lets flush
+                         * everything we can before we flag an error.
+                         */
+                        protected_entries++;
 
-                } else if ( entry_ptr->is_pinned ) {
+                    } else if ( entry_ptr->is_pinned ) {
 
-                    /* Test to see if we are can flush the entry now.
-                     * If we can, go ahead and flush, but don't tell
-                     * H5C_flush_single_entry() to destroy the entry
-                     * as pinned entries can't be evicted.
-                     */
-                    if(entry_ptr->flush_dep_height == curr_flush_dep_height ) {
-                        status = H5C_flush_single_entry(f,
-                                                        primary_dxpl_id,
-                                                        secondary_dxpl_id,
-                                                        NULL,
-                                                        entry_ptr->addr,
-                                                        H5C__NO_FLAGS_SET,
-                                                        &first_flush,
-                                                        FALSE);
-                        if ( status < 0 ) {
-
-                            /* This shouldn't happen -- if it does, we are toast
-                             * so just scream and die.
-                             */
+                        /* Test to see if we are can flush the entry now.
+                         * If we can, go ahead and flush, but don't tell
+                         * H5C_flush_single_entry() to destroy the entry
+                         * as pinned entries can't be evicted.
+                         */
+                        if(entry_ptr->flush_dep_height == curr_flush_dep_height ) {
+                            status = H5C_flush_single_entry(f,
+                                                            primary_dxpl_id,
+                                                            secondary_dxpl_id,
+                                                            NULL,
+                                                            entry_ptr->addr,
+                                                            H5C__NO_FLAGS_SET,
+                                                            &first_flush,
+                                                            FALSE);
+                            if ( status < 0 ) {
+
+                                /* This shouldn't happen -- if it does, we are toast
+                                 * so just scream and die.
+                                 */
 
-                            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
-                                        "dirty pinned entry flush failed.")
+                                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
+                                            "dirty pinned entry flush failed.")
+                            } /* end if */
+                            flushed_during_dep_loop = TRUE;
                         } /* end if */
-                        flushed_during_dep_loop = TRUE;
+                        else if(entry_ptr->flush_dep_height < curr_flush_dep_height)
+                            /* This shouldn't happen -- if it does, just scream and die.  */
+                            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL,  "dirty entry below current flush dep. height.")
                     } /* end if */
-                    else if(entry_ptr->flush_dep_height < curr_flush_dep_height)
-                        /* This shouldn't happen -- if it does, just scream and die.  */
-                        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL,  "dirty entry below current flush dep. height.")
-                } /* end if */
-                else {
-                    if(entry_ptr->flush_dep_height == curr_flush_dep_height ){
+                    else {
+                        if(entry_ptr->flush_dep_height == curr_flush_dep_height ){
 
-                        status = H5C_flush_single_entry(f,
-                                                        primary_dxpl_id,
-                                                        secondary_dxpl_id,
-                                                        NULL,
-                                                        entry_ptr->addr,
-                                                        (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG),
-                                                        &first_flush,
-                                                        TRUE);
-                        if ( status < 0 ) {
-
-                            /* This shouldn't happen -- if it does, we are toast so
-                             * just scream and die.
-                             */
+                            status = H5C_flush_single_entry(f,
+                                                            primary_dxpl_id,
+                                                            secondary_dxpl_id,
+                                                            NULL,
+                                                            entry_ptr->addr,
+                                                            (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG),
+                                                            &first_flush,
+                                                            TRUE);
+                            if ( status < 0 ) {
 
-                            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
-                                        "dirty entry flush destroy failed.")
+                                /* This shouldn't happen -- if it does, we are toast so
+                                 * just scream and die.
+                                 */
+
+                                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
+                                            "dirty entry flush destroy failed.")
+                            } /* end if */
+                            flushed_during_dep_loop = TRUE;
                         } /* end if */
-                        flushed_during_dep_loop = TRUE;
-                    } /* end if */
-                    else if(entry_ptr->flush_dep_height < curr_flush_dep_height)
-                        /* This shouldn't happen -- if it does, just scream and die.  */
-                        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL,  "dirty entry below current flush dep. height.")
-                } /* end else */
+                        else if(entry_ptr->flush_dep_height < curr_flush_dep_height)
+                            /* This shouldn't happen -- if it does, just scream and die.  */
+                            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL,  "dirty entry below current flush dep. height.")
+                    } /* end else */
+                } /* end if */
             } /* end while loop scanning skip list */
 
 #if H5C_DO_SANITY_CHECKS
@@ -7835,45 +7980,52 @@ H5C_flush_invalidate_cache(H5F_t * f,
                     next_entry_ptr = entry_ptr->ht_next;
                     HDassert ( ( next_entry_ptr == NULL ) ||
                                ( next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC ) );
-                    if ( entry_ptr->is_protected ) {
-
-                        /* we have major problems -- but lets flush and destroy
-                         * everything we can before we flag an error.
-                         */
-                        protected_entries++;
 
-                        if ( ! entry_ptr->in_slist ) {
+                    if ( ( ! entry_ptr->flush_me_last ) ||
+                         ( ( entry_ptr->flush_me_last ) &&
+                           ( cache_ptr->num_last_entries >=
+                             cache_ptr->slist_len ) ) ) {
 
-                            HDassert( !(entry_ptr->is_dirty) );
-                        }
-                    } else if ( ! ( entry_ptr->is_pinned ) ) {
+                        if ( entry_ptr->is_protected ) {
 
-                        /* Test to see if we are can flush the entry now.
-                         * If we can, go ahead and flush.
-                         */
-                        if(entry_ptr->flush_dep_height == curr_flush_dep_height ){
-                            status = H5C_flush_single_entry(f,
-                                                            primary_dxpl_id,
-                                                            secondary_dxpl_id,
-                                                            NULL,
-                                                            entry_ptr->addr,
-                                                            (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG),
-                                                            &first_flush,
-                                                            TRUE);
-                            if ( status < 0 ) {
+                            /* we have major problems -- but lets flush and destroy
+                             * everything we can before we flag an error.
+                             */
+                            protected_entries++;
 
-                                /* This shouldn't happen -- if it does, we are toast so
-                                 * just scream and die.
-                                 */
+                            if ( ! entry_ptr->in_slist ) {
 
-                                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
-                                            "Entry flush destroy failed.")
+                                HDassert( !(entry_ptr->is_dirty) );
                             }
-                            flushed_during_dep_loop = TRUE;
+                        } else if ( ! ( entry_ptr->is_pinned ) ) {
+
+                            /* Test to see if we are can flush the entry now.
+                             * If we can, go ahead and flush.
+                             */
+                            if(entry_ptr->flush_dep_height == curr_flush_dep_height ){
+                                status = H5C_flush_single_entry(f,
+                                                                primary_dxpl_id,
+                                                                secondary_dxpl_id,
+                                                                NULL,
+                                                                entry_ptr->addr,
+                                                                (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG),
+                                                                &first_flush,
+                                                                TRUE);
+                                if ( status < 0 ) {
+
+                                    /* This shouldn't happen -- if it does, we are toast so
+                                     * just scream and die.
+                                     */
+
+                                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
+                                                "Entry flush destroy failed.")
+                                }
+                                flushed_during_dep_loop = TRUE;
+                            } /* end if */
+                            else if(entry_ptr->flush_dep_height < curr_flush_dep_height)
+                                /* This shouldn't happen -- if it does, just scream and die.  */
+                                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL,  "dirty entry below current flush dep. height.")
                         } /* end if */
-                        else if(entry_ptr->flush_dep_height < curr_flush_dep_height)
-                            /* This shouldn't happen -- if it does, just scream and die.  */
-                            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL,  "dirty entry below current flush dep. height.")
                     } /* end if */
                     /* We can't do anything if the entry is pinned.  The
                      * hope is that the entry will be unpinned as the
diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h
index 37068f1..909578b 100644
--- a/src/H5Cpkg.h
+++ b/src/H5Cpkg.h
@@ -282,6 +282,16 @@
  *                 don't use this at present, I hope that this will allow
  *                 some optimizations when I get to it.
  *
+ * num_last_entries: The number of entries in the cache that can only be
+ *                   flushed after all other entries in the cache have
+ *                   been flushed. At this time, this will only ever be
+ *                   one entry (the superblock), and the code has been
+ *                   protected with HDasserts to enforce this. This restraint
+ *                   can certainly be relaxed in the future if the need for
+ *                   multiple entries being flushed last arises, though
+ *                   explicit tests for that case should be added when said
+ *                   HDasserts are removed.
+ *
  * With the addition of the fractal heap, the cache must now deal with
  * the case in which entries may be dirtied, moved, or have their sizes
  * changed during a flush.  To allow sanity checks in this situation, the
@@ -878,6 +888,7 @@ struct H5C_t
     int32_t                     slist_len;
     size_t                      slist_size;
     H5SL_t *                    slist_ptr;
+    int32_t                     num_last_entries;
 #if H5C_DO_SANITY_CHECKS
     int64_t			slist_len_increase;
     int64_t			slist_size_increase;
@@ -1964,6 +1975,10 @@ if ( (cache_ptr)->index_size !=                                             \
     } else {                                                 \
 	(cache_ptr)->clean_index_size += (entry_ptr)->size;  \
     }                                                        \
+    if ((entry_ptr)->flush_me_last) {                        \
+        (cache_ptr)->num_last_entries++;                     \
+        HDassert((cache_ptr)->num_last_entries == 1);        \
+    }                                                        \
     H5C__UPDATE_STATS_FOR_HT_INSERTION(cache_ptr)            \
 }
 
@@ -1993,6 +2008,10 @@ if ( (cache_ptr)->index_size !=                                             \
     } else {                                                  \
 	(cache_ptr)->clean_index_size -= (entry_ptr)->size;   \
     }                                                         \
+    if ((entry_ptr)->flush_me_last) {                         \
+        (cache_ptr)->num_last_entries--;                      \
+        HDassert((cache_ptr)->num_last_entries == 0);         \
+    }                                                         \
     H5C__UPDATE_STATS_FOR_HT_DELETION(cache_ptr)              \
 }
 
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index 0c7631a..7fde69b 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -371,6 +371,23 @@ typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr,
  *		H5C__FLUSH_MARKED_ENTRIES_FLAG.  The flag is reset when
  *		the entry is flushed for whatever reason.
  *
+ * flush_me_last:  Boolean flag indicating that this entry should not be
+ *                 flushed from the cache until all other entries without
+ *                 the flush_me_last flag set have been flushed.
+ *
+ * flush_me_collectively:  Boolean flag indicating that this entry needs
+ *                         to be flushed collectively when in a parallel
+ *                         situation.
+ * 
+ *      Note: At this time, the flush_me_last and flush_me_collectively
+ *            flags will only be applied to one entry, the superblock,
+ *            and the code utilizing these flags is protected with HDasserts
+ *            to enforce this. This restraint can certainly be relaxed in
+ *            the future if the the need for multiple entries getting flushed
+ *            last or collectively arises, though the code allowing for that
+ *            will need to be expanded and tested appropriately if that
+ *            functionality is desired.
+ *
  * clear_on_unprotect:  Boolean flag used only in PHDF5.  When H5C is used
  *		to implement the metadata cache In the parallel case, only
  *		the cache with mpi rank 0 is allowed to actually write to
@@ -578,7 +595,7 @@ typedef struct H5C_cache_entry_t
     haddr_t			addr;
     size_t			size;
     const H5C_class_t *		type;
-    haddr_t		    tag;
+    haddr_t		        tag;
     hbool_t			is_dirty;
     hbool_t			dirtied;
     hbool_t			is_protected;
@@ -587,13 +604,15 @@ typedef struct H5C_cache_entry_t
     hbool_t			is_pinned;
     hbool_t			in_slist;
     hbool_t			flush_marker;
+    hbool_t                     flush_me_last;
 #ifdef H5_HAVE_PARALLEL
+    hbool_t                     flush_me_collectively;
     hbool_t			clear_on_unprotect;
-    hbool_t		flush_immediately;
+    hbool_t		        flush_immediately;
 #endif /* H5_HAVE_PARALLEL */
     hbool_t			flush_in_progress;
     hbool_t			destroy_in_progress;
-    hbool_t		free_file_space_on_destroy;
+    hbool_t		        free_file_space_on_destroy;
 
     /* fields supporting the 'flush dependency' feature: */
 
@@ -1042,6 +1061,8 @@ typedef struct H5C_auto_size_ctl_t
 #define H5C__READ_ONLY_FLAG			0x0200
 #define H5C__FREE_FILE_SPACE_FLAG		0x0800
 #define H5C__TAKE_OWNERSHIP_FLAG		0x1000
+#define H5C__FLUSH_LAST_FLAG			0x2000
+#define H5C__FLUSH_COLLECTIVELY_FLAG		0x4000
 
 #ifdef H5_HAVE_PARALLEL
 H5_DLL herr_t H5C_apply_candidate_list(H5F_t * f,
diff --git a/src/H5FDmpiposix.c b/src/H5FDmpiposix.c
index e6c23f0..0ad39a9 100644
--- a/src/H5FDmpiposix.c
+++ b/src/H5FDmpiposix.c
@@ -89,12 +89,12 @@ typedef struct H5FD_mpiposix_t {
     MPI_Comm  comm;    /*communicator        */
     int         mpi_rank;       /* This process's rank                  */
     int         mpi_size;       /* Total number of processes            */
-    haddr_t  eof;    /*end-of-file marker      */
-    haddr_t  eoa;    /*end-of-address marker      */
-    haddr_t  last_eoa;  /* Last known end-of-address marker  */
-    haddr_t  pos;    /* Current file I/O position          */
-    int    op;    /* Last file I/O operation    */
-    hsize_t  naccess;  /* Number of (write) accesses to file   */
+    haddr_t	eof;		/*end-of-file marker			*/
+    haddr_t	eoa;		/*end-of-address marker			*/
+    haddr_t	last_eoa;	/* Last known end-of-address marker	*/
+    haddr_t	pos;		/* Current file I/O position	        */
+    int		op;		/* Last file I/O operation		*/
+    hsize_t	naccess;	/* Number of (write) accesses to file   */
 #ifdef H5_HAVE_GPFS
     size_t      blksize;        /* Block size of file system            */
 #endif
@@ -230,13 +230,13 @@ static const H5FD_class_mpi_t H5FD_mpiposix_g = {
     H5FD_mpiposix_set_eoa,       /*set_eoa    */
     H5FD_mpiposix_get_eof,      /*get_eof    */
     H5FD_mpiposix_get_handle,                   /*get_handle            */
-    H5FD_mpiposix_read,        /*read      */
-    H5FD_mpiposix_write,      /*write      */
+    H5FD_mpiposix_read,				/*read			*/
+    H5FD_mpiposix_write,			/*write			*/
     NULL,          /*flush      */
-    H5FD_mpiposix_truncate,      /*truncate    */
+    H5FD_mpiposix_truncate,			/*truncate		*/
     NULL,                                       /*lock                  */
     NULL,                                       /*unlock                */
-    H5FD_FLMAP_SINGLE         /*fl_map    */
+    H5FD_FLMAP_SINGLE 				/*fl_map		*/
     },  /* End of superclass information */
     H5FD_mpiposix_mpi_rank,                     /*get_rank              */
     H5FD_mpiposix_mpi_size,                     /*get_size              */
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index 8bf4405..8205392 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -516,7 +516,7 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id)
         HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to set EOA value for superblock")
 
     /* Insert superblock into cache, pinned */
-    if(H5AC_insert_entry(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, H5AC__PIN_ENTRY_FLAG) < 0)
+    if(H5AC_insert_entry(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, H5AC__PIN_ENTRY_FLAG | H5AC__FLUSH_LAST_FLAG | H5AC__FLUSH_COLLECTIVELY_FLAG) < 0)
         HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, "can't add superblock to cache")
     sblock_in_cache = TRUE;
 
diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c
index cc1137b..48fd139 100644
--- a/src/H5Fsuper_cache.c
+++ b/src/H5Fsuper_cache.c
@@ -153,6 +153,12 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata)
     if(NULL == (sblock = H5FL_CALLOC(H5F_super_t)))
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
 
+    /* The superblock must be flushed last (and collectively in parallel) */
+    sblock->cache_info.flush_me_last = TRUE;
+#ifdef H5_HAVE_PARALLEL
+    sblock->cache_info.flush_me_collectively = TRUE;
+#endif
+
     /* Read fixed-size portion of the superblock */
     p = sbuf;
     H5_CHECK_OVERFLOW(fixed_size, size_t, haddr_t);
@@ -638,6 +644,15 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr,
     HDassert(f);
     HDassert(H5F_addr_eq(addr, 0));
     HDassert(sblock);
+    
+    /* Assert that the superblock is marked as being flushed last (and 
+       collectively in parallel) */
+    /* (We'll rely on the cache to make sure it actually *is* flushed
+       last (and collectively in parallel), but this check doesn't hurt) */
+    HDassert(sblock->cache_info.flush_me_last);    
+#ifdef H5_HAVE_PARALLEL
+    HDassert(sblock->cache_info.flush_me_collectively);
+#endif
 
     if(sblock->cache_info.is_dirty) {
         uint8_t         buf[H5F_MAX_SUPERBLOCK_SIZE + H5F_MAX_DRVINFOBLOCK_SIZE];  /* Superblock & driver info blockencoding buffer */
@@ -677,10 +692,15 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr,
                 *p++ = 0;   /*reserved */
             } /* end if */
 
+            /* Encode the base address */
             H5F_addr_encode(f, &p, sblock->base_addr);
+
+            /* Encode the address of global free-space index */
             H5F_addr_encode(f, &p, sblock->ext_addr);
             rel_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER);
             H5F_addr_encode(f, &p, (rel_eoa + sblock->base_addr));
+
+            /* Encode the driver informaton block address */
             H5F_addr_encode(f, &p, sblock->driver_addr);
 
             /* Encode the root group object entry, including the cached stab info */
@@ -731,9 +751,12 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr,
             *p++ = (uint8_t)H5F_SIZEOF_SIZE(f);
             *p++ = sblock->status_flags;
 
-            /* Base, superblock extension & end of file addresses */
+            /* Encode the base address */
             H5F_addr_encode(f, &p, sblock->base_addr);
+
+            /* Encode the address of the superblock extension */
             H5F_addr_encode(f, &p, sblock->ext_addr);
+
             rel_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER);
             H5F_addr_encode(f, &p, (rel_eoa + sblock->base_addr));
 
@@ -767,13 +790,18 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr,
 
         /* Check for newer version of superblock format & superblock extension */
         if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 && H5F_addr_defined(sblock->ext_addr)) {
+            H5O_loc_t 	ext_loc;        /* "Object location" for superblock extension */
+
+            /* Open the superblock extension's object header */
+            if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0)
+                HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension")
+
             /* Check for ignoring the driver info for this file */
             if(!H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) {
                 /* Check for driver info message */
                 H5_ASSIGN_OVERFLOW(driver_size, H5FD_sb_size(f->shared->lf), hsize_t, size_t);
                 if(driver_size > 0) {
                     H5O_drvinfo_t drvinfo;      /* Driver info */
-                    H5O_loc_t 	ext_loc; 	/* "Object location" for superblock extension */
                     uint8_t dbuf[H5F_MAX_DRVINFOBLOCK_SIZE];  /* Driver info block encoding buffer */
 
                     /* Sanity check */
@@ -783,21 +811,19 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr,
                     if(H5FD_sb_encode(f->shared->lf, drvinfo.name, dbuf) < 0)
                         HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to encode driver information")
 
-                    /* Open the superblock extension's object header */
-                    if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0)
-                        HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension")
-
                     /* Write driver info information to the superblock extension */
                     drvinfo.len = driver_size;
                     drvinfo.buf = dbuf;
                     if(H5O_msg_write(&ext_loc, H5O_DRVINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &drvinfo, dxpl_id) < 0)
                         HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message")
 
-                    /* Close the superblock extension object header */
-                    if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0)
-                        HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension")
                 } /* end if */
+
             } /* end if */
+
+            /* Close the superblock extension object header */
+            if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0)
+                HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension")
         } /* end if */
 
         /* Reset the dirty flag.  */