[svn-r17411] Description:

	Make H5AC_flush just flush the cache and make H5AC_dest perform the
proper parallel synchronization before destroying the cache.  Also, further
discriminate between 'closing' and 'non-closing' actions in H5F_flush.

Tested on:
        FreeBSD/32 6.3 (duty) in debug mode
        FreeBSD/64 6.3 (liberty) w/C++ & FORTRAN, in debug mode
        Linux/32 2.6 (kagiso) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe,
                                in debug mode
        Linux/64-amd64 2.6 (smirom) w/Intel compilers w/default API=1.6.x,
                                w/C++ & FORTRAN, in production mode
        Solaris/32 2.10 (linew) w/deprecated symbols disabled, w/C++ & FORTRAN,
                                w/szip filter, in production mode
        Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN,
                                in production mode
        Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in production mode
        Linux/64-amd64 2.6 (abe) w/parallel, w/FORTRAN, in production mode
        Mac OS X/32 10.5.8 (amazon) in debug mode
        Mac OS X/32 10.5.8 (amazon) w/C++ & FORTRAN, w/threadsafe,
                                in production mode

8 files changed, 209 insertions, 340 deletions

diff --git a/src/H5AC.c b/src/H5AC.c
index 9bfc32a..9fd2e8f 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -193,6 +193,8 @@ static herr_t H5AC_receive_and_apply_clean_list(H5F_t  * f,
 static herr_t H5AC_log_renamed_entry(H5AC_t * cache_ptr,
                                      haddr_t old_addr,
                                      haddr_t new_addr);
+
+static herr_t H5AC_flush_entries(H5F_t *f);
 #endif /* H5_HAVE_PARALLEL */
 
 
@@ -765,64 +767,52 @@ done:
 herr_t
 H5AC_dest(H5F_t *f, hid_t dxpl_id)
 {
-    H5AC_t *cache = NULL;
-    herr_t ret_value=SUCCEED;      /* Return value */
 #ifdef H5_HAVE_PARALLEL
     H5AC_aux_t * aux_ptr = NULL;
 #endif /* H5_HAVE_PARALLEL */
+    herr_t ret_value = SUCCEED;      /* Return value */
 
     FUNC_ENTER_NOAPI(H5AC_dest, FAIL)
 
-    assert(f);
-    assert(f->shared->cache);
-    cache = f->shared->cache;
-#ifdef H5_HAVE_PARALLEL
-    aux_ptr = cache->aux_ptr;
-
-    if ( aux_ptr != NULL ) {
-
-        HDassert ( aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC );
-    }
-#endif /* H5_HAVE_PARALLEL */
+    /* Sanity check */
+    HDassert(f);
+    HDassert(f->shared->cache);
 
 #if H5AC__TRACE_FILE_ENABLED
-    if ( H5AC_close_trace_file(cache) < 0 ) {
-
-        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                    "H5AC_close_trace_file() failed.")
-    }
+    if(H5AC_close_trace_file(f->shared->cache) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_close_trace_file() failed.")
 #endif /* H5AC__TRACE_FILE_ENABLED */
 
-    if ( H5C_dest(f, dxpl_id, H5AC_noblock_dxpl_id, cache) < 0 ) {
+#ifdef H5_HAVE_PARALLEL
+    aux_ptr = f->shared->cache->aux_ptr;
+    if(aux_ptr)
+        /* Sanity check */
+        HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC);
 
-        HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "can't destroy cache")
-    }
+    /* Attempt to flush all entries from rank 0 & Bcast clean list to other ranks */
+    if(H5AC_flush_entries(f) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush.")
+#endif /* H5_HAVE_PARALLEL */
 
+    /* Destroy the cache */
+    if(H5C_dest(f, dxpl_id, H5AC_noblock_dxpl_id, f->shared->cache) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "can't destroy cache")
     f->shared->cache = NULL;
 
 #ifdef H5_HAVE_PARALLEL
-    if ( aux_ptr != NULL ) {
-
-        if ( aux_ptr->d_slist_ptr != NULL ) {
-
+    if(aux_ptr != NULL) {
+        if(aux_ptr->d_slist_ptr != NULL)
             H5SL_close(aux_ptr->d_slist_ptr);
-        }
-
-        if ( aux_ptr->c_slist_ptr != NULL ) {
-
+        if(aux_ptr->c_slist_ptr != NULL)
             H5SL_close(aux_ptr->c_slist_ptr);
-        }
-
         aux_ptr->magic = 0;
         H5FL_FREE(H5AC_aux_t, aux_ptr);
         aux_ptr = NULL;
-    }
+    } /* end if */
 #endif /* H5_HAVE_PARALLEL */
 
 done:
-
     FUNC_LEAVE_NOAPI(ret_value)
-
 } /* H5AC_dest() */
 
 
@@ -922,43 +912,11 @@ done:
  * Purpose:	Flush (and possibly destroy) the metadata cache associated
  *		with the specified file.
  *
- *		This is a re-write of an earlier version of the function
- *		which was reputedly capable of flushing (and destroying
- *		if requested) individual entries, individual entries if
- *		they match the supplied type, all entries of a given type,
- *		as well as all entries in the cache.
- *
- *		As only this last capability is actually used at present,
- *		I have not implemented the other capabilities in this
- *		version of the function.
- *
- *		The type and addr parameters are retained to avoid source
- *		code changed, but values other than NULL and HADDR_UNDEF
- *		respectively are errors.  If all goes well, they should
- *		be removed, and the function renamed to something more
- *		descriptive -- perhaps H5AC_flush_cache.
- *
  *		If the cache contains protected entries, the function will
  *		fail, as protected entries cannot be flushed.  However
  *		all unprotected entries should be flushed before the
  *		function returns failure.
  *
- *		For historical purposes, the original version of the
- *		purpose section is reproduced below:
- *
- *              ============ Original Version of "Purpose:" ============
- *
- *              Flushes (and destroys if DESTROY is non-zero) the specified
- *              entry from the cache.  If the entry TYPE is CACHE_FREE and
- *              ADDR is HADDR_UNDEF then all types of entries are
- *              flushed. If TYPE is CACHE_FREE and ADDR is defined then
- *              whatever is cached at ADDR is flushed.  Otherwise the thing
- *              at ADDR is flushed if it is the correct type.
- *
- *              If there are protected objects they will not be flushed.
- *              However, an attempt will be made to flush all non-protected
- *              items before this function returns failure.
- *
  * Return:      Non-negative on success/Negative on failure if there was a
  *              request to flush all items and something was protected.
  *
@@ -966,45 +924,16 @@ done:
  *              matzke@llnl.gov
  *              Jul  9 1997
  *
- * Modifications:
- * 		Robb Matzke, 1999-07-27
- *		The ADDR argument is passed by value.
- *
- *		Complete re-write. See above for details.  -- JRM 5/11/04
- *
- *		Abstracted the guts of the function to H5C_flush_cache()
- *		in H5C.c, and then re-wrote the function as a wrapper for
- *		H5C_flush_cache().
- *
- *                                                 JRM - 6/7/04
- *
- *		JRM - 7/5/05
- *		Modified function as part of a fix for a cache coherency
- *		bug in PHDF5.  See the header comments on the H5AC_aux_t
- *		structure for details.
- *
- *		JRM -- 5/11/06
- *		Added call to the write_done callback.
- *
- *		JRM -- 6/6/06
- * 		Added trace file support.
- *
  *-------------------------------------------------------------------------
  */
 herr_t
-H5AC_flush(H5F_t *f, hid_t dxpl_id, unsigned flags)
+H5AC_flush(H5F_t *f, hid_t dxpl_id)
 {
-    herr_t	  status;
-    herr_t	  ret_value = SUCCEED;      /* Return value */
-#ifdef H5_HAVE_PARALLEL
-    H5AC_aux_t	* aux_ptr = NULL;
-    int		  mpi_code;
-#endif /* H5_HAVE_PARALLEL */
 #if H5AC__TRACE_FILE_ENABLED
     char 	  trace[128] = "";
     FILE *	  trace_file_ptr = NULL;
 #endif /* H5AC__TRACE_FILE_ENABLED */
-
+    herr_t	  ret_value = SUCCEED;      /* Return value */
 
     FUNC_ENTER_NOAPI(H5AC_flush, FAIL)
 
@@ -1015,108 +944,31 @@ H5AC_flush(H5F_t *f, hid_t dxpl_id, unsigned flags)
     /* For the flush, only the flags are really necessary in the trace file.
      * Write the result to catch occult errors.
      */
-    if ( ( f != NULL ) &&
-         ( f->shared != NULL ) &&
-	 ( f->shared->cache != NULL ) &&
-	 ( H5C_get_trace_file_ptr(f->shared->cache, &trace_file_ptr) >= 0 ) &&
-	 ( trace_file_ptr != NULL ) ) {
-
-	sprintf(trace, "H5AC_flush 0x%x", flags);
-    }
+    if((f != NULL) &&
+            (f->shared != NULL) &&
+            (f->shared->cache != NULL) &&
+            (H5C_get_trace_file_ptr(f->shared->cache, &trace_file_ptr) >= 0) &&
+            (trace_file_ptr != NULL))
+	sprintf(trace, "H5AC_flush");
 #endif /* H5AC__TRACE_FILE_ENABLED */
 
 #ifdef H5_HAVE_PARALLEL
-    aux_ptr = f->shared->cache->aux_ptr;
-
-    if ( aux_ptr != NULL ) {
-
-#if H5AC_DEBUG_DIRTY_BYTES_CREATION
-        HDfprintf(stdout,
-                  "%d::H5AC_flush: (u/uu/i/iu/r/ru) = %d/%d/%d/%d/%d/%d\n",
-                  (int)(aux_ptr->mpi_rank),
-                  (int)(aux_ptr->unprotect_dirty_bytes),
-                  (int)(aux_ptr->unprotect_dirty_bytes_updates),
-                  (int)(aux_ptr->insert_dirty_bytes),
-                  (int)(aux_ptr->insert_dirty_bytes_updates),
-                  (int)(aux_ptr->rename_dirty_bytes),
-                  (int)(aux_ptr->rename_dirty_bytes_updates));
-#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */
-
-        /* to prevent "messages from the future" we must synchronize all
-         * processes before we start the flush.  Hence the following
-         * barrier.
-         */
-        if ( MPI_SUCCESS != (mpi_code = MPI_Barrier(aux_ptr->mpi_comm)) ) {
-
-            HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code)
-        }
-
-        /* if the clear only flag is set, this flush will not involve any
-         * disk I/O.  In such cases, it is not necessary to let process 0
-         * flush first.
-         */
-        if ( ( aux_ptr->mpi_rank == 0 ) &&
-             ( (flags & H5AC__FLUSH_CLEAR_ONLY_FLAG) == 0 ) ) {
-
-            unsigned init_flush_flags = H5AC__NO_FLAGS_SET;
-
-            if ( ( (flags & H5AC__FLUSH_MARKED_ENTRIES_FLAG) != 0 ) &&
-                 ( (flags & H5AC__FLUSH_INVALIDATE_FLAG) == 0 ) ) {
-
-                init_flush_flags |= H5AC__FLUSH_MARKED_ENTRIES_FLAG;
-            }
-
-	    aux_ptr->write_permitted = TRUE;
-
-            status = H5C_flush_cache(f,
-                                     H5AC_noblock_dxpl_id,
-                                     H5AC_noblock_dxpl_id,
-                                     f->shared->cache,
-                                     init_flush_flags);
-
-	    aux_ptr->write_permitted = FALSE;
-
-            if ( status < 0 ) {
-
-                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush.")
-            }
-
-            if ( aux_ptr->write_done != NULL ) {
-
-                (aux_ptr->write_done)();
-	    }
-
-        } /* end if ( aux_ptr->mpi_rank == 0 ) */
-
-        status = H5AC_propagate_flushed_and_still_clean_entries_list(f,
-                                                          H5AC_noblock_dxpl_id,
-                                                          f->shared->cache,
-                                                          FALSE);
-    } /* end if ( aux_ptr != NULL ) */
+    /* Attempt to flush all entries from rank 0 & Bcast clean list to other ranks */
+    if(H5AC_flush_entries(f) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush.")
 #endif /* H5_HAVE_PARALLEL */
 
-    status = H5C_flush_cache(f,
-                             dxpl_id,
-                             H5AC_noblock_dxpl_id,
-                             f->shared->cache,
-                             flags);
-
-    if ( status < 0 ) {
-
-        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.")
-    }
+    /* Flush the cache */
+    if(H5C_flush_cache(f, dxpl_id, H5AC_noblock_dxpl_id, f->shared->cache, H5AC__NO_FLAGS_SET) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush cache.")
 
 done:
-
 #if H5AC__TRACE_FILE_ENABLED
-    if ( trace_file_ptr != NULL ) {
-
+    if(trace_file_ptr != NULL)
         HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value);
-    }
 #endif /* H5AC__TRACE_FILE_ENABLED */
 
     FUNC_LEAVE_NOAPI(ret_value)
-
 } /* H5AC_flush() */
 
 
@@ -4885,5 +4737,89 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 
 } /* H5AC_receive_and_apply_clean_list() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5AC_flush_entries
+ *
+ * Purpose:	Flush the metadata cache associated with the specified file,
+ *		only writing from rank 0, but propagating the cleaned entries
+ *		to all ranks.
+ *
+ * Return:      Non-negative on success/Negative on failure if there was a
+ *              request to flush all items and something was protected.
+ *
+ * Programmer:  Quincey Koziol
+ *              koziol@hdfgroup.org
+ *              Aug 22 2009
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5AC_flush_entries(H5F_t *f)
+{
+    herr_t	  ret_value = SUCCEED;      /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT(H5AC_flush_entries)
+
+    HDassert(f);
+    HDassert(f->shared->cache);
+
+    /* Check if we have >1 ranks */
+    if(f->shared->cache->aux_ptr) {
+        H5AC_aux_t	* aux_ptr = f->shared->cache->aux_ptr;
+        int		  mpi_code;
+
+#if H5AC_DEBUG_DIRTY_BYTES_CREATION
+        HDfprintf(stdout,
+                  "%d::H5AC_flush: (u/uu/i/iu/r/ru) = %d/%d/%d/%d/%d/%d\n",
+                  (int)(aux_ptr->mpi_rank),
+                  (int)(aux_ptr->unprotect_dirty_bytes),
+                  (int)(aux_ptr->unprotect_dirty_bytes_updates),
+                  (int)(aux_ptr->insert_dirty_bytes),
+                  (int)(aux_ptr->insert_dirty_bytes_updates),
+                  (int)(aux_ptr->rename_dirty_bytes),
+                  (int)(aux_ptr->rename_dirty_bytes_updates));
+#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */
+
+        /* to prevent "messages from the future" we must synchronize all
+         * processes before we start the flush.  Hence the following
+         * barrier.
+         */
+        if(MPI_SUCCESS != (mpi_code = MPI_Barrier(aux_ptr->mpi_comm)))
+            HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code)
+
+        /* Flush data to disk, from rank 0 process */
+        if(aux_ptr->mpi_rank == 0 ) {
+            herr_t	  status;
+
+	    aux_ptr->write_permitted = TRUE;
+
+            status = H5C_flush_cache(f,
+                                     H5AC_noblock_dxpl_id,
+                                     H5AC_noblock_dxpl_id,
+                                     f->shared->cache,
+                                     H5AC__NO_FLAGS_SET);
+
+	    aux_ptr->write_permitted = FALSE;
+
+            if(status < 0)
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush.")
+
+            if(aux_ptr->write_done != NULL)
+                (aux_ptr->write_done)();
+        } /* end if ( aux_ptr->mpi_rank == 0 ) */
+
+        /* Propagate cleaned entries to other ranks */
+        if(H5AC_propagate_flushed_and_still_clean_entries_list(f,
+                                                          H5AC_noblock_dxpl_id,
+                                                          f->shared->cache,
+                                                          FALSE) < 0 )
+            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't propagate clean entries list.")
+    } /* end if ( aux_ptr != NULL ) */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_flush_entries() */
 #endif /* H5_HAVE_PARALLEL */
 
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index 99231ca..d36069a 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -343,7 +343,7 @@ H5_DLL herr_t H5AC_destroy_flush_dependency(H5F_t *f, void *parent_thing,
 H5_DLL herr_t H5AC_unprotect(H5F_t *f, hid_t dxpl_id,
                              const H5AC_class_t *type, haddr_t addr,
 			     void *thing, unsigned flags);
-H5_DLL herr_t H5AC_flush(H5F_t *f, hid_t dxpl_id, unsigned flags);
+H5_DLL herr_t H5AC_flush(H5F_t *f, hid_t dxpl_id);
 H5_DLL herr_t H5AC_mark_pinned_entry_dirty(H5F_t * f,
 		                           void *  thing,
 					   hbool_t size_changed,
diff --git a/src/H5C.c b/src/H5C.c
index c8e4d05..b90f963 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -3434,10 +3434,6 @@ H5C_def_auto_resize_rpt_fcn(H5C_t * cache_ptr,
  * Programmer:  John Mainzer
  *		6/2/04
  *
- * Modifications:
- *
- *              None.
- *
  *-------------------------------------------------------------------------
  */
 herr_t
@@ -3450,21 +3446,20 @@ H5C_dest(H5F_t * f,
 
     FUNC_ENTER_NOAPI(H5C_dest, FAIL)
 
-    HDassert( cache_ptr );
-    HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC );
-    HDassert( cache_ptr->skip_file_checks || f );
-
-    if ( H5C_flush_cache(f, primary_dxpl_id, secondary_dxpl_id,
-                         cache_ptr, H5C__FLUSH_INVALIDATE_FLAG) < 0 ) {
+    /* Sanity check */
+    HDassert(cache_ptr);
+    HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+    HDassert(cache_ptr->skip_file_checks || f);
 
+    /* Flush and invalidate all cache entries */
+    if(H5C_flush_invalidate_cache(f, primary_dxpl_id, secondary_dxpl_id,
+                cache_ptr, H5C__NO_FLAGS_SET) < 0 )
         HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
-    }
-
-    if ( cache_ptr->slist_ptr != NULL ) {
 
+    if(cache_ptr->slist_ptr != NULL) {
         H5SL_close(cache_ptr->slist_ptr);
         cache_ptr->slist_ptr = NULL;
-    }
+    } /* end if */
 
     cache_ptr->magic = 0;
 
@@ -3472,66 +3467,10 @@ H5C_dest(H5F_t * f,
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
-
 } /* H5C_dest() */
 
 
 /*-------------------------------------------------------------------------
- * Function:    H5C_dest_empty
- *
- * Purpose:     Destroy an empty cache.
- *
- *              This function fails if the cache is not empty on entry.
- *
- *		Note that *cache_ptr has been freed upon successful return.
- *
- * Return:      Non-negative on success/Negative on failure
- *
- * Programmer:  John Mainzer
- *		6/2/04
- *
- * Modifications:
- *
- *		None.
- *
- *-------------------------------------------------------------------------
- */
-herr_t
-H5C_dest_empty(H5C_t * cache_ptr)
-{
-    herr_t ret_value=SUCCEED;      /* Return value */
-
-    FUNC_ENTER_NOAPI(H5C_dest_empty, FAIL)
-
-    /* This would normally be an assert, but we need to use an HGOTO_ERROR
-     * call to shut up the compiler.
-     */
-    if ( ( ! cache_ptr ) ||
-         ( cache_ptr->magic != H5C__H5C_T_MAGIC ) ||
-         ( cache_ptr->index_len != 0 ) ) {
-
-        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                    "Bad cache_ptr or non-empty cache on entry.")
-    }
-
-
-    if ( cache_ptr->slist_ptr != NULL ) {
-
-        H5SL_close(cache_ptr->slist_ptr);
-        cache_ptr->slist_ptr = NULL;
-    }
-
-    cache_ptr->magic = 0;
-
-    (void)H5FL_FREE(H5C_t, cache_ptr);
-
-done:
-    FUNC_LEAVE_NOAPI(ret_value)
-
-} /* H5C_dest_empty() */
-
-
-/*-------------------------------------------------------------------------
  *
  * Function:    H5C_expunge_entry
  *
@@ -3928,8 +3867,7 @@ H5C_flush_cache(H5F_t *  f,
 #ifndef NDEBUG
                     if ( entry_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) {
 
-                        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                                    "entry_ptr->magic invalid ?!?!");
+                        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry_ptr->magic is invalid ?!?!")
 
                     } else
 #endif /* NDEBUG */
@@ -3937,9 +3875,8 @@ H5C_flush_cache(H5F_t *  f,
                          ( ! entry_ptr->in_slist ) ) {
 
                         /* the s-list has been modified out from under us.
-                         * set node_ptr to NULL and break out of the inner loop.
+                         * break out of the loop.
                          */
-                        node_ptr = NULL;
                         goto end_of_inner_loop;;
                     }
 
@@ -3950,11 +3887,8 @@ H5C_flush_cache(H5F_t *  f,
 
                     if ( node_ptr != NULL ) {
                         next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr);
-
-                        if ( next_entry_ptr == NULL ) {
-                            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                                        "next_entry_ptr == NULL 2 ?!?!");
-                        }
+                        if ( NULL == next_entry_ptr )
+                            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!")
                         HDassert( next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC );
                         HDassert( next_entry_ptr->is_dirty );
                         HDassert( next_entry_ptr->in_slist );
@@ -3999,18 +3933,19 @@ H5C_flush_cache(H5F_t *  f,
                                                                 FALSE);
                                 if ( status < 0 ) {
 
-                                    /* This shouldn't happen -- if it does, we are
-                                     * toast so just scream and die.
+                                    /* This shouldn't happen -- if it does, we are toast
+                                     * so just scream and die.
                                      */
                                     HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
                                                 "dirty pinned entry flush failed.")
-                                }
+                                } /* end if */
                                 flushed_during_dep_loop = TRUE;
                             } /* end if */
                             else if(entry_ptr->flush_dep_height < curr_flush_dep_height)
                                 /* This shouldn't happen -- if it does, just scream and die.  */
                                 HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL,  "dirty entry below current flush dep. height.")
-                        } else {
+                        } /* end if */
+                        else {
                             /* Test to see if we are can flush the entry now.
                              * If we can, go ahead and flush.  Note that we
                              * aren't trying to do a destroy here, so that
@@ -10335,7 +10270,6 @@ H5C_flush_invalidate_cache(H5F_t *  f,
 {
     herr_t              status;
     herr_t		ret_value = SUCCEED;
-    hbool_t		done = FALSE;
     hbool_t		first_flush = TRUE;
     int32_t		protected_entries = 0;
     int32_t		i;
@@ -10411,7 +10345,7 @@ H5C_flush_invalidate_cache(H5F_t *  f,
     cur_pel_len = cache_ptr->pel_len;
     old_pel_len = cache_ptr->pel_len;
 
-    while ( ! done )
+    while ( cache_ptr->index_len > 0 )
     {
         unsigned curr_flush_dep_height = 0;
         unsigned flush_dep_passes = 0;
@@ -10436,24 +10370,19 @@ H5C_flush_invalidate_cache(H5F_t *  f,
 
             } else {
 
+                /* Start at beginning of skip list each time */
                 node_ptr = H5SL_first(cache_ptr->slist_ptr);
+                HDassert( node_ptr != NULL );
 
-                if ( node_ptr == NULL ) {
-                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                                "slist_len != 0 && node_ptr == NULL");
-                }
-
+                /* Get cache entry for this node */
                 next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr);
-
-                if ( next_entry_ptr == NULL ) {
-                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                                "next_entry_ptr == NULL 1 ?!?!");
-                }
+                if ( NULL == next_entry_ptr )
+                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!")
                 HDassert( next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC );
                 HDassert( next_entry_ptr->is_dirty );
                 HDassert( next_entry_ptr->in_slist );
-
             }
+
 #if H5C_DO_SANITY_CHECKS
             /* Depending on circumstances, H5C_flush_single_entry() will
              * remove dirty entries from the slist as it flushes them.
@@ -10518,8 +10447,7 @@ H5C_flush_invalidate_cache(H5F_t *  f,
 #ifndef NDEBUG
                 if ( entry_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) {
 
-                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                                "entry_ptr->magic is invalid ?!?!");
+                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry_ptr->magic is invalid ?!?!")
 
                 } else
 #endif /* NDEBUG */
@@ -10538,19 +10466,13 @@ H5C_flush_invalidate_cache(H5F_t *  f,
 
                 node_ptr = H5SL_next(node_ptr);
                 if ( node_ptr != NULL ) {
-
                     next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr);
-
-                    if ( next_entry_ptr == NULL ) {
-                        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                                    "next_entry_ptr == NULL 2 ?!?!");
-                    }
+                    if ( NULL == next_entry_ptr )
+                        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!")
                     HDassert( next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC );
                     HDassert( next_entry_ptr->is_dirty );
                     HDassert( next_entry_ptr->in_slist );
-
                 } else {
-
                     next_entry_ptr = NULL;
                 }
 
@@ -10811,24 +10733,21 @@ end_of_inner_loop:
             HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
 	                "Maximum passes on flush exceeded.")
 	}
-
-	if ( cache_ptr->index_len <= 0 ) {
-
-	    done = TRUE;
-            HDassert( cache_ptr->index_size == 0 );
-            HDassert( cache_ptr->clean_index_size == 0 );
-            HDassert( cache_ptr->dirty_index_size == 0 );
-	    HDassert( cache_ptr->slist_len == 0 );
-	    HDassert( cache_ptr->slist_size == 0 );
-	    HDassert( cache_ptr->pel_len == 0 );
-	    HDassert( cache_ptr->pel_size == 0 );
-	    HDassert( cache_ptr->pl_len == 0 );
-	    HDassert( cache_ptr->pl_size == 0 );
-	    HDassert( cache_ptr->LRU_list_len == 0 );
-	    HDassert( cache_ptr->LRU_list_size == 0 );
-        }
     } /* main while loop */
 
+    /* Invariants, after destroying all entries in the hash table */
+    HDassert( cache_ptr->index_size == 0 );
+    HDassert( cache_ptr->clean_index_size == 0 );
+    HDassert( cache_ptr->dirty_index_size == 0 );
+    HDassert( cache_ptr->slist_len == 0 );
+    HDassert( cache_ptr->slist_size == 0 );
+    HDassert( cache_ptr->pel_len == 0 );
+    HDassert( cache_ptr->pel_size == 0 );
+    HDassert( cache_ptr->pl_len == 0 );
+    HDassert( cache_ptr->pl_size == 0 );
+    HDassert( cache_ptr->LRU_list_len == 0 );
+    HDassert( cache_ptr->LRU_list_size == 0 );
+
 
     HDassert( protected_entries <= cache_ptr->pl_len );
 
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index 5524f2b..5c7fe9f 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -1026,8 +1026,6 @@ H5_DLL herr_t H5C_dest(H5F_t * f,
                        hid_t   secondary_dxpl_id,
                        H5C_t * cache_ptr);
 
-H5_DLL herr_t H5C_dest_empty(H5C_t * cache_ptr);
-
 H5_DLL herr_t H5C_expunge_entry(H5F_t *             f,
 		                hid_t               primary_dxpl_id,
                                 hid_t               secondary_dxpl_id,
diff --git a/src/H5F.c b/src/H5F.c
index 352f97c..0d163f6 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -1032,6 +1032,11 @@ H5F_dest(H5F_t *f, hid_t dxpl_id)
             /* Push error, but keep going*/
             HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file")
 
+        /* Shutdown the metadata cache */
+        if(H5AC_dest(f, dxpl_id))
+            /* Push error, but keep going*/
+            HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file")
+
         /*
          * Do not close the root group since we didn't count it, but free
          * the memory associated with it.
@@ -1050,9 +1055,6 @@ H5F_dest(H5F_t *f, hid_t dxpl_id)
         } /* end if */
 
         /* Destroy other components of the file */
-        if(H5AC_dest(f, dxpl_id))
-            /* Push error, but keep going*/
-            HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file")
         if(H5F_accum_reset(f) < 0)
             /* Push error, but keep going*/
             HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file")
@@ -1672,7 +1674,6 @@ done:
 herr_t
 H5F_flush_real(H5F_t *f, hid_t dxpl_id, hbool_t closing)
 {
-    unsigned H5AC_flags = H5AC__NO_FLAGS_SET;   /* Flags for H5AC_flush() */
     herr_t   ret_value = SUCCEED;       /* Return value */
 
     FUNC_ENTER_NOAPI(H5F_flush_real, FAIL)
@@ -1709,18 +1710,17 @@ H5F_flush_real(H5F_t *f, hid_t dxpl_id, hbool_t closing)
             HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space")
     } /* end else */
 
-    /* Flush (and invalidate, if requested) the entire metadata cache */
     if(closing) {
-        H5AC_flags |= H5AC__FLUSH_INVALIDATE_FLAG;
-
         /* Unpin the superblock, since we're about to destroy the cache */
         if(H5AC_unpin_entry(f, f->shared->sblock) < 0)
-            /* Push error, but keep going*/
-            HDONE_ERROR(H5E_FSPACE, H5E_CANTUNPIN, FAIL, "unable to unpin superblock")
+            HGOTO_ERROR(H5E_FSPACE, H5E_CANTUNPIN, FAIL, "unable to unpin superblock")
         f->shared->sblock = NULL;
     } /* end if */
-    if(H5AC_flush(f, dxpl_id, H5AC_flags) < 0)
-        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush metadata cache")
+    else {
+        /* Flush the entire metadata cache */
+        if(H5AC_flush(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush metadata cache")
+    } /* end else */
 
     /* If we will be closing the file, we don't need to flush the accumulator info */
     if(!closing) {
diff --git a/test/ohdr.c b/test/ohdr.c
index 905551d..df5f63b 100644
--- a/test/ohdr.c
+++ b/test/ohdr.c
@@ -60,8 +60,8 @@ test_cont(char *filename, hid_t fapl)
     H5O_info_t  oinfo;               
     H5O_loc_t	oh_locA, oh_locB;
     time_t	time_new;
-    char	*short_name = "T";
-    char	*long_name = "This is the message";
+    const char	*short_name = "T";
+    const char	*long_name = "This is the message";
     size_t	nchunks;
 
     TESTING("object header continuation block");
@@ -100,7 +100,9 @@ test_cont(char *filename, hid_t fapl)
     if(H5O_msg_create(&oh_locA, H5O_NAME_ID, 0, 0, &short_name, H5P_DATASET_XFER_DEFAULT) < 0)
 	FAIL_STACK_ERROR
 
-    if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
+    if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT) < 0)
+	FAIL_STACK_ERROR
+    if(H5AC_expunge_entry(f, H5P_DATASET_XFER_DEFAULT, H5AC_OHDR, oh_locA.addr, H5AC__NO_FLAGS_SET) < 0)
 	FAIL_STACK_ERROR
 
     if(H5O_get_info(&oh_locA, H5P_DATASET_XFER_DEFAULT, FALSE, &oinfo) < 0)
@@ -210,7 +212,9 @@ main(void)
         time_new = 11111111;
         if(H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
             FAIL_STACK_ERROR
-        if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
+        if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT) < 0)
+            FAIL_STACK_ERROR
+        if(H5AC_expunge_entry(f, H5P_DATASET_XFER_DEFAULT, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0)
             FAIL_STACK_ERROR
         if(NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro, H5P_DATASET_XFER_DEFAULT))
             FAIL_STACK_ERROR
@@ -226,7 +230,9 @@ main(void)
         time_new = 33333333;
         if(H5O_msg_write(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
             FAIL_STACK_ERROR
-        if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
+        if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT) < 0)
+            FAIL_STACK_ERROR
+        if(H5AC_expunge_entry(f, H5P_DATASET_XFER_DEFAULT, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0)
             FAIL_STACK_ERROR
         if(NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro, H5P_DATASET_XFER_DEFAULT))
             FAIL_STACK_ERROR
@@ -255,7 +261,9 @@ main(void)
             if(H5O_msg_create(&oh_loc, H5O_MTIME_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
                 FAIL_STACK_ERROR
         } /* end for */
-        if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
+        if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT) < 0)
+            FAIL_STACK_ERROR
+        if(H5AC_expunge_entry(f, H5P_DATASET_XFER_DEFAULT, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0)
             FAIL_STACK_ERROR
 
         /* Make certain that chunk #0 in the object header will be encoded with a 2-byte size */
@@ -294,7 +302,9 @@ main(void)
             time_new = (i + 1) * 1000 + 10;
             if(H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
                 FAIL_STACK_ERROR
-            if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
+            if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT) < 0)
+                FAIL_STACK_ERROR
+            if(H5AC_expunge_entry(f, H5P_DATASET_XFER_DEFAULT, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0)
                 FAIL_STACK_ERROR
         } /* end for */
         PASSED();
@@ -322,7 +332,9 @@ main(void)
         time_new = 22222222;
         if(H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, H5O_MSG_FLAG_CONSTANT, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
             FAIL_STACK_ERROR
-        if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
+        if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT) < 0)
+            FAIL_STACK_ERROR
+        if(H5AC_expunge_entry(f, H5P_DATASET_XFER_DEFAULT, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0)
             FAIL_STACK_ERROR
         if(NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro, H5P_DATASET_XFER_DEFAULT))
             FAIL_STACK_ERROR
diff --git a/test/tsohm.c b/test/tsohm.c
index 348677d..b5b489b 100644
--- a/test/tsohm.c
+++ b/test/tsohm.c
@@ -544,8 +544,12 @@ error:
 static hid_t
 close_reopen_file(hid_t file, const char* filename, hid_t fapl_id)
 {
-    if(H5Fclose(file) < 0) goto error;
-    return H5Fopen(filename, H5F_ACC_RDWR, fapl_id);
+    hid_t fid;
+
+    if(H5Fclose(file) < 0) FAIL_STACK_ERROR
+    if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) FAIL_STACK_ERROR
+
+    return(fid);
 
 error:
     return -1;
@@ -598,19 +602,19 @@ size1_helper(hid_t file, const char* filename, hid_t fapl_id, int test_file_clos
     dim1[0] = 1;
     if((space_id=H5Screate_simple(1,dim1,NULL)) < 0) TEST_ERROR
 
-    if((dset_id = H5Dcreate2(file,DSETNAME[0],dtype1_id,space_id,H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR
+    if((dset_id = H5Dcreate2(file,DSETNAME[0],dtype1_id,space_id,H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR
 
     /* Test writing and reading */
-    if(H5Dwrite(dset_id,dtype1_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,&wdata) < 0) TEST_ERROR
+    if(H5Dwrite(dset_id,dtype1_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,&wdata) < 0) FAIL_STACK_ERROR
 
-    if(H5Dread(dset_id,dtype1_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,&rdata) < 0) TEST_ERROR
+    if(H5Dread(dset_id,dtype1_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,&rdata) < 0) FAIL_STACK_ERROR
 
     if(rdata.i1!=wdata.i1 || rdata.i2!=wdata.i2 || HDstrcmp(rdata.str, wdata.str)) {
         H5_FAILED(); AT();
         printf("incorrect read data\n");
         goto error;
     } /* end if */
-    if(H5Dclose(dset_id) < 0) TEST_ERROR
+    if(H5Dclose(dset_id) < 0) FAIL_STACK_ERROR
 
     /* Close and re-open the file if requested*/
     if(test_file_closing) {
@@ -618,8 +622,8 @@ size1_helper(hid_t file, const char* filename, hid_t fapl_id, int test_file_clos
     }
 
     /* Create more datasets with the same datatype */
-    if((dset_id = H5Dcreate2(file,DSETNAME[1],dtype1_id,space_id,H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR
-    if(H5Dclose(dset_id) < 0) TEST_ERROR
+    if((dset_id = H5Dcreate2(file,DSETNAME[1],dtype1_id,space_id,H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR
+    if(H5Dclose(dset_id) < 0) FAIL_STACK_ERROR
 
     /* Close and re-open the file if requested*/
     if(test_file_closing) {
diff --git a/testpar/t_cache.c b/testpar/t_cache.c
index 4e1e5ba..9bf3a8d 100644
--- a/testpar/t_cache.c
+++ b/testpar/t_cache.c
@@ -5449,7 +5449,7 @@ trace_file_check(void)
       "H5AC_unpin_entry 4 0\n",
       "H5AC_rename 0 8a65 15 0\n",
       "H5AC_rename 8a65 0 15 0\n",
-      "H5AC_flush 0x0 0\n",
+      "H5AC_flush 0\n",
       NULL
     };
     char buffer[256];