[svn-r27293] Description:

    Split parallel metadata cache code into separate source code modules.

Tested on:
    MacOSX/64 10.10.3 (amazon) w/serial & parallel
    (too minor for h5committest)

1 files changed, 65 insertions, 1239 deletions

diff --git a/src/H5C.c b/src/H5C.c
index fc4e8a5..5c96593 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -41,7 +41,7 @@
  *
  *	Code Changes:
  *
- *	 - Remove extra functionality in H5C_flush_single_entry()?
+ *	 - Remove extra functionality in H5C__flush_single_entry()?
  *
  *	 - Change protect/unprotect to lock/unlock.
  *
@@ -70,16 +70,22 @@
  *
  **************************************************************************/
 
+/****************/
+/* Module Setup */
+/****************/
+
 #define H5C_PACKAGE		/*suppress error about including H5Cpkg   */
 #define H5F_PACKAGE		/*suppress error about including H5Fpkg	  */
 
 
+/***********/
+/* Headers */
+/***********/
 #include "H5private.h"		/* Generic Functions			*/
 #ifdef H5_HAVE_PARALLEL
 #include "H5ACprivate.h"        /* Metadata cache                       */
 #endif /* H5_HAVE_PARALLEL */
 #include "H5Cpkg.h"		/* Cache				*/
-#include "H5Dprivate.h"		/* Dataset functions			*/
 #include "H5Eprivate.h"		/* Error handling		  	*/
 #include "H5Fpkg.h"		/* Files				*/
 #include "H5FDprivate.h"	/* File drivers				*/
@@ -91,9 +97,9 @@
 #include "H5SLprivate.h"	/* Skip lists				*/
 
 
-/*
- * Private macros.
- */
+/****************/
+/* Local Macros */
+/****************/
 #if H5C_DO_MEMORY_SANITY_CHECKS
 #define H5C_IMAGE_EXTRA_SPACE 8
 #define H5C_IMAGE_SANITY_VALUE "DeadBeef"
@@ -101,17 +107,14 @@
 #define H5C_IMAGE_EXTRA_SPACE 0
 #endif /* H5C_DO_MEMORY_SANITY_CHECKS */
 
-/*
- * Private file-scope variables.
- */
-
-/* Declare a free list to manage the H5C_t struct */
-H5FL_DEFINE_STATIC(H5C_t);
+/******************/
+/* Local Typedefs */
+/******************/
 
 
-/*
- * Private file-scope function declarations:
- */
+/********************/
+/* Local Prototypes */
+/********************/
 
 static herr_t H5C__auto_adjust_cache_size(H5F_t * f,
                                           hid_t dxpl_id,
@@ -140,13 +143,6 @@ static herr_t H5C__flash_increase_cache_size(H5C_t * cache_ptr,
                                              size_t old_entry_size,
                                              size_t new_entry_size);
 
-static herr_t H5C_flush_single_entry(const H5F_t *	 f,
-                                     hid_t 		 dxpl_id,
-                                     haddr_t		 addr,
-                                     unsigned	         flags,
-                                     hbool_t del_entry_from_slist_on_destroy,
-                                     int64_t		*entry_size_change_ptr);
-
 static herr_t H5C_flush_invalidate_cache(const H5F_t *  f,
                                           hid_t    dxpl_id,
 			                  unsigned flags);
@@ -198,6 +194,24 @@ herr_t H5C_dump_cache(H5C_t * cache_ptr, const char *  cache_name);
 herr_t H5C_dump_cache_skip_list(H5C_t * cache_ptr, char * calling_fcn);
 #endif /* debugging routines */
 
+/*********************/
+/* Package Variables */
+/*********************/
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/* Declare a free list to manage the H5C_t struct */
+H5FL_DEFINE_STATIC(H5C_t);
+
+
 
 /****************************************************************************
  *
@@ -386,882 +400,6 @@ H5C__epoch_marker_fsf_size(const void H5_ATTR_UNUSED * thing, size_t H5_ATTR_UNU
 
 
 /*-------------------------------------------------------------------------
- * Function:    H5C_apply_candidate_list
- *
- * Purpose:     Apply the supplied candidate list.
- *
- *		We used to do this by simply having each process write 
- *		every mpi_size-th entry in the candidate list, starting 
- *		at index mpi_rank, and mark all the others clean.  
- *
- *		However, this can cause unnecessary contention in a file 
- *		system by increasing the number of processes writing to 
- *		adjacent locations in the HDF5 file.
- *
- *		To attempt to minimize this, we now arange matters such 
- *		that each process writes n adjacent entries in the 
- *		candidate list, and marks all others clean.  We must do
- *		this in such a fashion as to guarantee that each entry 
- *		on the candidate list is written by exactly one process, 
- *		and marked clean by all others.  
- *
- *		To do this, first construct a table mapping mpi_rank
- *		to the index of the first entry in the candidate list to
- *		be written by the process of that mpi_rank, and then use
- *		the table to control which entries are written and which
- *		are marked as clean as a function of the mpi_rank.
- *
- *		Note that the table must be identical on all processes, as
- *		all see the same candidate list, mpi_size, and mpi_rank --
- *		the inputs used to construct the table.  
- *
- *		We construct the table as follows.  Let:
- *
- *			n = num_candidates / mpi_size;
- *
- *			m = num_candidates % mpi_size;
- *
- *		Now allocate an array of integers of length mpi_size + 1, 
- *		and call this array candidate_assignment_table. 
- *
- *		Conceptually, if the number of candidates is a multiple
- *		of the mpi_size, we simply pass through the candidate list
- *		and assign n entries to each process to flush, with the 
- *		index of the first entry to flush in the location in 
- *		the candidate_assignment_table indicated by the mpi_rank
- *		of the process.  
- *
- *		In the more common case in which the candidate list isn't 
- *		isn't a multiple of the mpi_size, we pretend it is, and 
- *		give num_candidates % mpi_size processes one extra entry
- *		each to make things work out.
- *
- *		Once the table is constructed, we determine the first and
- *		last entry this process is to flush as follows:
- *
- *	 	first_entry_to_flush = candidate_assignment_table[mpi_rank]
- *
- *		last_entry_to_flush = 
- *			candidate_assignment_table[mpi_rank + 1] - 1;
- *		
- *		With these values determined, we simply scan through the 
- *		candidate list, marking all entries in the range 
- *		[first_entry_to_flush, last_entry_to_flush] for flush,
- *		and all others to be cleaned.
- *
- *		Finally, we scan the LRU from tail to head, flushing 
- *		or marking clean the candidate entries as indicated.
- *		If necessary, we scan the pinned list as well.
- *
- *		Note that this function will fail if any protected or 
- *		clean entries appear on the candidate list.
- *
- *		This function is used in managing sync points, and 
- *		shouldn't be used elsewhere.
- *
- * Return:      Success:        SUCCEED
- *
- *              Failure:        FAIL
- *
- * Programmer:  John Mainzer
- *              3/17/10
- *
- * Changes:     Ported code to detect next entry status changes as the 
- *              the result of a flush from the serial code in the scan of 
- *              the LRU.  Also added code to detect and adapt to the 
- *              removal from the cache of the next entry in the scan of 
- *		the LRU.
- *
- *		Note that at present, all of these changes should not 
- *		be required as the operations on entries as they are 
- *		flushed that can cause these condiditions are not premitted
- *		in the parallel case.  However, Quincey indicates that 
- *		this may change, and thus has requested the modification.
- *
- *		Note the assert(FALSE) in the if statement whose body 
- *		restarts the scan of the LRU.  As the body of the if 
- *		statement should be unreachable, it should never be 
- *		triggered until the constraints on the parallel case 
- *		are relaxed.  Please remove the assertion at that time.
- *
- *		Also added warning on the Pinned Entry List scan, as it
- *		is potentially subject to the same issue.  As there is 
- *		no cognate of this scan in the serial code, I don't have
- *		a fix to port to it.
- *
- *						JRM -- 4/10/19
- *		
- *-------------------------------------------------------------------------
- */
-#ifdef H5_HAVE_PARALLEL
-#define H5C_APPLY_CANDIDATE_LIST__DEBUG 0
-herr_t
-H5C_apply_candidate_list(H5F_t * f,
-                         hid_t dxpl_id,
-                         H5C_t * cache_ptr,
-                         int num_candidates,
-                         haddr_t * candidates_list_ptr,
-                         int mpi_rank,
-                         int mpi_size)
-{
-    hbool_t		restart_scan;
-    hbool_t		prev_is_dirty;
-    int                 i;
-    int			m;
-    int			n;
-    int			first_entry_to_flush;
-    int			last_entry_to_flush;
-    int			entries_to_clear = 0;
-    int			entries_to_flush = 0;
-    int			entries_to_flush_or_clear_last = 0;
-    int			entries_to_flush_collectively = 0;
-    int			entries_cleared = 0;
-    int			entries_flushed = 0;
-    int			entries_delayed = 0;
-    int			entries_flushed_or_cleared_last = 0;
-    int			entries_flushed_collectively = 0;
-    int			entries_examined = 0;
-    int			initial_list_len;
-    int               * candidate_assignment_table = NULL;
-    haddr_t		addr;
-    H5C_cache_entry_t *	clear_ptr = NULL;
-    H5C_cache_entry_t *	next_ptr = NULL;
-    H5C_cache_entry_t *	entry_ptr = NULL;
-    H5C_cache_entry_t *	flush_ptr = NULL;
-    H5C_cache_entry_t * delayed_ptr = NULL;
-#if H5C_DO_SANITY_CHECKS
-    haddr_t		last_addr;
-#endif /* H5C_DO_SANITY_CHECKS */
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    char		tbl_buf[1024];
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-    herr_t              ret_value = SUCCEED;      /* Return value */
-
-    FUNC_ENTER_NOAPI(FAIL)
-
-    HDassert( cache_ptr != NULL );
-    HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC );
-    HDassert( num_candidates > 0 );
-    HDassert( num_candidates <= cache_ptr->slist_len );
-    HDassert( candidates_list_ptr != NULL );
-    HDassert( 0 <= mpi_rank );
-    HDassert( mpi_rank < mpi_size );
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    HDfprintf(stdout, "%s:%d: setting up candidate assignment table.\n", 
-              FUNC, mpi_rank);
-    for ( i = 0; i < 1024; i++ ) tbl_buf[i] = '\0';
-    sprintf(&(tbl_buf[0]), "candidate list = ");
-    for ( i = 0; i < num_candidates; i++ )
-    {
-        sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), " 0x%llx", 
-                (long long)(*(candidates_list_ptr + i)));
-    }
-    sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), "\n");
-    HDfprintf(stdout, "%s", tbl_buf);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-    n = num_candidates / mpi_size;
-    m = num_candidates % mpi_size;
-    HDassert(n >= 0);
-
-    if(NULL == (candidate_assignment_table = (int *)H5MM_malloc(sizeof(int) * (size_t)(mpi_size + 1))))
-        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for candidate assignment table")
-
-    candidate_assignment_table[0] = 0;
-    candidate_assignment_table[mpi_size] = num_candidates;
-
-    if(m == 0) { /* mpi_size is an even divisor of num_candidates */
-        HDassert(n > 0);
-        for(i = 1; i < mpi_size; i++)
-            candidate_assignment_table[i] = candidate_assignment_table[i - 1] + n;
-    } /* end if */
-    else { 
-        for(i = 1; i <= m; i++)
-            candidate_assignment_table[i] = candidate_assignment_table[i - 1] + n + 1;
-
-        if(num_candidates < mpi_size) {
-            for(i = m + 1; i < mpi_size; i++)
-                candidate_assignment_table[i] = num_candidates;
-        } /* end if */
-        else {
-            for(i = m + 1; i < mpi_size; i++)
-                candidate_assignment_table[i] = candidate_assignment_table[i - 1] + n;
-        } /* end else */
-    } /* end else */
-    HDassert((candidate_assignment_table[mpi_size - 1] + n) == num_candidates);
-
-#if H5C_DO_SANITY_CHECKS
-    /* verify that the candidate assignment table has the expected form */
-    for ( i = 1; i < mpi_size - 1; i++ ) 
-    {
-        int a, b;
-
-        a = candidate_assignment_table[i] - candidate_assignment_table[i - 1];
-        b = candidate_assignment_table[i + 1] - candidate_assignment_table[i];
-
-        HDassert( n + 1 >= a );
-        HDassert( a >= b );
-        HDassert( b >= n );
-    }
-#endif /* H5C_DO_SANITY_CHECKS */
-
-    first_entry_to_flush = candidate_assignment_table[mpi_rank];
-    last_entry_to_flush = candidate_assignment_table[mpi_rank + 1] - 1;
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    for ( i = 0; i < 1024; i++ )
-        tbl_buf[i] = '\0';
-    sprintf(&(tbl_buf[0]), "candidate assignment table = ");
-    for(i = 0; i <= mpi_size; i++)
-        sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), " %d", candidate_assignment_table[i]);
-    sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), "\n");
-    HDfprintf(stdout, "%s", tbl_buf);
-
-    HDfprintf(stdout, "%s:%d: flush entries [%d, %d].\n", 
-              FUNC, mpi_rank, first_entry_to_flush, last_entry_to_flush);
-
-    HDfprintf(stdout, "%s:%d: marking entries.\n", FUNC, mpi_rank);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-    for(i = 0; i < num_candidates; i++) {
-        addr = candidates_list_ptr[i];
-        HDassert( H5F_addr_defined(addr) );
-
-#if H5C_DO_SANITY_CHECKS
-        if ( i > 0 ) {
-            if ( last_addr == addr ) {
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Duplicate entry in cleaned list.\n")
-            } else if ( last_addr > addr ) {
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "candidate list not sorted.\n")
-            }
-        }
-
-        last_addr = addr;
-#endif /* H5C_DO_SANITY_CHECKS */
-
-        H5C__SEARCH_INDEX(cache_ptr, addr, entry_ptr, FAIL)
-        if(entry_ptr == NULL) {
-            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed candidate entry not in cache?!?!?.")
-        } else if(!entry_ptr->is_dirty) {
-            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed entry not dirty?!?!?.")
-        } else if ( entry_ptr->is_protected ) {
-            /* For now at least, we can't deal with protected entries.
-             * If we encounter one, scream and die.  If it becomes an
-             * issue, we should be able to work around this. 
-             */
-            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed entry is protected?!?!?.")
-        } else {
-            /* determine whether the entry is to be cleared or flushed,
-             * and mark it accordingly.  We will scan the protected and 
-             * pinned list shortly, and clear or flush according to these
-             * markings.  
-             */
-            if((i >= first_entry_to_flush) && (i <= last_entry_to_flush)) {
-                entries_to_flush++;
-                entry_ptr->flush_immediately = TRUE;
-            } /* end if */
-            else {
-                entries_to_clear++;
-                entry_ptr->clear_on_unprotect = TRUE;
-            } /* end else */
-        } /* end else */
-    } /* end for */
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    HDfprintf(stdout, "%s:%d: num candidates/to clear/to flush = %d/%d/%d.\n", 
-              FUNC, mpi_rank, (int)num_candidates, (int)entries_to_clear,
-              (int)entries_to_flush);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-
-    /* We have now marked all the entries on the candidate list for 
-     * either flush or clear -- now scan the LRU and the pinned list
-     * for these entries and do the deed.
-     *
-     * Note that we are doing things in this round about manner so as
-     * to preserve the order of the LRU list to the best of our ability.
-     * If we don't do this, my experiments indicate that we will have a
-     * noticably poorer hit ratio as a result.
-     */
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    HDfprintf(stdout, "%s:%d: scanning LRU list. len = %d.\n", FUNC, mpi_rank,
-              (int)(cache_ptr->LRU_list_len));
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-    /* ===================================================================== *
-     * Now scan the LRU and PEL lists, flushing or clearing entries as
-     * needed.
-     *
-     * The flush_me_last and flush_me_collectively flags may dictate how or
-     * when some entries can be flushed, and should be addressed here.
-     * However, in their initial implementation, these flags only apply to the
-     * superblock, so there's only a relatively small change to this function
-     * to account for this one case where they come into play. If these flags
-     * are ever expanded upon, this function and the following flushing steps
-     * should be reworked to account for additional cases.
-     * ===================================================================== */
-
-    HDassert(entries_to_flush >= 0);
-
-    restart_scan = FALSE;
-    entries_examined = 0;
-    initial_list_len = cache_ptr->LRU_list_len;
-    entry_ptr = cache_ptr->LRU_tail_ptr;
-
-    /* Examine each entry in the LRU list */
-    while ( ( entry_ptr != NULL ) 
-            && 
-            ( entries_examined <= (entries_to_flush + 1) * initial_list_len ) 
-            &&
-            ( (entries_cleared + entries_flushed) < num_candidates ) ) {
-
-        if ( entry_ptr->prev != NULL )
-            prev_is_dirty = entry_ptr->prev->is_dirty;
-
-        /* If this process needs to clear this entry. */
-        if(entry_ptr->clear_on_unprotect) {
-
-            HDassert(entry_ptr->is_dirty);
-
-            next_ptr = entry_ptr->next; 
-            entry_ptr->clear_on_unprotect = FALSE;
-            clear_ptr = entry_ptr;
-            entry_ptr = entry_ptr->prev;
-            entries_cleared++;
-
-#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
-    HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank,
-              (long long)clear_ptr->addr);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-	    /* No need to check for the next entry in the scan being 
-             * removed from the cache, as this call to H5C_flush_single_entry()
-             * will not call either the pre_serialize or serialize callbacks.
-             */
-
-            if(H5C_flush_single_entry(f,
-                                      dxpl_id,
-                                      clear_ptr->addr,
-                                      H5C__FLUSH_CLEAR_ONLY_FLAG,
-                                      TRUE, 
-                                      NULL) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
-        } /* end if */
-
-        /* Else, if this process needs to flush this entry. */
-        else if (entry_ptr->flush_immediately) {
-
-	    HDassert(entry_ptr->is_dirty);
-
-            next_ptr = entry_ptr->next; 
-            entry_ptr->flush_immediately = FALSE;
-            flush_ptr = entry_ptr;
-            entry_ptr = entry_ptr->prev;
-            entries_flushed++;
-
-#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
-    HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank,
-              (long long)flush_ptr->addr);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-            /* reset entries_removed_counter and
-             * last_entry_removed_ptr prior to the call to
-             * H5C_flush_single_entry() so that we can spot
-             * unexpected removals of entries from the cache,
-             * and set the restart_scan flag if proceeding
-             * would be likely to cause us to scan an entry
-             * that is no longer in the cache.
-             *
-             * Note that as of this writing (April 2015) this 
-             * case cannot occur in the parallel case.  However 
-             * Quincey is making noises about changing this, hence 
-             * the insertion of this test.
-             *
-             * Note also that there is no test code to verify 
-             * that this code actually works (although similar code
-             * in the serial version exists and is tested).  
-             * 
-             * Implementing a test will likely require implementing
-             * flush op like facilities in the parallel tests.  At
-             * a guess this will not be terribly painful, but it 
-             * will take a bit of time.
-             */
-            cache_ptr->entries_removed_counter = 0;
-            cache_ptr->last_entry_removed_ptr  = NULL;
-
-            if(H5C_flush_single_entry(f,
-                                      dxpl_id,
-                                      flush_ptr->addr,
-                                      H5C__NO_FLAGS_SET,
-                                      TRUE,
-                                      NULL) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry.")
-
-            if ( ( cache_ptr->entries_removed_counter > 1 ) ||
-                 ( cache_ptr->last_entry_removed_ptr == entry_ptr ) )
-
-                restart_scan = TRUE;
-
-        } /* end else-if */
-
-        /* Otherwise, no action to be taken on this entry. Grab the next. */
-        else {
-            entry_ptr = entry_ptr->prev;
-
-            if ( entry_ptr != NULL )
-                next_ptr = entry_ptr->next; 
-
-        } /* end else */
-
-        if ( ( entry_ptr != NULL ) 
-             &&
-             ( ( restart_scan )
-               ||
-               ( entry_ptr->is_dirty != prev_is_dirty )
-               ||
-               ( entry_ptr->next != next_ptr )
-               ||
-               ( entry_ptr->is_protected )
-               ||
-               ( entry_ptr->is_pinned ) 
-             ) 
-           ) {
-
-            /* something has happened to the LRU -- start over
-             * from the tail.
-             *
-             * Recall that this code should be un-reachable at present,
-             * as all the operations by entries on flush that could cause
-             * it to be reachable are disallowed in the parallel case at
-             * present.  Hence the following assertion which should be 
-             * removed if the above changes.
-             */
-
-	    HDassert( ! restart_scan );
-            HDassert( entry_ptr->is_dirty == prev_is_dirty );
-            HDassert( entry_ptr->next == next_ptr );
-            HDassert( ! entry_ptr->is_protected );
-            HDassert( ! entry_ptr->is_pinned );
-
-            HDassert(FALSE); /* see comment above */
-
-            restart_scan = FALSE;
-            entry_ptr = cache_ptr->LRU_tail_ptr;
-/* 
-	    H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr)
-*/
-        }
-
-        entries_examined++;
-    } /* end while */
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %d/%d/%d.\n",
-              FUNC, mpi_rank, entries_examined,
-              entries_cleared, entries_flushed);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-    /* It is also possible that some of the cleared entries are on the
-     * pinned list.  Must scan that also.
-     *
-     * WARNING:
-     *
-     *	As we now allow unpinning, and removal of other entries as a side 
-     *  effect of flushing an entry, it is possible that the next entry
-     *  in a PEL scan could either be no longer pinned, or no longer in
-     *  the cache by the time we get to it.
-     *
-     *  At present, this is not possible in this case, as we disallow such
-     *  operations in the parallel version of the library.  However, Quincey
-     *  has been making noises about relaxing this.  If and when he does,
-     *  we have a potential problem here.
-     *
-     *  The same issue exists in the serial cache, and there are tests 
-     *  to detect this problem when it occurs, and adjust to it.  As seen
-     *  above in the LRU scan, I have ported such tests to the parallel 
-     *  code where a close cognate exists in the serial code.  
-     *
-     *  I haven't done so here, as there are no PEL scans where the problem
-     *  can occur in the serial code.  Needless to say, this will have to 
-     *  be repaired if the constraints on pre_serialize and serialize 
-     *  callbacks are relaxed in the parallel version of the metadata cache.
-     *
-     *						JRM -- 4/1/15
-     */
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n",
-             FUNC, mpi_rank, (int)(cache_ptr->pel_len));
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-    entry_ptr = cache_ptr->pel_head_ptr;
-    while((entry_ptr != NULL) &&
-          ((entries_cleared + entries_flushed + entries_delayed)
-            < num_candidates)) {
-
-        /* If entry is marked for flush or for clear */
-        if((entry_ptr->clear_on_unprotect||entry_ptr->flush_immediately)) {
-
-            /* If this entry needs to be flushed last */
-            if (entry_ptr->flush_me_last) {
-
-                /* At this time, only the superblock supports being
-                   flushed last. Conveniently, it also happens to be the only
-                   entry that supports being flushed collectively, as well. Also
-                   conveniently, it's always pinned, so we only need to check
-                   for it while scanning the PEL here. Finally, it's never
-                   included in a candidate list that excludes other dirty
-                   entries in a cache, so we can handle this relatively simple
-                   case here.
-                   
-                   For now, this function asserts this and saves the entry
-                   to flush it after scanning the rest of the PEL list.
-
-                   If there are ever more entries that either need to be
-                   flushed last and/or flushed collectively, this whole routine
-                   will need to be reworked to handle all additional cases. As
-                   it is the simple case of a single pinned entry needing
-                   flushed last and collectively is just a minor addition to
-                   this routine, but signficantly buffing up the usage of
-                   flush_me_last or flush_me_collectively will require a more
-                   intense rework of this function and potentially the function
-                   of candidate lists as a whole. */
-
-                HDassert(entry_ptr->flush_me_collectively);
-                entries_to_flush_or_clear_last++;
-                entries_to_flush_collectively++;
-                HDassert(entries_to_flush_or_clear_last == 1);
-                HDassert(entries_to_flush_collectively == 1);
-
-                /* Delay the entry. It will be flushed later. */
-                delayed_ptr = entry_ptr;
-                entries_delayed++;
-                HDassert(entries_delayed == 1);
-
-            } /* end if */
-
-            /* Else, this process needs to clear this entry. */
-            else if (entry_ptr->clear_on_unprotect) {
-                HDassert(!entry_ptr->flush_immediately);
-                entry_ptr->clear_on_unprotect = FALSE;
-                clear_ptr = entry_ptr;
-                entry_ptr = entry_ptr->next;
-                entries_cleared++;
-
-#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
-            HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank,
-                      (long long)clear_ptr->addr);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-                if(H5C_flush_single_entry(f,
-                                      dxpl_id,
-                                      clear_ptr->addr,
-                                      H5C__FLUSH_CLEAR_ONLY_FLAG,
-                                      TRUE,
-                                      NULL) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
-            } /* end else-if */
-
-            /* Else, if this process needs to independently flush this entry. */
-            else if (entry_ptr->flush_immediately) {
-                entry_ptr->flush_immediately = FALSE;
-                flush_ptr = entry_ptr;
-                entry_ptr = entry_ptr->next;
-                entries_flushed++;
-
-#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
-                HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank,
-                      (long long)flush_ptr->addr);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-                if(H5C_flush_single_entry(f,
-                                      dxpl_id,
-                                      flush_ptr->addr,
-                                      H5C__NO_FLAGS_SET,
-                                      TRUE,
-                                      NULL) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
-            } /* end else-if */
-        } /* end if */
-
-        /* Otherwise, this entry is not marked for flush or clear. Grab the next. */
-        else {
-            entry_ptr = entry_ptr->next;
-        } /* end else */
-
-    } /* end while */
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
-    HDfprintf(stdout,
-              "%s:%d: pel entries examined/cleared/flushed = %d/%d/%d.\n",
-              FUNC, mpi_rank, entries_examined,
-              entries_cleared, entries_flushed);
-    HDfprintf(stdout, "%s:%d: done.\n", FUNC, mpi_rank);
-
-    HDfsync(stdout);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
-    /* ====================================================================== *
-     * Now, handle all delayed entries.                                       *
-     *                                                                        *
-     * This can *only* be the superblock at this time, so it's relatively     *
-     * easy to deal with. We're collectively flushing the entry saved from    *
-     * above. This will need to be handled differently if there are ever more *
-     * than one entry needing this special treatment.)                        *
-     * ====================================================================== */
-
-    if (delayed_ptr) {
-
-        if (delayed_ptr->clear_on_unprotect) {
-            entry_ptr->clear_on_unprotect = FALSE;
-            entries_cleared++;
-        } else if (delayed_ptr->flush_immediately) {
-            entry_ptr->flush_immediately = FALSE;
-            entries_flushed++;
-        } /* end if */
-
-        if(H5C_flush_single_entry(f,
-                                  dxpl_id,
-                                  delayed_ptr->addr,
-                                  H5C__NO_FLAGS_SET,
-                                  TRUE,
-                                  NULL) < 0)
-            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL,
-                        "Can't flush entry collectively.")
-
-        entries_flushed_collectively++;
-        entries_flushed_or_cleared_last++;
-    } /* end if */
-
-    /* ====================================================================== *
-     * Finished flushing everything.                                          *
-     * ====================================================================== */
-
-    HDassert((entries_flushed == entries_to_flush));
-    HDassert((entries_cleared == entries_to_clear));
-    HDassert((entries_flushed_or_cleared_last == entries_to_flush_or_clear_last));
-    HDassert((entries_flushed_collectively == entries_to_flush_collectively));
-    
-    if((entries_flushed != entries_to_flush) || 
-       (entries_cleared != entries_to_clear) ||
-       (entries_flushed_or_cleared_last != entries_to_flush_or_clear_last) ||
-       (entries_flushed_collectively != entries_to_flush_collectively))
-        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry count mismatch.")
-
-done:
-    if(candidate_assignment_table != NULL)
-        candidate_assignment_table = (int *)H5MM_xfree((void *)candidate_assignment_table);
-
-    FUNC_LEAVE_NOAPI(ret_value)
-} /* H5C_apply_candidate_list() */
-#endif /* H5_HAVE_PARALLEL */
-
-
-/*-------------------------------------------------------------------------
- * Function:    H5C_construct_candidate_list__clean_cache
- *
- * Purpose:     Construct the list of entries that should be flushed to 
- *		clean all entries in the cache.
- *
- *		This function is used in managing sync points, and 
- *		shouldn't be used elsewhere.
- *
- * Return:      Success:        SUCCEED
- *
- *              Failure:        FAIL
- *
- * Programmer:  John Mainzer
- *              3/17/10
- *
- *-------------------------------------------------------------------------
- */
-#ifdef H5_HAVE_PARALLEL
-herr_t
-H5C_construct_candidate_list__clean_cache(H5C_t * cache_ptr)
-{
-    size_t              space_needed;
-    herr_t              ret_value = SUCCEED;      /* Return value */
-
-    FUNC_ENTER_NOAPI(FAIL)
-
-    HDassert( cache_ptr != NULL );
-    HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC );
-
-    /* As a sanity check, set space needed to the size of the skip list.
-     * This should be the sum total of the sizes of all the dirty entries
-     * in the metadata cache.
-     */
-    space_needed = cache_ptr->slist_size;
-
-    /* Recall that while we shouldn't have any protected entries at this
-     * point, it is possible that some dirty entries may reside on the
-     * pinned list at this point.
-     */
-    HDassert( cache_ptr->slist_size <= 
-              (cache_ptr->dLRU_list_size + cache_ptr->pel_size) );
-    HDassert( cache_ptr->slist_len  <= 
-              (cache_ptr->dLRU_list_len + cache_ptr->pel_len) );
-
-    if(space_needed > 0) { /* we have work to do */
-        H5C_cache_entry_t *entry_ptr;
-        int     nominated_entries_count = 0;
-        size_t  nominated_entries_size = 0;
-        haddr_t	nominated_addr;
-
-        HDassert( cache_ptr->slist_len > 0 );
-
-        /* Scan the dirty LRU list from tail forward and nominate sufficient
-         * entries to free up the necessary space. 
-         */
-        entry_ptr = cache_ptr->dLRU_tail_ptr;
-        while((nominated_entries_size < space_needed) &&
-                (nominated_entries_count < cache_ptr->slist_len) &&
-                (entry_ptr != NULL)) {
-            HDassert( ! (entry_ptr->is_protected) );
-            HDassert( ! (entry_ptr->is_read_only) );
-            HDassert( entry_ptr->ro_ref_count == 0 );
-            HDassert( entry_ptr->is_dirty );
-            HDassert( entry_ptr->in_slist );
-
-            nominated_addr = entry_ptr->addr;
-            if(H5AC_add_candidate((H5AC_t *)cache_ptr, nominated_addr) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_add_candidate() failed(1).")
-
-            nominated_entries_size += entry_ptr->size;
-            nominated_entries_count++;
-            entry_ptr = entry_ptr->aux_prev;
-        } /* end while */
-        HDassert( entry_ptr == NULL );
-
-        /* it is possible that there are some dirty entries on the 
-         * protected entry list as well -- scan it too if necessary
-         */
-        entry_ptr = cache_ptr->pel_head_ptr;
-        while((nominated_entries_size < space_needed) &&
-                (nominated_entries_count < cache_ptr->slist_len) &&
-                (entry_ptr != NULL)) {
-            if(entry_ptr->is_dirty) {
-                HDassert( ! (entry_ptr->is_protected) );
-                HDassert( ! (entry_ptr->is_read_only) );
-                HDassert( entry_ptr->ro_ref_count == 0 );
-                HDassert( entry_ptr->is_dirty );
-                HDassert( entry_ptr->in_slist );
-
-                nominated_addr = entry_ptr->addr;
-                if(H5AC_add_candidate((H5AC_t *)cache_ptr, nominated_addr) < 0)
-                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_add_candidate() failed(2).")
-
-                nominated_entries_size += entry_ptr->size;
-                nominated_entries_count++;
-            } /* end if */
-
-            entry_ptr = entry_ptr->next;
-        } /* end while */
-
-        HDassert( nominated_entries_count == cache_ptr->slist_len );
-        HDassert( nominated_entries_size == space_needed );
-    } /* end if */
-
-done:
-    FUNC_LEAVE_NOAPI(ret_value)
-} /* H5C_construct_candidate_list__clean_cache() */
-#endif /* H5_HAVE_PARALLEL */
-
-
-/*-------------------------------------------------------------------------
- * Function:    H5C_construct_candidate_list__min_clean
- *
- * Purpose:     Construct the list of entries that should be flushed to 
- *		get the cache back within its min clean constraints.
- *
- *		This function is used in managing sync points, and 
- *		shouldn't be used elsewhere.
- *
- * Return:      Success:        SUCCEED
- *
- *              Failure:        FAIL
- *
- * Programmer:  John Mainzer
- *              3/17/10
- *
- *-------------------------------------------------------------------------
- */
-#ifdef H5_HAVE_PARALLEL
-herr_t
-H5C_construct_candidate_list__min_clean(H5C_t * cache_ptr)
-{
-    size_t              space_needed = 0;
-    herr_t              ret_value = SUCCEED;      /* Return value */
-
-    FUNC_ENTER_NOAPI(FAIL)
-
-    HDassert( cache_ptr != NULL );
-    HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC );
-
-    /* compute the number of bytes (if any) that must be flushed to get the 
-     * cache back within its min clean constraints.
-     */
-    if(cache_ptr->max_cache_size > cache_ptr->index_size) {
-        if(((cache_ptr->max_cache_size - cache_ptr->index_size) +
-               cache_ptr->cLRU_list_size) >= cache_ptr->min_clean_size)
-            space_needed = 0;
-        else
-            space_needed = cache_ptr->min_clean_size -
-                ((cache_ptr->max_cache_size - cache_ptr->index_size) +
-                 cache_ptr->cLRU_list_size);
-    } /* end if */
-    else {
-        if(cache_ptr->min_clean_size <= cache_ptr->cLRU_list_size)
-           space_needed = 0;
-        else
-            space_needed = cache_ptr->min_clean_size -
-                           cache_ptr->cLRU_list_size;
-    } /* end else */
-
-    if(space_needed > 0) { /* we have work to do */
-        H5C_cache_entry_t *entry_ptr;
-        int    nominated_entries_count = 0;
-        size_t nominated_entries_size = 0;
-
-        HDassert( cache_ptr->slist_len > 0 );
-
-        /* Scan the dirty LRU list from tail forward and nominate sufficient
-         * entries to free up the necessary space. 
-         */
-        entry_ptr = cache_ptr->dLRU_tail_ptr;
-        while((nominated_entries_size < space_needed) &&
-                (nominated_entries_count < cache_ptr->slist_len) &&
-                (entry_ptr != NULL) &&
-                (!entry_ptr->flush_me_last)) {
-            haddr_t		nominated_addr;
-
-            HDassert( ! (entry_ptr->is_protected) );
-            HDassert( ! (entry_ptr->is_read_only) );
-            HDassert( entry_ptr->ro_ref_count == 0 );
-            HDassert( entry_ptr->is_dirty );
-            HDassert( entry_ptr->in_slist );
-
-            nominated_addr = entry_ptr->addr;
-            if(H5AC_add_candidate((H5AC_t *)cache_ptr, nominated_addr) < 0)
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_add_candidate() failed.")
-
-            nominated_entries_size += entry_ptr->size;
-            nominated_entries_count++;
-            entry_ptr = entry_ptr->aux_prev;
-        } /* end while */
-        HDassert( nominated_entries_count <= cache_ptr->slist_len );
-        HDassert( nominated_entries_size >= space_needed );
-    } /* end if */
-
-done:
-    FUNC_LEAVE_NOAPI(ret_value)
-} /* H5C_construct_candidate_list__min_clean() */
-#endif /* H5_HAVE_PARALLEL */
-
-
-/*-------------------------------------------------------------------------
  * Function:    H5C_create
  *
  * Purpose:     Allocate, initialize, and return the address of a new
@@ -1818,7 +956,7 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type,
     if(entry_ptr->is_pinned)
         HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "Target entry is pinned.")
 
-    /* If we get this far, call H5C_flush_single_entry() with the
+    /* If we get this far, call H5C__flush_single_entry() with the
      * H5C__FLUSH_INVALIDATE_FLAG and the H5C__FLUSH_CLEAR_ONLY_FLAG.
      * This will clear the entry, and then delete it from the cache.
      */
@@ -1831,8 +969,8 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type,
     entry_size      = entry_ptr->size;
 #endif /* H5C_DO_EXTREME_SANITY_CHECKS */
 
-    if(H5C_flush_single_entry(f, dxpl_id, entry_ptr->addr, flush_flags, TRUE, NULL) < 0)
-        HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "H5C_flush_single_entry() failed.")
+    if(H5C__flush_single_entry(f, dxpl_id, entry_ptr->addr, flush_flags, TRUE, NULL) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "H5C__flush_single_entry() failed.")
 
 #if H5C_DO_SANITY_CHECKS
     if ( entry_was_dirty )
@@ -1982,7 +1120,7 @@ H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags)
         /* set the cache_ptr->slist_change_in_pre_serialize and
 	 * cache_ptr->slist_change_in_serialize to false.
          *
-         * These flags are set to TRUE by H5C_flush_single_entry if the 
+         * These flags are set to TRUE by H5C__flush_single_entry if the 
  	 * slist is modified by a pre_serialize or serialize call respectively.
          * H5C_flush_cache uses these flags to detect any modifications
          * to the slist that might corrupt the scan of the slist -- and 
@@ -2156,7 +1294,7 @@ H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags)
                                 flushed_entries_size += (int64_t)entry_ptr->size;
                                 entry_size_change = 0;
 #endif /* H5C_DO_SANITY_CHECKS */
-                                status = H5C_flush_single_entry(f,
+                                status = H5C__flush_single_entry(f,
                                                             dxpl_id,
                                                             entry_ptr->addr,
                                                             flags,
@@ -2216,7 +1354,7 @@ H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags)
                                 flushed_entries_size += (int64_t)entry_ptr->size;
                                 entry_size_change = 0;
 #endif /* H5C_DO_SANITY_CHECKS */
-                                status = H5C_flush_single_entry(f,
+                                status = H5C__flush_single_entry(f,
                                                             dxpl_id,
                                                             entry_ptr->addr,
                                                             flags,
@@ -3198,318 +2336,6 @@ done:
 
 
 /*-------------------------------------------------------------------------
- *
- * Function:    H5C_mark_entries_as_clean
- *
- * Purpose:     When the H5C code is used to implement the metadata caches
- *		in PHDF5, only the cache with MPI_rank 0 is allowed to
- *		actually write entries to disk -- all other caches must
- *		retain dirty entries until they are advised that the
- *		entries are clean.
- *
- *		This function exists to allow the H5C code to receive these
- *		notifications.
- *
- *		The function receives a list of entry base addresses
- *		which must refer to dirty entries in the cache.  If any
- *		of the entries are either clean or don't exist, the
- *		function flags an error.
- *
- *		The function scans the list of entries and flushes all
- *		those that are currently unprotected with the
- *		H5C__FLUSH_CLEAR_ONLY_FLAG.  Those that are currently
- *		protected are flagged for clearing when they are
- *		unprotected.
- *
- * Return:      Non-negative on success/Negative on failure
- *
- * Programmer:  John Mainzer
- *              7/5/05
- *
- * Changes:     Tidied up code, removeing some old commented out 
- *		code that had been left in pending success of the 
- *		new version.
- *
- *		Note that unlike H5C_apply_candidate_list(), 
- *		H5C_mark_entries_as_clean() makes all its calls to 
- *		H6C_flush_single_entry() with the 
- *		H5C__FLUSH_CLEAR_ONLY_FLAG set.  As a result, 
- *		the pre_serialize() and serialize calls are not made.
- *
- *		This then implies that (assuming such actions were 
- *		permitted in the parallel case) no loads, dirties, 
- *		resizes, or removals of other entries can occur as 
- *		a side effect of the flush.  Hence, there is no need
- *		for the checks for entry removal / status change 
- *		that I ported to H5C_apply_candidate_list().
- *
- *		However, if (in addition to allowing such operations
- *		in the parallel case), we allow such operations outside
- *		of the pre_serialize / serialize routines, this may 
- *		cease to be the case -- requiring a review of this 
- *		function.
- *
- *-------------------------------------------------------------------------
- */
-#ifdef H5_HAVE_PARALLEL
-herr_t
-H5C_mark_entries_as_clean(H5F_t *  f,
-                          hid_t     dxpl_id,
-                          int32_t   ce_array_len,
-                          haddr_t * ce_array_ptr)
-{
-    H5C_t *             cache_ptr;
-    int			entries_cleared;
-    int			entries_examined;
-    int                 i;
-    int			initial_list_len;
-    haddr_t		addr;
-#if H5C_DO_SANITY_CHECKS
-    int			pinned_entries_marked = 0;
-    int			protected_entries_marked = 0;
-    int			other_entries_marked = 0;
-    haddr_t		last_addr;
-#endif /* H5C_DO_SANITY_CHECKS */
-    H5C_cache_entry_t *	clear_ptr = NULL;
-    H5C_cache_entry_t *	entry_ptr = NULL;
-    herr_t		ret_value = SUCCEED;      /* Return value */
-
-    FUNC_ENTER_NOAPI(FAIL)
-
-    HDassert( f );
-    HDassert( f->shared );
-    cache_ptr = f->shared->cache;
-    HDassert( cache_ptr );
-    HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC );
-
-    HDassert( ce_array_len > 0 );
-    HDassert( ce_array_ptr != NULL );
-
-#if H5C_DO_EXTREME_SANITY_CHECKS
-    if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) ||
-         ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) ||
-         ( H5C_validate_lru_list(cache_ptr) < 0 ) ) {
-
-        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                    "an extreme sanity check failed on entry.\n");
-    }
-#endif /* H5C_DO_EXTREME_SANITY_CHECKS */
-
-    for ( i = 0; i < ce_array_len; i++ )
-    {
-        addr = ce_array_ptr[i];
-
-#if H5C_DO_SANITY_CHECKS
-        if ( i == 0 ) {
-
-            last_addr = addr;
-
-        } else {
-
-            if ( last_addr == addr ) {
-
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                            "Duplicate entry in cleaned list.\n");
-
-            } else if ( last_addr > addr ) {
-
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                            "cleaned list not sorted.\n");
-            }
-        }
-
-#if H5C_DO_EXTREME_SANITY_CHECKS
-        if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) ||
-             ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) ||
-             ( H5C_validate_lru_list(cache_ptr) < 0 ) ) {
-
-            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                        "an extreme sanity check failed in for loop.\n");
-        }
-#endif /* H5C_DO_EXTREME_SANITY_CHECKS */
-#endif /* H5C_DO_SANITY_CHECKS */
-
-        HDassert( H5F_addr_defined(addr) );
-
-        H5C__SEARCH_INDEX(cache_ptr, addr, entry_ptr, FAIL)
-
-        if ( entry_ptr == NULL ) {
-#if H5C_DO_SANITY_CHECKS
-	    HDfprintf(stdout,
-                  "H5C_mark_entries_as_clean: entry[%d] = %ld not in cache.\n",
-                      (int)i,
-                      (long)addr);
-#endif /* H5C_DO_SANITY_CHECKS */
-            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                        "Listed entry not in cache?!?!?.")
-
-        } else if ( ! entry_ptr->is_dirty ) {
-
-#if H5C_DO_SANITY_CHECKS
-	    HDfprintf(stdout,
-                      "H5C_mark_entries_as_clean: entry %ld is not dirty!?!\n",
-                      (long)addr);
-#endif /* H5C_DO_SANITY_CHECKS */
-            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                        "Listed entry not dirty?!?!?.")
-
-        } else {
-
-            /* Mark the entry to be cleared on unprotect.  We will
-             * scan the LRU list shortly, and clear all those entries
-             * not currently protected.
-             */
-            entry_ptr->clear_on_unprotect = TRUE;
-#if H5C_DO_SANITY_CHECKS
-	    if ( entry_ptr->is_protected ) {
-
-		protected_entries_marked++;
-
-	    } else if ( entry_ptr->is_pinned ) {
-
-		pinned_entries_marked++;
-
-	    } else {
-
-		other_entries_marked++;
-	    }
-#endif /* H5C_DO_SANITY_CHECKS */
-        }
-    }
-
-    /* Scan through the LRU list from back to front, and flush the
-     * entries whose clear_on_unprotect flags are set.  Observe that
-     * any protected entries will not be on the LRU, and therefore
-     * will not be flushed at this time.
-     *
-     * Note that unlike H5C_apply_candidate_list(), 
-     * H5C_mark_entries_as_clean() makes all its calls to 
-     * H6C_flush_single_entry() with the H5C__FLUSH_CLEAR_ONLY_FLAG 
-     * set.  As a result, the pre_serialize() and serialize calls are 
-     * not made.
-     *
-     * This then implies that (assuming such actions were 
-     * permitted in the parallel case) no loads, dirties, 
-     * resizes, or removals of other entries can occur as 
-     * a side effect of the flush.  Hence, there is no need
-     * for the checks for entry removal / status change 
-     * that I ported to H5C_apply_candidate_list().
-     *
-     * However, if (in addition to allowing such operations
-     * in the parallel case), we allow such operations outside
-     * of the pre_serialize / serialize routines, this may 
-     * cease to be the case -- requiring a review of this 
-     * point.
-     *					JRM -- 4/7/15
-     */
-
-    entries_cleared = 0;
-    entries_examined = 0;
-    initial_list_len = cache_ptr->LRU_list_len;
-    entry_ptr = cache_ptr->LRU_tail_ptr;
-
-    while ( ( entry_ptr != NULL ) &&
-            ( entries_examined <= initial_list_len ) &&
-            ( entries_cleared < ce_array_len ) )
-    {
-        if ( entry_ptr->clear_on_unprotect ) {
-
-            entry_ptr->clear_on_unprotect = FALSE;
-            clear_ptr = entry_ptr;
-            entry_ptr = entry_ptr->prev;
-            entries_cleared++;
-
-            if ( H5C_flush_single_entry(f,
-                                        dxpl_id,
-                                        clear_ptr->addr,
-                                        H5C__FLUSH_CLEAR_ONLY_FLAG,
-                                        TRUE,
-                                        NULL) < 0 ) {
-
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
-            }
-        } else {
-
-            entry_ptr = entry_ptr->prev;
-        }
-        entries_examined++;
-    }
-
-#if H5C_DO_SANITY_CHECKS
-    HDassert( entries_cleared == other_entries_marked );
-#endif /* H5C_DO_SANITY_CHECKS */
-
-    /* It is also possible that some of the cleared entries are on the
-     * pinned list.  Must scan that also.
-     */
-
-    entry_ptr = cache_ptr->pel_head_ptr;
-
-    while ( entry_ptr != NULL )
-    {
-        if ( entry_ptr->clear_on_unprotect ) {
-
-            entry_ptr->clear_on_unprotect = FALSE;
-            clear_ptr = entry_ptr;
-            entry_ptr = entry_ptr->next;
-            entries_cleared++;
-
-            if ( H5C_flush_single_entry(f,
-                                        dxpl_id,
-                                        clear_ptr->addr,
-                                        H5C__FLUSH_CLEAR_ONLY_FLAG,
-                                        TRUE,
-                                        NULL) < 0 ) {
-
-                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.")
-            }
-        } else {
-
-            entry_ptr = entry_ptr->next;
-        }
-    }
-
-#if H5C_DO_SANITY_CHECKS
-    HDassert( entries_cleared == pinned_entries_marked + other_entries_marked );
-    HDassert( entries_cleared + protected_entries_marked == ce_array_len );
-#endif /* H5C_DO_SANITY_CHECKS */
-
-    HDassert( ( entries_cleared == ce_array_len ) ||
-              ( (ce_array_len - entries_cleared) <= cache_ptr->pl_len ) );
-
-#if H5C_DO_SANITY_CHECKS
-    i = 0;
-    entry_ptr = cache_ptr->pl_head_ptr;
-    while ( entry_ptr != NULL )
-    {
-        if ( entry_ptr->clear_on_unprotect ) {
-
-            i++;
-        }
-        entry_ptr = entry_ptr->next;
-    }
-    HDassert( (entries_cleared + i) == ce_array_len );
-#endif /* H5C_DO_SANITY_CHECKS */
-
-done:
-
-#if H5C_DO_EXTREME_SANITY_CHECKS
-    if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) ||
-         ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) ||
-         ( H5C_validate_lru_list(cache_ptr) < 0 ) ) {
-
-        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
-                    "an extreme sanity check failed on exit.\n");
-    }
-#endif /* H5C_DO_EXTREME_SANITY_CHECKS */
-
-    FUNC_LEAVE_NOAPI(ret_value)
-
-} /* H5C_mark_entries_as_clean() */
-#endif /* H5_HAVE_PARALLEL */
-
-
-/*-------------------------------------------------------------------------
  * Function:    H5C_mark_entry_dirty
  *
  * Purpose:	Mark a pinned or protected entry as dirty.  The target entry
@@ -5891,7 +4717,7 @@ done:
  *		writes (secondary_dxpl_id).  Since an uprotect cannot
  *		occasion a write at present, all this is moot for now.
  *		However, things change, and in any case,
- *		H5C_flush_single_entry() needs primary_dxpl_id and
+ *		H5C__flush_single_entry() needs primary_dxpl_id and
  *		secondary_dxpl_id in its parameter list.
  *
  *		The function can't cause a read either, so the dxpl_id
@@ -6133,7 +4959,7 @@ H5C_unprotect(H5F_t *		  f,
             if(take_ownership)
                 flush_flags |= H5C__TAKE_OWNERSHIP_FLAG;
 
-            if ( H5C_flush_single_entry(f,
+            if ( H5C__flush_single_entry(f,
                                         dxpl_id,
                                         addr,
                                         flush_flags,
@@ -6174,7 +5000,7 @@ H5C_unprotect(H5F_t *		  f,
                             "hash table contains multiple entries for addr?!?.")
             }
 
-            if ( H5C_flush_single_entry(f,
+            if ( H5C__flush_single_entry(f,
                                         dxpl_id,
                                         addr,
                                         H5C__FLUSH_CLEAR_ONLY_FLAG,
@@ -7504,7 +6330,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
 
                 /* reset entries_removed_counter and
                  * last_entry_removed_ptr prior to the call to
-                 * H5C_flush_single_entry() so that we can spot
+                 * H5C__flush_single_entry() so that we can spot
                  * unexpected removals of entries from the cache,
                  * and set the restart_scan flag if proceeding
                  * would be likely to cause us to scan an entry
@@ -7513,7 +6339,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
                 cache_ptr->entries_removed_counter = 0;
                 cache_ptr->last_entry_removed_ptr  = NULL;
 
-                result = H5C_flush_single_entry(f,
+                result = H5C__flush_single_entry(f,
                                                 dxpl_id,
                                                 entry_ptr->addr,
                                                 H5C__NO_FLAGS_SET,
@@ -7529,7 +6355,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
 
                 bytes_evicted += entry_ptr->size;
 
-                result = H5C_flush_single_entry(f,
+                result = H5C__flush_single_entry(f,
                                                 dxpl_id,
                                                 entry_ptr->addr,
                                                 H5C__FLUSH_INVALIDATE_FLAG,
@@ -7620,7 +6446,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
 
             if ( ! (entry_ptr->is_dirty) ) {
 
-                result = H5C_flush_single_entry(f,
+                result = H5C__flush_single_entry(f,
                                                 dxpl_id,
                                                 entry_ptr->addr,
                                                 H5C__FLUSH_INVALIDATE_FLAG,
@@ -8126,7 +6952,7 @@ done:
  *		was largely removed from the cache data structures before
  *		the flush proper.  However, re-entrant calls to the cache
  *		in the parallel case required a re-factoring of the 
- *		H5C_flush_single_entry() function to keep entries fully
+ *		H5C__flush_single_entry() function to keep entries fully
  *		in the cache until after the pre-serialize and serialize
  *		calls.
  *                                               JRM -- 12/25/14
@@ -8239,7 +7065,7 @@ H5C_flush_invalidate_cache(const H5F_t * f,
              */
 
 #if H5C_DO_SANITY_CHECKS
-            /* Depending on circumstances, H5C_flush_single_entry() will
+            /* Depending on circumstances, H5C__flush_single_entry() will
              * remove dirty entries from the slist as it flushes them.
              * Thus for sanity checks we must make note of the initial
              * slist length and size before we do any flushes.
@@ -8272,7 +7098,7 @@ H5C_flush_invalidate_cache(const H5F_t * f,
             /* set the cache_ptr->slist_change_in_pre_serialize and
              * cache_ptr->slist_change_in_serialize to false.
              *
-             * These flags are set to TRUE by H5C_flush_single_entry if the
+             * These flags are set to TRUE by H5C__flush_single_entry if the
              * slist is modified by a pre_serialize or serialize call 
              * respectively.
              *
@@ -8381,7 +7207,7 @@ H5C_flush_invalidate_cache(const H5F_t * f,
 
                         /* Test to see if we are can flush the entry now.
                          * If we can, go ahead and flush, but don't tell
-                         * H5C_flush_single_entry() to destroy the entry
+                         * H5C__flush_single_entry() to destroy the entry
                          * as pinned entries can't be evicted.
                          */
                         if(entry_ptr->flush_dep_height == curr_flush_dep_height ) {
@@ -8399,7 +7225,7 @@ H5C_flush_invalidate_cache(const H5F_t * f,
 		            entry_size_change = 0;
 #endif /* H5C_DO_SANITY_CHECKS */
 
-                            status = H5C_flush_single_entry(f,
+                            status = H5C__flush_single_entry(f,
                                                             dxpl_id,
                                                             entry_ptr->addr,
                                                             H5C__NO_FLAGS_SET,
@@ -8461,7 +7287,7 @@ H5C_flush_invalidate_cache(const H5F_t * f,
 		            entry_size_change = 0;
 #endif /* H5C_DO_SANITY_CHECKS */
 
-                            status = H5C_flush_single_entry(f,
+                            status = H5C__flush_single_entry(f,
                                     dxpl_id,
                                     entry_ptr->addr,
                                     (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG),
@@ -8603,7 +7429,7 @@ H5C_flush_invalidate_cache(const H5F_t * f,
 
                                 entry_was_dirty = entry_ptr->is_dirty;
 
-                                status = H5C_flush_single_entry(f,
+                                status = H5C__flush_single_entry(f,
                                     dxpl_id,
                                     entry_ptr->addr,
                                     (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG),
@@ -8755,7 +7581,7 @@ done:
 
 /*-------------------------------------------------------------------------
  *
- * Function:    H5C_flush_single_entry
+ * Function:    H5C__flush_single_entry
  *
  * Purpose:     Flush or clear (and evict if requested) the cache entry
  *		with the specified address and type.  If the type is NULL,
@@ -8814,8 +7640,8 @@ done:
  *
  *-------------------------------------------------------------------------
  */
-static herr_t
-H5C_flush_single_entry(const H5F_t *	   f,
+herr_t
+H5C__flush_single_entry(const H5F_t *	   f,
                        hid_t 		   dxpl_id,
                        haddr_t		   addr,
                        unsigned	     	   flags,
@@ -8838,7 +7664,7 @@ H5C_flush_single_entry(const H5F_t *	   f,
     H5C_cache_entry_t * entry_ptr = NULL;
     herr_t		ret_value = SUCCEED;      /* Return value */
 
-    FUNC_ENTER_NOAPI_NOINIT
+    FUNC_ENTER_PACKAGE
 
     HDassert( f );
     HDassert( cache_ptr );
@@ -9620,7 +8446,7 @@ done:
               ( ( entry_ptr ) && ( ! entry_ptr->is_dirty ) ) );
 
     FUNC_LEAVE_NOAPI(ret_value)
-} /* H5C_flush_single_entry() */
+} /* H5C__flush_single_entry() */
 
 
 /*-------------------------------------------------------------------------
@@ -10099,7 +8925,7 @@ done:
  *		attempts to load another entry.
  *
  *		This error was exposed by a re-factor of the 
- *		H5C_flush_single_entry() routine.  However, it was 
+ *		H5C__flush_single_entry() routine.  However, it was 
  *		a potential bug from the moment that entries were 
  *		allowed to load other entries on flush.
  *
@@ -10221,7 +9047,7 @@ H5C_make_space_in_cache(H5F_t *	f,
 
 		    /* reset entries_removed_counter and 
                      * last_entry_removed_ptr prior to the call to 
-                     * H5C_flush_single_entry() so that we can spot 
+                     * H5C__flush_single_entry() so that we can spot 
                      * unexpected removals of entries from the cache,
                      * and set the restart_scan flag if proceeding 
                      * would be likely to cause us to scan an entry 
@@ -10230,7 +9056,7 @@ H5C_make_space_in_cache(H5F_t *	f,
                     cache_ptr->entries_removed_counter = 0;
                     cache_ptr->last_entry_removed_ptr  = NULL;
 
-                    result = H5C_flush_single_entry(f,
+                    result = H5C__flush_single_entry(f,
                                                     dxpl_id,
                                                     entry_ptr->addr,
                                                     H5C__NO_FLAGS_SET,
@@ -10249,7 +9075,7 @@ H5C_make_space_in_cache(H5F_t *	f,
                     cache_ptr->entries_scanned_to_make_space++;
 #endif /* H5C_COLLECT_CACHE_STATS */
 
-                    result = H5C_flush_single_entry(f,
+                    result = H5C__flush_single_entry(f,
                                                     dxpl_id,
                                                     entry_ptr->addr,
                                                     H5C__FLUSH_INVALIDATE_FLAG,
@@ -10404,7 +9230,7 @@ H5C_make_space_in_cache(H5F_t *	f,
 
             prev_ptr = entry_ptr->aux_prev;
 
-            result = H5C_flush_single_entry(f,
+            result = H5C__flush_single_entry(f,
                                             dxpl_id,
                                             entry_ptr->addr,
                                             H5C__FLUSH_INVALIDATE_FLAG,