From 3552beb08b10a9037691905b5dec644428a9ac35 Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Fri, 10 Aug 2012 21:26:34 -0500 Subject: [svn-r22666] Description: Merge "flush me last" & "flush me collectively" feature from the avoid_truncate branch back to the trunk. (To help enable allowing the free space section info to reside in temporary address space) Tested on: Mac OSX/64 10.7.4 (amazon) w/debug, gcc 4.7.x, C++, FORTRAN & threadsafe (h5committest forthcoming) --- src/H5ACprivate.h | 2 + src/H5C.c | 468 ++++++++++++++++++++++++++++++++++----------------- src/H5Cpkg.h | 19 +++ src/H5Cprivate.h | 27 ++- src/H5FDmpiposix.c | 20 +-- src/H5Fsuper.c | 2 +- src/H5Fsuper_cache.c | 44 ++++- 7 files changed, 401 insertions(+), 181 deletions(-) diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 456eb0d..26fa051 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -321,6 +321,8 @@ H5_DLLVAR hid_t H5AC_ind_dxpl_id; #define H5AC__FLUSH_IGNORE_PROTECTED_FLAG H5C__FLUSH_IGNORE_PROTECTED_FLAG #define H5AC__FREE_FILE_SPACE_FLAG H5C__FREE_FILE_SPACE_FLAG #define H5AC__TAKE_OWNERSHIP_FLAG H5C__TAKE_OWNERSHIP_FLAG +#define H5AC__FLUSH_LAST_FLAG H5C__FLUSH_LAST_FLAG +#define H5AC__FLUSH_COLLECTIVELY_FLAG H5C__FLUSH_COLLECTIVELY_FLAG /* #defines of flags used to report entry status in the diff --git a/src/H5C.c b/src/H5C.c index f651fb2..c229a46 100644 --- a/src/H5C.c +++ b/src/H5C.c @@ -414,12 +414,6 @@ done: * Programmer: John Mainzer * 3/17/10 * - * Modifications: - * - * Heavily reworked to have each process flush a group of - * adjacent entries. - * JRM -- 4/15/10 - * *------------------------------------------------------------------------- */ #ifdef H5_HAVE_PARALLEL @@ -442,8 +436,13 @@ H5C_apply_candidate_list(H5F_t * f, int last_entry_to_flush; int entries_to_clear = 0; int entries_to_flush = 0; + int entries_to_flush_or_clear_last = 0; + int entries_to_flush_collectively = 0; int entries_cleared = 0; int entries_flushed = 0; + int entries_delayed = 0; + int entries_flushed_or_cleared_last = 0; + int entries_flushed_collectively = 0; int entries_examined = 0; int initial_list_len; int * candidate_assignment_table = NULL; @@ -451,6 +450,7 @@ H5C_apply_candidate_list(H5F_t * f, H5C_cache_entry_t * clear_ptr = NULL; H5C_cache_entry_t * entry_ptr = NULL; H5C_cache_entry_t * flush_ptr = NULL; + H5C_cache_entry_t * delayed_ptr = NULL; #if H5C_DO_SANITY_CHECKS haddr_t last_addr; #endif /* H5C_DO_SANITY_CHECKS */ @@ -612,12 +612,28 @@ H5C_apply_candidate_list(H5F_t * f, (int)(cache_ptr->LRU_list_len)); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + /* ====================================================================== * + * Now scan the LRU and PEL lists, flushing or clearing entries as + * needed. + * + * The flush_me_last and flush_me_collectively flags may dictate how or + * when some entries can be flushed, and should be addressed here. + * However, in their initial implementation, these flags only apply to the + * superblock, so there's only a relatively small change to this function + * to account for this one case where they come into play. If these flags + * are ever expanded upon, this function and the following flushing steps + * should be reworked to account for additional cases. + * ====================================================================== */ + entries_examined = 0; initial_list_len = cache_ptr->LRU_list_len; entry_ptr = cache_ptr->LRU_tail_ptr; + /* Examine each entry in the LRU list */ while((entry_ptr != NULL) && (entries_examined <= initial_list_len) && ((entries_cleared + entries_flushed) < num_candidates)) { + + /* If this process needs to clear this entry. */ if(entry_ptr->clear_on_unprotect) { entry_ptr->clear_on_unprotect = FALSE; clear_ptr = entry_ptr; @@ -625,7 +641,7 @@ H5C_apply_candidate_list(H5F_t * f, entries_cleared++; #if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, + HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, (long long)clear_ptr->addr); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ @@ -638,14 +654,18 @@ H5C_apply_candidate_list(H5F_t * f, &first_flush, TRUE) < 0) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") - } else if(entry_ptr->flush_immediately) { + } /* end if */ + + /* Else, if this process needs to flush this entry. */ + else if (entry_ptr->flush_immediately) { + entry_ptr->flush_immediately = FALSE; flush_ptr = entry_ptr; entry_ptr = entry_ptr->prev; entries_flushed++; #if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, + HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, (long long)flush_ptr->addr); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ @@ -657,17 +677,20 @@ H5C_apply_candidate_list(H5F_t * f, H5C__NO_FLAGS_SET, &first_flush, TRUE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") - } else { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry.") + } /* end else-if */ + + /* Otherwise, no action to be taken on this entry. Grab the next. */ + else { entry_ptr = entry_ptr->prev; - } + } /* end else */ entries_examined++; } /* end while */ #if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %d/%d/%d.\n", - FUNC, mpi_rank, entries_examined, + HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %d/%d/%d.\n", + FUNC, mpi_rank, entries_examined, entries_cleared, entries_flushed); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ @@ -676,69 +699,168 @@ H5C_apply_candidate_list(H5F_t * f, */ #if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n", + HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n", FUNC, mpi_rank, (int)(cache_ptr->pel_len)); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ entry_ptr = cache_ptr->pel_head_ptr; while((entry_ptr != NULL) && - ((entries_cleared + entries_flushed) < num_candidates)) { - if(entry_ptr->clear_on_unprotect) { - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_cleared++; + ((entries_cleared + entries_flushed + entries_delayed) + < num_candidates)) { + + /* If entry is marked for flush or for clear */ + if((entry_ptr->clear_on_unprotect||entry_ptr->flush_immediately)) { + + /* If this entry needs to be flushed last */ + if (entry_ptr->flush_me_last) { + + /* At this time, only the superblock supports being + flushed last. Conveniently, it also happens to be the only + entry that supports being flushed collectively, as well. Also + conveniently, it's always pinned, so we only need to check + for it while scanning the PEL here. Finally, it's never + included in a candidate list that excludes other dirty + entries in a cache, so we can handle this relatively simple + case here. + + For now, this function asserts this and saves the entry + to flush it after scanning the rest of the PEL list. + + If there are ever more entries that either need to be + flushed last and/or flushed collectively, this whole routine + will need to be reworked to handle all additional cases. As + it is the simple case of a single pinned entry needing + flushed last and collectively is just a minor addition to + this routine, but signficantly buffing up the usage of + flush_me_last or flush_me_collectively will require a more + intense rework of this function and potentially the function + of candidate lists as a whole. */ + + HDassert(entry_ptr->flush_me_collectively); + entries_to_flush_or_clear_last++; + entries_to_flush_collectively++; + HDassert(entries_to_flush_or_clear_last == 1); + HDassert(entries_to_flush_collectively == 1); + + /* Delay the entry. It will be flushed later. */ + delayed_ptr = entry_ptr; + entries_delayed++; + HDassert(entries_delayed == 1); + + } /* end if */ + + /* Else, this process needs to clear this entry. */ + else if (entry_ptr->clear_on_unprotect) { + HDassert(!entry_ptr->flush_immediately); + entry_ptr->clear_on_unprotect = FALSE; + clear_ptr = entry_ptr; + entry_ptr = entry_ptr->next; + entries_cleared++; #if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, + HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, (long long)clear_ptr->addr); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - if(H5C_flush_single_entry(f, - primary_dxpl_id, - secondary_dxpl_id, - clear_ptr->type, - clear_ptr->addr, - H5C__FLUSH_CLEAR_ONLY_FLAG, - &first_flush, - TRUE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") - } else if(entry_ptr->flush_immediately) { - entry_ptr->flush_immediately = FALSE; - flush_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_flushed++; + if(H5C_flush_single_entry(f, + primary_dxpl_id, + secondary_dxpl_id, + clear_ptr->type, + clear_ptr->addr, + H5C__FLUSH_CLEAR_ONLY_FLAG, + &first_flush, + TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") + } /* end else-if */ + + /* Else, if this process needs to independently flush this entry. */ + else if (entry_ptr->flush_immediately) { + entry_ptr->flush_immediately = FALSE; + flush_ptr = entry_ptr; + entry_ptr = entry_ptr->next; + entries_flushed++; #if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, + HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, (long long)flush_ptr->addr); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - if(H5C_flush_single_entry(f, - primary_dxpl_id, - secondary_dxpl_id, - flush_ptr->type, - flush_ptr->addr, - H5C__NO_FLAGS_SET, - &first_flush, - TRUE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") - } else { + if(H5C_flush_single_entry(f, + primary_dxpl_id, + secondary_dxpl_id, + flush_ptr->type, + flush_ptr->addr, + H5C__NO_FLAGS_SET, + &first_flush, + TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry.") + } /* end else-if */ + } /* end if */ + + /* Otherwise, this entry is not marked for flush or clear. Grab the next. */ + else { entry_ptr = entry_ptr->next; - } + } /* end else */ + } /* end while */ #if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, - "%s:%d: pel entries examined/cleared/flushed = %d/%d/%d.\n", - FUNC, mpi_rank, entries_examined, + HDfprintf(stdout, + "%s:%d: pel entries examined/cleared/flushed = %d/%d/%d.\n", + FUNC, mpi_rank, entries_examined, entries_cleared, entries_flushed); HDfprintf(stdout, "%s:%d: done.\n", FUNC, mpi_rank); fsync(stdout); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - if((entries_flushed != entries_to_flush) || (entries_cleared != entries_to_clear)) + /* ====================================================================== * + * Now, handle all delayed entries. * + * * + * This can *only* be the superblock at this time, so it's relatively * + * easy to deal with. We're collectively flushing the entry saved from * + * above. This will need to be handled differently if there are ever more * + * than one entry needing this special treatment.) * + * ====================================================================== */ + + if (delayed_ptr) { + + if (delayed_ptr->clear_on_unprotect) { + entry_ptr->clear_on_unprotect = FALSE; + entries_cleared++; + } else if (delayed_ptr->flush_immediately) { + entry_ptr->flush_immediately = FALSE; + entries_flushed++; + } /* end if */ + + if(H5C_flush_single_entry(f, + primary_dxpl_id, + secondary_dxpl_id, + delayed_ptr->type, + delayed_ptr->addr, + H5C__NO_FLAGS_SET, + &first_flush, + TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, + "Can't flush entry collectively.") + + entries_flushed_collectively++; + entries_flushed_or_cleared_last++; + } /* end if */ + + /* ====================================================================== * + * Finished flushing everything. * + * ====================================================================== */ + + HDassert((entries_flushed == entries_to_flush)); + HDassert((entries_cleared == entries_to_clear)); + HDassert((entries_flushed_or_cleared_last == entries_to_flush_or_clear_last)); + HDassert((entries_flushed_collectively == entries_to_flush_collectively)); + + if((entries_flushed != entries_to_flush) || + (entries_cleared != entries_to_clear) || + (entries_flushed_or_cleared_last != entries_to_flush_or_clear_last) || + (entries_flushed_collectively != entries_to_flush_collectively)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry count mismatch.") done: @@ -924,7 +1046,8 @@ H5C_construct_candidate_list__min_clean(H5C_t * cache_ptr) entry_ptr = cache_ptr->dLRU_tail_ptr; while((nominated_entries_size < space_needed) && (nominated_entries_count < cache_ptr->slist_len) && - (entry_ptr != NULL)) { + (entry_ptr != NULL) && + (!entry_ptr->flush_me_last)) { haddr_t nominated_addr; HDassert( ! (entry_ptr->is_protected) ); @@ -1771,8 +1894,12 @@ H5C_flush_cache(H5F_t *f, hid_t primary_dxpl_id, hid_t secondary_dxpl_id, unsign HDassert( entry_ptr != NULL ); HDassert( entry_ptr->in_slist ); - if ( ( ! flush_marked_entries ) || - ( entry_ptr->flush_marker ) ) { + if ( ( ( ! flush_marked_entries ) || + ( entry_ptr->flush_marker ) ) && + ( ( ! entry_ptr->flush_me_last ) || + ( ( entry_ptr->flush_me_last ) && + ( cache_ptr->num_last_entries >= + cache_ptr->slist_len ) ) ) ) { if ( entry_ptr->is_protected ) { @@ -2524,6 +2651,10 @@ H5C_insert_entry(H5F_t * f, herr_t result; hbool_t first_flush = TRUE; hbool_t insert_pinned; + hbool_t flush_last; +#ifdef H5_HAVE_PARALLEL + hbool_t flush_collectively; +#endif hbool_t set_flush_marker; hbool_t write_permitted = TRUE; size_t empty_space; @@ -2562,8 +2693,12 @@ H5C_insert_entry(H5F_t * f, } #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ - set_flush_marker = ( (flags & H5C__SET_FLUSH_MARKER_FLAG) != 0 ); - insert_pinned = ( (flags & H5C__PIN_ENTRY_FLAG) != 0 ); + set_flush_marker = ( (flags & H5C__SET_FLUSH_MARKER_FLAG) != 0 ); + insert_pinned = ( (flags & H5C__PIN_ENTRY_FLAG) != 0 ); + flush_last = ( (flags & H5C__FLUSH_LAST_FLAG) != 0 ); +#ifdef H5_HAVE_PARALLEL + flush_collectively = ( (flags & H5C__FLUSH_COLLECTIVELY_FLAG) != 0 ); +#endif entry_ptr = (H5C_cache_entry_t *)thing; @@ -2604,6 +2739,10 @@ H5C_insert_entry(H5F_t * f, entry_ptr->is_pinned = insert_pinned; entry_ptr->pinned_from_client = insert_pinned; + entry_ptr->flush_me_last = flush_last; +#ifdef H5_HAVE_PARALLEL + entry_ptr->flush_me_collectively = flush_collectively; +#endif /* newly inserted entries are assumed to be dirty */ entry_ptr->is_dirty = TRUE; @@ -7710,85 +7849,91 @@ H5C_flush_invalidate_cache(H5F_t * f, HDassert( entry_ptr != NULL ); HDassert( entry_ptr->in_slist ); -#if H5C_DO_SANITY_CHECKS - /* update actual_slist_len & actual_slist_size before - * the flush. Note that the entry will be removed - * from the slist after the flush, and thus may be - * resized by the flush callback. This is OK, as - * we will catch the size delta in - * cache_ptr->slist_size_increase. - * - * Note that we include pinned entries in this count, even - * though we will not actually flush them. - */ - actual_slist_len++; - actual_slist_size += entry_ptr->size; -#endif /* H5C_DO_SANITY_CHECKS */ + if ( ( ! entry_ptr->flush_me_last ) || + ( ( entry_ptr->flush_me_last ) && + ( cache_ptr->num_last_entries >= + cache_ptr->slist_len ) ) ) { + + #if H5C_DO_SANITY_CHECKS + /* update actual_slist_len & actual_slist_size before + * the flush. Note that the entry will be removed + * from the slist after the flush, and thus may be + * resized by the flush callback. This is OK, as + * we will catch the size delta in + * cache_ptr->slist_size_increase. + * + * Note that we include pinned entries in this count, even + * though we will not actually flush them. + */ + actual_slist_len++; + actual_slist_size += entry_ptr->size; + #endif /* H5C_DO_SANITY_CHECKS */ - if ( entry_ptr->is_protected ) { + if ( entry_ptr->is_protected ) { - /* we have major problems -- but lets flush - * everything we can before we flag an error. - */ - protected_entries++; + /* we have major problems -- but lets flush + * everything we can before we flag an error. + */ + protected_entries++; - } else if ( entry_ptr->is_pinned ) { + } else if ( entry_ptr->is_pinned ) { - /* Test to see if we are can flush the entry now. - * If we can, go ahead and flush, but don't tell - * H5C_flush_single_entry() to destroy the entry - * as pinned entries can't be evicted. - */ - if(entry_ptr->flush_dep_height == curr_flush_dep_height ) { - status = H5C_flush_single_entry(f, - primary_dxpl_id, - secondary_dxpl_id, - NULL, - entry_ptr->addr, - H5C__NO_FLAGS_SET, - &first_flush, - FALSE); - if ( status < 0 ) { - - /* This shouldn't happen -- if it does, we are toast - * so just scream and die. - */ + /* Test to see if we are can flush the entry now. + * If we can, go ahead and flush, but don't tell + * H5C_flush_single_entry() to destroy the entry + * as pinned entries can't be evicted. + */ + if(entry_ptr->flush_dep_height == curr_flush_dep_height ) { + status = H5C_flush_single_entry(f, + primary_dxpl_id, + secondary_dxpl_id, + NULL, + entry_ptr->addr, + H5C__NO_FLAGS_SET, + &first_flush, + FALSE); + if ( status < 0 ) { + + /* This shouldn't happen -- if it does, we are toast + * so just scream and die. + */ - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "dirty pinned entry flush failed.") + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ + "dirty pinned entry flush failed.") + } /* end if */ + flushed_during_dep_loop = TRUE; } /* end if */ - flushed_during_dep_loop = TRUE; + else if(entry_ptr->flush_dep_height < curr_flush_dep_height) + /* This shouldn't happen -- if it does, just scream and die. */ + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry below current flush dep. height.") } /* end if */ - else if(entry_ptr->flush_dep_height < curr_flush_dep_height) - /* This shouldn't happen -- if it does, just scream and die. */ - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry below current flush dep. height.") - } /* end if */ - else { - if(entry_ptr->flush_dep_height == curr_flush_dep_height ){ + else { + if(entry_ptr->flush_dep_height == curr_flush_dep_height ){ - status = H5C_flush_single_entry(f, - primary_dxpl_id, - secondary_dxpl_id, - NULL, - entry_ptr->addr, - (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG), - &first_flush, - TRUE); - if ( status < 0 ) { - - /* This shouldn't happen -- if it does, we are toast so - * just scream and die. - */ + status = H5C_flush_single_entry(f, + primary_dxpl_id, + secondary_dxpl_id, + NULL, + entry_ptr->addr, + (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG), + &first_flush, + TRUE); + if ( status < 0 ) { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "dirty entry flush destroy failed.") + /* This shouldn't happen -- if it does, we are toast so + * just scream and die. + */ + + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ + "dirty entry flush destroy failed.") + } /* end if */ + flushed_during_dep_loop = TRUE; } /* end if */ - flushed_during_dep_loop = TRUE; - } /* end if */ - else if(entry_ptr->flush_dep_height < curr_flush_dep_height) - /* This shouldn't happen -- if it does, just scream and die. */ - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry below current flush dep. height.") - } /* end else */ + else if(entry_ptr->flush_dep_height < curr_flush_dep_height) + /* This shouldn't happen -- if it does, just scream and die. */ + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry below current flush dep. height.") + } /* end else */ + } /* end if */ } /* end while loop scanning skip list */ #if H5C_DO_SANITY_CHECKS @@ -7835,45 +7980,52 @@ H5C_flush_invalidate_cache(H5F_t * f, next_entry_ptr = entry_ptr->ht_next; HDassert ( ( next_entry_ptr == NULL ) || ( next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC ) ); - if ( entry_ptr->is_protected ) { - - /* we have major problems -- but lets flush and destroy - * everything we can before we flag an error. - */ - protected_entries++; - if ( ! entry_ptr->in_slist ) { + if ( ( ! entry_ptr->flush_me_last ) || + ( ( entry_ptr->flush_me_last ) && + ( cache_ptr->num_last_entries >= + cache_ptr->slist_len ) ) ) { - HDassert( !(entry_ptr->is_dirty) ); - } - } else if ( ! ( entry_ptr->is_pinned ) ) { + if ( entry_ptr->is_protected ) { - /* Test to see if we are can flush the entry now. - * If we can, go ahead and flush. - */ - if(entry_ptr->flush_dep_height == curr_flush_dep_height ){ - status = H5C_flush_single_entry(f, - primary_dxpl_id, - secondary_dxpl_id, - NULL, - entry_ptr->addr, - (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG), - &first_flush, - TRUE); - if ( status < 0 ) { + /* we have major problems -- but lets flush and destroy + * everything we can before we flag an error. + */ + protected_entries++; - /* This shouldn't happen -- if it does, we are toast so - * just scream and die. - */ + if ( ! entry_ptr->in_slist ) { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "Entry flush destroy failed.") + HDassert( !(entry_ptr->is_dirty) ); } - flushed_during_dep_loop = TRUE; + } else if ( ! ( entry_ptr->is_pinned ) ) { + + /* Test to see if we are can flush the entry now. + * If we can, go ahead and flush. + */ + if(entry_ptr->flush_dep_height == curr_flush_dep_height ){ + status = H5C_flush_single_entry(f, + primary_dxpl_id, + secondary_dxpl_id, + NULL, + entry_ptr->addr, + (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG), + &first_flush, + TRUE); + if ( status < 0 ) { + + /* This shouldn't happen -- if it does, we are toast so + * just scream and die. + */ + + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ + "Entry flush destroy failed.") + } + flushed_during_dep_loop = TRUE; + } /* end if */ + else if(entry_ptr->flush_dep_height < curr_flush_dep_height) + /* This shouldn't happen -- if it does, just scream and die. */ + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry below current flush dep. height.") } /* end if */ - else if(entry_ptr->flush_dep_height < curr_flush_dep_height) - /* This shouldn't happen -- if it does, just scream and die. */ - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry below current flush dep. height.") } /* end if */ /* We can't do anything if the entry is pinned. The * hope is that the entry will be unpinned as the diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 37068f1..909578b 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -282,6 +282,16 @@ * don't use this at present, I hope that this will allow * some optimizations when I get to it. * + * num_last_entries: The number of entries in the cache that can only be + * flushed after all other entries in the cache have + * been flushed. At this time, this will only ever be + * one entry (the superblock), and the code has been + * protected with HDasserts to enforce this. This restraint + * can certainly be relaxed in the future if the need for + * multiple entries being flushed last arises, though + * explicit tests for that case should be added when said + * HDasserts are removed. + * * With the addition of the fractal heap, the cache must now deal with * the case in which entries may be dirtied, moved, or have their sizes * changed during a flush. To allow sanity checks in this situation, the @@ -878,6 +888,7 @@ struct H5C_t int32_t slist_len; size_t slist_size; H5SL_t * slist_ptr; + int32_t num_last_entries; #if H5C_DO_SANITY_CHECKS int64_t slist_len_increase; int64_t slist_size_increase; @@ -1964,6 +1975,10 @@ if ( (cache_ptr)->index_size != \ } else { \ (cache_ptr)->clean_index_size += (entry_ptr)->size; \ } \ + if ((entry_ptr)->flush_me_last) { \ + (cache_ptr)->num_last_entries++; \ + HDassert((cache_ptr)->num_last_entries == 1); \ + } \ H5C__UPDATE_STATS_FOR_HT_INSERTION(cache_ptr) \ } @@ -1993,6 +2008,10 @@ if ( (cache_ptr)->index_size != \ } else { \ (cache_ptr)->clean_index_size -= (entry_ptr)->size; \ } \ + if ((entry_ptr)->flush_me_last) { \ + (cache_ptr)->num_last_entries--; \ + HDassert((cache_ptr)->num_last_entries == 0); \ + } \ H5C__UPDATE_STATS_FOR_HT_DELETION(cache_ptr) \ } diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 0c7631a..7fde69b 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -371,6 +371,23 @@ typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr, * H5C__FLUSH_MARKED_ENTRIES_FLAG. The flag is reset when * the entry is flushed for whatever reason. * + * flush_me_last: Boolean flag indicating that this entry should not be + * flushed from the cache until all other entries without + * the flush_me_last flag set have been flushed. + * + * flush_me_collectively: Boolean flag indicating that this entry needs + * to be flushed collectively when in a parallel + * situation. + * + * Note: At this time, the flush_me_last and flush_me_collectively + * flags will only be applied to one entry, the superblock, + * and the code utilizing these flags is protected with HDasserts + * to enforce this. This restraint can certainly be relaxed in + * the future if the the need for multiple entries getting flushed + * last or collectively arises, though the code allowing for that + * will need to be expanded and tested appropriately if that + * functionality is desired. + * * clear_on_unprotect: Boolean flag used only in PHDF5. When H5C is used * to implement the metadata cache In the parallel case, only * the cache with mpi rank 0 is allowed to actually write to @@ -578,7 +595,7 @@ typedef struct H5C_cache_entry_t haddr_t addr; size_t size; const H5C_class_t * type; - haddr_t tag; + haddr_t tag; hbool_t is_dirty; hbool_t dirtied; hbool_t is_protected; @@ -587,13 +604,15 @@ typedef struct H5C_cache_entry_t hbool_t is_pinned; hbool_t in_slist; hbool_t flush_marker; + hbool_t flush_me_last; #ifdef H5_HAVE_PARALLEL + hbool_t flush_me_collectively; hbool_t clear_on_unprotect; - hbool_t flush_immediately; + hbool_t flush_immediately; #endif /* H5_HAVE_PARALLEL */ hbool_t flush_in_progress; hbool_t destroy_in_progress; - hbool_t free_file_space_on_destroy; + hbool_t free_file_space_on_destroy; /* fields supporting the 'flush dependency' feature: */ @@ -1042,6 +1061,8 @@ typedef struct H5C_auto_size_ctl_t #define H5C__READ_ONLY_FLAG 0x0200 #define H5C__FREE_FILE_SPACE_FLAG 0x0800 #define H5C__TAKE_OWNERSHIP_FLAG 0x1000 +#define H5C__FLUSH_LAST_FLAG 0x2000 +#define H5C__FLUSH_COLLECTIVELY_FLAG 0x4000 #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5C_apply_candidate_list(H5F_t * f, diff --git a/src/H5FDmpiposix.c b/src/H5FDmpiposix.c index e6c23f0..0ad39a9 100644 --- a/src/H5FDmpiposix.c +++ b/src/H5FDmpiposix.c @@ -89,12 +89,12 @@ typedef struct H5FD_mpiposix_t { MPI_Comm comm; /*communicator */ int mpi_rank; /* This process's rank */ int mpi_size; /* Total number of processes */ - haddr_t eof; /*end-of-file marker */ - haddr_t eoa; /*end-of-address marker */ - haddr_t last_eoa; /* Last known end-of-address marker */ - haddr_t pos; /* Current file I/O position */ - int op; /* Last file I/O operation */ - hsize_t naccess; /* Number of (write) accesses to file */ + haddr_t eof; /*end-of-file marker */ + haddr_t eoa; /*end-of-address marker */ + haddr_t last_eoa; /* Last known end-of-address marker */ + haddr_t pos; /* Current file I/O position */ + int op; /* Last file I/O operation */ + hsize_t naccess; /* Number of (write) accesses to file */ #ifdef H5_HAVE_GPFS size_t blksize; /* Block size of file system */ #endif @@ -230,13 +230,13 @@ static const H5FD_class_mpi_t H5FD_mpiposix_g = { H5FD_mpiposix_set_eoa, /*set_eoa */ H5FD_mpiposix_get_eof, /*get_eof */ H5FD_mpiposix_get_handle, /*get_handle */ - H5FD_mpiposix_read, /*read */ - H5FD_mpiposix_write, /*write */ + H5FD_mpiposix_read, /*read */ + H5FD_mpiposix_write, /*write */ NULL, /*flush */ - H5FD_mpiposix_truncate, /*truncate */ + H5FD_mpiposix_truncate, /*truncate */ NULL, /*lock */ NULL, /*unlock */ - H5FD_FLMAP_SINGLE /*fl_map */ + H5FD_FLMAP_SINGLE /*fl_map */ }, /* End of superclass information */ H5FD_mpiposix_mpi_rank, /*get_rank */ H5FD_mpiposix_mpi_size, /*get_size */ diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 8bf4405..8205392 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -516,7 +516,7 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to set EOA value for superblock") /* Insert superblock into cache, pinned */ - if(H5AC_insert_entry(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, H5AC__PIN_ENTRY_FLAG) < 0) + if(H5AC_insert_entry(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, H5AC__PIN_ENTRY_FLAG | H5AC__FLUSH_LAST_FLAG | H5AC__FLUSH_COLLECTIVELY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, "can't add superblock to cache") sblock_in_cache = TRUE; diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index cc1137b..48fd139 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -153,6 +153,12 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) if(NULL == (sblock = H5FL_CALLOC(H5F_super_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + /* The superblock must be flushed last (and collectively in parallel) */ + sblock->cache_info.flush_me_last = TRUE; +#ifdef H5_HAVE_PARALLEL + sblock->cache_info.flush_me_collectively = TRUE; +#endif + /* Read fixed-size portion of the superblock */ p = sbuf; H5_CHECK_OVERFLOW(fixed_size, size_t, haddr_t); @@ -638,6 +644,15 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, HDassert(f); HDassert(H5F_addr_eq(addr, 0)); HDassert(sblock); + + /* Assert that the superblock is marked as being flushed last (and + collectively in parallel) */ + /* (We'll rely on the cache to make sure it actually *is* flushed + last (and collectively in parallel), but this check doesn't hurt) */ + HDassert(sblock->cache_info.flush_me_last); +#ifdef H5_HAVE_PARALLEL + HDassert(sblock->cache_info.flush_me_collectively); +#endif if(sblock->cache_info.is_dirty) { uint8_t buf[H5F_MAX_SUPERBLOCK_SIZE + H5F_MAX_DRVINFOBLOCK_SIZE]; /* Superblock & driver info blockencoding buffer */ @@ -677,10 +692,15 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, *p++ = 0; /*reserved */ } /* end if */ + /* Encode the base address */ H5F_addr_encode(f, &p, sblock->base_addr); + + /* Encode the address of global free-space index */ H5F_addr_encode(f, &p, sblock->ext_addr); rel_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER); H5F_addr_encode(f, &p, (rel_eoa + sblock->base_addr)); + + /* Encode the driver informaton block address */ H5F_addr_encode(f, &p, sblock->driver_addr); /* Encode the root group object entry, including the cached stab info */ @@ -731,9 +751,12 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, *p++ = (uint8_t)H5F_SIZEOF_SIZE(f); *p++ = sblock->status_flags; - /* Base, superblock extension & end of file addresses */ + /* Encode the base address */ H5F_addr_encode(f, &p, sblock->base_addr); + + /* Encode the address of the superblock extension */ H5F_addr_encode(f, &p, sblock->ext_addr); + rel_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER); H5F_addr_encode(f, &p, (rel_eoa + sblock->base_addr)); @@ -767,13 +790,18 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, /* Check for newer version of superblock format & superblock extension */ if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 && H5F_addr_defined(sblock->ext_addr)) { + H5O_loc_t ext_loc; /* "Object location" for superblock extension */ + + /* Open the superblock extension's object header */ + if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension") + /* Check for ignoring the driver info for this file */ if(!H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) { /* Check for driver info message */ H5_ASSIGN_OVERFLOW(driver_size, H5FD_sb_size(f->shared->lf), hsize_t, size_t); if(driver_size > 0) { H5O_drvinfo_t drvinfo; /* Driver info */ - H5O_loc_t ext_loc; /* "Object location" for superblock extension */ uint8_t dbuf[H5F_MAX_DRVINFOBLOCK_SIZE]; /* Driver info block encoding buffer */ /* Sanity check */ @@ -783,21 +811,19 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, if(H5FD_sb_encode(f->shared->lf, drvinfo.name, dbuf) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to encode driver information") - /* Open the superblock extension's object header */ - if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension") - /* Write driver info information to the superblock extension */ drvinfo.len = driver_size; drvinfo.buf = dbuf; if(H5O_msg_write(&ext_loc, H5O_DRVINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &drvinfo, dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message") - /* Close the superblock extension object header */ - if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") } /* end if */ + } /* end if */ + + /* Close the superblock extension object header */ + if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") } /* end if */ /* Reset the dirty flag. */ -- cgit v0.12