diff options
| -rw-r--r-- | src/H5ACprivate.h | 2 | ||||
| -rw-r--r-- | src/H5C.c | 21 | ||||
| -rw-r--r-- | src/H5Cdbg.c | 6 | ||||
| -rw-r--r-- | src/H5Cpkg.h | 440 | ||||
| -rw-r--r-- | src/H5Cprivate.h | 21 | ||||
| -rw-r--r-- | src/H5Fsuper_cache.c | 372 | ||||
| -rw-r--r-- | src/H5HFcache.c | 95 | ||||
| -rw-r--r-- | src/H5HFdblock.c | 4 | ||||
| -rw-r--r-- | src/H5HGcache.c | 108 | ||||
| -rw-r--r-- | src/H5HLcache.c | 122 | ||||
| -rw-r--r-- | src/H5Ocache.c | 324 | ||||
| -rw-r--r-- | src/H5Pdxpl.c | 2 | ||||
| -rw-r--r-- | src/H5SM.c | 4 | ||||
| -rw-r--r-- | test/cache.c | 48 | ||||
| -rw-r--r-- | test/cache_common.h | 36 |
15 files changed, 1001 insertions, 604 deletions
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index afd7817..07bb8fa 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -153,7 +153,7 @@ typedef enum { /* Aliases for the "ring" type and values */ typedef H5C_ring_t H5AC_ring_t; #define H5AC_RING_INV H5C_RING_UNDEFINED -#define H5AC_RING_US H5C_RING_USER +#define H5AC_RING_USER H5C_RING_USER #define H5AC_RING_FSM H5C_RING_FSM #define H5AC_RING_SBE H5C_RING_SBE #define H5AC_RING_SB H5C_RING_SB @@ -329,6 +329,11 @@ H5C_create(size_t max_cache_size, for(i = 0; i < H5C__HASH_TABLE_LEN; i++) (cache_ptr->index)[i] = NULL; + cache_ptr->il_len = 0; + cache_ptr->il_size = (size_t)0; + cache_ptr->il_head = NULL; + cache_ptr->il_tail = NULL; + /* Tagging Field Initializations */ cache_ptr->ignore_tags = FALSE; @@ -1330,6 +1335,8 @@ H5C_insert_entry(H5F_t * f, entry_ptr->ht_next = NULL; entry_ptr->ht_prev = NULL; + entry_ptr->il_next = NULL; + entry_ptr->il_prev = NULL; entry_ptr->next = NULL; entry_ptr->prev = NULL; @@ -1733,7 +1740,7 @@ H5C_move_entry(H5C_t * cache_ptr, * don't mark it as dirty either, lest we confuse the flush call back. */ if(!entry_ptr->destroy_in_progress) { - H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) + H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL) if(entry_ptr->in_slist) { HDassert(cache_ptr->slist_ptr); @@ -5464,7 +5471,7 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, * -- JRM */ next_entry_ptr = cache_ptr->index[i]; - H5C__UPDATE_STATS_FOR_HASH_BUCKET_SCAN_RESTART(cache_ptr) + H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) } /* end if */ else cache_ptr->entry_watched_for_removal = NULL; @@ -6059,7 +6066,7 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ * Finally, if the destroy_entry flag is set, discard the * entry. */ - H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) + H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL) if(entry_ptr->in_slist && del_from_slist_on_destroy) H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) @@ -6569,7 +6576,7 @@ H5C_load_entry(H5F_t * f, entry->size = len; HDassert(entry->size < H5C_MAX_ENTRY_SIZE); entry->image_ptr = image; - entry->image_up_to_date = TRUE; + entry->image_up_to_date = !dirty; entry->type = type; entry->is_dirty = dirty; entry->dirtied = FALSE; @@ -6597,6 +6604,8 @@ H5C_load_entry(H5F_t * f, entry->flush_dep_ndirty_children = 0; entry->ht_next = NULL; entry->ht_prev = NULL; + entry->il_next = NULL; + entry->il_prev = NULL; entry->next = NULL; entry->prev = NULL; @@ -7832,7 +7841,7 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p /* We must update cache data structures for the change in address */ if(entry_ptr->addr == old_addr) { /* Delete the entry from the hash table and the slist */ - H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr); + H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL); H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE); /* update the entry for its new address */ @@ -7928,7 +7937,7 @@ H5C_remove_entry(void *_entry) * 3) Remove it from the tag list for this object */ - H5C__DELETE_FROM_INDEX(cache, entry) + H5C__DELETE_FROM_INDEX(cache, entry, FAIL) H5C__UPDATE_RP_FOR_EVICTION(cache, entry, FAIL) diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index e7ed443..9266acd 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -642,11 +642,11 @@ H5C_stats(H5C_t * cache_ptr, cache_ptr->entries_scanned_to_make_space)); HDfprintf(stdout, - "%s slist/LRU/hash bkt scan restarts = %lld / %lld / %lld.\n", + "%s slist/LRU/index scan restarts = %lld / %lld / %lld.\n", cache_ptr->prefix, (long long)(cache_ptr->slist_scan_restarts), (long long)(cache_ptr->LRU_scan_restarts), - (long long)(cache_ptr->hash_bucket_scan_restarts)); + (long long)(cache_ptr->index_scan_restarts)); #if H5C_COLLECT_CACHE_ENTRY_STATS @@ -867,7 +867,7 @@ H5C_stats__reset(H5C_t H5_ATTR_UNUSED * cache_ptr) cache_ptr->slist_scan_restarts = 0; cache_ptr->LRU_scan_restarts = 0; - cache_ptr->hash_bucket_scan_restarts = 0; + cache_ptr->index_scan_restarts = 0; #if H5C_COLLECT_CACHE_ENTRY_STATS for(i = 0; i <= cache_ptr->max_type_id; i++) { diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index fd3f282..e30ca8d 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -150,6 +150,13 @@ * * JRM - 9/8/05 * + * - Added macros supporting the index list -- a doubly liked list of + * all entries in the index. This list is necessary to reduce the + * cost of visiting all entries in the cache, which was previously + * done via a scan of the hash table. + * + * JRM - 10/15/15 + * ****************************************************************************/ #if H5C_DO_SANITY_CHECKS @@ -456,6 +463,130 @@ if ( ( (entry_ptr) == NULL ) || \ } \ } /* H5C__AUX_DLL_REMOVE() */ +#if H5C_DO_SANITY_CHECKS + +#define H5C__IL_DLL_PRE_REMOVE_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) \ +if ( ( (hd_ptr) == NULL ) || \ + ( (tail_ptr) == NULL ) || \ + ( (entry_ptr) == NULL ) || \ + ( (len) <= 0 ) || \ + ( (Size) < (entry_ptr)->size ) || \ + ( ( (Size) == (entry_ptr)->size ) && ( ! ( (len) == 1 ) ) ) || \ + ( ( (entry_ptr)->il_prev == NULL ) && ( (hd_ptr) != (entry_ptr) ) ) || \ + ( ( (entry_ptr)->il_next == NULL ) && ( (tail_ptr) != (entry_ptr) ) ) || \ + ( ( (len) == 1 ) && \ + ( ! ( ( (hd_ptr) == (entry_ptr) ) && ( (tail_ptr) == (entry_ptr) ) && \ + ( (entry_ptr)->il_next == NULL ) && \ + ( (entry_ptr)->il_prev == NULL ) && \ + ( (Size) == (entry_ptr)->size ) \ + ) \ + ) \ + ) \ + ) { \ + HDassert(0 && "il DLL pre remove SC failed"); \ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, (fv), "il DLL pre remove SC failed") \ +} + +#define H5C__IL_DLL_PRE_INSERT_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) \ +if ( ( (entry_ptr) == NULL ) || \ + ( (entry_ptr)->il_next != NULL ) || \ + ( (entry_ptr)->il_prev != NULL ) || \ + ( ( ( (hd_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ + ( (hd_ptr) != (tail_ptr) ) \ + ) || \ + ( (len) < 0 ) || \ + ( ( (len) == 1 ) && \ + ( ( (hd_ptr) != (tail_ptr) ) || ( (Size) <= 0 ) || \ + ( (hd_ptr) == NULL ) || ( (hd_ptr)->size != (Size) ) \ + ) \ + ) || \ + ( ( (len) >= 1 ) && \ + ( ( (hd_ptr) == NULL ) || ( (hd_ptr)->il_prev != NULL ) || \ + ( (tail_ptr) == NULL ) || ( (tail_ptr)->il_next != NULL ) \ + ) \ + ) \ + ) { \ + HDassert(0 && "IL DLL pre insert SC failed"); \ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, (fv), "IL DLL pre insert SC failed") \ +} + +#define H5C__IL_DLL_SC(head_ptr, tail_ptr, len, Size, fv) \ +if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ + ( (head_ptr) != (tail_ptr) ) \ + ) || \ + ( (len) < 0 ) || \ + ( ( (len) == 1 ) && \ + ( ( (head_ptr) != (tail_ptr) ) || \ + ( (head_ptr) == NULL ) || ( (head_ptr)->size != (Size) ) \ + ) \ + ) || \ + ( ( (len) >= 1 ) && \ + ( ( (head_ptr) == NULL ) || ( (head_ptr)->il_prev != NULL ) || \ + ( (tail_ptr) == NULL ) || ( (tail_ptr)->il_next != NULL ) \ + ) \ + ) \ + ) { \ + HDassert(0 && "IL DLL sanity check failed"); \ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, (fv), "IL DLL sanity check failed") \ +} + +#else /* H5C_DO_SANITY_CHECKS */ + +#define H5C__IL_DLL_PRE_REMOVE_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) +#define H5C__IL_DLL_PRE_INSERT_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) +#define H5C__IL_DLL_SC(head_ptr, tail_ptr, len, Size, fv) + +#endif /* H5C_DO_SANITY_CHECKS */ + + +#define H5C__IL_DLL_APPEND(entry_ptr, head_ptr, tail_ptr, len, Size, fail_val)\ +{ \ + H5C__IL_DLL_PRE_INSERT_SC(entry_ptr, head_ptr, tail_ptr, len, Size, \ + fail_val) \ + if ( (head_ptr) == NULL ) \ + { \ + (head_ptr) = (entry_ptr); \ + (tail_ptr) = (entry_ptr); \ + } \ + else \ + { \ + (tail_ptr)->il_next = (entry_ptr); \ + (entry_ptr)->il_prev = (tail_ptr); \ + (tail_ptr) = (entry_ptr); \ + } \ + (len)++; \ + (Size) += entry_ptr->size; \ + H5C__IL_DLL_SC(head_ptr, tail_ptr, len, Size, fail_val) \ +} /* H5C__IL_DLL_APPEND() */ + +#define H5C__IL_DLL_REMOVE(entry_ptr, head_ptr, tail_ptr, len, Size, fv) \ +{ \ + H5C__IL_DLL_PRE_REMOVE_SC(entry_ptr, head_ptr, tail_ptr, len, Size, fv) \ + { \ + if ( (head_ptr) == (entry_ptr) ) \ + { \ + (head_ptr) = (entry_ptr)->il_next; \ + if ( (head_ptr) != NULL ) \ + (head_ptr)->il_prev = NULL; \ + } \ + else \ + (entry_ptr)->il_prev->il_next = (entry_ptr)->il_next; \ + if ( (tail_ptr) == (entry_ptr) ) \ + { \ + (tail_ptr) = (entry_ptr)->il_prev; \ + if ( (tail_ptr) != NULL ) \ + (tail_ptr)->il_next = NULL; \ + } \ + else \ + (entry_ptr)->il_next->il_prev = (entry_ptr)->il_prev; \ + entry_ptr->il_next = NULL; \ + entry_ptr->il_prev = NULL; \ + (len)--; \ + (Size) -= entry_ptr->size; \ + } \ + H5C__IL_DLL_SC(head_ptr, tail_ptr, len, Size, fv) \ +} /* H5C__IL_DLL_REMOVE() */ + /*********************************************************************** * @@ -567,8 +698,8 @@ if ( ( (entry_ptr) == NULL ) || \ #define H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) \ ((cache_ptr)->LRU_scan_restarts)++; -#define H5C__UPDATE_STATS_FOR_HASH_BUCKET_SCAN_RESTART(cache_ptr) \ - ((cache_ptr)->hash_bucket_scan_restarts)++; +#define H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) \ + ((cache_ptr)->index_scan_restarts)++; #if H5C_COLLECT_CACHE_ENTRY_STATS @@ -791,7 +922,7 @@ if ( ( (entry_ptr) == NULL ) || \ #define H5C__UPDATE_STATS_FOR_UNPIN(cache_ptr, entry_ptr) #define H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr) #define H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) -#define H5C__UPDATE_STATS_FOR_HASH_BUCKET_SCAN_RESTART(cache_ptr) +#define H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) #endif /* H5C_COLLECT_CACHE_STATS */ @@ -820,6 +951,14 @@ if ( ( (entry_ptr) == NULL ) || \ * * JRM -- 9/1/15 * + * - Updated existing index macros and sanity checks macros to + * maintain an doubly linked list of all entries in the index. + * This is necessary to reduce the computational cost of visiting + * all entries in the index, which used to be done by scanning + * the hash table. + * + * JRM -- 10/15/15 + * ***********************************************************************/ /* H5C__HASH_TABLE_LEN is defined in H5Cpkg.h. It mut be a power of two. */ @@ -853,13 +992,15 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, fail_val, \ "Pre HT insert SC failed") \ } -#define H5C__POST_HT_INSERT_SC(cache_ptr, fail_val) \ +#define H5C__POST_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->magic != H5C__H5C_T_MAGIC ) || \ ( (cache_ptr)->index_size != \ @@ -874,7 +1015,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, fail_val, \ "Post HT insert SC failed") \ @@ -914,7 +1057,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Pre HT remove SC failed") \ } @@ -938,7 +1083,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Post HT remove SC failed") \ } @@ -956,7 +1103,7 @@ if ( ( (cache_ptr) == NULL ) || \ } /* (Keep in sync w/H5C_TEST__POST_SUC_HT_SEARCH_SC macro in test/cache_common.h -QAK) */ -#define H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k, fail_val) \ +#define H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k, fail_val) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->magic != H5C__H5C_T_MAGIC ) || \ ( (cache_ptr)->index_len < 1 ) || \ @@ -964,7 +1111,6 @@ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_size < (entry_ptr)->size ) || \ ( (cache_ptr)->index_size != \ ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ) || \ - ( H5F_addr_ne((entry_ptr)->addr, (Addr)) ) || \ ( (entry_ptr)->size <= 0 ) || \ ( ((cache_ptr)->index)[k] == NULL ) || \ ( ( ((cache_ptr)->index)[k] != (entry_ptr) ) && \ @@ -1016,7 +1162,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "Pre HT entry size change SC failed") \ @@ -1045,7 +1193,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "Post HT entry size change SC failed") \ @@ -1144,7 +1294,7 @@ if ( ( (cache_ptr)->index_size != \ #else /* H5C_DO_SANITY_CHECKS */ #define H5C__PRE_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) -#define H5C__POST_HT_INSERT_SC(cache_ptr, fail_val) +#define H5C__POST_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) #define H5C__PRE_HT_REMOVE_SC(cache_ptr, entry_ptr) #define H5C__POST_HT_REMOVE_SC(cache_ptr, entry_ptr) #define H5C__PRE_HT_SEARCH_SC(cache_ptr, Addr, fail_val) @@ -1162,71 +1312,77 @@ if ( ( (cache_ptr)->index_size != \ #endif /* H5C_DO_SANITY_CHECKS */ -#define H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, fail_val) \ -{ \ - int k; \ - H5C__PRE_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) \ - k = H5C__HASH_FCN((entry_ptr)->addr); \ - if(((cache_ptr)->index)[k] != NULL) { \ - (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr); \ - } \ - ((cache_ptr)->index)[k] = (entry_ptr); \ - (cache_ptr)->index_len++; \ - (cache_ptr)->index_size += (entry_ptr)->size; \ - ((cache_ptr)->index_ring_len[entry_ptr->ring])++; \ - ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ - += (entry_ptr)->size; \ - if((entry_ptr)->is_dirty) { \ - (cache_ptr)->dirty_index_size += (entry_ptr)->size; \ - ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ - += (entry_ptr)->size; \ - } else { \ - (cache_ptr)->clean_index_size += (entry_ptr)->size; \ - ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ - += (entry_ptr)->size; \ - } \ - if ((entry_ptr)->flush_me_last) { \ - (cache_ptr)->num_last_entries++; \ - HDassert((cache_ptr)->num_last_entries <= 2); \ - } \ - H5C__UPDATE_STATS_FOR_HT_INSERTION(cache_ptr) \ - H5C__POST_HT_INSERT_SC(cache_ptr, fail_val) \ +#define H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, fail_val) \ +{ \ + int k; \ + H5C__PRE_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) \ + k = H5C__HASH_FCN((entry_ptr)->addr); \ + if(((cache_ptr)->index)[k] != NULL) { \ + (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr); \ + } \ + ((cache_ptr)->index)[k] = (entry_ptr); \ + (cache_ptr)->index_len++; \ + (cache_ptr)->index_size += (entry_ptr)->size; \ + ((cache_ptr)->index_ring_len[entry_ptr->ring])++; \ + ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ + += (entry_ptr)->size; \ + if((entry_ptr)->is_dirty) { \ + (cache_ptr)->dirty_index_size += (entry_ptr)->size; \ + ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ + += (entry_ptr)->size; \ + } else { \ + (cache_ptr)->clean_index_size += (entry_ptr)->size; \ + ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ + += (entry_ptr)->size; \ + } \ + if((entry_ptr)->flush_me_last) { \ + (cache_ptr)->num_last_entries++; \ + HDassert((cache_ptr)->num_last_entries <= 2); \ + } \ + H5C__IL_DLL_APPEND((entry_ptr), (cache_ptr)->il_head, \ + (cache_ptr)->il_tail, (cache_ptr)->il_len, \ + (cache_ptr)->il_size, fail_val) \ + H5C__UPDATE_STATS_FOR_HT_INSERTION(cache_ptr) \ + H5C__POST_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) \ } -#define H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) \ -{ \ - int k; \ - H5C__PRE_HT_REMOVE_SC(cache_ptr, entry_ptr) \ - k = H5C__HASH_FCN((entry_ptr)->addr); \ - if ( (entry_ptr)->ht_next ) \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ - if ( (entry_ptr)->ht_prev ) \ - (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ - if ( ((cache_ptr)->index)[k] == (entry_ptr) ) \ - ((cache_ptr)->index)[k] = (entry_ptr)->ht_next; \ - (entry_ptr)->ht_next = NULL; \ - (entry_ptr)->ht_prev = NULL; \ - (cache_ptr)->index_len--; \ - (cache_ptr)->index_size -= (entry_ptr)->size; \ - ((cache_ptr)->index_ring_len[entry_ptr->ring])--; \ - ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ - -= (entry_ptr)->size; \ - if((entry_ptr)->is_dirty) { \ - (cache_ptr)->dirty_index_size -= (entry_ptr)->size; \ - ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ - -= (entry_ptr)->size; \ - } else { \ - (cache_ptr)->clean_index_size -= (entry_ptr)->size; \ - ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ - -= (entry_ptr)->size; \ - } \ - if((entry_ptr)->flush_me_last) { \ - (cache_ptr)->num_last_entries--; \ - HDassert((cache_ptr)->num_last_entries <= 1); \ - } \ - H5C__UPDATE_STATS_FOR_HT_DELETION(cache_ptr) \ - H5C__POST_HT_REMOVE_SC(cache_ptr, entry_ptr) \ +#define H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, fail_val) \ +{ \ + int k; \ + H5C__PRE_HT_REMOVE_SC(cache_ptr, entry_ptr) \ + k = H5C__HASH_FCN((entry_ptr)->addr); \ + if((entry_ptr)->ht_next) \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + if((entry_ptr)->ht_prev) \ + (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ + if(((cache_ptr)->index)[k] == (entry_ptr)) \ + ((cache_ptr)->index)[k] = (entry_ptr)->ht_next; \ + (entry_ptr)->ht_next = NULL; \ + (entry_ptr)->ht_prev = NULL; \ + (cache_ptr)->index_len--; \ + (cache_ptr)->index_size -= (entry_ptr)->size; \ + ((cache_ptr)->index_ring_len[entry_ptr->ring])--; \ + ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ + -= (entry_ptr)->size; \ + if((entry_ptr)->is_dirty) { \ + (cache_ptr)->dirty_index_size -= (entry_ptr)->size; \ + ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ + -= (entry_ptr)->size; \ + } else { \ + (cache_ptr)->clean_index_size -= (entry_ptr)->size; \ + ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ + -= (entry_ptr)->size; \ + } \ + if((entry_ptr)->flush_me_last) { \ + (cache_ptr)->num_last_entries--; \ + HDassert((cache_ptr)->num_last_entries <= 1); \ + } \ + H5C__IL_DLL_REMOVE((entry_ptr), (cache_ptr)->il_head, \ + (cache_ptr)->il_tail, (cache_ptr)->il_len, \ + (cache_ptr)->il_size, fail_val) \ + H5C__UPDATE_STATS_FOR_HT_DELETION(cache_ptr) \ + H5C__POST_HT_REMOVE_SC(cache_ptr, entry_ptr) \ } #define H5C__SEARCH_INDEX(cache_ptr, Addr, entry_ptr, fail_val) \ @@ -1236,51 +1392,51 @@ if ( ( (cache_ptr)->index_size != \ H5C__PRE_HT_SEARCH_SC(cache_ptr, Addr, fail_val) \ k = H5C__HASH_FCN(Addr); \ entry_ptr = ((cache_ptr)->index)[k]; \ - while ( ( entry_ptr ) && ( H5F_addr_ne(Addr, (entry_ptr)->addr) ) ) { \ + while(entry_ptr) { \ + if(H5F_addr_eq(Addr, (entry_ptr)->addr)) { \ + H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k, fail_val) \ + if(entry_ptr != ((cache_ptr)->index)[k]) { \ + if((entry_ptr)->ht_next) \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + HDassert((entry_ptr)->ht_prev != NULL); \ + (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ + ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ + (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ + (entry_ptr)->ht_prev = NULL; \ + ((cache_ptr)->index)[k] = (entry_ptr); \ + H5C__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k, fail_val) \ + } \ + break; \ + } \ (entry_ptr) = (entry_ptr)->ht_next; \ (depth)++; \ } \ - if ( entry_ptr ) { \ - H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k, fail_val) \ - if ( entry_ptr != ((cache_ptr)->index)[k] ) { \ - if ( (entry_ptr)->ht_next ) \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ - HDassert( (entry_ptr)->ht_prev != NULL ); \ - (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ - ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ - (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ - (entry_ptr)->ht_prev = NULL; \ - ((cache_ptr)->index)[k] = (entry_ptr); \ - H5C__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k, fail_val) \ - } \ - } \ H5C__UPDATE_STATS_FOR_HT_SEARCH(cache_ptr, (entry_ptr != NULL), depth) \ } #define H5C__SEARCH_INDEX_NO_STATS(cache_ptr, Addr, entry_ptr, fail_val) \ { \ int k; \ - int depth = 0; \ H5C__PRE_HT_SEARCH_SC(cache_ptr, Addr, fail_val) \ k = H5C__HASH_FCN(Addr); \ entry_ptr = ((cache_ptr)->index)[k]; \ - while ( ( entry_ptr ) && ( H5F_addr_ne(Addr, (entry_ptr)->addr) ) ) { \ - (entry_ptr) = (entry_ptr)->ht_next; \ - (depth)++; \ - } \ - if ( entry_ptr ) { \ - H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k, fail_val) \ - if ( entry_ptr != ((cache_ptr)->index)[k] ) { \ - if ( (entry_ptr)->ht_next ) \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ - HDassert( (entry_ptr)->ht_prev != NULL ); \ - (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ - ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ - (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ - (entry_ptr)->ht_prev = NULL; \ - ((cache_ptr)->index)[k] = (entry_ptr); \ - H5C__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k, fail_val) \ + while(entry_ptr) { \ + if(H5F_addr_eq(Addr, (entry_ptr)->addr)) { \ + H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k, fail_val) \ + if(entry_ptr != ((cache_ptr)->index)[k]) { \ + if((entry_ptr)->ht_next) \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + HDassert((entry_ptr)->ht_prev != NULL); \ + (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ + ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ + (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ + (entry_ptr)->ht_prev = NULL; \ + ((cache_ptr)->index)[k] = (entry_ptr); \ + H5C__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k, fail_val) \ + } \ + break; \ } \ + (entry_ptr) = (entry_ptr)->ht_next; \ } \ } @@ -1317,20 +1473,23 @@ if ( ( (cache_ptr)->index_size != \ (cache_ptr)->index_size += (new_size); \ ((cache_ptr)->index_ring_size[entry_ptr->ring]) -= (old_size); \ ((cache_ptr)->index_ring_size[entry_ptr->ring]) += (new_size); \ - if ( was_clean ) { \ + if(was_clean) { \ (cache_ptr)->clean_index_size -= (old_size); \ ((cache_ptr)->clean_index_ring_size[entry_ptr->ring])-= (old_size); \ } else { \ (cache_ptr)->dirty_index_size -= (old_size); \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring])-= (old_size); \ } \ - if ( (entry_ptr)->is_dirty ) { \ + if((entry_ptr)->is_dirty) { \ (cache_ptr)->dirty_index_size += (new_size); \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring])+= (new_size); \ } else { \ (cache_ptr)->clean_index_size += (new_size); \ ((cache_ptr)->clean_index_ring_size[entry_ptr->ring])+= (new_size); \ } \ + H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->il_len, \ + (cache_ptr)->il_size, \ + (old_size), (new_size)) \ H5C__POST_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ entry_ptr) \ } @@ -3342,6 +3501,17 @@ typedef struct H5C_tag_info_t { * The cache requires an index to facilitate searching for entries. The * following fields support that index. * + * Addendum: JRM -- 10/14/15 + * + * We sometimes need to visit all entries in the cache. In the past, this + * was done by scanning the hash table. However, this is expensive, and + * we have come to scan the hash table often enough that it has become a + * performance issue. To repair this, I have added code to maintain a + * list of all entries in the index -- call this list the index list. + * + * The index list is maintained by the same macros that maintain the + * index, and must have the same length and size as the index proper. + * * index_len: Number of entries currently in the hash table used to index * the cache. * @@ -3409,6 +3579,36 @@ typedef struct H5C_tag_info_t { * changing the H5C__HASH_FCN macro and the deletion of the * H5C__HASH_MASK #define. No other changes should be required. * + * il_len: Number of entries on the index list. + * + * This must always be equal to index_len. As such, this + * field is redundant. However, the existing linked list + * management macros expect to maintain a length field, so + * this field exists primarily to avoid adding complexity to + * these macros. + * + * il_size: Number of bytes of cache entries currently stored in the + * index list. + * + * This must always be equal to index_size. As such, this + * field is redundant. However, the existing linked list + * management macros expect to maintain a size field, so + * this field exists primarily to avoid adding complexity to + * these macros. + * + * il_head: Pointer to the head of the doubly linked list of entries in + * the index list. Note that cache entries on this list are + * linked by their il_next and il_prev fields. + * + * This field is NULL if the index is empty. + * + * il_tail: Pointer to the tail of the doubly linked list of entries in + * the index list. Note that cache entries on this list are + * linked by their il_next and il_prev fields. + * + * This field is NULL if the index is empty. + * + * * With the addition of the take ownership flag, it is possible that * an entry may be removed from the cache as the result of the flush of * a second entry. In general, this causes little trouble, but it is @@ -4071,10 +4271,14 @@ typedef struct H5C_tag_info_t { * avoid potential issues with change of status of the next * entry in the scan. * - * hash_bucket_scan_restarts: Number of times a scan of a hash bucket list - * (that contains calls to H5C__flush_single_entry()) has been - * restarted to avoid potential issues with change of status - * of the next entry in the scan. + * index_scan_restarts: Number of times a scan of the index has been + * restarted to avoid potential issues with load, insertion + * or change in flush dependency height of an entry other + * than the target entry as the result of call(s) to the + * pre_serialize or serialize callbacks. + * + * Note that at present, this condition can only be triggerd + * by a call to H5C_serialize_single_entry(). * * The remaining stats are collected only when both H5C_COLLECT_CACHE_STATS * and H5C_COLLECT_CACHE_ENTRY_STATS are true. @@ -4143,7 +4347,11 @@ struct H5C_t { size_t clean_index_ring_size[H5C_RING_NTYPES]; size_t dirty_index_size; size_t dirty_index_ring_size[H5C_RING_NTYPES]; - H5C_cache_entry_t * index[H5C__HASH_TABLE_LEN]; + H5C_cache_entry_t * index[H5C__HASH_TABLE_LEN]; + int32_t il_len; + size_t il_size; + H5C_cache_entry_t * il_head; + H5C_cache_entry_t * il_tail; /* Fields to detect entries removed during scans */ int64_t entries_removed_counter; @@ -4292,7 +4500,7 @@ struct H5C_t { /* Fields for tracking skip list scan restarts */ int64_t slist_scan_restarts; int64_t LRU_scan_restarts; - int64_t hash_bucket_scan_restarts; + int64_t index_scan_restarts; #if H5C_COLLECT_CACHE_ENTRY_STATS int32_t max_accesses[H5C__MAX_NUM_TYPE_IDS + 1]; diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 77ebe8a..365fd6c 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -1218,6 +1218,15 @@ typedef int H5C_ring_t; * If there are multiple entries in any hash bin, they are stored in a doubly * linked list. * + * Addendum: JRM -- 10/14/15 + * + * We have come to scan all entries in the cache frequently enough that + * the cost of doing so by scanning the hash table has become unacceptable. + * To reduce this cost, the index now also maintains a doubly linked list + * of all entries in the index. This list is known as the index list. + * The il_next and il_prev fields discussed below were added to support + * the index list. + * * ht_next: Next pointer used by the hash table to store multiple * entries in a single hash bin. This field points to the * next entry in the doubly linked list of entries in the @@ -1228,6 +1237,16 @@ typedef int H5C_ring_t; * previous entry in the doubly linked list of entries in * the hash bin, or NULL if there is no previuos entry. * + * il_next: Next pointer used by the index to maintain a doubly linked + * list of all entries in the index (and thus in the cache). + * This field contains a pointer to the next entry in the + * index list, or NULL if there is no next entry. + * + * il_prev: Prev pointer used by the index to maintain a doubly linked + * list of all entries in the index (and thus in the cache). + * This field contains a pointer to the previous entry in the + * index list, or NULL if there is no previous entry. + * * * Fields supporting replacement policies: * @@ -1382,6 +1401,8 @@ typedef struct H5C_cache_entry_t { /* fields supporting the hash table: */ struct H5C_cache_entry_t *ht_next; struct H5C_cache_entry_t *ht_prev; + struct H5C_cache_entry_t *il_next; + struct H5C_cache_entry_t *il_prev; /* fields supporting replacement policies: */ struct H5C_cache_entry_t *next; diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index 0dc504e..8e832d7 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -91,6 +91,14 @@ static herr_t H5F__cache_drvrinfo_serialize(const H5F_t *f, void *image, size_t void *thing); static herr_t H5F__cache_drvrinfo_free_icr(void *thing); +/* Local encode/decode routines */ +static herr_t H5F__superblock_prefix_decode(H5F_super_t *sblock, + const uint8_t **image_ref, const H5F_superblock_cache_ud_t *udata, + hbool_t extend_eoa); +static herr_t H5F__drvrinfo_prefix_decode(H5O_drvinfo_t *drvinfo, char *drv_name, + const uint8_t **image_ref, H5F_drvrinfo_cache_ud_t *udata, + hbool_t extend_eoa); + /*********************/ /* Package Variables */ @@ -148,6 +156,158 @@ H5FL_EXTERN(H5F_super_t); /*------------------------------------------------------------------------- + * Function: H5F__superblock_prefix_decode + * + * Purpose: Decode a superblock prefix + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__superblock_prefix_decode(H5F_super_t *sblock, const uint8_t **image_ref, + const H5F_superblock_cache_ud_t *udata, hbool_t extend_eoa) +{ + const uint8_t *image = (const uint8_t *)*image_ref; /* Pointer into raw data buffer */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check arguments */ + HDassert(sblock); + HDassert(image_ref); + HDassert(image); + HDassert(udata); + HDassert(udata->f); + + /* Skip over signature (already checked when locating the superblock) */ + image += H5F_SIGNATURE_LEN; + + /* Superblock version */ + sblock->super_vers = *image++; + if(sblock->super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad superblock version number") + + /* Sanity check */ + HDassert(((size_t)(image - (const uint8_t *)*image_ref)) == H5F_SUPERBLOCK_FIXED_SIZE); + + /* Determine the size of addresses & size of offsets, for computing the + * variable-sized portion of the superblock. + */ + if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_2) { + sblock->sizeof_addr = image[4]; + sblock->sizeof_size = image[5]; + } /* end if */ + else { + sblock->sizeof_addr = image[0]; + sblock->sizeof_size = image[1]; + } /* end else */ + if(sblock->sizeof_addr != 2 && sblock->sizeof_addr != 4 && + sblock->sizeof_addr != 8 && sblock->sizeof_addr != 16 && sblock->sizeof_addr != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number in an address") + if(sblock->sizeof_size != 2 && sblock->sizeof_size != 4 && + sblock->sizeof_size != 8 && sblock->sizeof_size != 16 && sblock->sizeof_size != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number for object size") + + /* Check for extending the EOA for the file */ + if(extend_eoa) { + size_t variable_size; /* Variable size of superblock */ + + /* Determine the size of the variable-length part of the superblock */ + variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(sblock->super_vers, sblock->sizeof_addr, sblock->sizeof_size); + HDassert(variable_size > 0); + + /* Make certain we can read the variable-sized portion of the superblock */ + if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + } /* end if */ + + /* Update the image buffer pointer */ + *image_ref = image; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__superblock_prefix_decode() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__drvrinfo_prefix_decode + * + * Purpose: Decode a driver info prefix + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__drvrinfo_prefix_decode(H5O_drvinfo_t *drvrinfo, char *drv_name, + const uint8_t **image_ref, H5F_drvrinfo_cache_ud_t *udata, + hbool_t extend_eoa) +{ + const uint8_t *image = (const uint8_t *)*image_ref; /* Pointer into raw data buffer */ + unsigned drv_vers; /* Version of driver info block */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(drvrinfo); + HDassert(image_ref); + HDassert(image); + HDassert(udata); + HDassert(udata->f); + + /* Version number */ + drv_vers = *image++; + if(drv_vers != HDF5_DRIVERINFO_VERSION_0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad driver information block version number") + + image += 3; /* reserved bytes */ + + /* Driver info size */ + UINT32DECODE(image, drvrinfo->len); + + /* Driver name and/or version */ + if(drv_name) { + HDmemcpy(drv_name, (const char *)image, (size_t)8); + drv_name[8] = '\0'; + image += 8; /* advance past name/version */ + } /* end if */ + + /* Extend the EOA if required so that we can read the complete driver info block */ + if(extend_eoa) { + haddr_t eoa; /* Current EOA for the file */ + haddr_t min_eoa; /* Minimum EOA needed for reading the driver info */ + + /* Get current EOA... */ + eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER); + if(!H5F_addr_defined(eoa)) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* ... if it is too small, extend it. */ + min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvrinfo->len; + + /* If it grew, set it */ + if(H5F_addr_gt(min_eoa, eoa)) + if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + } /* end if */ + + /* Update the image buffer pointer */ + *image_ref = image; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__drvrinfo_prefix_decode() */ + + +/*------------------------------------------------------------------------- * Function: H5F__cache_superblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. @@ -195,10 +355,7 @@ H5F__cache_superblock_get_final_load_size(const void *_image, size_t image_len, { const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ - unsigned super_vers; /* Superblock version */ - uint8_t sizeof_addr; /* Size of offsets in the file (in bytes) */ - uint8_t sizeof_size; /* Size of lengths in the file (in bytes) */ - size_t variable_size; /* Variable size of superblock */ + H5F_super_t sblock; /* Temporary file superblock */ htri_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -206,56 +363,20 @@ H5F__cache_superblock_get_final_load_size(const void *_image, size_t image_len, /* Check arguments */ HDassert(image); HDassert(udata); - HDassert(udata->f); HDassert(actual_len); HDassert(*actual_len == image_len); - - /* Skip over file signature */ - image += H5F_SIGNATURE_LEN; - - /* Superblock version */ - super_vers = *image++; - if(super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad superblock version number") - - /* Save the version to be used in verify_chksum callback */ - udata->super_vers = super_vers; - - /* Sanity check */ - HDassert(((size_t)(image - (const uint8_t *)_image)) == H5F_SUPERBLOCK_FIXED_SIZE); HDassert(image_len >= H5F_SUPERBLOCK_FIXED_SIZE + 6); - /* Determine the size of addresses & size of offsets, for computing the - * variable-sized portion of the superblock. - */ - if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { - sizeof_addr = image[4]; - sizeof_size = image[5]; - } /* end if */ - else { - sizeof_addr = image[0]; - sizeof_size = image[1]; - } /* end else */ - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number in an address") - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number for object size") - - /* Determine the size of the variable-length part of the superblock */ - variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size); - HDassert(variable_size > 0); + /* Deserialize the file superblock's prefix */ + if(H5F__superblock_prefix_decode(&sblock, &image, udata, TRUE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "can't decode file superblock prefix") - /* Sanity check */ - HDassert(image_len == (H5F_SUPERBLOCK_FIXED_SIZE + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE)); - - /* Make certain we can read the variable-sized portion of the superblock */ - if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + /* Save the version to be used in verify_chksum callback */ + udata->super_vers = sblock.super_vers; /* Set the final size for the cache image */ - *actual_len = H5F_SUPERBLOCK_FIXED_SIZE + variable_size; + *actual_len = H5F_SUPERBLOCK_FIXED_SIZE + + (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(sblock.super_vers, sblock.sizeof_addr, sblock.sizeof_size); done: FUNC_LEAVE_NOAPI(ret_value) @@ -325,10 +446,6 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, H5F_super_t *sblock = NULL; /* File's superblock */ H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ - size_t variable_size; /* Variable size of superblock */ - unsigned super_vers; /* Superblock version */ - uint8_t sizeof_addr; /* Size of offsets in the file (in bytes) */ - uint8_t sizeof_size; /* Size of lengths in the file (in bytes) */ H5F_super_t *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -337,54 +454,18 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, HDassert(image); HDassert(udata); HDassert(udata->f); + HDassert(len >= H5F_SUPERBLOCK_FIXED_SIZE + 6); /* Allocate space for the superblock */ if(NULL == (sblock = H5FL_CALLOC(H5F_super_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Skip over signature (already checked when locating the superblock) */ - image += H5F_SIGNATURE_LEN; - - /* Superblock version */ - super_vers = *image++; - if(super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad superblock version number") - - /* Record the superblock version */ - sblock->super_vers = super_vers; - - /* Sanity check */ - HDassert(((size_t)(image - (const uint8_t *)_image)) == H5F_SUPERBLOCK_FIXED_SIZE); - HDassert(len >= H5F_SUPERBLOCK_FIXED_SIZE + 6); - - /* Determine the size of addresses & size of offsets, for computing the - * variable-sized portion of the superblock. - */ - if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { - sizeof_addr = image[4]; - sizeof_size = image[5]; - } /* end if */ - else { - sizeof_addr = image[0]; - sizeof_size = image[1]; - } /* end else */ - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_addr = sizeof_addr; - sblock->sizeof_size = sizeof_size; - - /* Determine the size of the variable-length part of the superblock */ - variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size); - HDassert(variable_size > 0); - - HDassert(len == (H5F_SUPERBLOCK_FIXED_SIZE + variable_size)); + /* Deserialize the file superblock's prefix */ + if(H5F__superblock_prefix_decode(sblock, &image, udata, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode file superblock prefix") /* Check for older version of superblock format */ - if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { + if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_2) { uint32_t status_flags; /* File status flags */ unsigned sym_leaf_k; /* Symbol table leaf node's 'K' value */ unsigned snode_btree_k; /* B-tree symbol table internal node 'K' value */ @@ -405,21 +486,13 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, if(HDF5_SHAREDHEADER_VERSION != *image++) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad shared-header format version number") - /* Size of file addresses */ - sizeof_addr = *image++; - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - sblock->sizeof_addr = sizeof_addr; - udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ - - /* Size of file sizes */ - sizeof_size = *image++; - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_size = sizeof_size; - udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ + /* Skip over size of file addresses (already decoded) */ + image++; + udata->f->shared->sizeof_addr = sblock->sizeof_addr; /* Keep a local copy also */ + + /* Skip over size of file sizes (already decoded) */ + image++; + udata->f->shared->sizeof_size = sblock->sizeof_size; /* Keep a local copy also */ /* Skip over reserved byte */ image++; @@ -452,11 +525,11 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, * If the superblock version # is greater than 0, read in the indexed * storage B-tree internal 'K' value */ - if(super_vers > HDF5_SUPERBLOCK_VERSION_DEF) { + if(sblock->super_vers > HDF5_SUPERBLOCK_VERSION_DEF) { UINT16DECODE(image, chunk_btree_k); /* Reserved bytes are present only in version 1 */ - if(super_vers == HDF5_SUPERBLOCK_VERSION_1) + if(sblock->super_vers == HDF5_SUPERBLOCK_VERSION_1) image += 2; /* reserved */ } /* end if */ else @@ -498,21 +571,13 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, else { uint32_t read_chksum; /* Checksum read from file */ - /* Size of file addresses */ - sizeof_addr = *image++; - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - sblock->sizeof_addr = sizeof_addr; - udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ - - /* Size of file sizes */ - sizeof_size = *image++; - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_size = sizeof_size; - udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ + /* Skip over size of file addresses (already decoded) */ + image++; + udata->f->shared->sizeof_addr = sblock->sizeof_addr; /* Keep a local copy also */ + + /* Skip over size of file sizes (already decoded) */ + image++; + udata->f->shared->sizeof_size = sblock->sizeof_size; /* Keep a local copy also */ /* File status flags (not really used yet) */ sblock->status_flags = *image++; @@ -588,7 +653,7 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- - * Function: H5FS__cache_hdf_pre_serialize + * Function: H5FS__cache_superblock_pre_serialize * * Purpose: The current use of this function is a cludge to repair an * oversight in the conversion of the superblock code to use the @@ -927,50 +992,24 @@ H5F__cache_drvrinfo_get_final_load_size(const void *_image, size_t image_len, { const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5F_drvrinfo_cache_ud_t *udata = (H5F_drvrinfo_cache_ud_t *)_udata; /* User data */ - unsigned drv_vers; /* Version of driver info block */ - size_t drvinfo_len; /* Length of encoded buffer */ - haddr_t eoa; /* Current EOA for the file */ - haddr_t min_eoa; /* Minimum EOA needed for reading the driver info */ - htri_t ret_value = SUCCEED; /* Return value */ + H5O_drvinfo_t drvrinfo; /* Driver info */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Check arguments */ HDassert(image); HDassert(udata); - HDassert(udata->f); HDassert(actual_len); HDassert(*actual_len == image_len); - - /* Version number */ - drv_vers = *image++; - if(drv_vers != HDF5_DRIVERINFO_VERSION_0) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad driver information block version number") - - image += 3; /* reserved bytes */ - - /* Driver info size */ - UINT32DECODE(image, drvinfo_len); - - /* Sanity check */ HDassert(image_len == H5F_DRVINFOBLOCK_HDR_SIZE); - /* Extend the EOA if required so that we can read the complete driver info block */ - - /* Get current EOA... */ - if((eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver get_eoa request failed") - - /* ... if it is too small, extend it. */ - min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len; - - /* If it grew, set it */ - if(H5F_addr_gt(min_eoa, eoa)) - if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + /* Deserialize the file driver info's prefix */ + if(H5F__drvrinfo_prefix_decode(&drvrinfo, NULL, &image, udata, TRUE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "can't decode file driver info prefix") /* Set the final size for the cache image */ - *actual_len = H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len; + *actual_len = H5F_DRVINFOBLOCK_HDR_SIZE + drvrinfo.len; done: FUNC_LEAVE_NOAPI(ret_value) @@ -999,7 +1038,6 @@ H5F__cache_drvrinfo_deserialize(const void *_image, size_t len, void *_udata, H5F_drvrinfo_cache_ud_t *udata = (H5F_drvrinfo_cache_ud_t *)_udata; /* User data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ char drv_name[9]; /* Name of driver */ - unsigned drv_vers; /* Version of driver info block */ H5O_drvinfo_t *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -1014,21 +1052,11 @@ H5F__cache_drvrinfo_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (drvinfo = (H5O_drvinfo_t *)H5MM_calloc(sizeof(H5O_drvinfo_t)))) HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "memory allocation failed for driver info message") - /* Version number */ - drv_vers = *image++; - if(drv_vers != HDF5_DRIVERINFO_VERSION_0) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad driver information block version number") - - image += 3; /* reserved bytes */ - - /* Driver info size */ - UINT32DECODE(image, drvinfo->len); - - /* Driver name and/or version */ - HDstrncpy(drv_name, (const char *)image, (size_t)8); - drv_name[8] = '\0'; - image += 8; /* advance past name/version */ + /* Deserialize the file driver info's prefix */ + if(H5F__drvrinfo_prefix_decode(drvinfo, drv_name, &image, udata, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode file driver info prefix") + /* Sanity check */ HDassert(len == (H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len)); /* Validate and decode driver information */ diff --git a/src/H5HFcache.c b/src/H5HFcache.c index f85b344..1e2eb8f 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -69,6 +69,7 @@ /********************/ /* Local encode/decode routines */ +static herr_t H5HF__hdr_prefix_decode(H5HF_hdr_t *hdr, const uint8_t **image_ref); static herr_t H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable); static herr_t H5HF__dtable_decode(H5F_t *f, const uint8_t **pp, H5HF_dtable_t *dtable); @@ -200,6 +201,52 @@ H5FL_BLK_DEFINE(direct_block); /*------------------------------------------------------------------------- + * Function: H5HF__hdr_prefix_decode() + * + * Purpose: Decode a fractal heap header's prefix + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HF__hdr_prefix_decode(H5HF_hdr_t *hdr, const uint8_t **image_ref) +{ + const uint8_t *image = *image_ref; /* Pointer into into supplied image */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(hdr); + HDassert(image); + + /* Magic number */ + if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "wrong fractal heap header signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(*image++ != H5HF_HDR_VERSION) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong fractal heap header version") + + /* General heap information */ + UINT16DECODE(image, hdr->id_len); /* Heap ID length */ + UINT16DECODE(image, hdr->filter_len); /* I/O filters' encoded length */ + + /* Update the image buffer pointer */ + *image_ref = image; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__hdr_prefix_decode() */ + + +/*------------------------------------------------------------------------- * Function: H5HF__dtable_decode * * Purpose: Decodes the metadata for a doubling table @@ -360,10 +407,10 @@ static herr_t H5HF__cache_hdr_get_final_load_size(const void *_image, size_t image_len, void *_udata, size_t *actual_len) { - const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ - H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* pointer to user data */ - unsigned filter_len; /* Size of I/O filter information (in bytes) */ - htri_t ret_value = SUCCEED; /* Return value */ + H5HF_hdr_t hdr; /* Temporary fractal heap header */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into into supplied image */ + H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -373,25 +420,16 @@ H5HF__cache_hdr_get_final_load_size(const void *_image, size_t image_len, HDassert(actual_len); HDassert(*actual_len == image_len); - /* Magic number */ - if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "wrong fractal heap header signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(*image++ != H5HF_HDR_VERSION) - HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong fractal heap header version") - - /* General heap information */ - image += 2; /* Heap ID length */ - UINT16DECODE(image, filter_len); /* I/O filters' encoded length */ + /* Deserialize the fractal heap header's prefix */ + if(H5HF__hdr_prefix_decode(&hdr, &image) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode fractal heap header prefix") /* Check for I/O filter info on this heap */ - if(filter_len > 0) + if(hdr.filter_len > 0) /* Compute the extra heap header size */ *actual_len += (size_t)(H5F_SIZEOF_SIZE(udata->f) /* Size of size for filtered root direct block */ + (unsigned)4 /* Size of filter mask for filtered root direct block */ - + filter_len); /* Size of encoded I/O filter info */ + + hdr.filter_len); /* Size of encoded I/O filter info */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -473,18 +511,9 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (hdr = H5HF_hdr_alloc(udata->f))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Magic number */ - if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "wrong fractal heap header signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(*image++ != H5HF_HDR_VERSION) - HGOTO_ERROR(H5E_HEAP, H5E_VERSION, NULL, "wrong fractal heap header version") - - /* General heap information */ - UINT16DECODE(image, hdr->id_len); /* Heap ID length */ - UINT16DECODE(image, hdr->filter_len); /* I/O filters' encoded length */ + /* Deserialize the fractal heap header's prefix */ + if(H5HF__hdr_prefix_decode(hdr, &image) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode fractal heap header prefix") /* Heap status flags */ /* (bit 0: "huge" object IDs have wrapped) */ @@ -1618,10 +1647,6 @@ H5HF__cache_dblock_verify_chksum(const void *_image, size_t len, void *_udata) hdr = par_info->hdr; HDassert(hdr); - /* Reset callback context info */ - udata->decompressed = FALSE; - udata->dblk = NULL; - /* Get out if data block is not checksummed */ if(!(hdr->checksum_dblocks)) HGOTO_DONE(TRUE); @@ -1728,7 +1753,7 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, H5HF_dblock_cache_ud_t *udata = (H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */ H5HF_parent_t *par_info; /* Pointer to parent information */ H5HF_direct_t *dblock = NULL; /* Direct block info */ - const uint8_t *image = _image;/* Pointer into raw data buffer */ + const uint8_t *image = (const uint8_t *)_image;/* Pointer into raw data buffer */ void *read_buf = NULL; /* Pointer to buffer to decompress */ haddr_t heap_addr; /* Address of heap header in the file */ void * ret_value = NULL; /* Return value */ diff --git a/src/H5HFdblock.c b/src/H5HFdblock.c index 1272ab0..b34534c 100644 --- a/src/H5HFdblock.c +++ b/src/H5HFdblock.c @@ -493,6 +493,10 @@ H5HF_man_dblock_protect(H5HF_hdr_t *hdr, hid_t dxpl_id, haddr_t dblock_addr, udata.filter_mask = 0; } /* end else */ + /* Reset compression context info */ + udata.decompressed = FALSE; + udata.dblk = NULL; + /* Protect the direct block */ if(NULL == (dblock = (H5HF_direct_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_FHEAP_DBLOCK, dblock_addr, &udata, flags))) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, NULL, "unable to protect fractal heap direct block") diff --git a/src/H5HGcache.c b/src/H5HGcache.c index 6bc9860..a83c22b 100644 --- a/src/H5HGcache.c +++ b/src/H5HGcache.c @@ -72,6 +72,10 @@ static herr_t H5HG__cache_heap_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HG__cache_heap_free_icr(void *thing); +/* Prefix deserialization */ +static herr_t H5HG__hdr_deserialize(H5HG_heap_t *heap, const uint8_t *image, + const H5F_t *f); + /*********************/ /* Package Variables */ @@ -108,6 +112,52 @@ const H5AC_class_t H5AC_GHEAP[1] = {{ /*------------------------------------------------------------------------- + * Function: H5HG__hdr_deserialize() + * + * Purpose: Decode a global heap's header + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HG__hdr_deserialize(H5HG_heap_t *heap, const uint8_t *image, const H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(heap); + HDassert(image); + HDassert(f); + + /* Magic number */ + if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad global heap collection signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(H5HG_VERSION != *image++) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in global heap") + + /* Reserved */ + image += 3; + + /* Size */ + H5F_DECODE_LENGTH(f, image, heap->size); + HDassert(heap->size >= H5HG_MINSIZE); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HG__hdr_deserialize() */ + + +/*------------------------------------------------------------------------- * Function: H5HG__cache_heap_get_initial_load_size() * * Purpose: Return the initial speculative read size to the metadata @@ -154,41 +204,27 @@ H5HG__cache_heap_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *imag *------------------------------------------------------------------------- */ static herr_t -H5HG__cache_heap_get_final_load_size(const void *_image, size_t image_len, - void *_udata, size_t *actual_len) +H5HG__cache_heap_get_final_load_size(const void *image, size_t image_len, + void *udata, size_t *actual_len) { - const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ - H5F_t *f = (H5F_t *)_udata; /* File pointer -- obtained from user data */ - size_t heap_size = 0; /* Total size of collection */ - herr_t ret_value = SUCCEED; /* Return value */ + H5HG_heap_t heap; /* Global heap */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Sanity check */ HDassert(image); - HDassert(f); + HDassert(udata); HDassert(actual_len); HDassert(*actual_len == image_len); - - /* Magic number */ - if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad global heap collection signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(H5HG_VERSION != *image++) - HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in global heap") - - /* Reserved */ - image += 3; - - /* Size */ - H5F_DECODE_LENGTH(f, image, heap_size); - HDassert(heap_size >= H5HG_MINSIZE); HDassert(image_len == H5HG_MINSIZE); + /* Deserialize the heap's header */ + if(H5HG__hdr_deserialize(&heap, (const uint8_t *)image, (const H5F_t *)udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode global heap prefix") + /* Set the final size for the cache image */ - *actual_len = heap_size; + *actual_len = heap.size; done: FUNC_LEAVE_NOAPI(ret_value) @@ -236,29 +272,13 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (heap->chunk = H5FL_BLK_MALLOC(gheap_chunk, len))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* copy the image buffer into the newly allocate chunk */ + /* Copy the image buffer into the newly allocate chunk */ HDmemcpy(heap->chunk, _image, len); - image = heap->chunk; - - /* Magic number */ - if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "bad global heap collection signature") - image += H5_SIZEOF_MAGIC; + /* Deserialize the heap's header */ + if(H5HG__hdr_deserialize(heap, (const uint8_t *)heap->chunk, f) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode global heap header") - /* Version */ - if(H5HG_VERSION != *image++) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "wrong version number in global heap") - - /* Reserved */ - image += 3; - - /* Size */ - H5F_DECODE_LENGTH(f, image, heap->size); - HDassert(heap->size >= H5HG_MINSIZE); - HDassert((len == H5HG_MINSIZE) /* first try */ || - ((len == heap->size) && (len > H5HG_MINSIZE))); /* second try */ - /* Decode each object */ image = heap->chunk + H5HG_SIZEOF_HDR(f); nalloc = H5HG_NOBJS(f, heap->size); diff --git a/src/H5HLcache.c b/src/H5HLcache.c index 3db239e..b2ca839 100644 --- a/src/H5HLcache.c +++ b/src/H5HLcache.c @@ -91,6 +91,10 @@ static herr_t H5HL__cache_datablock_serialize(const H5F_t *f, void *image, static herr_t H5HL__cache_datablock_notify(H5C_notify_action_t action, void *_thing); static herr_t H5HL__cache_datablock_free_icr(void *thing); +/* Header deserialization */ +static herr_t H5HL__hdr_deserialize(H5HL_t *heap, const uint8_t *image, + H5HL_cache_prfx_ud_t *udata); + /* Free list de/serialization */ static herr_t H5HL__fl_deserialize(H5HL_t *heap); static void H5HL__fl_serialize(const H5HL_t *heap); @@ -147,6 +151,64 @@ const H5AC_class_t H5AC_LHEAP_DBLK[1] = {{ /*------------------------------------------------------------------------- + * Function: H5HL__hdr_deserialize() + * + * Purpose: Decode a local heap's header + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HL__hdr_deserialize( H5HL_t *heap, const uint8_t *image, + H5HL_cache_prfx_ud_t *udata) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(heap); + HDassert(image); + HDassert(udata); + + /* Check magic number */ + if(HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad local heap signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(H5HL_VERSION != *image++) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in local heap") + + /* Reserved */ + image += 3; + + /* Store the prefix's address & length */ + heap->prfx_addr = udata->prfx_addr; + heap->prfx_size = udata->sizeof_prfx; + + /* Heap data size */ + H5F_DECODE_LENGTH_LEN(image, heap->dblk_size, udata->sizeof_size); + + /* Free list head */ + H5F_DECODE_LENGTH_LEN(image, heap->free_block, udata->sizeof_size); + if(heap->free_block != H5HL_FREE_NULL && heap->free_block >= heap->dblk_size) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap free list") + + /* Heap data address */ + H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap->dblk_addr)); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HL__hdr_deserialize() */ + + +/*------------------------------------------------------------------------- * Function: H5HL__fl_deserialize * * Purpose: Deserialize the free list for a heap data block @@ -319,32 +381,9 @@ H5HL__cache_prefix_get_final_load_size(const void *_image, size_t image_len, HDassert(actual_len); HDassert(*actual_len == image_len); - /* Check magic number */ - if(HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad local heap signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(H5HL_VERSION != *image++) - HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in local heap") - - /* Reserved */ - image += 3; - - /* Store the prefix's address & length */ - heap.prfx_addr = udata->prfx_addr; /* NEED */ - heap.prfx_size = udata->sizeof_prfx; /* NEED */ - - /* Heap data size */ - H5F_DECODE_LENGTH_LEN(image, heap.dblk_size, udata->sizeof_size); /* NEED */ - - /* Free list head */ - H5F_DECODE_LENGTH_LEN(image, heap.free_block, udata->sizeof_size); - if(heap.free_block != H5HL_FREE_NULL && heap.free_block >= heap.dblk_size) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap free list"); - - /* Heap data address */ - H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap.dblk_addr)); /* NEED */ + /* Deserialize the heap's header */ + if(H5HL__hdr_deserialize(&heap, (const uint8_t *)image, udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode local heap header") /* Set the final size for the cache image */ *actual_len = heap.prfx_size; @@ -398,41 +437,18 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t len, void *_udata, HDassert(H5F_addr_defined(udata->prfx_addr)); HDassert(dirty); - /* Check magic number */ - if(HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad local heap signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(H5HL_VERSION != *image++) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "wrong version number in local heap") - - /* Reserved */ - image += 3; - /* Allocate space in memory for the heap */ if(NULL == (heap = H5HL__new(udata->sizeof_size, udata->sizeof_addr, udata->sizeof_prfx))) HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "can't allocate local heap structure"); + /* Deserialize the heap's header */ + if(H5HL__hdr_deserialize(heap, (const uint8_t *)image, udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode local heap header") + /* Allocate the heap prefix */ if(NULL == (prfx = H5HL__prfx_new(heap))) HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "can't allocate local heap prefix"); - /* Store the prefix's address & length */ - heap->prfx_addr = udata->prfx_addr; - heap->prfx_size = udata->sizeof_prfx; - - /* Heap data size */ - H5F_DECODE_LENGTH_LEN(image, heap->dblk_size, udata->sizeof_size); - - /* Free list head */ - H5F_DECODE_LENGTH_LEN(image, heap->free_block, udata->sizeof_size); - if((heap->free_block != H5HL_FREE_NULL) && (heap->free_block >= heap->dblk_size)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap free list") - - /* Heap data address */ - H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap->dblk_addr)); - /* Check if heap block exists */ if(heap->dblk_size) { /* Check if heap data block is contiguous with header */ diff --git a/src/H5Ocache.c b/src/H5Ocache.c index d398e41..1067b8c 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -84,6 +84,10 @@ static herr_t H5O__cache_chk_serialize(const H5F_t *f, void *image, size_t len, static herr_t H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing); static herr_t H5O__cache_chk_free_icr(void *thing); +/* Prefix routines */ +static herr_t H5O__prefix_deserialize(const uint8_t *image, + H5O_cache_ud_t *udata); + /* Chunk routines */ static herr_t H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image, H5O_common_cache_ud_t *udata, hbool_t *dirty); @@ -198,13 +202,11 @@ H5O__cache_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) *------------------------------------------------------------------------- */ static herr_t -H5O__cache_get_final_load_size(const void *_image, size_t image_len, +H5O__cache_get_final_load_size(const void *image, size_t image_len, void *_udata, size_t *actual_len) { - const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - H5O_t *oh = NULL; /* Object header read in */ - htri_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -214,136 +216,17 @@ H5O__cache_get_final_load_size(const void *_image, size_t image_len, HDassert(actual_len); HDassert(*actual_len == image_len); - /* Allocate space for the new object header data structure */ - if(NULL == (oh = H5FL_CALLOC(H5O_t))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") - - /* File-specific, non-stored information */ - oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); - oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f); - - /* Check for presence of magic number */ - /* (indicates version 2 or later) */ - if(!HDmemcmp(image, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { - /* Magic number */ - image += H5_SIZEOF_MAGIC; - - /* Version */ - oh->version = *image++; - if(H5O_VERSION_2 != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") - - /* Flags */ - oh->flags = *image++; - if(oh->flags & ~H5O_HDR_ALL_FLAGS) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown object header status flag(s)") - - /* Number of links to object (unless overridden by refcount message) */ - oh->nlink = 1; - - /* Time fields */ - if(oh->flags & H5O_HDR_STORE_TIMES) { - uint32_t tmp; /* Temporary value */ - - UINT32DECODE(image, tmp); - oh->atime = (time_t)tmp; - UINT32DECODE(image, tmp); - oh->mtime = (time_t)tmp; - UINT32DECODE(image, tmp); - oh->ctime = (time_t)tmp; - UINT32DECODE(image, tmp); - oh->btime = (time_t)tmp; - } /* end if */ - else - oh->atime = oh->mtime = oh->ctime = oh->btime = 0; - - /* Attribute fields */ - if(oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { - UINT16DECODE(image, oh->max_compact); - UINT16DECODE(image, oh->min_dense); - if(oh->max_compact < oh->min_dense) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header attribute phase change values") - } /* end if */ - else { - oh->max_compact = H5O_CRT_ATTR_MAX_COMPACT_DEF; - oh->min_dense = H5O_CRT_ATTR_MIN_DENSE_DEF; - } /* end else */ - - /* First chunk size */ - switch(oh->flags & H5O_HDR_CHUNK0_SIZE) { - case 0: /* 1 byte size */ - udata->chunk0_size = *image++; - break; - - case 1: /* 2 byte size */ - UINT16DECODE(image, udata->chunk0_size); - break; - - case 2: /* 4 byte size */ - UINT32DECODE(image, udata->chunk0_size); - break; - - case 3: /* 8 byte size */ - UINT64DECODE(image, udata->chunk0_size); - break; - - default: - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") - } /* end switch */ - if(udata->chunk0_size > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") - } /* end if */ - else { - /* Version */ - oh->version = *image++; - if(H5O_VERSION_1 != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") - - /* Flags */ - oh->flags = H5O_CRT_OHDR_FLAGS_DEF; - - /* Reserved */ - image++; - - /* Number of messages */ - UINT16DECODE(image, udata->v1_pfx_nmesgs); - - /* Link count */ - UINT32DECODE(image, oh->nlink); - - /* Reset unused time fields */ - oh->atime = oh->mtime = oh->ctime = oh->btime = 0; - - /* Reset unused attribute fields */ - oh->max_compact = 0; - oh->min_dense = 0; - - /* First chunk size */ - UINT32DECODE(image, udata->chunk0_size); - if((udata->v1_pfx_nmesgs > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || - (udata->v1_pfx_nmesgs == 0 && udata->chunk0_size > 0)) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") - - /* Reserved, in version 1 (for 8-byte alignment padding) */ - image += 4; - } /* end else */ + /* Deserialize the object header prefix */ + if(H5O__prefix_deserialize((const uint8_t *)image, udata) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "can't deserialize object header prefix") - /* Determine object header prefix length */ - HDassert((size_t)((const uint8_t *)image - (const uint8_t *)_image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + /* Sanity check */ + HDassert(udata->oh); /* Set the final size for the cache image */ - *actual_len = udata->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh); - - /* Save the object header for later use in 'deserialize' callback */ - udata->oh = oh; - oh = NULL; + *actual_len = udata->chunk0_size + (size_t)H5O_SIZEOF_HDR(udata->oh); done: - /* Release the [possibly partially initialized] object header on errors */ - if(ret_value < 0 && oh) - if(H5O__free(oh) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header data") - FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__cache_get_final_load_size() */ @@ -413,12 +296,11 @@ H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) *------------------------------------------------------------------------- */ static void * -H5O__cache_deserialize(const void *_image, size_t len, void *_udata, +H5O__cache_deserialize(const void *image, size_t len, void *_udata, hbool_t *dirty) { - H5O_t *oh; /* Object header read in */ + H5O_t *oh = NULL; /* Object header read in */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - const uint8_t *image = (const uint8_t *)_image; /* Pointer into buffer to decode */ void * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -427,11 +309,24 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, HDassert(image); HDassert(len > 0); HDassert(udata); - HDassert(udata->oh); HDassert(udata->common.f); HDassert(udata->common.cont_msg_info); HDassert(dirty); + /* Check for partially deserialized object header */ + /* (Object header prefix will be deserialized if the object header came + * through the 'get_final_load_size' callback and not deserialized if + * the object header is coming from a cache image - QAK, 2016/12/14) + */ + if(NULL == udata->oh) { + /* Deserialize the object header prefix */ + if(H5O__prefix_deserialize((const uint8_t *)image, udata) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, NULL, "can't deserialize object header prefix") + + /* Sanity check */ + HDassert(udata->oh); + } /* end if */ + /* Retrieve partially deserialized object header from user data */ oh = udata->oh; @@ -448,7 +343,7 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, oh->proxy = NULL; /* Parse the first chunk */ - if(H5O__chunk_deserialize(oh, udata->common.addr, udata->chunk0_size, (const uint8_t *)_image, &(udata->common), dirty) < 0) + if(H5O__chunk_deserialize(oh, udata->common.addr, udata->chunk0_size, (const uint8_t *)image, &(udata->common), dirty) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk") /* Note that we've loaded the object header from the file */ @@ -717,11 +612,7 @@ H5O__cache_notify(H5AC_notify_action_t action, void *_thing) break; default: -#ifdef NDEBUG HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown action from metadata cache") -#else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); -#endif /* NDEBUG */ } /* end switch */ done: @@ -1228,6 +1119,163 @@ done: /*------------------------------------------------------------------------- + * Function: H5O__prefix_deserialize() + * + * Purpose: Deserialize an object header prefix + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * December 14, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O__prefix_deserialize(const uint8_t *_image, H5O_cache_ud_t *udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5O_t *oh = NULL; /* Object header read in */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* Allocate space for the new object header data structure */ + if(NULL == (oh = H5FL_CALLOC(H5O_t))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") + + /* File-specific, non-stored information */ + oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); + oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f); + + /* Check for presence of magic number */ + /* (indicates version 2 or later) */ + if(!HDmemcmp(image, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + /* Magic number */ + image += H5_SIZEOF_MAGIC; + + /* Version */ + oh->version = *image++; + if(H5O_VERSION_2 != oh->version) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") + + /* Flags */ + oh->flags = *image++; + if(oh->flags & ~H5O_HDR_ALL_FLAGS) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown object header status flag(s)") + + /* Number of links to object (unless overridden by refcount message) */ + oh->nlink = 1; + + /* Time fields */ + if(oh->flags & H5O_HDR_STORE_TIMES) { + uint32_t tmp; /* Temporary value */ + + UINT32DECODE(image, tmp); + oh->atime = (time_t)tmp; + UINT32DECODE(image, tmp); + oh->mtime = (time_t)tmp; + UINT32DECODE(image, tmp); + oh->ctime = (time_t)tmp; + UINT32DECODE(image, tmp); + oh->btime = (time_t)tmp; + } /* end if */ + else + oh->atime = oh->mtime = oh->ctime = oh->btime = 0; + + /* Attribute fields */ + if(oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { + UINT16DECODE(image, oh->max_compact); + UINT16DECODE(image, oh->min_dense); + if(oh->max_compact < oh->min_dense) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header attribute phase change values") + } /* end if */ + else { + oh->max_compact = H5O_CRT_ATTR_MAX_COMPACT_DEF; + oh->min_dense = H5O_CRT_ATTR_MIN_DENSE_DEF; + } /* end else */ + + /* First chunk size */ + switch(oh->flags & H5O_HDR_CHUNK0_SIZE) { + case 0: /* 1 byte size */ + udata->chunk0_size = *image++; + break; + + case 1: /* 2 byte size */ + UINT16DECODE(image, udata->chunk0_size); + break; + + case 2: /* 4 byte size */ + UINT32DECODE(image, udata->chunk0_size); + break; + + case 3: /* 8 byte size */ + UINT64DECODE(image, udata->chunk0_size); + break; + + default: + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") + } /* end switch */ + if(udata->chunk0_size > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") + } /* end if */ + else { + /* Version */ + oh->version = *image++; + if(H5O_VERSION_1 != oh->version) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") + + /* Flags */ + oh->flags = H5O_CRT_OHDR_FLAGS_DEF; + + /* Reserved */ + image++; + + /* Number of messages */ + UINT16DECODE(image, udata->v1_pfx_nmesgs); + + /* Link count */ + UINT32DECODE(image, oh->nlink); + + /* Reset unused time fields */ + oh->atime = oh->mtime = oh->ctime = oh->btime = 0; + + /* Reset unused attribute fields */ + oh->max_compact = 0; + oh->min_dense = 0; + + /* First chunk size */ + UINT32DECODE(image, udata->chunk0_size); + if((udata->v1_pfx_nmesgs > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || + (udata->v1_pfx_nmesgs == 0 && udata->chunk0_size > 0)) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") + + /* Reserved, in version 1 (for 8-byte alignment padding) */ + image += 4; + } /* end else */ + + /* Verify object header prefix length */ + HDassert((size_t)(image - _image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + + /* Save the object header for later use in 'deserialize' callback */ + udata->oh = oh; + oh = NULL; + +done: + /* Release the [possibly partially initialized] object header on errors */ + if(ret_value < 0 && oh) + if(H5O__free(oh) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header data") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__prefix_deserialize() */ + + +/*------------------------------------------------------------------------- * Function: H5O__chunk_deserialize * * Purpose: Deserialize a chunk for an object header diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index c094c20..4a3eb72 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -173,7 +173,7 @@ #define H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_DEF 0 /* Ring type - private property */ #define H5AC_XFER_RING_SIZE sizeof(unsigned) -#define H5AC_XFER_RING_DEF H5AC_RING_US +#define H5AC_XFER_RING_DEF H5AC_RING_USER #define H5AC_XFER_RING_ENC H5P__encode_unsigned #define H5AC_XFER_RING_DEC H5P__decode_unsigned #ifdef H5_DEBUG_BUILD @@ -144,7 +144,7 @@ H5SM_init(H5F_t *f, H5P_genplist_t * fc_plist, const H5O_loc_t *ext_loc, hid_t d HDassert(!H5F_addr_defined(H5F_SOHM_ADDR(f))); /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_US, &dxpl, &orig_ring) < 0) + if(H5AC_set_ring(dxpl_id, H5AC_RING_USER, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_SOHM, H5E_CANTSET, FAIL, "unable to set ring value") /* Initialize master table */ @@ -2021,7 +2021,7 @@ H5SM_get_info(const H5O_loc_t *ext_loc, H5P_genplist_t *fc_plist, hid_t dxpl_id) cache_udata.f = f; /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_US, &dxpl, &orig_ring) < 0) + if(H5AC_set_ring(dxpl_id, H5AC_RING_USER, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_SOHM, H5E_CANTSET, FAIL, "unable to set ring value") /* Read the rest of the SOHM table information from the cache */ diff --git a/test/cache.c b/test/cache.c index 0cd0c99..ed4fceb 100644 --- a/test/cache.c +++ b/test/cache.c @@ -15913,7 +15913,6 @@ check_destroy_pinned_err(void) unprotect_entry(file_ptr, 0, 0, H5C__PIN_ENTRY_FLAG); if(H5C_dest(file_ptr, H5AC_ind_read_dxpl_id) >= 0) { - pass = FALSE; failure_mssg = "destroy succeeded on cache with pinned entry.\n"; } /* end if */ @@ -34541,7 +34540,7 @@ cedds__expunge_dirty_entry_in_flush_test(H5F_t * file_ptr) if(pass) if((cache_ptr->slist_scan_restarts != 1) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "unexpected scan restart stats in cedds__expunge_dirty_entry_in_flush_test()."; } /* end if */ @@ -34920,7 +34919,7 @@ cedds__H5C_make_space_in_cache(H5F_t * file_ptr) if(pass) if((cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 1) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "unexpected scan restart stats in cedds__H5C_make_space_in_cache()."; @@ -35330,7 +35329,7 @@ cedds__H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * file_ptr) if(pass) if((cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 1) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "unexpected scan restart stats in cedds__H5C__autoadjust__ageout__evict_aged_out_entries()."; @@ -35353,7 +35352,26 @@ cedds__H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * file_ptr) /*------------------------------------------------------------------------- * Function: cedds__H5C_flush_invalidate_cache__bucket_scan() * - * Purpose: Verify that H5C_flush_invalidate_cache() can handle + * Purpose: Note: We now use the index list when we scan the + * contents of the metadata cache, so in principal, + * this test is obsolete. However, even using the + * index list, restarts are possible, and must be + * handled gracefully. + * + * As it turns out, this test triggers index list + * scan restarts, and thus with minor changes is + * still a useful test. + * + * For this reason, with the exception of changing + * to check the index_scan_restart stat instead of + * hash bucket restarts, I'm leaving the test + * alone. If and when it starts to fail due to + * other changes, we can re-work it to test + * index list scan restarts explicitly. + * + * JRM -- 11/2/16 + * + * Verify that H5C_flush_invalidate_cache() can handle * the removal from the cache of the next item in * its scans of hash buckets. * @@ -35455,20 +35473,20 @@ cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t * file_ptr) if(cache_ptr == NULL) { pass = FALSE; - failure_mssg = "cache_ptr NULL on entry to cedds for H5C__autoadjust__ageout__evict_aged_out_entries() test."; + failure_mssg = "cache_ptr NULL on entry to cedds for cedds__H5C_flush_invalidate_cache__bucket_scan() test."; } else if((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0)) { pass = FALSE; - failure_mssg = "cache not empty at start cedds for H5C__autoadjust__ageout__evict_aged_out_entries() test."; + failure_mssg = "cache not empty at start cedds for cedds__H5C_flush_invalidate_cache__bucket_scan() test."; } else if((cache_ptr->max_cache_size != (2 * 1024 * 1024)) || (cache_ptr->min_clean_size != (1 * 1024 * 1024))) { pass = FALSE; failure_mssg = - "unexpected cache config at start of cedds H5C__autoadjust__ageout__evict_aged_out_entries() test."; + "unexpected cache config at start of cedds cedds__H5C_flush_invalidate_cache__bucket_scan() test."; } else { @@ -35696,9 +35714,13 @@ cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t * file_ptr) } /* end if */ if(pass) + /* as this test is now checking for index list scan restarts, + * the following has been modified to check this instead of + * hash bucket scan restarts. + */ if((cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 1)) { + (cache_ptr->index_scan_restarts != 1)) { pass = FALSE; failure_mssg = "unexpected scan restart stats in cedds__H5C_flush_invalidate_cache__bucket_scan()."; } @@ -35912,7 +35934,7 @@ check_stats__smoke_check_1(H5F_t * file_ptr) (cache_ptr->entries_scanned_to_make_space != 0) || (cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(1)."; @@ -35997,7 +36019,7 @@ check_stats__smoke_check_1(H5F_t * file_ptr) (cache_ptr->entries_scanned_to_make_space != 0) || (cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(2)."; @@ -36082,7 +36104,7 @@ check_stats__smoke_check_1(H5F_t * file_ptr) (cache_ptr->entries_scanned_to_make_space != 33) || (cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(3)."; @@ -36186,7 +36208,7 @@ check_stats__smoke_check_1(H5F_t * file_ptr) (cache_ptr->entries_scanned_to_make_space != 33) || (cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(4)."; diff --git a/test/cache_common.h b/test/cache_common.h index 1ab04cb..befcee4 100644 --- a/test/cache_common.h +++ b/test/cache_common.h @@ -384,7 +384,7 @@ if ( ( (cache_ptr) == NULL ) || \ HDfprintf(stdout, "Pre HT search SC failed.\n"); \ } -#define H5C_TEST__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k) \ +#define H5C_TEST__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->magic != H5C__H5C_T_MAGIC ) || \ ( (cache_ptr)->index_len < 1 ) || \ @@ -392,7 +392,6 @@ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_size < (entry_ptr)->size ) || \ ( (cache_ptr)->index_size != \ ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ) || \ - ( H5F_addr_ne((entry_ptr)->addr, (Addr)) ) || \ ( (entry_ptr)->size <= 0 ) || \ ( ((cache_ptr)->index)[k] == NULL ) || \ ( ( ((cache_ptr)->index)[k] != (entry_ptr) ) && \ @@ -416,32 +415,29 @@ if ( ( (cache_ptr) == NULL ) || \ #define H5C_TEST__SEARCH_INDEX(cache_ptr, Addr, entry_ptr) \ { \ int k; \ - int depth = 0; \ H5C_TEST__PRE_HT_SEARCH_SC(cache_ptr, Addr) \ k = H5C__HASH_FCN(Addr); \ entry_ptr = ((cache_ptr)->index)[k]; \ - while ( ( entry_ptr ) && ( H5F_addr_ne(Addr, (entry_ptr)->addr) ) ) \ + while ( entry_ptr ) \ { \ - (entry_ptr) = (entry_ptr)->ht_next; \ - (depth)++; \ - } \ - if ( entry_ptr ) \ - { \ - H5C_TEST__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k) \ - if ( entry_ptr != ((cache_ptr)->index)[k] ) \ + if ( H5F_addr_eq(Addr, (entry_ptr)->addr) ) \ { \ - if ( (entry_ptr)->ht_next ) \ + H5C_TEST__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k) \ + if ( entry_ptr != ((cache_ptr)->index)[k] ) \ { \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + if ( (entry_ptr)->ht_next ) \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + HDassert( (entry_ptr)->ht_prev != NULL ); \ + (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ + ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ + (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ + (entry_ptr)->ht_prev = NULL; \ + ((cache_ptr)->index)[k] = (entry_ptr); \ + H5C_TEST__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k) \ } \ - HDassert( (entry_ptr)->ht_prev != NULL ); \ - (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ - ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ - (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ - (entry_ptr)->ht_prev = NULL; \ - ((cache_ptr)->index)[k] = (entry_ptr); \ - H5C_TEST__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k) \ + break; \ } \ + (entry_ptr) = (entry_ptr)->ht_next; \ } \ } |