diff options
author | Dana Robinson <derobins@hdfgroup.org> | 2016-12-30 13:44:15 (GMT) |
---|---|---|
committer | Dana Robinson <derobins@hdfgroup.org> | 2016-12-30 13:44:15 (GMT) |
commit | 8215920678c09238ecc86d9eecc03a634f96d2d0 (patch) | |
tree | 4012f77323e147b1883a77cb0f0a17605720a1c0 | |
parent | 5e26cd2d824ea05e5ae5ca3aebaa439a9b80e7fc (diff) | |
parent | 0610c41c75920b145a144e62df142977b605fc97 (diff) | |
download | hdf5-8215920678c09238ecc86d9eecc03a634f96d2d0.zip hdf5-8215920678c09238ecc86d9eecc03a634f96d2d0.tar.gz hdf5-8215920678c09238ecc86d9eecc03a634f96d2d0.tar.bz2 |
Merge pull request #225 in HDFFV/hdf5 from develop_merge_cache_image_03 to develop
Merged incoming changes from cache_image.
* commit '0610c41c75920b145a144e62df142977b605fc97':
Bring support for doubly-linked list of all entries in cache from incoming cache image branch.
Align with name change of "user" ring in incoming cache image changes.
Refactor several cache clients to have only one copy of their deserialization code.
-rw-r--r-- | src/H5ACprivate.h | 2 | ||||
-rw-r--r-- | src/H5C.c | 21 | ||||
-rw-r--r-- | src/H5Cdbg.c | 6 | ||||
-rw-r--r-- | src/H5Cpkg.h | 440 | ||||
-rw-r--r-- | src/H5Cprivate.h | 21 | ||||
-rw-r--r-- | src/H5Fsuper_cache.c | 372 | ||||
-rw-r--r-- | src/H5HFcache.c | 95 | ||||
-rw-r--r-- | src/H5HFdblock.c | 4 | ||||
-rw-r--r-- | src/H5HGcache.c | 108 | ||||
-rw-r--r-- | src/H5HLcache.c | 122 | ||||
-rw-r--r-- | src/H5Ocache.c | 324 | ||||
-rw-r--r-- | src/H5Pdxpl.c | 2 | ||||
-rw-r--r-- | src/H5SM.c | 4 | ||||
-rw-r--r-- | test/cache.c | 48 | ||||
-rw-r--r-- | test/cache_common.h | 36 |
15 files changed, 1001 insertions, 604 deletions
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index afd7817..07bb8fa 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -153,7 +153,7 @@ typedef enum { /* Aliases for the "ring" type and values */ typedef H5C_ring_t H5AC_ring_t; #define H5AC_RING_INV H5C_RING_UNDEFINED -#define H5AC_RING_US H5C_RING_USER +#define H5AC_RING_USER H5C_RING_USER #define H5AC_RING_FSM H5C_RING_FSM #define H5AC_RING_SBE H5C_RING_SBE #define H5AC_RING_SB H5C_RING_SB @@ -329,6 +329,11 @@ H5C_create(size_t max_cache_size, for(i = 0; i < H5C__HASH_TABLE_LEN; i++) (cache_ptr->index)[i] = NULL; + cache_ptr->il_len = 0; + cache_ptr->il_size = (size_t)0; + cache_ptr->il_head = NULL; + cache_ptr->il_tail = NULL; + /* Tagging Field Initializations */ cache_ptr->ignore_tags = FALSE; @@ -1330,6 +1335,8 @@ H5C_insert_entry(H5F_t * f, entry_ptr->ht_next = NULL; entry_ptr->ht_prev = NULL; + entry_ptr->il_next = NULL; + entry_ptr->il_prev = NULL; entry_ptr->next = NULL; entry_ptr->prev = NULL; @@ -1733,7 +1740,7 @@ H5C_move_entry(H5C_t * cache_ptr, * don't mark it as dirty either, lest we confuse the flush call back. */ if(!entry_ptr->destroy_in_progress) { - H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) + H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL) if(entry_ptr->in_slist) { HDassert(cache_ptr->slist_ptr); @@ -5464,7 +5471,7 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, * -- JRM */ next_entry_ptr = cache_ptr->index[i]; - H5C__UPDATE_STATS_FOR_HASH_BUCKET_SCAN_RESTART(cache_ptr) + H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) } /* end if */ else cache_ptr->entry_watched_for_removal = NULL; @@ -6059,7 +6066,7 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ * Finally, if the destroy_entry flag is set, discard the * entry. */ - H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) + H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL) if(entry_ptr->in_slist && del_from_slist_on_destroy) H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) @@ -6569,7 +6576,7 @@ H5C_load_entry(H5F_t * f, entry->size = len; HDassert(entry->size < H5C_MAX_ENTRY_SIZE); entry->image_ptr = image; - entry->image_up_to_date = TRUE; + entry->image_up_to_date = !dirty; entry->type = type; entry->is_dirty = dirty; entry->dirtied = FALSE; @@ -6597,6 +6604,8 @@ H5C_load_entry(H5F_t * f, entry->flush_dep_ndirty_children = 0; entry->ht_next = NULL; entry->ht_prev = NULL; + entry->il_next = NULL; + entry->il_prev = NULL; entry->next = NULL; entry->prev = NULL; @@ -7832,7 +7841,7 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p /* We must update cache data structures for the change in address */ if(entry_ptr->addr == old_addr) { /* Delete the entry from the hash table and the slist */ - H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr); + H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL); H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE); /* update the entry for its new address */ @@ -7928,7 +7937,7 @@ H5C_remove_entry(void *_entry) * 3) Remove it from the tag list for this object */ - H5C__DELETE_FROM_INDEX(cache, entry) + H5C__DELETE_FROM_INDEX(cache, entry, FAIL) H5C__UPDATE_RP_FOR_EVICTION(cache, entry, FAIL) diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index e7ed443..9266acd 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -642,11 +642,11 @@ H5C_stats(H5C_t * cache_ptr, cache_ptr->entries_scanned_to_make_space)); HDfprintf(stdout, - "%s slist/LRU/hash bkt scan restarts = %lld / %lld / %lld.\n", + "%s slist/LRU/index scan restarts = %lld / %lld / %lld.\n", cache_ptr->prefix, (long long)(cache_ptr->slist_scan_restarts), (long long)(cache_ptr->LRU_scan_restarts), - (long long)(cache_ptr->hash_bucket_scan_restarts)); + (long long)(cache_ptr->index_scan_restarts)); #if H5C_COLLECT_CACHE_ENTRY_STATS @@ -867,7 +867,7 @@ H5C_stats__reset(H5C_t H5_ATTR_UNUSED * cache_ptr) cache_ptr->slist_scan_restarts = 0; cache_ptr->LRU_scan_restarts = 0; - cache_ptr->hash_bucket_scan_restarts = 0; + cache_ptr->index_scan_restarts = 0; #if H5C_COLLECT_CACHE_ENTRY_STATS for(i = 0; i <= cache_ptr->max_type_id; i++) { diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index fd3f282..e30ca8d 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -150,6 +150,13 @@ * * JRM - 9/8/05 * + * - Added macros supporting the index list -- a doubly liked list of + * all entries in the index. This list is necessary to reduce the + * cost of visiting all entries in the cache, which was previously + * done via a scan of the hash table. + * + * JRM - 10/15/15 + * ****************************************************************************/ #if H5C_DO_SANITY_CHECKS @@ -456,6 +463,130 @@ if ( ( (entry_ptr) == NULL ) || \ } \ } /* H5C__AUX_DLL_REMOVE() */ +#if H5C_DO_SANITY_CHECKS + +#define H5C__IL_DLL_PRE_REMOVE_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) \ +if ( ( (hd_ptr) == NULL ) || \ + ( (tail_ptr) == NULL ) || \ + ( (entry_ptr) == NULL ) || \ + ( (len) <= 0 ) || \ + ( (Size) < (entry_ptr)->size ) || \ + ( ( (Size) == (entry_ptr)->size ) && ( ! ( (len) == 1 ) ) ) || \ + ( ( (entry_ptr)->il_prev == NULL ) && ( (hd_ptr) != (entry_ptr) ) ) || \ + ( ( (entry_ptr)->il_next == NULL ) && ( (tail_ptr) != (entry_ptr) ) ) || \ + ( ( (len) == 1 ) && \ + ( ! ( ( (hd_ptr) == (entry_ptr) ) && ( (tail_ptr) == (entry_ptr) ) && \ + ( (entry_ptr)->il_next == NULL ) && \ + ( (entry_ptr)->il_prev == NULL ) && \ + ( (Size) == (entry_ptr)->size ) \ + ) \ + ) \ + ) \ + ) { \ + HDassert(0 && "il DLL pre remove SC failed"); \ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, (fv), "il DLL pre remove SC failed") \ +} + +#define H5C__IL_DLL_PRE_INSERT_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) \ +if ( ( (entry_ptr) == NULL ) || \ + ( (entry_ptr)->il_next != NULL ) || \ + ( (entry_ptr)->il_prev != NULL ) || \ + ( ( ( (hd_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ + ( (hd_ptr) != (tail_ptr) ) \ + ) || \ + ( (len) < 0 ) || \ + ( ( (len) == 1 ) && \ + ( ( (hd_ptr) != (tail_ptr) ) || ( (Size) <= 0 ) || \ + ( (hd_ptr) == NULL ) || ( (hd_ptr)->size != (Size) ) \ + ) \ + ) || \ + ( ( (len) >= 1 ) && \ + ( ( (hd_ptr) == NULL ) || ( (hd_ptr)->il_prev != NULL ) || \ + ( (tail_ptr) == NULL ) || ( (tail_ptr)->il_next != NULL ) \ + ) \ + ) \ + ) { \ + HDassert(0 && "IL DLL pre insert SC failed"); \ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, (fv), "IL DLL pre insert SC failed") \ +} + +#define H5C__IL_DLL_SC(head_ptr, tail_ptr, len, Size, fv) \ +if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ + ( (head_ptr) != (tail_ptr) ) \ + ) || \ + ( (len) < 0 ) || \ + ( ( (len) == 1 ) && \ + ( ( (head_ptr) != (tail_ptr) ) || \ + ( (head_ptr) == NULL ) || ( (head_ptr)->size != (Size) ) \ + ) \ + ) || \ + ( ( (len) >= 1 ) && \ + ( ( (head_ptr) == NULL ) || ( (head_ptr)->il_prev != NULL ) || \ + ( (tail_ptr) == NULL ) || ( (tail_ptr)->il_next != NULL ) \ + ) \ + ) \ + ) { \ + HDassert(0 && "IL DLL sanity check failed"); \ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, (fv), "IL DLL sanity check failed") \ +} + +#else /* H5C_DO_SANITY_CHECKS */ + +#define H5C__IL_DLL_PRE_REMOVE_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) +#define H5C__IL_DLL_PRE_INSERT_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) +#define H5C__IL_DLL_SC(head_ptr, tail_ptr, len, Size, fv) + +#endif /* H5C_DO_SANITY_CHECKS */ + + +#define H5C__IL_DLL_APPEND(entry_ptr, head_ptr, tail_ptr, len, Size, fail_val)\ +{ \ + H5C__IL_DLL_PRE_INSERT_SC(entry_ptr, head_ptr, tail_ptr, len, Size, \ + fail_val) \ + if ( (head_ptr) == NULL ) \ + { \ + (head_ptr) = (entry_ptr); \ + (tail_ptr) = (entry_ptr); \ + } \ + else \ + { \ + (tail_ptr)->il_next = (entry_ptr); \ + (entry_ptr)->il_prev = (tail_ptr); \ + (tail_ptr) = (entry_ptr); \ + } \ + (len)++; \ + (Size) += entry_ptr->size; \ + H5C__IL_DLL_SC(head_ptr, tail_ptr, len, Size, fail_val) \ +} /* H5C__IL_DLL_APPEND() */ + +#define H5C__IL_DLL_REMOVE(entry_ptr, head_ptr, tail_ptr, len, Size, fv) \ +{ \ + H5C__IL_DLL_PRE_REMOVE_SC(entry_ptr, head_ptr, tail_ptr, len, Size, fv) \ + { \ + if ( (head_ptr) == (entry_ptr) ) \ + { \ + (head_ptr) = (entry_ptr)->il_next; \ + if ( (head_ptr) != NULL ) \ + (head_ptr)->il_prev = NULL; \ + } \ + else \ + (entry_ptr)->il_prev->il_next = (entry_ptr)->il_next; \ + if ( (tail_ptr) == (entry_ptr) ) \ + { \ + (tail_ptr) = (entry_ptr)->il_prev; \ + if ( (tail_ptr) != NULL ) \ + (tail_ptr)->il_next = NULL; \ + } \ + else \ + (entry_ptr)->il_next->il_prev = (entry_ptr)->il_prev; \ + entry_ptr->il_next = NULL; \ + entry_ptr->il_prev = NULL; \ + (len)--; \ + (Size) -= entry_ptr->size; \ + } \ + H5C__IL_DLL_SC(head_ptr, tail_ptr, len, Size, fv) \ +} /* H5C__IL_DLL_REMOVE() */ + /*********************************************************************** * @@ -567,8 +698,8 @@ if ( ( (entry_ptr) == NULL ) || \ #define H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) \ ((cache_ptr)->LRU_scan_restarts)++; -#define H5C__UPDATE_STATS_FOR_HASH_BUCKET_SCAN_RESTART(cache_ptr) \ - ((cache_ptr)->hash_bucket_scan_restarts)++; +#define H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) \ + ((cache_ptr)->index_scan_restarts)++; #if H5C_COLLECT_CACHE_ENTRY_STATS @@ -791,7 +922,7 @@ if ( ( (entry_ptr) == NULL ) || \ #define H5C__UPDATE_STATS_FOR_UNPIN(cache_ptr, entry_ptr) #define H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr) #define H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) -#define H5C__UPDATE_STATS_FOR_HASH_BUCKET_SCAN_RESTART(cache_ptr) +#define H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) #endif /* H5C_COLLECT_CACHE_STATS */ @@ -820,6 +951,14 @@ if ( ( (entry_ptr) == NULL ) || \ * * JRM -- 9/1/15 * + * - Updated existing index macros and sanity checks macros to + * maintain an doubly linked list of all entries in the index. + * This is necessary to reduce the computational cost of visiting + * all entries in the index, which used to be done by scanning + * the hash table. + * + * JRM -- 10/15/15 + * ***********************************************************************/ /* H5C__HASH_TABLE_LEN is defined in H5Cpkg.h. It mut be a power of two. */ @@ -853,13 +992,15 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, fail_val, \ "Pre HT insert SC failed") \ } -#define H5C__POST_HT_INSERT_SC(cache_ptr, fail_val) \ +#define H5C__POST_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->magic != H5C__H5C_T_MAGIC ) || \ ( (cache_ptr)->index_size != \ @@ -874,7 +1015,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, fail_val, \ "Post HT insert SC failed") \ @@ -914,7 +1057,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Pre HT remove SC failed") \ } @@ -938,7 +1083,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Post HT remove SC failed") \ } @@ -956,7 +1103,7 @@ if ( ( (cache_ptr) == NULL ) || \ } /* (Keep in sync w/H5C_TEST__POST_SUC_HT_SEARCH_SC macro in test/cache_common.h -QAK) */ -#define H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k, fail_val) \ +#define H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k, fail_val) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->magic != H5C__H5C_T_MAGIC ) || \ ( (cache_ptr)->index_len < 1 ) || \ @@ -964,7 +1111,6 @@ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_size < (entry_ptr)->size ) || \ ( (cache_ptr)->index_size != \ ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ) || \ - ( H5F_addr_ne((entry_ptr)->addr, (Addr)) ) || \ ( (entry_ptr)->size <= 0 ) || \ ( ((cache_ptr)->index)[k] == NULL ) || \ ( ( ((cache_ptr)->index)[k] != (entry_ptr) ) && \ @@ -1016,7 +1162,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "Pre HT entry size change SC failed") \ @@ -1045,7 +1193,9 @@ if ( ( (cache_ptr) == NULL ) || \ (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_ring_size[(entry_ptr)->ring] != \ ((cache_ptr)->clean_index_ring_size[(entry_ptr)->ring] + \ - (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) ) { \ + (cache_ptr)->dirty_index_ring_size[(entry_ptr)->ring]) ) || \ + ( (cache_ptr)->index_len != (cache_ptr)->il_len ) || \ + ( (cache_ptr)->index_size != (cache_ptr)->il_size ) ) { \ HDassert(FALSE); \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "Post HT entry size change SC failed") \ @@ -1144,7 +1294,7 @@ if ( ( (cache_ptr)->index_size != \ #else /* H5C_DO_SANITY_CHECKS */ #define H5C__PRE_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) -#define H5C__POST_HT_INSERT_SC(cache_ptr, fail_val) +#define H5C__POST_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) #define H5C__PRE_HT_REMOVE_SC(cache_ptr, entry_ptr) #define H5C__POST_HT_REMOVE_SC(cache_ptr, entry_ptr) #define H5C__PRE_HT_SEARCH_SC(cache_ptr, Addr, fail_val) @@ -1162,71 +1312,77 @@ if ( ( (cache_ptr)->index_size != \ #endif /* H5C_DO_SANITY_CHECKS */ -#define H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, fail_val) \ -{ \ - int k; \ - H5C__PRE_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) \ - k = H5C__HASH_FCN((entry_ptr)->addr); \ - if(((cache_ptr)->index)[k] != NULL) { \ - (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr); \ - } \ - ((cache_ptr)->index)[k] = (entry_ptr); \ - (cache_ptr)->index_len++; \ - (cache_ptr)->index_size += (entry_ptr)->size; \ - ((cache_ptr)->index_ring_len[entry_ptr->ring])++; \ - ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ - += (entry_ptr)->size; \ - if((entry_ptr)->is_dirty) { \ - (cache_ptr)->dirty_index_size += (entry_ptr)->size; \ - ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ - += (entry_ptr)->size; \ - } else { \ - (cache_ptr)->clean_index_size += (entry_ptr)->size; \ - ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ - += (entry_ptr)->size; \ - } \ - if ((entry_ptr)->flush_me_last) { \ - (cache_ptr)->num_last_entries++; \ - HDassert((cache_ptr)->num_last_entries <= 2); \ - } \ - H5C__UPDATE_STATS_FOR_HT_INSERTION(cache_ptr) \ - H5C__POST_HT_INSERT_SC(cache_ptr, fail_val) \ +#define H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, fail_val) \ +{ \ + int k; \ + H5C__PRE_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) \ + k = H5C__HASH_FCN((entry_ptr)->addr); \ + if(((cache_ptr)->index)[k] != NULL) { \ + (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr); \ + } \ + ((cache_ptr)->index)[k] = (entry_ptr); \ + (cache_ptr)->index_len++; \ + (cache_ptr)->index_size += (entry_ptr)->size; \ + ((cache_ptr)->index_ring_len[entry_ptr->ring])++; \ + ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ + += (entry_ptr)->size; \ + if((entry_ptr)->is_dirty) { \ + (cache_ptr)->dirty_index_size += (entry_ptr)->size; \ + ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ + += (entry_ptr)->size; \ + } else { \ + (cache_ptr)->clean_index_size += (entry_ptr)->size; \ + ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ + += (entry_ptr)->size; \ + } \ + if((entry_ptr)->flush_me_last) { \ + (cache_ptr)->num_last_entries++; \ + HDassert((cache_ptr)->num_last_entries <= 2); \ + } \ + H5C__IL_DLL_APPEND((entry_ptr), (cache_ptr)->il_head, \ + (cache_ptr)->il_tail, (cache_ptr)->il_len, \ + (cache_ptr)->il_size, fail_val) \ + H5C__UPDATE_STATS_FOR_HT_INSERTION(cache_ptr) \ + H5C__POST_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) \ } -#define H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) \ -{ \ - int k; \ - H5C__PRE_HT_REMOVE_SC(cache_ptr, entry_ptr) \ - k = H5C__HASH_FCN((entry_ptr)->addr); \ - if ( (entry_ptr)->ht_next ) \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ - if ( (entry_ptr)->ht_prev ) \ - (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ - if ( ((cache_ptr)->index)[k] == (entry_ptr) ) \ - ((cache_ptr)->index)[k] = (entry_ptr)->ht_next; \ - (entry_ptr)->ht_next = NULL; \ - (entry_ptr)->ht_prev = NULL; \ - (cache_ptr)->index_len--; \ - (cache_ptr)->index_size -= (entry_ptr)->size; \ - ((cache_ptr)->index_ring_len[entry_ptr->ring])--; \ - ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ - -= (entry_ptr)->size; \ - if((entry_ptr)->is_dirty) { \ - (cache_ptr)->dirty_index_size -= (entry_ptr)->size; \ - ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ - -= (entry_ptr)->size; \ - } else { \ - (cache_ptr)->clean_index_size -= (entry_ptr)->size; \ - ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ - -= (entry_ptr)->size; \ - } \ - if((entry_ptr)->flush_me_last) { \ - (cache_ptr)->num_last_entries--; \ - HDassert((cache_ptr)->num_last_entries <= 1); \ - } \ - H5C__UPDATE_STATS_FOR_HT_DELETION(cache_ptr) \ - H5C__POST_HT_REMOVE_SC(cache_ptr, entry_ptr) \ +#define H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, fail_val) \ +{ \ + int k; \ + H5C__PRE_HT_REMOVE_SC(cache_ptr, entry_ptr) \ + k = H5C__HASH_FCN((entry_ptr)->addr); \ + if((entry_ptr)->ht_next) \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + if((entry_ptr)->ht_prev) \ + (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ + if(((cache_ptr)->index)[k] == (entry_ptr)) \ + ((cache_ptr)->index)[k] = (entry_ptr)->ht_next; \ + (entry_ptr)->ht_next = NULL; \ + (entry_ptr)->ht_prev = NULL; \ + (cache_ptr)->index_len--; \ + (cache_ptr)->index_size -= (entry_ptr)->size; \ + ((cache_ptr)->index_ring_len[entry_ptr->ring])--; \ + ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ + -= (entry_ptr)->size; \ + if((entry_ptr)->is_dirty) { \ + (cache_ptr)->dirty_index_size -= (entry_ptr)->size; \ + ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ + -= (entry_ptr)->size; \ + } else { \ + (cache_ptr)->clean_index_size -= (entry_ptr)->size; \ + ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ + -= (entry_ptr)->size; \ + } \ + if((entry_ptr)->flush_me_last) { \ + (cache_ptr)->num_last_entries--; \ + HDassert((cache_ptr)->num_last_entries <= 1); \ + } \ + H5C__IL_DLL_REMOVE((entry_ptr), (cache_ptr)->il_head, \ + (cache_ptr)->il_tail, (cache_ptr)->il_len, \ + (cache_ptr)->il_size, fail_val) \ + H5C__UPDATE_STATS_FOR_HT_DELETION(cache_ptr) \ + H5C__POST_HT_REMOVE_SC(cache_ptr, entry_ptr) \ } #define H5C__SEARCH_INDEX(cache_ptr, Addr, entry_ptr, fail_val) \ @@ -1236,51 +1392,51 @@ if ( ( (cache_ptr)->index_size != \ H5C__PRE_HT_SEARCH_SC(cache_ptr, Addr, fail_val) \ k = H5C__HASH_FCN(Addr); \ entry_ptr = ((cache_ptr)->index)[k]; \ - while ( ( entry_ptr ) && ( H5F_addr_ne(Addr, (entry_ptr)->addr) ) ) { \ + while(entry_ptr) { \ + if(H5F_addr_eq(Addr, (entry_ptr)->addr)) { \ + H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k, fail_val) \ + if(entry_ptr != ((cache_ptr)->index)[k]) { \ + if((entry_ptr)->ht_next) \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + HDassert((entry_ptr)->ht_prev != NULL); \ + (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ + ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ + (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ + (entry_ptr)->ht_prev = NULL; \ + ((cache_ptr)->index)[k] = (entry_ptr); \ + H5C__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k, fail_val) \ + } \ + break; \ + } \ (entry_ptr) = (entry_ptr)->ht_next; \ (depth)++; \ } \ - if ( entry_ptr ) { \ - H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k, fail_val) \ - if ( entry_ptr != ((cache_ptr)->index)[k] ) { \ - if ( (entry_ptr)->ht_next ) \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ - HDassert( (entry_ptr)->ht_prev != NULL ); \ - (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ - ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ - (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ - (entry_ptr)->ht_prev = NULL; \ - ((cache_ptr)->index)[k] = (entry_ptr); \ - H5C__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k, fail_val) \ - } \ - } \ H5C__UPDATE_STATS_FOR_HT_SEARCH(cache_ptr, (entry_ptr != NULL), depth) \ } #define H5C__SEARCH_INDEX_NO_STATS(cache_ptr, Addr, entry_ptr, fail_val) \ { \ int k; \ - int depth = 0; \ H5C__PRE_HT_SEARCH_SC(cache_ptr, Addr, fail_val) \ k = H5C__HASH_FCN(Addr); \ entry_ptr = ((cache_ptr)->index)[k]; \ - while ( ( entry_ptr ) && ( H5F_addr_ne(Addr, (entry_ptr)->addr) ) ) { \ - (entry_ptr) = (entry_ptr)->ht_next; \ - (depth)++; \ - } \ - if ( entry_ptr ) { \ - H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k, fail_val) \ - if ( entry_ptr != ((cache_ptr)->index)[k] ) { \ - if ( (entry_ptr)->ht_next ) \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ - HDassert( (entry_ptr)->ht_prev != NULL ); \ - (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ - ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ - (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ - (entry_ptr)->ht_prev = NULL; \ - ((cache_ptr)->index)[k] = (entry_ptr); \ - H5C__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k, fail_val) \ + while(entry_ptr) { \ + if(H5F_addr_eq(Addr, (entry_ptr)->addr)) { \ + H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k, fail_val) \ + if(entry_ptr != ((cache_ptr)->index)[k]) { \ + if((entry_ptr)->ht_next) \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + HDassert((entry_ptr)->ht_prev != NULL); \ + (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ + ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ + (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ + (entry_ptr)->ht_prev = NULL; \ + ((cache_ptr)->index)[k] = (entry_ptr); \ + H5C__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k, fail_val) \ + } \ + break; \ } \ + (entry_ptr) = (entry_ptr)->ht_next; \ } \ } @@ -1317,20 +1473,23 @@ if ( ( (cache_ptr)->index_size != \ (cache_ptr)->index_size += (new_size); \ ((cache_ptr)->index_ring_size[entry_ptr->ring]) -= (old_size); \ ((cache_ptr)->index_ring_size[entry_ptr->ring]) += (new_size); \ - if ( was_clean ) { \ + if(was_clean) { \ (cache_ptr)->clean_index_size -= (old_size); \ ((cache_ptr)->clean_index_ring_size[entry_ptr->ring])-= (old_size); \ } else { \ (cache_ptr)->dirty_index_size -= (old_size); \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring])-= (old_size); \ } \ - if ( (entry_ptr)->is_dirty ) { \ + if((entry_ptr)->is_dirty) { \ (cache_ptr)->dirty_index_size += (new_size); \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring])+= (new_size); \ } else { \ (cache_ptr)->clean_index_size += (new_size); \ ((cache_ptr)->clean_index_ring_size[entry_ptr->ring])+= (new_size); \ } \ + H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->il_len, \ + (cache_ptr)->il_size, \ + (old_size), (new_size)) \ H5C__POST_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ entry_ptr) \ } @@ -3342,6 +3501,17 @@ typedef struct H5C_tag_info_t { * The cache requires an index to facilitate searching for entries. The * following fields support that index. * + * Addendum: JRM -- 10/14/15 + * + * We sometimes need to visit all entries in the cache. In the past, this + * was done by scanning the hash table. However, this is expensive, and + * we have come to scan the hash table often enough that it has become a + * performance issue. To repair this, I have added code to maintain a + * list of all entries in the index -- call this list the index list. + * + * The index list is maintained by the same macros that maintain the + * index, and must have the same length and size as the index proper. + * * index_len: Number of entries currently in the hash table used to index * the cache. * @@ -3409,6 +3579,36 @@ typedef struct H5C_tag_info_t { * changing the H5C__HASH_FCN macro and the deletion of the * H5C__HASH_MASK #define. No other changes should be required. * + * il_len: Number of entries on the index list. + * + * This must always be equal to index_len. As such, this + * field is redundant. However, the existing linked list + * management macros expect to maintain a length field, so + * this field exists primarily to avoid adding complexity to + * these macros. + * + * il_size: Number of bytes of cache entries currently stored in the + * index list. + * + * This must always be equal to index_size. As such, this + * field is redundant. However, the existing linked list + * management macros expect to maintain a size field, so + * this field exists primarily to avoid adding complexity to + * these macros. + * + * il_head: Pointer to the head of the doubly linked list of entries in + * the index list. Note that cache entries on this list are + * linked by their il_next and il_prev fields. + * + * This field is NULL if the index is empty. + * + * il_tail: Pointer to the tail of the doubly linked list of entries in + * the index list. Note that cache entries on this list are + * linked by their il_next and il_prev fields. + * + * This field is NULL if the index is empty. + * + * * With the addition of the take ownership flag, it is possible that * an entry may be removed from the cache as the result of the flush of * a second entry. In general, this causes little trouble, but it is @@ -4071,10 +4271,14 @@ typedef struct H5C_tag_info_t { * avoid potential issues with change of status of the next * entry in the scan. * - * hash_bucket_scan_restarts: Number of times a scan of a hash bucket list - * (that contains calls to H5C__flush_single_entry()) has been - * restarted to avoid potential issues with change of status - * of the next entry in the scan. + * index_scan_restarts: Number of times a scan of the index has been + * restarted to avoid potential issues with load, insertion + * or change in flush dependency height of an entry other + * than the target entry as the result of call(s) to the + * pre_serialize or serialize callbacks. + * + * Note that at present, this condition can only be triggerd + * by a call to H5C_serialize_single_entry(). * * The remaining stats are collected only when both H5C_COLLECT_CACHE_STATS * and H5C_COLLECT_CACHE_ENTRY_STATS are true. @@ -4143,7 +4347,11 @@ struct H5C_t { size_t clean_index_ring_size[H5C_RING_NTYPES]; size_t dirty_index_size; size_t dirty_index_ring_size[H5C_RING_NTYPES]; - H5C_cache_entry_t * index[H5C__HASH_TABLE_LEN]; + H5C_cache_entry_t * index[H5C__HASH_TABLE_LEN]; + int32_t il_len; + size_t il_size; + H5C_cache_entry_t * il_head; + H5C_cache_entry_t * il_tail; /* Fields to detect entries removed during scans */ int64_t entries_removed_counter; @@ -4292,7 +4500,7 @@ struct H5C_t { /* Fields for tracking skip list scan restarts */ int64_t slist_scan_restarts; int64_t LRU_scan_restarts; - int64_t hash_bucket_scan_restarts; + int64_t index_scan_restarts; #if H5C_COLLECT_CACHE_ENTRY_STATS int32_t max_accesses[H5C__MAX_NUM_TYPE_IDS + 1]; diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 77ebe8a..365fd6c 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -1218,6 +1218,15 @@ typedef int H5C_ring_t; * If there are multiple entries in any hash bin, they are stored in a doubly * linked list. * + * Addendum: JRM -- 10/14/15 + * + * We have come to scan all entries in the cache frequently enough that + * the cost of doing so by scanning the hash table has become unacceptable. + * To reduce this cost, the index now also maintains a doubly linked list + * of all entries in the index. This list is known as the index list. + * The il_next and il_prev fields discussed below were added to support + * the index list. + * * ht_next: Next pointer used by the hash table to store multiple * entries in a single hash bin. This field points to the * next entry in the doubly linked list of entries in the @@ -1228,6 +1237,16 @@ typedef int H5C_ring_t; * previous entry in the doubly linked list of entries in * the hash bin, or NULL if there is no previuos entry. * + * il_next: Next pointer used by the index to maintain a doubly linked + * list of all entries in the index (and thus in the cache). + * This field contains a pointer to the next entry in the + * index list, or NULL if there is no next entry. + * + * il_prev: Prev pointer used by the index to maintain a doubly linked + * list of all entries in the index (and thus in the cache). + * This field contains a pointer to the previous entry in the + * index list, or NULL if there is no previous entry. + * * * Fields supporting replacement policies: * @@ -1382,6 +1401,8 @@ typedef struct H5C_cache_entry_t { /* fields supporting the hash table: */ struct H5C_cache_entry_t *ht_next; struct H5C_cache_entry_t *ht_prev; + struct H5C_cache_entry_t *il_next; + struct H5C_cache_entry_t *il_prev; /* fields supporting replacement policies: */ struct H5C_cache_entry_t *next; diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index 0dc504e..8e832d7 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -91,6 +91,14 @@ static herr_t H5F__cache_drvrinfo_serialize(const H5F_t *f, void *image, size_t void *thing); static herr_t H5F__cache_drvrinfo_free_icr(void *thing); +/* Local encode/decode routines */ +static herr_t H5F__superblock_prefix_decode(H5F_super_t *sblock, + const uint8_t **image_ref, const H5F_superblock_cache_ud_t *udata, + hbool_t extend_eoa); +static herr_t H5F__drvrinfo_prefix_decode(H5O_drvinfo_t *drvinfo, char *drv_name, + const uint8_t **image_ref, H5F_drvrinfo_cache_ud_t *udata, + hbool_t extend_eoa); + /*********************/ /* Package Variables */ @@ -148,6 +156,158 @@ H5FL_EXTERN(H5F_super_t); /*------------------------------------------------------------------------- + * Function: H5F__superblock_prefix_decode + * + * Purpose: Decode a superblock prefix + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__superblock_prefix_decode(H5F_super_t *sblock, const uint8_t **image_ref, + const H5F_superblock_cache_ud_t *udata, hbool_t extend_eoa) +{ + const uint8_t *image = (const uint8_t *)*image_ref; /* Pointer into raw data buffer */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check arguments */ + HDassert(sblock); + HDassert(image_ref); + HDassert(image); + HDassert(udata); + HDassert(udata->f); + + /* Skip over signature (already checked when locating the superblock) */ + image += H5F_SIGNATURE_LEN; + + /* Superblock version */ + sblock->super_vers = *image++; + if(sblock->super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad superblock version number") + + /* Sanity check */ + HDassert(((size_t)(image - (const uint8_t *)*image_ref)) == H5F_SUPERBLOCK_FIXED_SIZE); + + /* Determine the size of addresses & size of offsets, for computing the + * variable-sized portion of the superblock. + */ + if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_2) { + sblock->sizeof_addr = image[4]; + sblock->sizeof_size = image[5]; + } /* end if */ + else { + sblock->sizeof_addr = image[0]; + sblock->sizeof_size = image[1]; + } /* end else */ + if(sblock->sizeof_addr != 2 && sblock->sizeof_addr != 4 && + sblock->sizeof_addr != 8 && sblock->sizeof_addr != 16 && sblock->sizeof_addr != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number in an address") + if(sblock->sizeof_size != 2 && sblock->sizeof_size != 4 && + sblock->sizeof_size != 8 && sblock->sizeof_size != 16 && sblock->sizeof_size != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number for object size") + + /* Check for extending the EOA for the file */ + if(extend_eoa) { + size_t variable_size; /* Variable size of superblock */ + + /* Determine the size of the variable-length part of the superblock */ + variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(sblock->super_vers, sblock->sizeof_addr, sblock->sizeof_size); + HDassert(variable_size > 0); + + /* Make certain we can read the variable-sized portion of the superblock */ + if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + } /* end if */ + + /* Update the image buffer pointer */ + *image_ref = image; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__superblock_prefix_decode() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__drvrinfo_prefix_decode + * + * Purpose: Decode a driver info prefix + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__drvrinfo_prefix_decode(H5O_drvinfo_t *drvrinfo, char *drv_name, + const uint8_t **image_ref, H5F_drvrinfo_cache_ud_t *udata, + hbool_t extend_eoa) +{ + const uint8_t *image = (const uint8_t *)*image_ref; /* Pointer into raw data buffer */ + unsigned drv_vers; /* Version of driver info block */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(drvrinfo); + HDassert(image_ref); + HDassert(image); + HDassert(udata); + HDassert(udata->f); + + /* Version number */ + drv_vers = *image++; + if(drv_vers != HDF5_DRIVERINFO_VERSION_0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad driver information block version number") + + image += 3; /* reserved bytes */ + + /* Driver info size */ + UINT32DECODE(image, drvrinfo->len); + + /* Driver name and/or version */ + if(drv_name) { + HDmemcpy(drv_name, (const char *)image, (size_t)8); + drv_name[8] = '\0'; + image += 8; /* advance past name/version */ + } /* end if */ + + /* Extend the EOA if required so that we can read the complete driver info block */ + if(extend_eoa) { + haddr_t eoa; /* Current EOA for the file */ + haddr_t min_eoa; /* Minimum EOA needed for reading the driver info */ + + /* Get current EOA... */ + eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER); + if(!H5F_addr_defined(eoa)) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* ... if it is too small, extend it. */ + min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvrinfo->len; + + /* If it grew, set it */ + if(H5F_addr_gt(min_eoa, eoa)) + if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + } /* end if */ + + /* Update the image buffer pointer */ + *image_ref = image; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__drvrinfo_prefix_decode() */ + + +/*------------------------------------------------------------------------- * Function: H5F__cache_superblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. @@ -195,10 +355,7 @@ H5F__cache_superblock_get_final_load_size(const void *_image, size_t image_len, { const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ - unsigned super_vers; /* Superblock version */ - uint8_t sizeof_addr; /* Size of offsets in the file (in bytes) */ - uint8_t sizeof_size; /* Size of lengths in the file (in bytes) */ - size_t variable_size; /* Variable size of superblock */ + H5F_super_t sblock; /* Temporary file superblock */ htri_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -206,56 +363,20 @@ H5F__cache_superblock_get_final_load_size(const void *_image, size_t image_len, /* Check arguments */ HDassert(image); HDassert(udata); - HDassert(udata->f); HDassert(actual_len); HDassert(*actual_len == image_len); - - /* Skip over file signature */ - image += H5F_SIGNATURE_LEN; - - /* Superblock version */ - super_vers = *image++; - if(super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad superblock version number") - - /* Save the version to be used in verify_chksum callback */ - udata->super_vers = super_vers; - - /* Sanity check */ - HDassert(((size_t)(image - (const uint8_t *)_image)) == H5F_SUPERBLOCK_FIXED_SIZE); HDassert(image_len >= H5F_SUPERBLOCK_FIXED_SIZE + 6); - /* Determine the size of addresses & size of offsets, for computing the - * variable-sized portion of the superblock. - */ - if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { - sizeof_addr = image[4]; - sizeof_size = image[5]; - } /* end if */ - else { - sizeof_addr = image[0]; - sizeof_size = image[1]; - } /* end else */ - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number in an address") - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number for object size") - - /* Determine the size of the variable-length part of the superblock */ - variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size); - HDassert(variable_size > 0); + /* Deserialize the file superblock's prefix */ + if(H5F__superblock_prefix_decode(&sblock, &image, udata, TRUE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "can't decode file superblock prefix") - /* Sanity check */ - HDassert(image_len == (H5F_SUPERBLOCK_FIXED_SIZE + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE)); - - /* Make certain we can read the variable-sized portion of the superblock */ - if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + /* Save the version to be used in verify_chksum callback */ + udata->super_vers = sblock.super_vers; /* Set the final size for the cache image */ - *actual_len = H5F_SUPERBLOCK_FIXED_SIZE + variable_size; + *actual_len = H5F_SUPERBLOCK_FIXED_SIZE + + (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(sblock.super_vers, sblock.sizeof_addr, sblock.sizeof_size); done: FUNC_LEAVE_NOAPI(ret_value) @@ -325,10 +446,6 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, H5F_super_t *sblock = NULL; /* File's superblock */ H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ - size_t variable_size; /* Variable size of superblock */ - unsigned super_vers; /* Superblock version */ - uint8_t sizeof_addr; /* Size of offsets in the file (in bytes) */ - uint8_t sizeof_size; /* Size of lengths in the file (in bytes) */ H5F_super_t *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -337,54 +454,18 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, HDassert(image); HDassert(udata); HDassert(udata->f); + HDassert(len >= H5F_SUPERBLOCK_FIXED_SIZE + 6); /* Allocate space for the superblock */ if(NULL == (sblock = H5FL_CALLOC(H5F_super_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Skip over signature (already checked when locating the superblock) */ - image += H5F_SIGNATURE_LEN; - - /* Superblock version */ - super_vers = *image++; - if(super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad superblock version number") - - /* Record the superblock version */ - sblock->super_vers = super_vers; - - /* Sanity check */ - HDassert(((size_t)(image - (const uint8_t *)_image)) == H5F_SUPERBLOCK_FIXED_SIZE); - HDassert(len >= H5F_SUPERBLOCK_FIXED_SIZE + 6); - - /* Determine the size of addresses & size of offsets, for computing the - * variable-sized portion of the superblock. - */ - if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { - sizeof_addr = image[4]; - sizeof_size = image[5]; - } /* end if */ - else { - sizeof_addr = image[0]; - sizeof_size = image[1]; - } /* end else */ - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_addr = sizeof_addr; - sblock->sizeof_size = sizeof_size; - - /* Determine the size of the variable-length part of the superblock */ - variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size); - HDassert(variable_size > 0); - - HDassert(len == (H5F_SUPERBLOCK_FIXED_SIZE + variable_size)); + /* Deserialize the file superblock's prefix */ + if(H5F__superblock_prefix_decode(sblock, &image, udata, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode file superblock prefix") /* Check for older version of superblock format */ - if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { + if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_2) { uint32_t status_flags; /* File status flags */ unsigned sym_leaf_k; /* Symbol table leaf node's 'K' value */ unsigned snode_btree_k; /* B-tree symbol table internal node 'K' value */ @@ -405,21 +486,13 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, if(HDF5_SHAREDHEADER_VERSION != *image++) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad shared-header format version number") - /* Size of file addresses */ - sizeof_addr = *image++; - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - sblock->sizeof_addr = sizeof_addr; - udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ - - /* Size of file sizes */ - sizeof_size = *image++; - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_size = sizeof_size; - udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ + /* Skip over size of file addresses (already decoded) */ + image++; + udata->f->shared->sizeof_addr = sblock->sizeof_addr; /* Keep a local copy also */ + + /* Skip over size of file sizes (already decoded) */ + image++; + udata->f->shared->sizeof_size = sblock->sizeof_size; /* Keep a local copy also */ /* Skip over reserved byte */ image++; @@ -452,11 +525,11 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, * If the superblock version # is greater than 0, read in the indexed * storage B-tree internal 'K' value */ - if(super_vers > HDF5_SUPERBLOCK_VERSION_DEF) { + if(sblock->super_vers > HDF5_SUPERBLOCK_VERSION_DEF) { UINT16DECODE(image, chunk_btree_k); /* Reserved bytes are present only in version 1 */ - if(super_vers == HDF5_SUPERBLOCK_VERSION_1) + if(sblock->super_vers == HDF5_SUPERBLOCK_VERSION_1) image += 2; /* reserved */ } /* end if */ else @@ -498,21 +571,13 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, else { uint32_t read_chksum; /* Checksum read from file */ - /* Size of file addresses */ - sizeof_addr = *image++; - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - sblock->sizeof_addr = sizeof_addr; - udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ - - /* Size of file sizes */ - sizeof_size = *image++; - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_size = sizeof_size; - udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ + /* Skip over size of file addresses (already decoded) */ + image++; + udata->f->shared->sizeof_addr = sblock->sizeof_addr; /* Keep a local copy also */ + + /* Skip over size of file sizes (already decoded) */ + image++; + udata->f->shared->sizeof_size = sblock->sizeof_size; /* Keep a local copy also */ /* File status flags (not really used yet) */ sblock->status_flags = *image++; @@ -588,7 +653,7 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- - * Function: H5FS__cache_hdf_pre_serialize + * Function: H5FS__cache_superblock_pre_serialize * * Purpose: The current use of this function is a cludge to repair an * oversight in the conversion of the superblock code to use the @@ -927,50 +992,24 @@ H5F__cache_drvrinfo_get_final_load_size(const void *_image, size_t image_len, { const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5F_drvrinfo_cache_ud_t *udata = (H5F_drvrinfo_cache_ud_t *)_udata; /* User data */ - unsigned drv_vers; /* Version of driver info block */ - size_t drvinfo_len; /* Length of encoded buffer */ - haddr_t eoa; /* Current EOA for the file */ - haddr_t min_eoa; /* Minimum EOA needed for reading the driver info */ - htri_t ret_value = SUCCEED; /* Return value */ + H5O_drvinfo_t drvrinfo; /* Driver info */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Check arguments */ HDassert(image); HDassert(udata); - HDassert(udata->f); HDassert(actual_len); HDassert(*actual_len == image_len); - - /* Version number */ - drv_vers = *image++; - if(drv_vers != HDF5_DRIVERINFO_VERSION_0) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad driver information block version number") - - image += 3; /* reserved bytes */ - - /* Driver info size */ - UINT32DECODE(image, drvinfo_len); - - /* Sanity check */ HDassert(image_len == H5F_DRVINFOBLOCK_HDR_SIZE); - /* Extend the EOA if required so that we can read the complete driver info block */ - - /* Get current EOA... */ - if((eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver get_eoa request failed") - - /* ... if it is too small, extend it. */ - min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len; - - /* If it grew, set it */ - if(H5F_addr_gt(min_eoa, eoa)) - if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + /* Deserialize the file driver info's prefix */ + if(H5F__drvrinfo_prefix_decode(&drvrinfo, NULL, &image, udata, TRUE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "can't decode file driver info prefix") /* Set the final size for the cache image */ - *actual_len = H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len; + *actual_len = H5F_DRVINFOBLOCK_HDR_SIZE + drvrinfo.len; done: FUNC_LEAVE_NOAPI(ret_value) @@ -999,7 +1038,6 @@ H5F__cache_drvrinfo_deserialize(const void *_image, size_t len, void *_udata, H5F_drvrinfo_cache_ud_t *udata = (H5F_drvrinfo_cache_ud_t *)_udata; /* User data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ char drv_name[9]; /* Name of driver */ - unsigned drv_vers; /* Version of driver info block */ H5O_drvinfo_t *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -1014,21 +1052,11 @@ H5F__cache_drvrinfo_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (drvinfo = (H5O_drvinfo_t *)H5MM_calloc(sizeof(H5O_drvinfo_t)))) HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "memory allocation failed for driver info message") - /* Version number */ - drv_vers = *image++; - if(drv_vers != HDF5_DRIVERINFO_VERSION_0) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad driver information block version number") - - image += 3; /* reserved bytes */ - - /* Driver info size */ - UINT32DECODE(image, drvinfo->len); - - /* Driver name and/or version */ - HDstrncpy(drv_name, (const char *)image, (size_t)8); - drv_name[8] = '\0'; - image += 8; /* advance past name/version */ + /* Deserialize the file driver info's prefix */ + if(H5F__drvrinfo_prefix_decode(drvinfo, drv_name, &image, udata, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode file driver info prefix") + /* Sanity check */ HDassert(len == (H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len)); /* Validate and decode driver information */ diff --git a/src/H5HFcache.c b/src/H5HFcache.c index f85b344..1e2eb8f 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -69,6 +69,7 @@ /********************/ /* Local encode/decode routines */ +static herr_t H5HF__hdr_prefix_decode(H5HF_hdr_t *hdr, const uint8_t **image_ref); static herr_t H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable); static herr_t H5HF__dtable_decode(H5F_t *f, const uint8_t **pp, H5HF_dtable_t *dtable); @@ -200,6 +201,52 @@ H5FL_BLK_DEFINE(direct_block); /*------------------------------------------------------------------------- + * Function: H5HF__hdr_prefix_decode() + * + * Purpose: Decode a fractal heap header's prefix + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HF__hdr_prefix_decode(H5HF_hdr_t *hdr, const uint8_t **image_ref) +{ + const uint8_t *image = *image_ref; /* Pointer into into supplied image */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(hdr); + HDassert(image); + + /* Magic number */ + if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "wrong fractal heap header signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(*image++ != H5HF_HDR_VERSION) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong fractal heap header version") + + /* General heap information */ + UINT16DECODE(image, hdr->id_len); /* Heap ID length */ + UINT16DECODE(image, hdr->filter_len); /* I/O filters' encoded length */ + + /* Update the image buffer pointer */ + *image_ref = image; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__hdr_prefix_decode() */ + + +/*------------------------------------------------------------------------- * Function: H5HF__dtable_decode * * Purpose: Decodes the metadata for a doubling table @@ -360,10 +407,10 @@ static herr_t H5HF__cache_hdr_get_final_load_size(const void *_image, size_t image_len, void *_udata, size_t *actual_len) { - const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ - H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* pointer to user data */ - unsigned filter_len; /* Size of I/O filter information (in bytes) */ - htri_t ret_value = SUCCEED; /* Return value */ + H5HF_hdr_t hdr; /* Temporary fractal heap header */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into into supplied image */ + H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -373,25 +420,16 @@ H5HF__cache_hdr_get_final_load_size(const void *_image, size_t image_len, HDassert(actual_len); HDassert(*actual_len == image_len); - /* Magic number */ - if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "wrong fractal heap header signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(*image++ != H5HF_HDR_VERSION) - HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong fractal heap header version") - - /* General heap information */ - image += 2; /* Heap ID length */ - UINT16DECODE(image, filter_len); /* I/O filters' encoded length */ + /* Deserialize the fractal heap header's prefix */ + if(H5HF__hdr_prefix_decode(&hdr, &image) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode fractal heap header prefix") /* Check for I/O filter info on this heap */ - if(filter_len > 0) + if(hdr.filter_len > 0) /* Compute the extra heap header size */ *actual_len += (size_t)(H5F_SIZEOF_SIZE(udata->f) /* Size of size for filtered root direct block */ + (unsigned)4 /* Size of filter mask for filtered root direct block */ - + filter_len); /* Size of encoded I/O filter info */ + + hdr.filter_len); /* Size of encoded I/O filter info */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -473,18 +511,9 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (hdr = H5HF_hdr_alloc(udata->f))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Magic number */ - if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "wrong fractal heap header signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(*image++ != H5HF_HDR_VERSION) - HGOTO_ERROR(H5E_HEAP, H5E_VERSION, NULL, "wrong fractal heap header version") - - /* General heap information */ - UINT16DECODE(image, hdr->id_len); /* Heap ID length */ - UINT16DECODE(image, hdr->filter_len); /* I/O filters' encoded length */ + /* Deserialize the fractal heap header's prefix */ + if(H5HF__hdr_prefix_decode(hdr, &image) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode fractal heap header prefix") /* Heap status flags */ /* (bit 0: "huge" object IDs have wrapped) */ @@ -1618,10 +1647,6 @@ H5HF__cache_dblock_verify_chksum(const void *_image, size_t len, void *_udata) hdr = par_info->hdr; HDassert(hdr); - /* Reset callback context info */ - udata->decompressed = FALSE; - udata->dblk = NULL; - /* Get out if data block is not checksummed */ if(!(hdr->checksum_dblocks)) HGOTO_DONE(TRUE); @@ -1728,7 +1753,7 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, H5HF_dblock_cache_ud_t *udata = (H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */ H5HF_parent_t *par_info; /* Pointer to parent information */ H5HF_direct_t *dblock = NULL; /* Direct block info */ - const uint8_t *image = _image;/* Pointer into raw data buffer */ + const uint8_t *image = (const uint8_t *)_image;/* Pointer into raw data buffer */ void *read_buf = NULL; /* Pointer to buffer to decompress */ haddr_t heap_addr; /* Address of heap header in the file */ void * ret_value = NULL; /* Return value */ diff --git a/src/H5HFdblock.c b/src/H5HFdblock.c index 1272ab0..b34534c 100644 --- a/src/H5HFdblock.c +++ b/src/H5HFdblock.c @@ -493,6 +493,10 @@ H5HF_man_dblock_protect(H5HF_hdr_t *hdr, hid_t dxpl_id, haddr_t dblock_addr, udata.filter_mask = 0; } /* end else */ + /* Reset compression context info */ + udata.decompressed = FALSE; + udata.dblk = NULL; + /* Protect the direct block */ if(NULL == (dblock = (H5HF_direct_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_FHEAP_DBLOCK, dblock_addr, &udata, flags))) HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, NULL, "unable to protect fractal heap direct block") diff --git a/src/H5HGcache.c b/src/H5HGcache.c index 6bc9860..a83c22b 100644 --- a/src/H5HGcache.c +++ b/src/H5HGcache.c @@ -72,6 +72,10 @@ static herr_t H5HG__cache_heap_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HG__cache_heap_free_icr(void *thing); +/* Prefix deserialization */ +static herr_t H5HG__hdr_deserialize(H5HG_heap_t *heap, const uint8_t *image, + const H5F_t *f); + /*********************/ /* Package Variables */ @@ -108,6 +112,52 @@ const H5AC_class_t H5AC_GHEAP[1] = {{ /*------------------------------------------------------------------------- + * Function: H5HG__hdr_deserialize() + * + * Purpose: Decode a global heap's header + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HG__hdr_deserialize(H5HG_heap_t *heap, const uint8_t *image, const H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(heap); + HDassert(image); + HDassert(f); + + /* Magic number */ + if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad global heap collection signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(H5HG_VERSION != *image++) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in global heap") + + /* Reserved */ + image += 3; + + /* Size */ + H5F_DECODE_LENGTH(f, image, heap->size); + HDassert(heap->size >= H5HG_MINSIZE); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HG__hdr_deserialize() */ + + +/*------------------------------------------------------------------------- * Function: H5HG__cache_heap_get_initial_load_size() * * Purpose: Return the initial speculative read size to the metadata @@ -154,41 +204,27 @@ H5HG__cache_heap_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *imag *------------------------------------------------------------------------- */ static herr_t -H5HG__cache_heap_get_final_load_size(const void *_image, size_t image_len, - void *_udata, size_t *actual_len) +H5HG__cache_heap_get_final_load_size(const void *image, size_t image_len, + void *udata, size_t *actual_len) { - const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ - H5F_t *f = (H5F_t *)_udata; /* File pointer -- obtained from user data */ - size_t heap_size = 0; /* Total size of collection */ - herr_t ret_value = SUCCEED; /* Return value */ + H5HG_heap_t heap; /* Global heap */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Sanity check */ HDassert(image); - HDassert(f); + HDassert(udata); HDassert(actual_len); HDassert(*actual_len == image_len); - - /* Magic number */ - if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad global heap collection signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(H5HG_VERSION != *image++) - HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in global heap") - - /* Reserved */ - image += 3; - - /* Size */ - H5F_DECODE_LENGTH(f, image, heap_size); - HDassert(heap_size >= H5HG_MINSIZE); HDassert(image_len == H5HG_MINSIZE); + /* Deserialize the heap's header */ + if(H5HG__hdr_deserialize(&heap, (const uint8_t *)image, (const H5F_t *)udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode global heap prefix") + /* Set the final size for the cache image */ - *actual_len = heap_size; + *actual_len = heap.size; done: FUNC_LEAVE_NOAPI(ret_value) @@ -236,29 +272,13 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (heap->chunk = H5FL_BLK_MALLOC(gheap_chunk, len))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* copy the image buffer into the newly allocate chunk */ + /* Copy the image buffer into the newly allocate chunk */ HDmemcpy(heap->chunk, _image, len); - image = heap->chunk; - - /* Magic number */ - if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "bad global heap collection signature") - image += H5_SIZEOF_MAGIC; + /* Deserialize the heap's header */ + if(H5HG__hdr_deserialize(heap, (const uint8_t *)heap->chunk, f) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode global heap header") - /* Version */ - if(H5HG_VERSION != *image++) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "wrong version number in global heap") - - /* Reserved */ - image += 3; - - /* Size */ - H5F_DECODE_LENGTH(f, image, heap->size); - HDassert(heap->size >= H5HG_MINSIZE); - HDassert((len == H5HG_MINSIZE) /* first try */ || - ((len == heap->size) && (len > H5HG_MINSIZE))); /* second try */ - /* Decode each object */ image = heap->chunk + H5HG_SIZEOF_HDR(f); nalloc = H5HG_NOBJS(f, heap->size); diff --git a/src/H5HLcache.c b/src/H5HLcache.c index 3db239e..b2ca839 100644 --- a/src/H5HLcache.c +++ b/src/H5HLcache.c @@ -91,6 +91,10 @@ static herr_t H5HL__cache_datablock_serialize(const H5F_t *f, void *image, static herr_t H5HL__cache_datablock_notify(H5C_notify_action_t action, void *_thing); static herr_t H5HL__cache_datablock_free_icr(void *thing); +/* Header deserialization */ +static herr_t H5HL__hdr_deserialize(H5HL_t *heap, const uint8_t *image, + H5HL_cache_prfx_ud_t *udata); + /* Free list de/serialization */ static herr_t H5HL__fl_deserialize(H5HL_t *heap); static void H5HL__fl_serialize(const H5HL_t *heap); @@ -147,6 +151,64 @@ const H5AC_class_t H5AC_LHEAP_DBLK[1] = {{ /*------------------------------------------------------------------------- + * Function: H5HL__hdr_deserialize() + * + * Purpose: Decode a local heap's header + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * December 15, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HL__hdr_deserialize( H5HL_t *heap, const uint8_t *image, + H5HL_cache_prfx_ud_t *udata) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(heap); + HDassert(image); + HDassert(udata); + + /* Check magic number */ + if(HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad local heap signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(H5HL_VERSION != *image++) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in local heap") + + /* Reserved */ + image += 3; + + /* Store the prefix's address & length */ + heap->prfx_addr = udata->prfx_addr; + heap->prfx_size = udata->sizeof_prfx; + + /* Heap data size */ + H5F_DECODE_LENGTH_LEN(image, heap->dblk_size, udata->sizeof_size); + + /* Free list head */ + H5F_DECODE_LENGTH_LEN(image, heap->free_block, udata->sizeof_size); + if(heap->free_block != H5HL_FREE_NULL && heap->free_block >= heap->dblk_size) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap free list") + + /* Heap data address */ + H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap->dblk_addr)); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HL__hdr_deserialize() */ + + +/*------------------------------------------------------------------------- * Function: H5HL__fl_deserialize * * Purpose: Deserialize the free list for a heap data block @@ -319,32 +381,9 @@ H5HL__cache_prefix_get_final_load_size(const void *_image, size_t image_len, HDassert(actual_len); HDassert(*actual_len == image_len); - /* Check magic number */ - if(HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad local heap signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(H5HL_VERSION != *image++) - HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in local heap") - - /* Reserved */ - image += 3; - - /* Store the prefix's address & length */ - heap.prfx_addr = udata->prfx_addr; /* NEED */ - heap.prfx_size = udata->sizeof_prfx; /* NEED */ - - /* Heap data size */ - H5F_DECODE_LENGTH_LEN(image, heap.dblk_size, udata->sizeof_size); /* NEED */ - - /* Free list head */ - H5F_DECODE_LENGTH_LEN(image, heap.free_block, udata->sizeof_size); - if(heap.free_block != H5HL_FREE_NULL && heap.free_block >= heap.dblk_size) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap free list"); - - /* Heap data address */ - H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap.dblk_addr)); /* NEED */ + /* Deserialize the heap's header */ + if(H5HL__hdr_deserialize(&heap, (const uint8_t *)image, udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode local heap header") /* Set the final size for the cache image */ *actual_len = heap.prfx_size; @@ -398,41 +437,18 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t len, void *_udata, HDassert(H5F_addr_defined(udata->prfx_addr)); HDassert(dirty); - /* Check magic number */ - if(HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad local heap signature") - image += H5_SIZEOF_MAGIC; - - /* Version */ - if(H5HL_VERSION != *image++) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "wrong version number in local heap") - - /* Reserved */ - image += 3; - /* Allocate space in memory for the heap */ if(NULL == (heap = H5HL__new(udata->sizeof_size, udata->sizeof_addr, udata->sizeof_prfx))) HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "can't allocate local heap structure"); + /* Deserialize the heap's header */ + if(H5HL__hdr_deserialize(heap, (const uint8_t *)image, udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode local heap header") + /* Allocate the heap prefix */ if(NULL == (prfx = H5HL__prfx_new(heap))) HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "can't allocate local heap prefix"); - /* Store the prefix's address & length */ - heap->prfx_addr = udata->prfx_addr; - heap->prfx_size = udata->sizeof_prfx; - - /* Heap data size */ - H5F_DECODE_LENGTH_LEN(image, heap->dblk_size, udata->sizeof_size); - - /* Free list head */ - H5F_DECODE_LENGTH_LEN(image, heap->free_block, udata->sizeof_size); - if((heap->free_block != H5HL_FREE_NULL) && (heap->free_block >= heap->dblk_size)) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap free list") - - /* Heap data address */ - H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap->dblk_addr)); - /* Check if heap block exists */ if(heap->dblk_size) { /* Check if heap data block is contiguous with header */ diff --git a/src/H5Ocache.c b/src/H5Ocache.c index d398e41..1067b8c 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -84,6 +84,10 @@ static herr_t H5O__cache_chk_serialize(const H5F_t *f, void *image, size_t len, static herr_t H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing); static herr_t H5O__cache_chk_free_icr(void *thing); +/* Prefix routines */ +static herr_t H5O__prefix_deserialize(const uint8_t *image, + H5O_cache_ud_t *udata); + /* Chunk routines */ static herr_t H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image, H5O_common_cache_ud_t *udata, hbool_t *dirty); @@ -198,13 +202,11 @@ H5O__cache_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) *------------------------------------------------------------------------- */ static herr_t -H5O__cache_get_final_load_size(const void *_image, size_t image_len, +H5O__cache_get_final_load_size(const void *image, size_t image_len, void *_udata, size_t *actual_len) { - const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - H5O_t *oh = NULL; /* Object header read in */ - htri_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -214,136 +216,17 @@ H5O__cache_get_final_load_size(const void *_image, size_t image_len, HDassert(actual_len); HDassert(*actual_len == image_len); - /* Allocate space for the new object header data structure */ - if(NULL == (oh = H5FL_CALLOC(H5O_t))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") - - /* File-specific, non-stored information */ - oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); - oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f); - - /* Check for presence of magic number */ - /* (indicates version 2 or later) */ - if(!HDmemcmp(image, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { - /* Magic number */ - image += H5_SIZEOF_MAGIC; - - /* Version */ - oh->version = *image++; - if(H5O_VERSION_2 != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") - - /* Flags */ - oh->flags = *image++; - if(oh->flags & ~H5O_HDR_ALL_FLAGS) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown object header status flag(s)") - - /* Number of links to object (unless overridden by refcount message) */ - oh->nlink = 1; - - /* Time fields */ - if(oh->flags & H5O_HDR_STORE_TIMES) { - uint32_t tmp; /* Temporary value */ - - UINT32DECODE(image, tmp); - oh->atime = (time_t)tmp; - UINT32DECODE(image, tmp); - oh->mtime = (time_t)tmp; - UINT32DECODE(image, tmp); - oh->ctime = (time_t)tmp; - UINT32DECODE(image, tmp); - oh->btime = (time_t)tmp; - } /* end if */ - else - oh->atime = oh->mtime = oh->ctime = oh->btime = 0; - - /* Attribute fields */ - if(oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { - UINT16DECODE(image, oh->max_compact); - UINT16DECODE(image, oh->min_dense); - if(oh->max_compact < oh->min_dense) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header attribute phase change values") - } /* end if */ - else { - oh->max_compact = H5O_CRT_ATTR_MAX_COMPACT_DEF; - oh->min_dense = H5O_CRT_ATTR_MIN_DENSE_DEF; - } /* end else */ - - /* First chunk size */ - switch(oh->flags & H5O_HDR_CHUNK0_SIZE) { - case 0: /* 1 byte size */ - udata->chunk0_size = *image++; - break; - - case 1: /* 2 byte size */ - UINT16DECODE(image, udata->chunk0_size); - break; - - case 2: /* 4 byte size */ - UINT32DECODE(image, udata->chunk0_size); - break; - - case 3: /* 8 byte size */ - UINT64DECODE(image, udata->chunk0_size); - break; - - default: - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") - } /* end switch */ - if(udata->chunk0_size > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") - } /* end if */ - else { - /* Version */ - oh->version = *image++; - if(H5O_VERSION_1 != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") - - /* Flags */ - oh->flags = H5O_CRT_OHDR_FLAGS_DEF; - - /* Reserved */ - image++; - - /* Number of messages */ - UINT16DECODE(image, udata->v1_pfx_nmesgs); - - /* Link count */ - UINT32DECODE(image, oh->nlink); - - /* Reset unused time fields */ - oh->atime = oh->mtime = oh->ctime = oh->btime = 0; - - /* Reset unused attribute fields */ - oh->max_compact = 0; - oh->min_dense = 0; - - /* First chunk size */ - UINT32DECODE(image, udata->chunk0_size); - if((udata->v1_pfx_nmesgs > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || - (udata->v1_pfx_nmesgs == 0 && udata->chunk0_size > 0)) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") - - /* Reserved, in version 1 (for 8-byte alignment padding) */ - image += 4; - } /* end else */ + /* Deserialize the object header prefix */ + if(H5O__prefix_deserialize((const uint8_t *)image, udata) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "can't deserialize object header prefix") - /* Determine object header prefix length */ - HDassert((size_t)((const uint8_t *)image - (const uint8_t *)_image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + /* Sanity check */ + HDassert(udata->oh); /* Set the final size for the cache image */ - *actual_len = udata->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh); - - /* Save the object header for later use in 'deserialize' callback */ - udata->oh = oh; - oh = NULL; + *actual_len = udata->chunk0_size + (size_t)H5O_SIZEOF_HDR(udata->oh); done: - /* Release the [possibly partially initialized] object header on errors */ - if(ret_value < 0 && oh) - if(H5O__free(oh) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header data") - FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__cache_get_final_load_size() */ @@ -413,12 +296,11 @@ H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) *------------------------------------------------------------------------- */ static void * -H5O__cache_deserialize(const void *_image, size_t len, void *_udata, +H5O__cache_deserialize(const void *image, size_t len, void *_udata, hbool_t *dirty) { - H5O_t *oh; /* Object header read in */ + H5O_t *oh = NULL; /* Object header read in */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - const uint8_t *image = (const uint8_t *)_image; /* Pointer into buffer to decode */ void * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -427,11 +309,24 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, HDassert(image); HDassert(len > 0); HDassert(udata); - HDassert(udata->oh); HDassert(udata->common.f); HDassert(udata->common.cont_msg_info); HDassert(dirty); + /* Check for partially deserialized object header */ + /* (Object header prefix will be deserialized if the object header came + * through the 'get_final_load_size' callback and not deserialized if + * the object header is coming from a cache image - QAK, 2016/12/14) + */ + if(NULL == udata->oh) { + /* Deserialize the object header prefix */ + if(H5O__prefix_deserialize((const uint8_t *)image, udata) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, NULL, "can't deserialize object header prefix") + + /* Sanity check */ + HDassert(udata->oh); + } /* end if */ + /* Retrieve partially deserialized object header from user data */ oh = udata->oh; @@ -448,7 +343,7 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, oh->proxy = NULL; /* Parse the first chunk */ - if(H5O__chunk_deserialize(oh, udata->common.addr, udata->chunk0_size, (const uint8_t *)_image, &(udata->common), dirty) < 0) + if(H5O__chunk_deserialize(oh, udata->common.addr, udata->chunk0_size, (const uint8_t *)image, &(udata->common), dirty) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk") /* Note that we've loaded the object header from the file */ @@ -717,11 +612,7 @@ H5O__cache_notify(H5AC_notify_action_t action, void *_thing) break; default: -#ifdef NDEBUG HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown action from metadata cache") -#else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); -#endif /* NDEBUG */ } /* end switch */ done: @@ -1228,6 +1119,163 @@ done: /*------------------------------------------------------------------------- + * Function: H5O__prefix_deserialize() + * + * Purpose: Deserialize an object header prefix + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * December 14, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O__prefix_deserialize(const uint8_t *_image, H5O_cache_ud_t *udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5O_t *oh = NULL; /* Object header read in */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* Allocate space for the new object header data structure */ + if(NULL == (oh = H5FL_CALLOC(H5O_t))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") + + /* File-specific, non-stored information */ + oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); + oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f); + + /* Check for presence of magic number */ + /* (indicates version 2 or later) */ + if(!HDmemcmp(image, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + /* Magic number */ + image += H5_SIZEOF_MAGIC; + + /* Version */ + oh->version = *image++; + if(H5O_VERSION_2 != oh->version) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") + + /* Flags */ + oh->flags = *image++; + if(oh->flags & ~H5O_HDR_ALL_FLAGS) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown object header status flag(s)") + + /* Number of links to object (unless overridden by refcount message) */ + oh->nlink = 1; + + /* Time fields */ + if(oh->flags & H5O_HDR_STORE_TIMES) { + uint32_t tmp; /* Temporary value */ + + UINT32DECODE(image, tmp); + oh->atime = (time_t)tmp; + UINT32DECODE(image, tmp); + oh->mtime = (time_t)tmp; + UINT32DECODE(image, tmp); + oh->ctime = (time_t)tmp; + UINT32DECODE(image, tmp); + oh->btime = (time_t)tmp; + } /* end if */ + else + oh->atime = oh->mtime = oh->ctime = oh->btime = 0; + + /* Attribute fields */ + if(oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { + UINT16DECODE(image, oh->max_compact); + UINT16DECODE(image, oh->min_dense); + if(oh->max_compact < oh->min_dense) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header attribute phase change values") + } /* end if */ + else { + oh->max_compact = H5O_CRT_ATTR_MAX_COMPACT_DEF; + oh->min_dense = H5O_CRT_ATTR_MIN_DENSE_DEF; + } /* end else */ + + /* First chunk size */ + switch(oh->flags & H5O_HDR_CHUNK0_SIZE) { + case 0: /* 1 byte size */ + udata->chunk0_size = *image++; + break; + + case 1: /* 2 byte size */ + UINT16DECODE(image, udata->chunk0_size); + break; + + case 2: /* 4 byte size */ + UINT32DECODE(image, udata->chunk0_size); + break; + + case 3: /* 8 byte size */ + UINT64DECODE(image, udata->chunk0_size); + break; + + default: + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") + } /* end switch */ + if(udata->chunk0_size > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") + } /* end if */ + else { + /* Version */ + oh->version = *image++; + if(H5O_VERSION_1 != oh->version) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") + + /* Flags */ + oh->flags = H5O_CRT_OHDR_FLAGS_DEF; + + /* Reserved */ + image++; + + /* Number of messages */ + UINT16DECODE(image, udata->v1_pfx_nmesgs); + + /* Link count */ + UINT32DECODE(image, oh->nlink); + + /* Reset unused time fields */ + oh->atime = oh->mtime = oh->ctime = oh->btime = 0; + + /* Reset unused attribute fields */ + oh->max_compact = 0; + oh->min_dense = 0; + + /* First chunk size */ + UINT32DECODE(image, udata->chunk0_size); + if((udata->v1_pfx_nmesgs > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || + (udata->v1_pfx_nmesgs == 0 && udata->chunk0_size > 0)) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") + + /* Reserved, in version 1 (for 8-byte alignment padding) */ + image += 4; + } /* end else */ + + /* Verify object header prefix length */ + HDassert((size_t)(image - _image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + + /* Save the object header for later use in 'deserialize' callback */ + udata->oh = oh; + oh = NULL; + +done: + /* Release the [possibly partially initialized] object header on errors */ + if(ret_value < 0 && oh) + if(H5O__free(oh) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header data") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__prefix_deserialize() */ + + +/*------------------------------------------------------------------------- * Function: H5O__chunk_deserialize * * Purpose: Deserialize a chunk for an object header diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index c094c20..4a3eb72 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -173,7 +173,7 @@ #define H5D_XFER_DIRECT_CHUNK_WRITE_DATASIZE_DEF 0 /* Ring type - private property */ #define H5AC_XFER_RING_SIZE sizeof(unsigned) -#define H5AC_XFER_RING_DEF H5AC_RING_US +#define H5AC_XFER_RING_DEF H5AC_RING_USER #define H5AC_XFER_RING_ENC H5P__encode_unsigned #define H5AC_XFER_RING_DEC H5P__decode_unsigned #ifdef H5_DEBUG_BUILD @@ -144,7 +144,7 @@ H5SM_init(H5F_t *f, H5P_genplist_t * fc_plist, const H5O_loc_t *ext_loc, hid_t d HDassert(!H5F_addr_defined(H5F_SOHM_ADDR(f))); /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_US, &dxpl, &orig_ring) < 0) + if(H5AC_set_ring(dxpl_id, H5AC_RING_USER, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_SOHM, H5E_CANTSET, FAIL, "unable to set ring value") /* Initialize master table */ @@ -2021,7 +2021,7 @@ H5SM_get_info(const H5O_loc_t *ext_loc, H5P_genplist_t *fc_plist, hid_t dxpl_id) cache_udata.f = f; /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_US, &dxpl, &orig_ring) < 0) + if(H5AC_set_ring(dxpl_id, H5AC_RING_USER, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_SOHM, H5E_CANTSET, FAIL, "unable to set ring value") /* Read the rest of the SOHM table information from the cache */ diff --git a/test/cache.c b/test/cache.c index 0cd0c99..ed4fceb 100644 --- a/test/cache.c +++ b/test/cache.c @@ -15913,7 +15913,6 @@ check_destroy_pinned_err(void) unprotect_entry(file_ptr, 0, 0, H5C__PIN_ENTRY_FLAG); if(H5C_dest(file_ptr, H5AC_ind_read_dxpl_id) >= 0) { - pass = FALSE; failure_mssg = "destroy succeeded on cache with pinned entry.\n"; } /* end if */ @@ -34541,7 +34540,7 @@ cedds__expunge_dirty_entry_in_flush_test(H5F_t * file_ptr) if(pass) if((cache_ptr->slist_scan_restarts != 1) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "unexpected scan restart stats in cedds__expunge_dirty_entry_in_flush_test()."; } /* end if */ @@ -34920,7 +34919,7 @@ cedds__H5C_make_space_in_cache(H5F_t * file_ptr) if(pass) if((cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 1) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "unexpected scan restart stats in cedds__H5C_make_space_in_cache()."; @@ -35330,7 +35329,7 @@ cedds__H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * file_ptr) if(pass) if((cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 1) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "unexpected scan restart stats in cedds__H5C__autoadjust__ageout__evict_aged_out_entries()."; @@ -35353,7 +35352,26 @@ cedds__H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * file_ptr) /*------------------------------------------------------------------------- * Function: cedds__H5C_flush_invalidate_cache__bucket_scan() * - * Purpose: Verify that H5C_flush_invalidate_cache() can handle + * Purpose: Note: We now use the index list when we scan the + * contents of the metadata cache, so in principal, + * this test is obsolete. However, even using the + * index list, restarts are possible, and must be + * handled gracefully. + * + * As it turns out, this test triggers index list + * scan restarts, and thus with minor changes is + * still a useful test. + * + * For this reason, with the exception of changing + * to check the index_scan_restart stat instead of + * hash bucket restarts, I'm leaving the test + * alone. If and when it starts to fail due to + * other changes, we can re-work it to test + * index list scan restarts explicitly. + * + * JRM -- 11/2/16 + * + * Verify that H5C_flush_invalidate_cache() can handle * the removal from the cache of the next item in * its scans of hash buckets. * @@ -35455,20 +35473,20 @@ cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t * file_ptr) if(cache_ptr == NULL) { pass = FALSE; - failure_mssg = "cache_ptr NULL on entry to cedds for H5C__autoadjust__ageout__evict_aged_out_entries() test."; + failure_mssg = "cache_ptr NULL on entry to cedds for cedds__H5C_flush_invalidate_cache__bucket_scan() test."; } else if((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0)) { pass = FALSE; - failure_mssg = "cache not empty at start cedds for H5C__autoadjust__ageout__evict_aged_out_entries() test."; + failure_mssg = "cache not empty at start cedds for cedds__H5C_flush_invalidate_cache__bucket_scan() test."; } else if((cache_ptr->max_cache_size != (2 * 1024 * 1024)) || (cache_ptr->min_clean_size != (1 * 1024 * 1024))) { pass = FALSE; failure_mssg = - "unexpected cache config at start of cedds H5C__autoadjust__ageout__evict_aged_out_entries() test."; + "unexpected cache config at start of cedds cedds__H5C_flush_invalidate_cache__bucket_scan() test."; } else { @@ -35696,9 +35714,13 @@ cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t * file_ptr) } /* end if */ if(pass) + /* as this test is now checking for index list scan restarts, + * the following has been modified to check this instead of + * hash bucket scan restarts. + */ if((cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 1)) { + (cache_ptr->index_scan_restarts != 1)) { pass = FALSE; failure_mssg = "unexpected scan restart stats in cedds__H5C_flush_invalidate_cache__bucket_scan()."; } @@ -35912,7 +35934,7 @@ check_stats__smoke_check_1(H5F_t * file_ptr) (cache_ptr->entries_scanned_to_make_space != 0) || (cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(1)."; @@ -35997,7 +36019,7 @@ check_stats__smoke_check_1(H5F_t * file_ptr) (cache_ptr->entries_scanned_to_make_space != 0) || (cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(2)."; @@ -36082,7 +36104,7 @@ check_stats__smoke_check_1(H5F_t * file_ptr) (cache_ptr->entries_scanned_to_make_space != 33) || (cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(3)."; @@ -36186,7 +36208,7 @@ check_stats__smoke_check_1(H5F_t * file_ptr) (cache_ptr->entries_scanned_to_make_space != 33) || (cache_ptr->slist_scan_restarts != 0) || (cache_ptr->LRU_scan_restarts != 0) || - (cache_ptr->hash_bucket_scan_restarts != 0)) { + (cache_ptr->index_scan_restarts != 0)) { pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(4)."; diff --git a/test/cache_common.h b/test/cache_common.h index 1ab04cb..befcee4 100644 --- a/test/cache_common.h +++ b/test/cache_common.h @@ -384,7 +384,7 @@ if ( ( (cache_ptr) == NULL ) || \ HDfprintf(stdout, "Pre HT search SC failed.\n"); \ } -#define H5C_TEST__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k) \ +#define H5C_TEST__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->magic != H5C__H5C_T_MAGIC ) || \ ( (cache_ptr)->index_len < 1 ) || \ @@ -392,7 +392,6 @@ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_size < (entry_ptr)->size ) || \ ( (cache_ptr)->index_size != \ ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ) || \ - ( H5F_addr_ne((entry_ptr)->addr, (Addr)) ) || \ ( (entry_ptr)->size <= 0 ) || \ ( ((cache_ptr)->index)[k] == NULL ) || \ ( ( ((cache_ptr)->index)[k] != (entry_ptr) ) && \ @@ -416,32 +415,29 @@ if ( ( (cache_ptr) == NULL ) || \ #define H5C_TEST__SEARCH_INDEX(cache_ptr, Addr, entry_ptr) \ { \ int k; \ - int depth = 0; \ H5C_TEST__PRE_HT_SEARCH_SC(cache_ptr, Addr) \ k = H5C__HASH_FCN(Addr); \ entry_ptr = ((cache_ptr)->index)[k]; \ - while ( ( entry_ptr ) && ( H5F_addr_ne(Addr, (entry_ptr)->addr) ) ) \ + while ( entry_ptr ) \ { \ - (entry_ptr) = (entry_ptr)->ht_next; \ - (depth)++; \ - } \ - if ( entry_ptr ) \ - { \ - H5C_TEST__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k) \ - if ( entry_ptr != ((cache_ptr)->index)[k] ) \ + if ( H5F_addr_eq(Addr, (entry_ptr)->addr) ) \ { \ - if ( (entry_ptr)->ht_next ) \ + H5C_TEST__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, k) \ + if ( entry_ptr != ((cache_ptr)->index)[k] ) \ { \ - (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + if ( (entry_ptr)->ht_next ) \ + (entry_ptr)->ht_next->ht_prev = (entry_ptr)->ht_prev; \ + HDassert( (entry_ptr)->ht_prev != NULL ); \ + (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ + ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ + (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ + (entry_ptr)->ht_prev = NULL; \ + ((cache_ptr)->index)[k] = (entry_ptr); \ + H5C_TEST__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k) \ } \ - HDassert( (entry_ptr)->ht_prev != NULL ); \ - (entry_ptr)->ht_prev->ht_next = (entry_ptr)->ht_next; \ - ((cache_ptr)->index)[k]->ht_prev = (entry_ptr); \ - (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ - (entry_ptr)->ht_prev = NULL; \ - ((cache_ptr)->index)[k] = (entry_ptr); \ - H5C_TEST__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k) \ + break; \ } \ + (entry_ptr) = (entry_ptr)->ht_next; \ } \ } |