diff options
| author | Quincey Koziol <koziol@hdfgroup.org> | 2009-01-14 16:29:10 (GMT) |
|---|---|---|
| committer | Quincey Koziol <koziol@hdfgroup.org> | 2009-01-14 16:29:10 (GMT) |
| commit | 0894356a3c6e17b00c7911d8199b64eefca7aa3a (patch) | |
| tree | 9e9ab954ab103b64aa6ba87da4fbe30127f4035f /src | |
| parent | e0bb475838f0c31743018e5c1c770b30b9373440 (diff) | |
| download | hdf5-0894356a3c6e17b00c7911d8199b64eefca7aa3a.zip hdf5-0894356a3c6e17b00c7911d8199b64eefca7aa3a.tar.gz hdf5-0894356a3c6e17b00c7911d8199b64eefca7aa3a.tar.bz2 | |
[svn-r16311] Description:
Bring revisions 16280:16307 back from trunk.
Tested on:
FreeBSD/32 6.3 (duty) in debug mode
FreeBSD/64 6.3 (liberty) w/C++ & FORTRAN, in debug mode
Linux/32 2.6 (jam) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe,
in debug mode
Linux/64-amd64 2.6 (smirom) w/Intel compilers w/default API=1.6.x,
w/C++ & FORTRAN, in production mode
Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN,
in production mode
Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in production mode
Linux/64-amd64 2.6 (abe) w/parallel, w/FORTRAN, in production mode
Mac OS X/32 10.5.6 (amazon) in debug mode
Mac OS X/32 10.5.6 (amazon) w/C++ & FORTRAN, w/threadsafe,
in production mode
Diffstat (limited to 'src')
| -rw-r--r-- | src/H5ACprivate.h | 4 | ||||
| -rw-r--r-- | src/H5C.c | 819 | ||||
| -rw-r--r-- | src/H5Cpkg.h | 66 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 3 | ||||
| -rw-r--r-- | src/H5Fpublic.h | 4 | ||||
| -rw-r--r-- | src/H5Fquery.c | 26 | ||||
| -rw-r--r-- | src/H5Glink.c | 1 | ||||
| -rw-r--r-- | src/H5Gnode.c | 1 | ||||
| -rw-r--r-- | src/H5Lexternal.c | 90 | ||||
| -rw-r--r-- | src/H5Lprivate.h | 14 | ||||
| -rw-r--r-- | src/H5Lpublic.h | 7 | ||||
| -rw-r--r-- | src/H5Odbg.c | 1 | ||||
| -rw-r--r-- | src/H5Plapl.c | 185 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 8 | ||||
| -rw-r--r-- | src/H5Tdeprec.c | 1 | ||||
| -rw-r--r-- | src/H5public.h | 3 |
16 files changed, 1024 insertions, 209 deletions
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 4a6f9f3..66bdf2d 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -208,8 +208,8 @@ extern hid_t H5AC_ind_dxpl_id; /* char trace_file_name[] = */ "", \ /* hbool_t evictions_enabled = */ TRUE, \ /* hbool_t set_initial_size = */ TRUE, \ - /* size_t initial_size = */ ( 1 * 1024 * 1024), \ - /* double min_clean_fraction = */ 0.5, \ + /* size_t initial_size = */ ( 2 * 1024 * 1024), \ + /* double min_clean_fraction = */ 0.3, \ /* size_t max_size = */ (16 * 1024 * 1024), \ /* size_t min_size = */ ( 1 * 1024 * 1024), \ /* long int epoch_length = */ 50000, \ @@ -584,6 +584,14 @@ if ( ( (entry_ptr) == NULL ) || \ ((cache_ptr)->size_increases[(entry_ptr)->type->id])++; \ if ( (cache_ptr)->index_size > (cache_ptr)->max_index_size ) \ (cache_ptr)->max_index_size = (cache_ptr)->index_size; \ + if ( (cache_ptr)->clean_index_size > \ + (cache_ptr)->max_clean_index_size ) \ + (cache_ptr)->max_clean_index_size = \ + (cache_ptr)->clean_index_size; \ + if ( (cache_ptr)->dirty_index_size > \ + (cache_ptr)->max_dirty_index_size ) \ + (cache_ptr)->max_dirty_index_size = \ + (cache_ptr)->dirty_index_size; \ if ( (cache_ptr)->slist_size > (cache_ptr)->max_slist_size ) \ (cache_ptr)->max_slist_size = (cache_ptr)->slist_size; \ if ( (cache_ptr)->pl_size > (cache_ptr)->max_pl_size ) \ @@ -680,6 +688,14 @@ if ( ( (entry_ptr) == NULL ) || \ (cache_ptr)->max_index_len = (cache_ptr)->index_len; \ if ( (cache_ptr)->index_size > (cache_ptr)->max_index_size ) \ (cache_ptr)->max_index_size = (cache_ptr)->index_size; \ + if ( (cache_ptr)->clean_index_size > \ + (cache_ptr)->max_clean_index_size ) \ + (cache_ptr)->max_clean_index_size = \ + (cache_ptr)->clean_index_size; \ + if ( (cache_ptr)->dirty_index_size > \ + (cache_ptr)->max_dirty_index_size ) \ + (cache_ptr)->max_dirty_index_size = \ + (cache_ptr)->dirty_index_size; \ if ( (cache_ptr)->slist_len > (cache_ptr)->max_slist_len ) \ (cache_ptr)->max_slist_len = (cache_ptr)->slist_len; \ if ( (cache_ptr)->slist_size > (cache_ptr)->max_slist_size ) \ @@ -709,6 +725,14 @@ if ( ( (entry_ptr) == NULL ) || \ (cache_ptr)->max_index_len = (cache_ptr)->index_len; \ if ( (cache_ptr)->index_size > (cache_ptr)->max_index_size ) \ (cache_ptr)->max_index_size = (cache_ptr)->index_size; \ + if ( (cache_ptr)->clean_index_size > \ + (cache_ptr)->max_clean_index_size ) \ + (cache_ptr)->max_clean_index_size = \ + (cache_ptr)->clean_index_size; \ + if ( (cache_ptr)->dirty_index_size > \ + (cache_ptr)->max_dirty_index_size ) \ + (cache_ptr)->max_dirty_index_size = \ + (cache_ptr)->dirty_index_size; \ if ( (cache_ptr)->pl_len > (cache_ptr)->max_pl_len ) \ (cache_ptr)->max_pl_len = (cache_ptr)->pl_len; \ if ( (cache_ptr)->pl_size > (cache_ptr)->max_pl_size ) \ @@ -761,6 +785,14 @@ if ( ( (entry_ptr) == NULL ) || \ (cache_ptr)->max_index_len = (cache_ptr)->index_len; \ if ( (cache_ptr)->index_size > (cache_ptr)->max_index_size ) \ (cache_ptr)->max_index_size = (cache_ptr)->index_size; \ + if ( (cache_ptr)->clean_index_size > \ + (cache_ptr)->max_clean_index_size ) \ + (cache_ptr)->max_clean_index_size = \ + (cache_ptr)->clean_index_size; \ + if ( (cache_ptr)->dirty_index_size > \ + (cache_ptr)->max_dirty_index_size ) \ + (cache_ptr)->max_dirty_index_size = \ + (cache_ptr)->dirty_index_size; \ if ( (cache_ptr)->slist_len > (cache_ptr)->max_slist_len ) \ (cache_ptr)->max_slist_len = (cache_ptr)->slist_len; \ if ( (cache_ptr)->slist_size > (cache_ptr)->max_slist_size ) \ @@ -784,6 +816,14 @@ if ( ( (entry_ptr) == NULL ) || \ if ( (cache_ptr)->index_len > (cache_ptr)->max_index_len ) \ (cache_ptr)->max_index_len = (cache_ptr)->index_len; \ if ( (cache_ptr)->index_size > (cache_ptr)->max_index_size ) \ + if ( (cache_ptr)->clean_index_size > \ + (cache_ptr)->max_clean_index_size ) \ + (cache_ptr)->max_clean_index_size = \ + (cache_ptr)->clean_index_size; \ + if ( (cache_ptr)->dirty_index_size > \ + (cache_ptr)->max_dirty_index_size ) \ + (cache_ptr)->max_dirty_index_size = \ + (cache_ptr)->dirty_index_size; \ (cache_ptr)->max_index_size = (cache_ptr)->index_size; \ if ( (cache_ptr)->pl_len > (cache_ptr)->max_pl_len ) \ (cache_ptr)->max_pl_len = (cache_ptr)->pl_len; \ @@ -830,6 +870,14 @@ if ( ( (entry_ptr) == NULL ) || \ * When modifying these macros, remember to modify the similar macros * in tst/cache.c * + * Changes: + * + * - Updated existing index macros and sanity check macros to maintain + * the clean_index_size and dirty_index_size fields of H5C_t. Also + * added macros to allow us to track entry cleans and dirties. + * + * JRM -- 11/5/08 + * ***********************************************************************/ /* H5C__HASH_TABLE_LEN is defined in H5Cpkg.h. It mut be a power of two. */ @@ -849,7 +897,11 @@ if ( ( (cache_ptr) == NULL ) || \ ( (entry_ptr)->ht_prev != NULL ) || \ ( (entry_ptr)->size <= 0 ) || \ ( (k = H5C__HASH_FCN((entry_ptr)->addr)) < 0 ) || \ - ( k >= H5C__HASH_TABLE_LEN ) ) { \ + ( k >= H5C__HASH_TABLE_LEN ) || \ + ( (cache_ptr)->index_size != \ + ((cache_ptr)->clean_index_size + \ + (cache_ptr)->dirty_index_size) ) ) { \ + HDassert(0); /* JRM */ \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, fail_val, \ "Pre HT insert SC failed") \ } @@ -871,13 +923,19 @@ if ( ( (cache_ptr) == NULL ) || \ ( (entry_ptr)->ht_prev == NULL ) ) || \ ( ( ((cache_ptr)->index)[(H5C__HASH_FCN((entry_ptr)->addr))] == \ (entry_ptr) ) && \ - ( (entry_ptr)->ht_prev != NULL ) ) ) { \ + ( (entry_ptr)->ht_prev != NULL ) ) || \ + ( (cache_ptr)->index_size != \ + ((cache_ptr)->clean_index_size + \ + (cache_ptr)->dirty_index_size) ) ) { \ + HDassert(0); /* JRM */ \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Pre HT remove SC failed") \ } #define H5C__PRE_HT_SEARCH_SC(cache_ptr, Addr, fail_val) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->magic != H5C__H5C_T_MAGIC ) || \ + ( (cache_ptr)->index_size != \ + ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ) || \ ( ! H5F_addr_defined(Addr) ) || \ ( H5C__HASH_FCN(Addr) < 0 ) || \ ( H5C__HASH_FCN(Addr) >= H5C__HASH_TABLE_LEN ) ) { \ @@ -890,6 +948,8 @@ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_len < 1 ) || \ ( (entry_ptr) == NULL ) || \ ( (cache_ptr)->index_size < (entry_ptr)->size ) || \ + ( (cache_ptr)->index_size != \ + ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ) || \ ( H5F_addr_ne((entry_ptr)->addr, (Addr)) ) || \ ( (entry_ptr)->size <= 0 ) || \ ( ((cache_ptr)->index)[k] == NULL ) || \ @@ -913,7 +973,8 @@ if ( ( (cache_ptr) == NULL ) || \ "Post HT shift to front SC failed") \ } -#define H5C__PRE_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size) \ +#define H5C__PRE_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ + entry_ptr, was_clean) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_len <= 0 ) || \ ( (cache_ptr)->index_size <= 0 ) || \ @@ -921,22 +982,78 @@ if ( ( (cache_ptr) == NULL ) || \ ( (old_size) > (cache_ptr)->index_size ) || \ ( (new_size) <= 0 ) || \ ( ( (cache_ptr)->index_len == 1 ) && \ - ( (cache_ptr)->index_size != (old_size) ) ) ) { \ + ( (cache_ptr)->index_size != (old_size) ) ) || \ + ( (entry_ptr) == NULL ) ) { \ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "Pre HT entry size change SC failed") \ +} \ +HDassert( (cache_ptr)->index_size == \ + ((cache_ptr)->clean_index_size + \ + (cache_ptr)->dirty_index_size) ); \ +HDassert( (entry_ptr) != NULL ); \ +HDassert( ( ( was_clean ) && \ + ( (cache_ptr)->clean_index_size >= (old_size) ) ) || \ + ( ( ! (was_clean) ) && \ + ( (cache_ptr)->dirty_index_size >= (old_size) ) ) ); + +#define H5C__POST_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ + entry_ptr) \ +if ( ( (cache_ptr) == NULL ) || \ + ( (cache_ptr)->index_len <= 0 ) || \ + ( (cache_ptr)->index_size <= 0 ) || \ + ( (new_size) > (cache_ptr)->index_size ) || \ + ( ( (cache_ptr)->index_len == 1 ) && \ + ( (cache_ptr)->index_size != (new_size) ) ) ) { \ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Post HT entry size change SC failed") \ +} \ +HDassert( (cache_ptr)->index_size == \ + ((cache_ptr)->clean_index_size + \ + (cache_ptr)->dirty_index_size) ); \ +HDassert( ( ( (entry_ptr)->is_dirty ) && \ + ( (cache_ptr)->dirty_index_size >= (new_size) ) ) || \ + ( ( ! ((entry_ptr)->is_dirty) ) && \ + ( (cache_ptr)->clean_index_size >= (new_size) ) ) ); + +#define H5C__PRE_HT_UPDATE_FOR_ENTRY_CLEAN_SC(cache_ptr, entry_ptr) \ +{ \ + HDassert( (cache_ptr) != NULL ); \ + HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ + HDassert( (cache_ptr)->index_len > 0 ); \ + HDassert( (entry_ptr) != NULL ); \ + HDassert( (entry_ptr)->is_dirty == FALSE ); \ + HDassert( (cache_ptr)->index_size >= (entry_ptr)->size ); \ + HDassert( (cache_ptr)->dirty_index_size >= (entry_ptr)->size ); \ + HDassert( (cache_ptr)->index_size == \ + ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ); \ +} +/* JRM */ +#define H5C__PRE_HT_UPDATE_FOR_ENTRY_DIRTY_SC(cache_ptr, entry_ptr) \ +{ \ + HDassert( (cache_ptr) != NULL ); \ + HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ + HDassert( (cache_ptr)->index_len > 0 ); \ + HDassert( (entry_ptr) != NULL ); \ + HDassert( (entry_ptr)->is_dirty == TRUE ); \ + HDassert( (cache_ptr)->index_size >= (entry_ptr)->size ); \ + HDassert( (cache_ptr)->clean_index_size >= (entry_ptr)->size ); \ + HDassert( (cache_ptr)->index_size == \ + ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ); \ } -#define H5C__POST_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size) \ -if ( ( (cache_ptr) == NULL ) || \ - ( (cache_ptr)->index_len <= 0 ) || \ - ( (cache_ptr)->index_size <= 0 ) || \ - ( (new_size) > (cache_ptr)->index_size ) || \ - ( ( (cache_ptr)->index_len == 1 ) && \ - ( (cache_ptr)->index_size != (new_size) ) ) ) { \ - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "Post HT entry size change SC failed") \ +#define H5C__POST_HT_UPDATE_FOR_ENTRY_CLEAN_SC(cache_ptr, entry_ptr) \ +{ \ + HDassert( (cache_ptr)->index_size == \ + ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ); \ } +#define H5C__POST_HT_UPDATE_FOR_ENTRY_DIRTY_SC(cache_ptr, entry_ptr) \ +{ \ + HDassert( (cache_ptr)->index_size == \ + ((cache_ptr)->clean_index_size + (cache_ptr)->dirty_index_size) ); \ +} + + #else /* H5C_DO_SANITY_CHECKS */ #define H5C__PRE_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) @@ -944,8 +1061,14 @@ if ( ( (cache_ptr) == NULL ) || \ #define H5C__PRE_HT_SEARCH_SC(cache_ptr, Addr, fail_val) #define H5C__POST_SUC_HT_SEARCH_SC(cache_ptr, entry_ptr, Addr, k, fail_val) #define H5C__POST_HT_SHIFT_TO_FRONT(cache_ptr, entry_ptr, k, fail_val) -#define H5C__PRE_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size) -#define H5C__POST_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size) +#define H5C__PRE_HT_UPDATE_FOR_ENTRY_CLEAN_SC(cache_ptr, entry_ptr) +#define H5C__PRE_HT_UPDATE_FOR_ENTRY_DIRTY_SC(cache_ptr, entry_ptr) +#define H5C__PRE_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ + entry_ptr, was_clean) +#define H5C__POST_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ + entry_ptr) +#define H5C__POST_HT_UPDATE_FOR_ENTRY_CLEAN_SC(cache_ptr, entry_ptr) +#define H5C__POST_HT_UPDATE_FOR_ENTRY_DIRTY_SC(cache_ptr, entry_ptr) #endif /* H5C_DO_SANITY_CHECKS */ @@ -967,6 +1090,11 @@ if ( ( (cache_ptr) == NULL ) || \ } \ (cache_ptr)->index_len++; \ (cache_ptr)->index_size += (entry_ptr)->size; \ + if ( (entry_ptr)->is_dirty ) { \ + (cache_ptr)->dirty_index_size += (entry_ptr)->size; \ + } else { \ + (cache_ptr)->clean_index_size += (entry_ptr)->size; \ + } \ H5C__UPDATE_STATS_FOR_HT_INSERTION(cache_ptr) \ } @@ -991,6 +1119,11 @@ if ( ( (cache_ptr) == NULL ) || \ (entry_ptr)->ht_prev = NULL; \ (cache_ptr)->index_len--; \ (cache_ptr)->index_size -= (entry_ptr)->size; \ + if ( (entry_ptr)->is_dirty ) { \ + (cache_ptr)->dirty_index_size -= (entry_ptr)->size; \ + } else { \ + (cache_ptr)->clean_index_size -= (entry_ptr)->size; \ + } \ H5C__UPDATE_STATS_FOR_HT_DELETION(cache_ptr) \ } @@ -1059,12 +1192,40 @@ if ( ( (cache_ptr) == NULL ) || \ } \ } -#define H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(cache_ptr, old_size, new_size) \ -{ \ - H5C__PRE_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size) \ - (cache_ptr)->index_size -= old_size; \ - (cache_ptr)->index_size += new_size; \ - H5C__POST_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size) \ +#define H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN(cache_ptr, entry_ptr) \ +{ \ + H5C__PRE_HT_UPDATE_FOR_ENTRY_CLEAN_SC(cache_ptr, entry_ptr); \ + (cache_ptr)->dirty_index_size -= (entry_ptr)->size; \ + (cache_ptr)->clean_index_size += (entry_ptr)->size; \ + H5C__POST_HT_UPDATE_FOR_ENTRY_CLEAN_SC(cache_ptr, entry_ptr); \ +} + +#define H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(cache_ptr, entry_ptr) \ +{ \ + H5C__PRE_HT_UPDATE_FOR_ENTRY_DIRTY_SC(cache_ptr, entry_ptr); \ + (cache_ptr)->clean_index_size -= (entry_ptr)->size; \ + (cache_ptr)->dirty_index_size += (entry_ptr)->size; \ + H5C__POST_HT_UPDATE_FOR_ENTRY_DIRTY_SC(cache_ptr, entry_ptr); \ +} + +#define H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(cache_ptr, old_size, new_size, \ + entry_ptr, was_clean) \ +{ \ + H5C__PRE_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ + entry_ptr, was_clean) \ + (cache_ptr)->index_size -= (old_size); \ + (cache_ptr)->index_size += (new_size); \ + if ( was_clean ) { \ + (cache_ptr)->clean_index_size -= (old_size); \ + } else { \ + (cache_ptr)->dirty_index_size -= (old_size); \ + } \ + if ( (entry_ptr)->is_dirty ) { \ + (cache_ptr)->dirty_index_size += (new_size); \ + } else { \ + (cache_ptr)->clean_index_size += (new_size); \ + } \ + H5C__POST_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, entry_ptr) \ } @@ -2847,6 +3008,10 @@ done: * Added initialization for the new flash cache size increase * related fields of H5C_t. * + * JRM -- 11/5/08 + * Added initialization for the new clean_index_size and + * dirty_index_size fields of H5C_t. + * *------------------------------------------------------------------------- */ @@ -2922,6 +3087,8 @@ H5C_create(size_t max_cache_size, cache_ptr->index_len = 0; cache_ptr->index_size = (size_t)0; + cache_ptr->clean_index_size = (size_t)0; + cache_ptr->dirty_index_size = (size_t)0; cache_ptr->slist_len = 0; cache_ptr->slist_size = (size_t)0; @@ -4624,6 +4791,29 @@ done: * Added initialization for the new free_file_space_on_destroy * field. * + * JRM -- 11/13/08 + * Moved test to see if we already have an entry with the + * specified address in the cache. This was necessary as + * we used to modify some fields in the entry to be inserted + * priort to this test, which got the cache confused if the + * insertion failed because the entry was already present. + * + * Also revised the function to call H5C_make_space_in_cache() + * if the min_clean_size is not met at present, not just if + * there is insufficient space in the cache for the new + * entry. + * + * The purpose of this modification is to avoid "metadata + * blizzards" in the write only case. In such instances, + * the cache was allowed to fill with dirty metadata. When + * we finally needed to evict an entry to make space, we had + * to flush out a whole cache full of metadata -- which has + * interesting performance effects. We hope to avoid (or + * perhaps more accurately hide) this effect by maintaining + * the min_clean_size, which should force us to start flushing + * entries long before we actually have to evict something + * to make space. + * *------------------------------------------------------------------------- */ @@ -4644,6 +4834,7 @@ H5C_insert_entry(H5F_t * f, hbool_t insert_pinned; hbool_t set_flush_marker; hbool_t write_permitted = TRUE; + size_t empty_space; H5C_cache_entry_t * entry_ptr; H5C_cache_entry_t * test_entry_ptr; @@ -4677,12 +4868,39 @@ H5C_insert_entry(H5F_t * f, insert_pinned = ( (flags & H5C__PIN_ENTRY_FLAG) != 0 ); entry_ptr = (H5C_cache_entry_t *)thing; + + /* verify that the new entry isn't already in the hash table -- scream + * and die if it is. + */ + + H5C__SEARCH_INDEX(cache_ptr, addr, test_entry_ptr, FAIL) + + if ( test_entry_ptr != NULL ) { + + if ( test_entry_ptr == entry_ptr ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, \ + "entry already in cache.") + + } else { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, \ + "duplicate entry in cache.") + } + } + #ifndef NDEBUG entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC; #endif /* NDEBUG */ entry_ptr->addr = addr; entry_ptr->type = type; + entry_ptr->is_protected = FALSE; + entry_ptr->is_read_only = FALSE; + entry_ptr->ro_ref_count = 0; + + entry_ptr->is_pinned = insert_pinned; + /* newly inserted entries are assumed to be dirty */ entry_ptr->is_dirty = TRUE; @@ -4730,13 +4948,35 @@ H5C_insert_entry(H5F_t * f, } } - if ( ( cache_ptr->evictions_enabled ) && - ( (cache_ptr->index_size + entry_ptr->size) > - cache_ptr->max_cache_size ) ) { + if ( cache_ptr->index_size >= cache_ptr->max_cache_size ) { + + empty_space = 0; + + } else { + + empty_space = cache_ptr->max_cache_size - cache_ptr->index_size; + + } + + if ( ( cache_ptr->evictions_enabled ) + && + ( ( (cache_ptr->index_size + entry_ptr->size) > + cache_ptr->max_cache_size + ) + || + ( + ( ( empty_space + cache_ptr->clean_index_size ) < + cache_ptr->min_clean_size ) + ) + ) + ) { size_t space_needed; - cache_ptr->cache_full = TRUE; + if ( empty_space <= entry_ptr->size ) { + + cache_ptr->cache_full = TRUE; + } if ( cache_ptr->check_write_permitted != NULL ) { @@ -4803,38 +5043,6 @@ H5C_insert_entry(H5F_t * f, } } - /* verify that the new entry isn't already in the hash table -- scream - * and die if it is. - */ - - H5C__SEARCH_INDEX(cache_ptr, addr, test_entry_ptr, FAIL) - - if ( test_entry_ptr != NULL ) { - - if ( test_entry_ptr == entry_ptr ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, \ - "entry already in cache.") - - } else { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, \ - "duplicate entry in cache.") - } - } - - /* we don't initialize the protected field until here as it is - * possible that the entry is already in the cache, and already - * protected. If it is, we don't want to make things worse by - * marking it unprotected. - */ - - entry_ptr->is_protected = FALSE; - entry_ptr->is_read_only = FALSE; - entry_ptr->ro_ref_count = 0; - - entry_ptr->is_pinned = insert_pinned; - H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, FAIL) /* New entries are presumed to be dirty, so this if statement is @@ -5209,6 +5417,19 @@ done: * appropriate. * JRM -- 1/11/08 * + * + * Added code to update the clean_index_size and + * dirty_index_size fields of H5C_t in cases where the + * the entry was clean on protect, was marked dirty in + * this call, and did not change its size. Do this via + * a call to H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(). + * + * If the size changed, this case is already dealt with by + * by the pre-existing call to + * H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(). + * + * JRM -- 11/5/08 + * *------------------------------------------------------------------------- */ herr_t @@ -5219,6 +5440,7 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr, { herr_t ret_value = SUCCEED; /* Return value */ herr_t result; + hbool_t was_clean; size_t size_increase; H5C_cache_entry_t * entry_ptr; @@ -5243,6 +5465,9 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr, "Entry is protected??") } + /* make note of whether the entry was dirty to begin with */ + was_clean = ! ( entry_ptr->is_dirty ); + /* mark the entry as dirty if it isn't already */ entry_ptr->is_dirty = TRUE; @@ -5278,8 +5503,8 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr, (entry_ptr->size), (new_size)); /* update the hash table */ - H5C__UPDATE_INDEX_FOR_SIZE_CHANGE((cache_ptr), (entry_ptr->size),\ - (new_size)); + H5C__UPDATE_INDEX_FOR_SIZE_CHANGE((cache_ptr), (entry_ptr->size), \ + (new_size), (entry_ptr), (was_clean)); /* if the entry is in the skip list, update that too */ if ( entry_ptr->in_slist ) { @@ -5294,6 +5519,10 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr, /* finally, update the entry size proper */ entry_ptr->size = new_size; + + } else if ( ( was_clean ) && ( entry_ptr->is_dirty ) ) { + + H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(cache_ptr, entry_ptr) } if ( ! (entry_ptr->in_slist) ) { @@ -5341,6 +5570,12 @@ done: * it once we deal with the problem of entries being protected * read only, and then dirtied. * + * JRM -- 11/5/08 + * Added call to H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY() to + * update the new clean_index_size and dirty_index_size + * fields of H5C_t in the case that the entry was clean + * prior to this call, and is pinned and not protected. + * *------------------------------------------------------------------------- */ herr_t @@ -5348,6 +5583,7 @@ H5C_mark_pinned_or_protected_entry_dirty(H5C_t * cache_ptr, void * thing) { herr_t ret_value = SUCCEED; /* Return value */ + hbool_t was_pinned_unprotected_and_clean; H5C_cache_entry_t * entry_ptr; FUNC_ENTER_NOAPI(H5C_mark_pinned_or_protected_entry_dirty, FAIL) @@ -5367,9 +5603,15 @@ H5C_mark_pinned_or_protected_entry_dirty(H5C_t * cache_ptr, } else if ( entry_ptr->is_pinned ) { + was_pinned_unprotected_and_clean = ! ( entry_ptr->is_dirty ); + /* mark the entry as dirty if it isn't already */ entry_ptr->is_dirty = TRUE; + if ( was_pinned_unprotected_and_clean ) { + + H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(cache_ptr, entry_ptr); + } if ( ! (entry_ptr->in_slist) ) { @@ -5429,6 +5671,11 @@ done: * Note that in this case H5C_flush_single_entry() will handle * all these details for us. * + * JRM -- 11/5/08 + * On review this function looks like no change is needed to + * support the new clean_index_size and dirty_index_size + * fields of H5C_t. + * *------------------------------------------------------------------------- */ @@ -5591,7 +5838,7 @@ done: * Function: H5C_resize_pinned_entry * * Purpose: Resize a pinned entry. The target entry MUST be - * be pinned, and MUST not be unprotected. + * be pinned, and MUST be unprotected. * * Resizing an entry dirties it, so if the entry is not * already dirty, the function places the entry on the @@ -5608,6 +5855,19 @@ done: * appropriate. * JRM -- 1/11/08 * + * Added code to update the clean_index_size and + * dirty_index_size fields of H5C_t in cases where the + * the entry was clean prior to this call, was marked dirty, + * and did not change its size. Do this via a call to + * H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(). + * + * If the size changed, this case is already dealt with by + * by the pre-existing call to + * H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(). + * + * JRM -- 11/5/08 + * + * *------------------------------------------------------------------------- */ herr_t @@ -5618,6 +5878,7 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr, /* const char * fcn_name = "H5C_resize_pinned_entry()"; */ herr_t ret_value = SUCCEED; /* Return value */ herr_t result; + hbool_t was_clean; H5C_cache_entry_t * entry_ptr; size_t size_increase; @@ -5647,6 +5908,9 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr, "Entry is protected??") } + /* make note of whether the entry was clean to begin with */ + was_clean = ! ( entry_ptr->is_dirty ); + /* resizing dirties entries -- mark the entry as dirty if it * isn't already */ @@ -5685,7 +5949,7 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr, /* update the hash table */ H5C__UPDATE_INDEX_FOR_SIZE_CHANGE((cache_ptr), (entry_ptr->size),\ - (new_size)); + (new_size), (entry_ptr), (was_clean)); /* if the entry is in the skip list, update that too */ if ( entry_ptr->in_slist ) { @@ -5700,8 +5964,13 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr, /* finally, update the entry size proper */ entry_ptr->size = new_size; + + } else if ( was_clean ) { + + H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(cache_ptr, entry_ptr) } + if ( ! (entry_ptr->in_slist) ) { H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) @@ -5872,6 +6141,22 @@ done: * Added to do a flash cache size increase if appropriate * when a large entry is loaded. * + * JRM -- 11/13/08 + * Modified function to call H5C_make_space_in_cache() when + * the min_clean_size is violated, not just when there isn't + * enough space for and entry that has just been loaded. + * + * The purpose of this modification is to avoid "metadata + * blizzards" in the write only case. In such instances, + * the cache was allowed to fill with dirty metadata. When + * we finally needed to evict an entry to make space, we had + * to flush out a whole cache full of metadata -- which has + * interesting performance effects. We hope to avoid (or + * perhaps more accurately hide) this effect by maintaining + * the min_clean_size, which should force us to start flushing + * entries long before we actually have to evict something + * to make space. + * *------------------------------------------------------------------------- */ @@ -5893,6 +6178,7 @@ H5C_protect(H5F_t * f, hbool_t read_only = FALSE; hbool_t write_permitted; herr_t result; + size_t empty_space; void * thing; H5C_cache_entry_t * entry_ptr; void * ret_value; /* Return value */ @@ -5960,16 +6246,41 @@ H5C_protect(H5F_t * f, } } - /* try to free up some space if necessary and if evictions are - * permitted + if ( cache_ptr->index_size >= cache_ptr->max_cache_size ) { + + empty_space = 0; + + } else { + + empty_space = cache_ptr->max_cache_size - cache_ptr->index_size; + + } + + /* try to free up if necceary and if evictions are permitted. Note + * that if evictions are enabled, we will call H5C_make_space_in_cache() + * regardless if the min_free_space requirement is not met. */ - if ( ( cache_ptr->evictions_enabled ) && - ( (cache_ptr->index_size + entry_ptr->size) > - cache_ptr->max_cache_size ) ) { + + if ( ( cache_ptr->evictions_enabled ) + && + ( ( (cache_ptr->index_size + entry_ptr->size) > + cache_ptr->max_cache_size + ) + || + ( + ( ( empty_space + cache_ptr->clean_index_size ) < + cache_ptr->min_clean_size ) + ) + ) + ) { size_t space_needed; - cache_ptr->cache_full = TRUE; + if ( empty_space <= entry_ptr->size ) { + + cache_ptr->cache_full = TRUE; + + } if ( cache_ptr->check_write_permitted != NULL ) { @@ -6160,10 +6471,31 @@ H5C_protect(H5F_t * f, /* check to see if the cache is now oversized due to the cache * size reduction. If it is, try to evict enough entries to * bring the cache size down to the current maximum cache size. + * + * Also, if the min_clean_size requirement is not met, we + * should also call H5C_make_space_in_cache() to bring us + * into complience. */ - if ( cache_ptr->index_size > cache_ptr->max_cache_size ) { - cache_ptr->cache_full = TRUE; + if ( cache_ptr->index_size >= cache_ptr->max_cache_size ) { + + empty_space = 0; + + } else { + + empty_space = cache_ptr->max_cache_size - cache_ptr->index_size; + + } + + if ( ( cache_ptr->index_size > cache_ptr->max_cache_size ) + || + ( ( empty_space + cache_ptr->clean_index_size ) < + cache_ptr->min_clean_size) ) { + + if ( cache_ptr->index_size > cache_ptr->max_cache_size ) { + + cache_ptr->cache_full = TRUE; + } result = H5C_make_space_in_cache(f, primary_dxpl_id, secondary_dxpl_id, cache_ptr, @@ -6761,6 +7093,15 @@ done: * Added code supporting the new write_protects, * read_protects, and max_read_protects fields. * + * JRM -- 11/13/08 + * Added code displaying the max_clean_index_size and + * max_dirty_index_size. + * + * MAM -- 01/06/09 + * Added code displaying the calls_to_msic, + * total_entries_skipped_in_msic, total_entries_scanned_in_msic, + * and max_entries_skipped_in_msic fields. + * *------------------------------------------------------------------------- */ @@ -6808,6 +7149,8 @@ H5C_stats(H5C_t * cache_ptr, double hit_rate; double average_successful_search_depth = 0.0; double average_failed_search_depth = 0.0; + double average_entries_skipped_per_calls_to_msic = 0.0; + double average_entries_scanned_per_calls_to_msic = 0.0; #endif /* H5C_COLLECT_CACHE_STATS */ FUNC_ENTER_NOAPI(H5C_stats, FAIL) @@ -6927,6 +7270,14 @@ H5C_stats(H5C_t * cache_ptr, (long)(cache_ptr->max_index_len)); HDfprintf(stdout, + "%s current (max) clean/dirty idx size = %ld (%ld) / %ld (%ld)\n", + cache_ptr->prefix, + (long)(cache_ptr->clean_index_size), + (long)(cache_ptr->max_clean_index_size), + (long)(cache_ptr->dirty_index_size), + (long)(cache_ptr->max_dirty_index_size)); + + HDfprintf(stdout, "%s current (max) slist size / length = %ld (%ld) / %ld (%ld)\n", cache_ptr->prefix, (long)(cache_ptr->slist_size), @@ -7024,6 +7375,41 @@ H5C_stats(H5C_t * cache_ptr, (long)total_pinned_flushes, (long)total_pinned_clears); + HDfprintf(stdout, "%s MSIC: (make space in cache) calls = %lld\n", + cache_ptr->prefix, + (long long)(cache_ptr->calls_to_msic)); + + if (cache_ptr->calls_to_msic > 0) { + average_entries_skipped_per_calls_to_msic = + (((double)(cache_ptr->total_entries_skipped_in_msic)) / + ((double)(cache_ptr->calls_to_msic))); + } + + HDfprintf(stdout, "%s MSIC: Average/max entries skipped = %lf / %ld\n", + cache_ptr->prefix, + (float)average_entries_skipped_per_calls_to_msic, + (long)(cache_ptr->max_entries_skipped_in_msic)); + + if (cache_ptr->calls_to_msic > 0) { + average_entries_scanned_per_calls_to_msic = + (((double)(cache_ptr->total_entries_scanned_in_msic)) / + ((double)(cache_ptr->calls_to_msic))); + } + + HDfprintf(stdout, "%s MSIC: Average/max entries scanned = %lf / %ld\n", + cache_ptr->prefix, + (float)average_entries_scanned_per_calls_to_msic, + (long)(cache_ptr->max_entries_scanned_in_msic)); + + HDfprintf(stdout, "%s MSIC: Scanned to make space(evict) = %lld\n", + cache_ptr->prefix, + (long long)(cache_ptr->entries_scanned_to_make_space)); + + HDfprintf(stdout, "%s MSIC: Scanned to satisfy min_clean = %lld\n", + cache_ptr->prefix, + (long long)(cache_ptr->total_entries_scanned_in_msic - + cache_ptr->entries_scanned_to_make_space)); + #if H5C_COLLECT_CACHE_ENTRY_STATS HDfprintf(stdout, "%s aggregate max / min accesses = %d / %d\n", @@ -7192,6 +7578,15 @@ done: * Added initialization for the new write_protects, * read_protects, and max_read_protects fields. * + * JRM 11/13/08 + * Added initialization for the new max_clean_index_size and + * max_dirty_index_size fields. + * + * MAM -- 01/06/09 + * Added code to initalize the calls_to_msic, + * total_entries_skipped_in_msic, total_entries_scanned_in_msic, + * and max_entries_skipped_in_msic fields. + * *------------------------------------------------------------------------- */ @@ -7249,6 +7644,8 @@ H5C_stats__reset(H5C_t UNUSED * cache_ptr) cache_ptr->max_index_len = 0; cache_ptr->max_index_size = (size_t)0; + cache_ptr->max_clean_index_size = (size_t)0; + cache_ptr->max_dirty_index_size = (size_t)0; cache_ptr->max_slist_len = 0; cache_ptr->max_slist_size = (size_t)0; @@ -7259,6 +7656,13 @@ H5C_stats__reset(H5C_t UNUSED * cache_ptr) cache_ptr->max_pel_len = 0; cache_ptr->max_pel_size = (size_t)0; + cache_ptr->calls_to_msic = 0; + cache_ptr->total_entries_skipped_in_msic = 0; + cache_ptr->total_entries_scanned_in_msic = 0; + cache_ptr->max_entries_skipped_in_msic = 0; + cache_ptr->max_entries_scanned_in_msic = 0; + cache_ptr->entries_scanned_to_make_space = 0; + #if H5C_COLLECT_CACHE_ENTRY_STATS for ( i = 0; i <= cache_ptr->max_type_id; i++ ) @@ -7437,6 +7841,17 @@ done: * Separated "destroy entry" concept from "remove entry from * cache" concept, by adding the 'take_ownership' flag. * + * JRM -- 11/5/08 + * Added code to update the clean_index_size and + * dirty_index_size fields of H5C_t in cases where the + * the entry was clean on protect, was marked dirty on + * unprotect, and did not change its size. Do this via + * a call to H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(). + * + * If the size changed, this case is already dealt with by + * by the pre-existing call to + * H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(). + * *------------------------------------------------------------------------- */ herr_t @@ -7459,6 +7874,7 @@ H5C_unprotect(H5F_t * f, hbool_t unpin_entry; hbool_t free_file_space; hbool_t take_ownership; + hbool_t was_clean; #ifdef H5_HAVE_PARALLEL hbool_t clear_entry = FALSE; #endif /* H5_HAVE_PARALLEL */ @@ -7510,6 +7926,7 @@ H5C_unprotect(H5F_t * f, * the entry. */ dirtied |= entry_ptr->dirtied; + was_clean = ! ( entry_ptr->is_dirty ); #if H5C_DO_EXTREME_SANITY_CHECKS if ( H5C_validate_lru_list(cache_ptr) < 0 ) { @@ -7646,8 +8063,9 @@ H5C_unprotect(H5F_t * f, (entry_ptr->size), (new_size)); /* update the hash table */ - H5C__UPDATE_INDEX_FOR_SIZE_CHANGE((cache_ptr), (entry_ptr->size),\ - (new_size)); + H5C__UPDATE_INDEX_FOR_SIZE_CHANGE((cache_ptr), (entry_ptr->size), \ + (new_size), (entry_ptr), \ + (was_clean)); /* if the entry is in the skip list, update that too */ if ( entry_ptr->in_slist ) { @@ -7663,7 +8081,11 @@ H5C_unprotect(H5F_t * f, /* finally, update the entry size proper */ entry_ptr->size = new_size; - } + + } else if ( ( was_clean ) && ( entry_ptr->is_dirty ) ) { + + H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(cache_ptr, entry_ptr) + } /* Pin or unpin the entry as requested. */ if ( pin_entry ) { @@ -9929,6 +10351,8 @@ H5C_flush_invalidate_cache(H5F_t * f, done = TRUE; HDassert( cache_ptr->index_size == 0 ); + HDassert( cache_ptr->clean_index_size == 0 ); + HDassert( cache_ptr->dirty_index_size == 0 ); HDassert( cache_ptr->slist_len == 0 ); HDassert( cache_ptr->slist_size == 0 ); HDassert( cache_ptr->pel_len == 0 ); @@ -10053,6 +10477,11 @@ done: * cache" concept, by adding the 'take_ownership' flag and * the "destroy_entry" variable. * + * JRM -- 11/5/08 + * Added call to H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN() to + * maintain the new clean_index_size and clean_index_size + * fields of H5C_t. + * *------------------------------------------------------------------------- */ static herr_t @@ -10413,6 +10842,7 @@ H5C_flush_single_entry(H5F_t * f, HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ "unable to flush entry") } + #ifdef H5_HAVE_PARALLEL if ( flush_flags != H5C_CALLBACK__NO_FLAGS_SET ) { @@ -10455,6 +10885,12 @@ H5C_flush_single_entry(H5F_t * f, if ( ( ! destroy ) && ( entry_ptr->in_slist ) ) { H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr) + + } + + if ( ( ! destroy ) && ( was_dirty ) ) { + + H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN(cache_ptr, entry_ptr); } if ( ! destroy ) { /* i.e. if the entry still exists */ @@ -10489,10 +10925,18 @@ H5C_flush_single_entry(H5F_t * f, HDassert( entry_ptr->size < H5C_MAX_ENTRY_SIZE ); - /* update the hash table for the size change*/ + /* update the hash table for the size change + * We pass TRUE as the was_clean parameter, as we + * have already updated the clean and dirty index + * size fields for the fact that the entry has + * been flushed. (See above call to + * H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN()). + */ H5C__UPDATE_INDEX_FOR_SIZE_CHANGE((cache_ptr), \ (entry_ptr->size),\ - (new_size)); + (new_size), \ + (entry_ptr), \ + (TRUE)); /* The entry can't be protected since we just flushed it. * Thus we must update the replacement policy data @@ -10769,6 +11213,27 @@ done: * but one can argue that I should just scream and die if I * ever detect the condidtion. * + * JRM -- 11/13/08 + * Modified function to always observe the min_clean_size + * whether we are maintaining the clean and dirt LRU lists + * or not. To do this, we had to add the new clean_index_size + * and dirty_index_size fields to H5C_t, and supporting code + * as needed throughout the cache. + * + * The purpose of this modification is to avoid "metadata + * blizzards" in the write only case. In such instances, + * the cache was allowed to fill with dirty metadata. When + * we finally needed to evict an entry to make space, we had + * to flush out a whole cache full of metadata -- which has + * interesting performance effects. We hope to avoid (or + * perhaps more accurately hide) this effect by maintaining + * the min_clean_size, which should force us to start flushing + * entries long before we actually have to evict something + * to make space. + * + * MAM -- 01/06/09 + * Added code to maintain clean_entries_skipped and total_entries + * scanned statistics. *------------------------------------------------------------------------- */ @@ -10783,13 +11248,15 @@ H5C_make_space_in_cache(H5F_t * f, { herr_t ret_value = SUCCEED; /* Return value */ herr_t result; +#if H5C_COLLECT_CACHE_STATS + int32_t clean_entries_skipped = 0; + int32_t total_entries_scanned = 0; +#endif /* H5C_COLLECT_CACHE_STATS */ int32_t entries_examined = 0; int32_t initial_list_len; -#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS size_t empty_space; -#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ hbool_t prev_is_dirty = FALSE; - hbool_t entry_is_epoch_maker = FALSE; + hbool_t didnt_flush_entry = FALSE; H5C_cache_entry_t * entry_ptr; H5C_cache_entry_t * next_ptr; H5C_cache_entry_t * prev_ptr; @@ -10800,16 +11267,36 @@ H5C_make_space_in_cache(H5F_t * f, HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); HDassert( first_flush_ptr != NULL ); HDassert( ( *first_flush_ptr == TRUE ) || ( *first_flush_ptr == FALSE ) ); + HDassert( cache_ptr->index_size == + (cache_ptr->clean_index_size + cache_ptr->dirty_index_size) ); if ( write_permitted ) { initial_list_len = cache_ptr->LRU_list_len; + entry_ptr = cache_ptr->LRU_tail_ptr; - while ( ( (cache_ptr->index_size + space_needed) - > - cache_ptr->max_cache_size - ) + if ( cache_ptr->index_size >= cache_ptr->max_cache_size ) { + + empty_space = 0; + + } else { + + empty_space = cache_ptr->max_cache_size - cache_ptr->index_size; + + } + + while ( ( ( (cache_ptr->index_size + space_needed) + > + cache_ptr->max_cache_size + ) + || + ( + ( empty_space + cache_ptr->clean_index_size ) + < + ( cache_ptr->min_clean_size ) + ) + ) && ( entries_examined <= (2 * initial_list_len) ) && @@ -10830,10 +11317,18 @@ H5C_make_space_in_cache(H5F_t * f, if ( (entry_ptr->type)->id != H5C__EPOCH_MARKER_TYPE ) { - entry_is_epoch_maker = FALSE; + didnt_flush_entry = FALSE; if ( entry_ptr->is_dirty ) { +#if H5C_COLLECT_CACHE_STATS + if ( (cache_ptr->index_size + space_needed) + > + cache_ptr->max_cache_size ) { + + cache_ptr->entries_scanned_to_make_space++; + } +#endif /* H5C_COLLECT_CACHE_STATS */ result = H5C_flush_single_entry(f, primary_dxpl_id, secondary_dxpl_id, @@ -10843,7 +11338,12 @@ H5C_make_space_in_cache(H5F_t * f, H5C__NO_FLAGS_SET, first_flush_ptr, FALSE); - } else { + } else if ( (cache_ptr->index_size + space_needed) + > + cache_ptr->max_cache_size ) { +#if H5C_COLLECT_CACHE_STATS + cache_ptr->entries_scanned_to_make_space++; +#endif /* H5C_COLLECT_CACHE_STATS */ result = H5C_flush_single_entry(f, primary_dxpl_id, @@ -10854,13 +11354,31 @@ H5C_make_space_in_cache(H5F_t * f, H5C__FLUSH_INVALIDATE_FLAG, first_flush_ptr, TRUE); + } else { + + /* We have enough space so don't flush clean entry. + * Set result to SUCCEED to avoid triggering the error + * code below. + */ +#if H5C_COLLECT_CACHE_STATS + clean_entries_skipped++; +#endif /* H5C_COLLECT_CACHE_STATS */ + didnt_flush_entry = TRUE; + result = SUCCEED; + } + +#if H5C_COLLECT_CACHE_STATS + total_entries_scanned++; +#endif /* H5C_COLLECT_CACHE_STATS */ + + } else { /* Skip epoch markers. Set result to SUCCEED to avoid * triggering the error code below. */ - entry_is_epoch_maker = TRUE; + didnt_flush_entry = TRUE; result = SUCCEED; } @@ -10882,7 +11400,7 @@ H5C_make_space_in_cache(H5F_t * f, } #endif /* NDEBUG */ - if ( entry_is_epoch_maker ) { + if ( didnt_flush_entry ) { entry_ptr = prev_ptr; @@ -10913,121 +11431,52 @@ H5C_make_space_in_cache(H5F_t * f, entries_examined++; - } - -#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS - - entries_examined = 0; - initial_list_len = cache_ptr->dLRU_list_len; - entry_ptr = cache_ptr->dLRU_tail_ptr; - - if ( cache_ptr->index_size < cache_ptr->max_cache_size ) { - - empty_space = cache_ptr->max_cache_size - cache_ptr->index_size; - - } else { + if ( cache_ptr->index_size >= cache_ptr->max_cache_size ) { - empty_space = 0; - } - - while ( ( (cache_ptr->cLRU_list_size + empty_space) - < cache_ptr->min_clean_size ) && - ( entries_examined <= initial_list_len ) && - ( entry_ptr != NULL ) - ) - { - HDassert( ! (entry_ptr->is_protected) ); - HDassert( ! (entry_ptr->is_read_only) ); - HDassert( (entry_ptr->ro_ref_count) == 0 ); - HDassert( entry_ptr->is_dirty ); - HDassert( entry_ptr->in_slist ); - - prev_ptr = entry_ptr->aux_prev; - - next_ptr = entry_ptr->aux_next; - - if ( prev_ptr != NULL ) { - - HDassert( prev_ptr->is_dirty ); - } - - result = H5C_flush_single_entry(f, - primary_dxpl_id, - secondary_dxpl_id, - cache_ptr, - entry_ptr->type, - entry_ptr->addr, - H5C__NO_FLAGS_SET, - first_flush_ptr, - FALSE); + empty_space = 0; - if ( result < 0 ) { + } else { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unable to flush entry") - } + empty_space = cache_ptr->max_cache_size - cache_ptr->index_size; - if ( prev_ptr != NULL ) { -#ifndef NDEBUG - if (prev_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC) { + } + + HDassert( cache_ptr->index_size == + (cache_ptr->clean_index_size + + cache_ptr->dirty_index_size) ); - /* something horrible has happened to *prev_ptr -- - * scream and die. - */ + } - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "*prev_ptr corrupt 2") +#if H5C_COLLECT_CACHE_STATS + cache_ptr->calls_to_msic++; - } else -#endif /* #ifndef NDEBUG */ - if ( ( ! ( prev_ptr->is_dirty ) ) - || - ( prev_ptr->aux_next != next_ptr ) - || - ( prev_ptr->is_protected ) - || - ( prev_ptr->is_pinned ) ) { - - /* something has happened to the dirty LRU -- start over - * from the tail. - */ -#if 0 /* This debuging code may be useful in the future -- keep it for now. */ - if ( ! ( prev_ptr->is_dirty ) ) { - HDfprintf(stdout, "%s: ! prev_ptr->is_dirty\n", - fcn_name); - } - if ( prev_ptr->aux_next != next_ptr ) { - HDfprintf(stdout, "%s: prev_ptr->next != next_ptr\n", - fcn_name); - } - if ( prev_ptr->is_protected ) { - HDfprintf(stdout, "%s: prev_ptr->is_protected\n", - fcn_name); - } - if ( prev_ptr->is_pinned ) { - HDfprintf(stdout, "%s:prev_ptr->is_pinned\n", - fcn_name); - } + cache_ptr->total_entries_skipped_in_msic += clean_entries_skipped; + cache_ptr->total_entries_scanned_in_msic += total_entries_scanned; - HDfprintf(stdout, "%s: re-starting scan of dirty list\n", - fcn_name); -#endif /* JRM */ - entry_ptr = cache_ptr->dLRU_tail_ptr; + if ( clean_entries_skipped > cache_ptr->max_entries_skipped_in_msic ) { - } else { + cache_ptr->max_entries_skipped_in_msic = clean_entries_skipped; + } - entry_ptr = prev_ptr; + if ( total_entries_scanned > cache_ptr->max_entries_scanned_in_msic ) { - } - } else { + cache_ptr->max_entries_scanned_in_msic = total_entries_scanned; + } +#endif /* H5C_COLLECT_CACHE_STATS */ - entry_ptr = NULL; + HDassert( ( entries_examined > (2 * initial_list_len) ) || + ( (cache_ptr->pl_size + cache_ptr->min_clean_size) > + cache_ptr->max_cache_size ) || + ( ( cache_ptr->clean_index_size + empty_space ) + >= cache_ptr->min_clean_size ) ); - } +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS - entries_examined++; - } + HDassert( ( entries_examined > (2 * initial_list_len) ) || + ( cache_ptr->cLRU_list_size <= cache_ptr->clean_index_size ) ); + HDassert( ( entries_examined > (2 * initial_list_len) ) || + ( cache_ptr->dLRU_list_size <= cache_ptr->dirty_index_size ) ); #endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index bdbb25e..5e7b1a7 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -203,6 +203,38 @@ * index_size by two should yield a conservative estimate * of the cache's memory footprint. * + * clean_index_size: Number of bytes of clean entries currently stored in + * the hash table. Note that the index_size field (above) + * is also the sum of the sizes of all entries in the cache. + * Thus we should have the invarient that clean_index_size + + * dirty_index_size == index_size. + * + * WARNING: + * + * 1) The clean_index_size field is not maintained by the + * index macros, as the hash table doesn't care whether + * the entry is clean or dirty. Instead the field is + * maintained in the H5C__UPDATE_RP macros. + * + * 2) The value of the clean_index_size must not be mistaken + * for the current clean size of the cache. Rather, the + * clean size of the cache is the current value of + * clean_index_size plus the amount of empty space (if any) + * in the cache. + * + * dirty_index_size: Number of bytes of dirty entries currently stored in + * the hash table. Note that the index_size field (above) + * is also the sum of the sizes of all entries in the cache. + * Thus we should have the invarient that clean_index_size + + * dirty_index_size == index_size. + * + * WARNING: + * + * 1) The dirty_index_size field is not maintained by the + * index macros, as the hash table doesn't care whether + * the entry is clean or dirty. Instead the field is + * maintained in the H5C__UPDATE_RP macros. + * * index: Array of pointer to H5C_cache_entry_t of size * H5C__HASH_TABLE_LEN. At present, this value is a power * of two, not the usual prime number. @@ -722,6 +754,12 @@ * max_index_size: Largest value attained by the index_size field in the * current epoch. * + * max_clean_index_size: Largest value attained by the clean_index_size field + * in the current epoch. + * + * max_dirty_index_size: Largest value attained by the dirty_index_size field + * in the current epoch. + * * max_slist_len: Largest value attained by the slist_len field in the * current epoch. * @@ -740,6 +778,23 @@ * max_pel_size: Largest value attained by the pel_size field in the * current epoch. * + * calls_to_msic: Total number of calls to H5C_make_space_in_cache + * + * total_entries_skipped_in_msic: Number of clean entries skipped while + * enforcing the min_clean_fraction in H5C_make_space_in_cache(). + * + * total_entries_scanned_in_msic: Number of clean entries skipped while + * enforcing the min_clean_fraction in H5C_make_space_in_cache(). + * + * max_entries_skipped_in_msic: Maximum number of clean entries skipped + * in any one call to H5C_make_space_in_cache(). + * + * max_entries_scanned_in_msic: Maximum number of entries scanned over + * in any one call to H5C_make_space_in_cache(). + * + * entries_scanned_to_make_space: Number of entries scanned only when looking + * for entries to evict in order to make space in cache. + * The remaining stats are collected only when both H5C_COLLECT_CACHE_STATS * and H5C_COLLECT_CACHE_ENTRY_STATS are true. * @@ -830,6 +885,8 @@ struct H5C_t int32_t index_len; size_t index_size; + size_t clean_index_size; + size_t dirty_index_size; H5C_cache_entry_t * (index[H5C__HASH_TABLE_LEN]); @@ -923,6 +980,8 @@ struct H5C_t int32_t max_index_len; size_t max_index_size; + size_t max_clean_index_size; + size_t max_dirty_index_size; int32_t max_slist_len; size_t max_slist_size; @@ -933,6 +992,13 @@ struct H5C_t int32_t max_pel_len; size_t max_pel_size; + int64_t calls_to_msic; + int64_t total_entries_skipped_in_msic; + int64_t total_entries_scanned_in_msic; + int32_t max_entries_skipped_in_msic; + int32_t max_entries_scanned_in_msic; + int64_t entries_scanned_to_make_space; + #if H5C_COLLECT_CACHE_ENTRY_STATS int32_t max_accesses[H5C__MAX_NUM_TYPE_IDS + 1]; diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index fb28c4e..e717b99 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -259,6 +259,7 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; #define H5F_GC_REF(F) ((F)->shared->gc_ref) #define H5F_USE_LATEST_FORMAT(F) ((F)->shared->latest_format) #define H5F_EXTPATH(F) ((F)->extpath) +#define H5F_NAME(F) ((F)->name) #define H5F_GET_FC_DEGREE(F) ((F)->shared->fc_degree) #define H5F_STORE_MSG_CRT_IDX(F) ((F)->shared->store_msg_crt_idx) #define H5F_HAS_FEATURE(F,FL) ((F)->shared->lf->feature_flags & (FL)) @@ -280,6 +281,7 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; #define H5F_GC_REF(F) (H5F_gc_ref(F)) #define H5F_USE_LATEST_FORMAT(F) (H5F_use_latest_format(F)) #define H5F_EXTPATH(F) (H5F_get_extpath(F)) +#define H5F_NAME(F) (H5F_get_name(F)) #define H5F_GET_FC_DEGREE(F) (H5F_get_fc_degree(F)) #define H5F_STORE_MSG_CRT_IDX(F) (H5F_store_msg_crt_idx(F)) #define H5F_HAS_FEATURE(F,FL) (H5F_has_feature(F,FL)) @@ -466,6 +468,7 @@ H5_DLL unsigned H5F_decr_nopen_objs(H5F_t *f); H5_DLL unsigned H5F_get_intent(const H5F_t *f); H5_DLL hid_t H5F_get_access_plist(H5F_t *f, hbool_t app_ref); H5_DLL char *H5F_get_extpath(const H5F_t *f); +H5_DLL char *H5F_get_name(const H5F_t *f); H5_DLL hid_t H5F_get_id(H5F_t *file, hbool_t app_ref); H5_DLL size_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref); H5_DLL size_t H5F_get_obj_ids(const H5F_t *f, unsigned types, size_t max_objs, hid_t *obj_id_list, hbool_t app_ref); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 312d92e..287e9b5 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -50,6 +50,10 @@ #define H5F_ACC_DEBUG (H5CHECK 0x0008u) /*print debug info */ #define H5F_ACC_CREAT (H5CHECK 0x0010u) /*create non-existing files */ +/* Value passed to H5Pset_elink_acc_flags to cause flags to be taken from the + * parent file. */ +#define H5F_ACC_DEFAULT (H5CHECK 0xffffu) /*ignore setting on lapl */ + /* Flags for H5Fget_obj_count() & H5Fget_obj_ids() calls */ #define H5F_OBJ_FILE (0x0001u) /* File objects */ #define H5F_OBJ_DATASET (0x0002u) /* Dataset objects */ diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 550a507..275061d 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -130,6 +130,32 @@ H5F_get_extpath(const H5F_t *f) /*------------------------------------------------------------------------- + * Function: H5F_get_name + * + * Purpose: Retrieve the name of a file. + * + * Return: Success: The name of the file. + * + * Failure: ? (should not happen) + * + * Programmer: Neil Fortner + * December 15 2008 + * + *------------------------------------------------------------------------- + */ +char * +H5F_get_name(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOFUNC here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5F_get_name) + + HDassert(f); + + FUNC_LEAVE_NOAPI(f->name) +} /* end H5F_get_name() */ + + +/*------------------------------------------------------------------------- * Function: H5F_get_fcpl * * Purpose: Retrieve the value of a file's FCPL. diff --git a/src/H5Glink.c b/src/H5Glink.c index 9527521..7051453 100644 --- a/src/H5Glink.c +++ b/src/H5Glink.c @@ -41,6 +41,7 @@ #include "H5Iprivate.h" /* IDs */ #include "H5Lprivate.h" /* Links */ #include "H5MMprivate.h" /* Memory management */ +#include "H5Ppublic.h" /* Property Lists */ /****************/ diff --git a/src/H5Gnode.c b/src/H5Gnode.c index 9468302..6b16544 100644 --- a/src/H5Gnode.c +++ b/src/H5Gnode.c @@ -39,6 +39,7 @@ #include "H5HLprivate.h" /* Local Heaps */ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ +#include "H5Ppublic.h" /* Property Lists */ /* Private typedefs */ diff --git a/src/H5Lexternal.c b/src/H5Lexternal.c index 46ed467..2c6b3a3 100644 --- a/src/H5Lexternal.c +++ b/src/H5Lexternal.c @@ -197,6 +197,9 @@ H5L_extern_traverse(const char UNUSED *link_name, hid_t cur_group, const char *obj_name; /* Name external link's object */ size_t fname_len; /* Length of external link file name */ unsigned intent; /* File access permissions */ + H5L_elink_cb_t cb_info; /* Callback info struct */ + const char *parent_file_name = NULL; /* Parent file name */ + const char *parent_group_name = NULL; /* Parent group name */ hid_t fapl_id = -1; /* File access property list for external link's file */ hid_t ext_obj = -1; /* ID for external link's object */ hid_t ret_value; /* Return value */ @@ -237,15 +240,68 @@ H5L_extern_traverse(const char UNUSED *link_name, hid_t cur_group, if(H5G_loc(cur_group, &loc) < 0) HGOTO_ERROR(H5E_LINK, H5E_CANTGET, FAIL, "can't get object location") - /* get the file access mode flags for the parent file */ - intent = H5F_INTENT(loc.oloc->file); + /* get the access flags set for lapl_id if any */ + if(H5P_get(plist, H5L_ACS_ELINK_FLAGS_NAME, &intent) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get elink file access flags") - if ((fapl_id == H5P_DEFAULT) && ((fapl_id = H5F_get_access_plist(loc.oloc->file, FALSE)) < 0)) + /* get the file access mode flags for the parent file, if they were not set + * on lapl_id */ + if(intent == H5F_ACC_DEFAULT) + intent = H5F_INTENT(loc.oloc->file); + + if((fapl_id == H5P_DEFAULT) && ((fapl_id = H5F_get_access_plist(loc.oloc->file, FALSE)) < 0)) HGOTO_ERROR(H5E_LINK, H5E_CANTGET, FAIL, "can't get parent's file access property list") - /* Set file close degree for new file to "weak" */ + /* Get callback_info */ + if(H5P_get(plist, H5L_ACS_ELINK_CB_NAME, &cb_info)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get elink callback info") + + /* Get file access property list */ if(NULL == (fa_plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Make callback if it exists */ + if(cb_info.func) { + /* Get parent file name */ + parent_file_name = H5F_NAME(loc.oloc->file); + + /* Get parent group name */ + if(loc.path->user_path_r != NULL && loc.path->obj_hidden == 0) + /* Use user_path_r if possible */ + parent_group_name = H5RS_get_str(loc.path->user_path_r); + else { + /* Otherwise use H5G_get_name */ + ssize_t group_name_len; /* Length of parent group name */ + + /* Get length of parent group name */ + if((group_name_len = H5G_get_name(cur_group, NULL, (size_t) 0, lapl_id, H5AC_ind_dxpl_id)) < 0) + HGOTO_ERROR(H5E_LINK, H5E_CANTGET, FAIL, "unable to retrieve length of group name") + + /* account for null terminator */ + group_name_len++; + + /* Copy parent group name */ + if(NULL == (tempname = (char *) H5MM_malloc((size_t) group_name_len))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + if(H5G_get_name(cur_group, tempname, (size_t) group_name_len, lapl_id, H5AC_ind_dxpl_id) < 0) + HGOTO_ERROR(H5E_LINK, H5E_CANTGET, FAIL, "unable to retrieve group name") + parent_group_name = tempname; + } /* end else */ + + /* Make callback */ + if((cb_info.func)(parent_file_name, parent_group_name, file_name, obj_name, + &intent, fapl_id, cb_info.user_data) < 0) + HGOTO_ERROR(H5E_LINK, H5E_CALLBACK, FAIL, "traversal operator failed") + + /* Free tempname */ + tempname = (char *)H5MM_xfree(tempname); + + /* Check access flags */ + if((intent & H5F_ACC_TRUNC) || (intent & H5F_ACC_EXCL)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file open flags") + } /* end if */ + + /* Set file close degree for new file to "weak" */ if(H5P_set(fa_plist, H5F_ACS_CLOSE_DEGREE_NAME, &fc_degree) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file close degree") @@ -277,7 +333,8 @@ H5L_extern_traverse(const char UNUSED *link_name, hid_t cur_group, /* try searching from paths set in the environment variable */ if ((ext_file == NULL) && (env_prefix=HDgetenv("HDF5_EXT_PREFIX"))) { - tmp_env_prefix = H5MM_strdup(env_prefix); + if ((tmp_env_prefix = H5MM_strdup(env_prefix)) == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") pp = tmp_env_prefix; while ((tmp_env_prefix) && (*tmp_env_prefix)) { @@ -290,14 +347,13 @@ H5L_extern_traverse(const char UNUSED *link_name, hid_t cur_group, ext_file = H5F_open(full_name, ((intent & H5F_ACC_RDWR) ? H5F_ACC_RDWR : H5F_ACC_RDONLY), H5P_FILE_CREATE_DEFAULT, fapl_id, H5AC_dxpl_id); if (full_name) - H5MM_xfree(full_name); + H5MM_free(full_name); if (ext_file != NULL) break; H5E_clear_stack(NULL); } } /* end while */ - if (pp) - H5MM_xfree(pp); + pp = (char *)H5MM_xfree(pp); } /* try searching from property list */ @@ -311,7 +367,7 @@ H5L_extern_traverse(const char UNUSED *link_name, hid_t cur_group, H5P_FILE_CREATE_DEFAULT, fapl_id, H5AC_dxpl_id)) == NULL) H5E_clear_stack(NULL); if (full_name) - H5MM_xfree(full_name); + H5MM_free(full_name); } } @@ -323,7 +379,7 @@ H5L_extern_traverse(const char UNUSED *link_name, hid_t cur_group, H5P_FILE_CREATE_DEFAULT, fapl_id, H5AC_dxpl_id)) == NULL) H5E_clear_stack(NULL); if (full_name) - H5MM_xfree(full_name); + H5MM_free(full_name); } /* try the relative file_name stored in tempname */ @@ -333,8 +389,7 @@ H5L_extern_traverse(const char UNUSED *link_name, hid_t cur_group, HGOTO_ERROR(H5E_LINK, H5E_CANTOPENFILE, FAIL, "unable to open external file") } - if (tempname) - H5MM_xfree(tempname); + tempname = (char *)H5MM_xfree(tempname); /* Increment the number of open objects, to hold the file open */ H5F_incr_nopen_objs(ext_file); @@ -367,9 +422,14 @@ done: if(ext_file && H5F_try_close(ext_file) < 0) HDONE_ERROR(H5E_LINK, H5E_CANTCLOSEFILE, FAIL, "problem closing external file") - /* Close object if it's open and something failed */ - if(ret_value < 0 && ext_obj >= 0 && H5I_dec_ref(ext_obj, FALSE) < 0) - HDONE_ERROR(H5E_ATOM, H5E_CANTRELEASE, FAIL, "unable to close atom for external object") + if(ret_value < 0) { + H5MM_xfree(tempname); + H5MM_xfree(pp); + + /* Close object if it's open and something failed */ + if(ext_obj >= 0 && H5I_dec_ref(ext_obj, FALSE) < 0) + HDONE_ERROR(H5E_ATOM, H5E_CANTRELEASE, FAIL, "unable to close atom for external object") + } /* end if */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5L_extern_traverse() */ diff --git a/src/H5Lprivate.h b/src/H5Lprivate.h index cd17a87..f3079bc 100644 --- a/src/H5Lprivate.h +++ b/src/H5Lprivate.h @@ -39,15 +39,23 @@ #define H5L_CRT_INTERMEDIATE_GROUP_NAME "intermediate_group" /* Create intermediate groups flag */ /* ======== Link access property names ======== */ -#define H5L_ACS_NLINKS_NAME "max soft links" /* Number of soft links to traverse */ -#define H5L_ACS_ELINK_PREFIX_NAME "external link prefix" /* External link prefix */ -#define H5L_ACS_ELINK_FAPL_NAME "external link fapl" /* file access property list for external link access */ +#define H5L_ACS_NLINKS_NAME "max soft links" /* Number of soft links to traverse */ +#define H5L_ACS_ELINK_PREFIX_NAME "external link prefix" /* External link prefix */ +#define H5L_ACS_ELINK_FAPL_NAME "external link fapl" /* file access property list for external link access */ +#define H5L_ACS_ELINK_FLAGS_NAME "external link flags" /* file access flags for external link traversal */ +#define H5L_ACS_ELINK_CB_NAME "external link callback" /* callback function for external link traversal */ /****************************/ /* Library Private Typedefs */ /****************************/ +/* Structure for external link traversal callback property */ +typedef struct H5L_elink_cb_t { + H5L_elink_traverse_t func; + void *user_data; +} H5L_elink_cb_t; + /*****************************/ /* Library Private Variables */ diff --git a/src/H5Lpublic.h b/src/H5Lpublic.h index eba09b2..620d2e9 100644 --- a/src/H5Lpublic.h +++ b/src/H5Lpublic.h @@ -29,7 +29,6 @@ /* Public headers needed by this file */ #include "H5public.h" /* Generic Functions */ #include "H5Ipublic.h" /* IDs */ -#include "H5Ppublic.h" /* Property lists */ #include "H5Tpublic.h" /* Datatypes */ /*****************/ @@ -131,6 +130,12 @@ typedef struct { typedef herr_t (*H5L_iterate_t)(hid_t group, const char *name, const H5L_info_t *info, void *op_data); +/* Callback for external link traversal */ +typedef herr_t (*H5L_elink_traverse_t)(const char *parent_file_name, + const char *parent_group_name, const char *child_file_name, + const char *child_object_name, unsigned *acc_flags, hid_t fapl_id, + void *op_data); + /********************/ /* Public Variables */ diff --git a/src/H5Odbg.c b/src/H5Odbg.c index 7a49f75..8a8b783 100644 --- a/src/H5Odbg.c +++ b/src/H5Odbg.c @@ -37,6 +37,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ +#include "H5Ppublic.h" /* Property Lists */ /****************/ /* Local Macros */ diff --git a/src/H5Plapl.c b/src/H5Plapl.c index e7b4381..608240f 100644 --- a/src/H5Plapl.c +++ b/src/H5Plapl.c @@ -62,6 +62,14 @@ #define H5L_ACS_ELINK_FAPL_COPY H5P_lacc_elink_fapl_copy #define H5L_ACS_ELINK_FAPL_CLOSE H5P_lacc_elink_fapl_close +/* Definitions for file access flags for external link traversal */ +#define H5L_ACS_ELINK_FLAGS_SIZE sizeof(unsigned) +#define H5L_ACS_ELINK_FLAGS_DEF H5F_ACC_DEFAULT + +/* Definitions for callback function for external link traversal */ +#define H5L_ACS_ELINK_CB_SIZE sizeof(H5L_elink_cb_t) +#define H5L_ACS_ELINK_CB_DEF {NULL,NULL} + /******************/ /* Local Typedefs */ /******************/ @@ -142,6 +150,9 @@ H5P_lacc_reg_prop(H5P_genclass_t *pclass) size_t nlinks = H5L_ACS_NLINKS_DEF; /* Default number of soft links to traverse */ char *elink_prefix = H5L_ACS_ELINK_PREFIX_DEF; /* Default external link prefix string */ hid_t def_fapl_id = H5L_ACS_ELINK_FAPL_DEF; /* Default fapl for external link access */ + unsigned elink_flags = H5L_ACS_ELINK_FLAGS_DEF; /* Default file access flags for external link traversal */ + H5L_elink_cb_t elink_cb = H5L_ACS_ELINK_CB_DEF; /* Default external link traversal callback */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5P_lacc_reg_prop) @@ -160,6 +171,16 @@ H5P_lacc_reg_prop(H5P_genclass_t *pclass) if(H5P_register(pclass, H5L_ACS_ELINK_FAPL_NAME, H5L_ACS_ELINK_FAPL_SIZE, &def_fapl_id, NULL, NULL, NULL, H5L_ACS_ELINK_FAPL_DEL, H5L_ACS_ELINK_FAPL_COPY, NULL, H5L_ACS_ELINK_FAPL_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register property for external link file access flags */ + if(H5P_register(pclass, H5L_ACS_ELINK_FLAGS_NAME, H5L_ACS_ELINK_FLAGS_SIZE, + &elink_flags, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register property for external link file traversal callback */ + if(H5P_register(pclass, H5L_ACS_ELINK_CB_NAME, H5L_ACS_ELINK_CB_SIZE, + &elink_cb, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P_lacc_reg_prop() */ @@ -634,3 +655,167 @@ done: FUNC_LEAVE_API(ret_value); } /* end H5Pget_elink_fapl() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_elink_acc_flags + * + * Purpose: Sets the file access flags to be used when traversing an + * external link. This should be either H5F_ACC_RDONLY or + * H5F_ACC_RDWR, or H5F_ACC_DEFAULT to unset the value. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * Tuesday, December 9, 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_elink_acc_flags(hid_t lapl_id, unsigned flags) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(H5Pset_elink_acc_flags, FAIL) + H5TRACE2("e", "iIu", lapl_id, flags); + + /* Check that flags are valid */ + if((flags != H5F_ACC_RDWR) && (flags != H5F_ACC_RDONLY) && (flags != H5F_ACC_DEFAULT)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file open flags") + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set flags */ + if(H5P_set(plist, H5L_ACS_ELINK_FLAGS_NAME, &flags) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set access flags") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_elink_acc_flags() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_elink_acc_flags + * + * Purpose: Gets the file access flags to be used when traversing an + * external link. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * Tuesday, December 9, 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_elink_acc_flags(hid_t lapl_id, unsigned *flags) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(H5Pget_elink_acc_flags, FAIL) + H5TRACE2("e", "i*Iu", lapl_id, flags); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get flags */ + if (flags) + if(H5P_get(plist, H5L_ACS_ELINK_FLAGS_NAME, flags)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, 0, "can't get access flags") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_elink_acc_flags() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_elink_cb + * + * Purpose: Sets the file access flags to be used when traversing an + * external link. This should be either H5F_ACC_RDONLY or + * H5F_ACC_RDWR. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * Tuesday, December 15, 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_elink_cb(hid_t lapl_id, H5L_elink_traverse_t func, void *op_data) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5L_elink_cb_t cb_info; /* Callback info struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(H5Pset_elink_cb, FAIL) + H5TRACE3("e", "ix*x", lapl_id, func, op_data); + + /* Check if the callback function is NULL and the user data is non-NULL. + * This is almost certainly an error as the user data will not be used. */ + if(!func && op_data) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "callback is NULL while user data is not") + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Populate the callback info struct */ + cb_info.func = func; + cb_info.user_data = op_data; + + /* Set callback info */ + if(H5P_set(plist, H5L_ACS_ELINK_CB_NAME, &cb_info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set callback info") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_elink_acc_flags() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_elink_cb + * + * Purpose: Gets the file access flags to be used when traversing an + * external link. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * Tuesday, December 15, 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_elink_cb(hid_t lapl_id, H5L_elink_traverse_t *func, void **op_data) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5L_elink_cb_t cb_info; /* Callback info struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(H5Pget_elink_cb, FAIL) + H5TRACE3("e", "i*x**x", lapl_id, func, op_data); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(lapl_id, H5P_LINK_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get callback_info */ + if(H5P_get(plist, H5L_ACS_ELINK_CB_NAME, &cb_info)<0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get callback info") + + if(func) + *func = cb_info.func; + + if(op_data) + *op_data = cb_info.user_data; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_elink_cb() */ + diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 498eecc..a67ea71 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -29,6 +29,7 @@ #include "H5Fpublic.h" #include "H5FDpublic.h" #include "H5Ipublic.h" +#include "H5Lpublic.h" #include "H5MMpublic.h" #include "H5Tpublic.h" #include "H5Zpublic.h" @@ -46,9 +47,6 @@ #define H5OPEN #endif /* _H5private_H */ -/* Default value for all property list classes */ -#define H5P_DEFAULT 0 - /* * The library's property list classes */ @@ -384,6 +382,10 @@ H5_DLL herr_t H5Pset_elink_prefix(hid_t plist_id, const char *prefix); H5_DLL ssize_t H5Pget_elink_prefix(hid_t plist_id, char *prefix, size_t size); H5_DLL hid_t H5Pget_elink_fapl(hid_t lapl_id); H5_DLL herr_t H5Pset_elink_fapl(hid_t lapl_id, hid_t fapl_id); +H5_DLL herr_t H5Pset_elink_acc_flags(hid_t lapl_id, unsigned flags); +H5_DLL herr_t H5Pget_elink_acc_flags(hid_t lapl_id, unsigned *flags); +H5_DLL herr_t H5Pset_elink_cb(hid_t lapl_id, H5L_elink_traverse_t func, void *op_data); +H5_DLL herr_t H5Pget_elink_cb(hid_t lapl_id, H5L_elink_traverse_t *func, void **op_data); /* Object copy property list (OCPYPL) routines */ H5_DLL herr_t H5Pset_copy_object(hid_t plist_id, unsigned crt_intmd); diff --git a/src/H5Tdeprec.c b/src/H5Tdeprec.c index df6238c..0314a38 100644 --- a/src/H5Tdeprec.c +++ b/src/H5Tdeprec.c @@ -44,6 +44,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5FOprivate.h" /* File objects */ #include "H5Iprivate.h" /* IDs */ +#include "H5Ppublic.h" /* Property Lists */ #include "H5Tpkg.h" /* Datatypes */ diff --git a/src/H5public.h b/src/H5public.h index e926473..cf9ff14 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -264,6 +264,9 @@ typedef signed long_long hssize_t; # error "nothing appropriate for uint64_t" #endif +/* Default value for all property list classes */ +#define H5P_DEFAULT 0 + /* Common iteration orders */ typedef enum { H5_ITER_UNKNOWN = -1, /* Unknown order */ |