diff options
| author | Quincey Koziol <koziol@hdfgroup.org> | 2017-01-29 07:19:37 (GMT) |
|---|---|---|
| committer | Quincey Koziol <koziol@hdfgroup.org> | 2017-01-29 07:19:37 (GMT) |
| commit | 1949d5756d424be0b23fc127fc99140374a7dd13 (patch) | |
| tree | e7d368b8bbcb589e48e88c1b781b08e16655a832 | |
| parent | 5a776edb8156c36d3a7e2f9e23c6fb349eaa90e0 (diff) | |
| download | hdf5-1949d5756d424be0b23fc127fc99140374a7dd13.zip hdf5-1949d5756d424be0b23fc127fc99140374a7dd13.tar.gz hdf5-1949d5756d424be0b23fc127fc99140374a7dd13.tar.bz2 | |
Cache image feature and testing
64 files changed, 3015 insertions, 138 deletions
@@ -504,10 +504,12 @@ ./src/H5C.c ./src/H5Cdbg.c ./src/H5Cepoch.c +./src/H5Cimage.c ./src/H5Clog.c ./src/H5Cmodule.h ./src/H5Cmpio.c ./src/H5Cpkg.h +./src/H5Cprefetched.c ./src/H5Cprivate.h ./src/H5Cpublic.h ./src/H5Cquery.c @@ -737,6 +739,7 @@ ./src/H5Obogus.c ./src/H5Obtreek.c ./src/H5Ocache.c +./src/H5Ocache_image.c ./src/H5Ochunk.c ./src/H5Ocont.c ./src/H5Ocopy.c @@ -901,6 +904,7 @@ ./test/cache_api.c ./test/cache_common.c ./test/cache_common.h +./test/cache_image.c ./test/cache_logging.c ./test/cache_tagging.c ./test/cmpd_dset.c @@ -946,6 +950,8 @@ ./test/flush1.c ./test/flush2.c ./test/flushrefresh.c +./test/genall5.c +./test/genall5.h ./test/gen_bad_ohdr.c ./test/gen_bad_compound.c ./test/gen_bogus.c @@ -102,6 +102,7 @@ $Source = ""; "H5A_operator2_t" => "x", "H5A_info_t" => "x", "H5AC_cache_config_t" => "x", + "H5AC_cache_image_config_t" => "x", "H5D_append_cb_t" => "x", "H5D_gather_func_t" => "x", "H5D_operator_t" => "x", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 510db1e..b7cfeb2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -90,8 +90,10 @@ set (H5C_SOURCES ${HDF5_SRC_DIR}/H5C.c ${HDF5_SRC_DIR}/H5Cdbg.c ${HDF5_SRC_DIR}/H5Cepoch.c + ${HDF5_SRC_DIR}/H5Cimage.c ${HDF5_SRC_DIR}/H5Clog.c ${HDF5_SRC_DIR}/H5Cmpio.c + ${HDF5_SRC_DIR}/H5Cprefetched.c ${HDF5_SRC_DIR}/H5Cquery.c ${HDF5_SRC_DIR}/H5Ctag.c ${HDF5_SRC_DIR}/H5Ctest.c @@ -439,6 +441,7 @@ set (H5O_SOURCES ${HDF5_SRC_DIR}/H5Obogus.c ${HDF5_SRC_DIR}/H5Obtreek.c ${HDF5_SRC_DIR}/H5Ocache.c + ${HDF5_SRC_DIR}/H5Ocache_image.c ${HDF5_SRC_DIR}/H5Ochunk.c ${HDF5_SRC_DIR}/H5Ocont.c ${HDF5_SRC_DIR}/H5Ocopy.c @@ -142,7 +142,8 @@ static const H5AC_class_t *const H5AC_class_s[] = { H5AC_SUPERBLOCK, /* (25) file superblock */ H5AC_DRVRINFO, /* (26) driver info block (supplements superblock) */ H5AC_EPOCH_MARKER, /* (27) epoch marker - always internal to cache */ - H5AC_PROXY_ENTRY /* (28) cache entry proxy */ + H5AC_PROXY_ENTRY, /* (28) cache entry proxy */ + H5AC_PREFETCHED_ENTRY /* (29) prefetched entry - always internal to cache */ }; @@ -370,12 +371,13 @@ H5AC_term_package(void) *------------------------------------------------------------------------- */ herr_t -H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr) +H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_config_t * image_config_ptr) { #ifdef H5_HAVE_PARALLEL char prefix[H5C__PREFIX_LEN] = ""; H5AC_aux_t * aux_ptr = NULL; #endif /* H5_HAVE_PARALLEL */ + struct H5C_cache_image_ctl_t int_ci_config = H5C__DEFAULT_CACHE_IMAGE_CTL; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -384,11 +386,16 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr) HDassert(f); HDassert(NULL == f->shared->cache); HDassert(config_ptr != NULL) ; + HDassert(image_config_ptr != NULL) ; + HDassert(image_config_ptr->version == H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION); HDcompile_assert(NELMTS(H5AC_class_s) == H5AC_NTYPES); HDcompile_assert(H5C__MAX_NUM_TYPE_IDS == H5AC_NTYPES); + /* Validate configurations */ if(H5AC_validate_config(config_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Bad cache configuration") + if(H5AC_validate_cache_image_config(image_config_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Bad cache image configuration") #ifdef H5_HAVE_PARALLEL if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) { @@ -430,6 +437,7 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr) aux_ptr->candidate_slist_ptr = NULL; aux_ptr->write_done = NULL; aux_ptr->sync_point_done = NULL; + aux_ptr->p0_image_len = 0; sprintf(prefix, "%d:", mpi_rank); @@ -504,6 +512,20 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr) if(H5AC_set_cache_auto_resize_config(f->shared->cache, config_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "auto resize configuration failed") + /* don't need to get the current H5C image config here since the + * cache has just been created, and thus f->shared->cache->image_ctl + * must still set to its initial value (H5C__DEFAULT_CACHE_IMAGE_CTL). + * Note that this not true as soon as control returns to the application + * program, as some test code modifies f->shared->cache->image_ctl. + */ + int_ci_config.version = image_config_ptr->version; + int_ci_config.generate_image = image_config_ptr->generate_image; + int_ci_config.save_resize_status = image_config_ptr->save_resize_status; + int_ci_config.entry_ageout = image_config_ptr->entry_ageout; + + if(H5C_set_cache_image_config(f, f->shared->cache, &int_ci_config) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "auto resize configuration failed") + done: #ifdef H5_HAVE_PARALLEL /* if there is a failure, try to tidy up the auxilary structure */ @@ -982,6 +1004,41 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_load_cache_image_on_next_protect + * + * Purpose: Load the cache image block at the specified location, + * decode it, and insert its contents into the metadata + * cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 7/6/15 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_load_cache_image_on_next_protect(H5F_t * f, haddr_t addr, hsize_t len, + hbool_t rw) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if(H5C_load_cache_image_on_next_protect(f, addr, len, rw) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "call to H5C_load_cache_image_on_next_protect failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_load_cache_image_on_next_protect() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_mark_entry_dirty * * Purpose: Mark a pinned or protected entry as dirty. The target @@ -2395,6 +2452,61 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_validate_cache_image_config() + * + * Purpose: Run a sanity check on the contents of the supplied + * instance of H5AC_cache_image_config_t. + * + * Do nothing and return SUCCEED if no errors are detected, + * and flag an error and return FAIL otherwise. + * + * At present, this function operates by packing the data + * from the instance of H5AC_cache_image_config_t into an + * instance of H5C_cache_image_ctl_t, and then calling + * H5C_validate_cache_image_config(). If and when + * H5AC_cache_image_config_t and H5C_cache_image_ctl_t + * diverge, we may have to change this. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 6/25/15 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_validate_cache_image_config(H5AC_cache_image_config_t *config_ptr) +{ + H5C_cache_image_ctl_t internal_config = H5C__DEFAULT_CACHE_IMAGE_CTL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Check args */ + if(config_ptr == NULL) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "NULL config_ptr on entry") + + if(config_ptr->version != H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Unknown image config version") + + /* don't need to get the current H5C image config here since the + * default values of fields not in the H5AC config will always be + * valid. + */ + internal_config.version = config_ptr->version; + internal_config.generate_image = config_ptr->generate_image; + internal_config.save_resize_status = config_ptr->save_resize_status; + internal_config.entry_ageout = config_ptr->entry_ageout; + + if(H5C_validate_cache_image_config(&internal_config) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "error(s) in new cache image config") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_validate_cache_image_config() */ + + +/*------------------------------------------------------------------------- * * Function: H5AC__check_if_write_permitted * diff --git a/src/H5ACdbg.c b/src/H5ACdbg.c index 8d99c6f..e2f7b13 100644 --- a/src/H5ACdbg.c +++ b/src/H5ACdbg.c @@ -252,6 +252,207 @@ done: /*------------------------------------------------------------------------- * + * Function: H5AC_get_entry_ptr_from_addr() + * + * Purpose: Debugging function that attempts to look up an entry in the + * cache by its file address, and if found, returns a pointer + * to the entry in *entry_ptr_ptr. If the entry is not in the + * cache, *entry_ptr_ptr is set to NULL. + * + * WARNING: This call should be used only in debugging + * routines, and it should be avoided when + * possible. + * + * Further, if we ever multi-thread the cache, + * this routine will have to be either discarded + * or heavily re-worked. + * + * Finally, keep in mind that the entry whose + * pointer is obtained in this fashion may not + * be in a stable state. + * + * Note that this function is only defined if NDEBUG + * is not defined. + * + * As heavy use of this function is almost certainly a + * bad idea, the metadata cache tracks the number of + * successful calls to this function, and (if + * H5C_DO_SANITY_CHECKS is defined) displays any + * non-zero count on cache shutdown. + * + * This function is just a wrapper that calls the H5C + * version of the function. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: John Mainzer, 5/30/14 + * + *------------------------------------------------------------------------- + */ +#ifndef NDEBUG +herr_t +H5AC_get_entry_ptr_from_addr(const H5F_t *f, haddr_t addr, void **entry_ptr_ptr) +{ + H5C_t *cache_ptr; /* Ptr to cache */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + + if(H5C_get_entry_ptr_from_addr(cache_ptr, addr, entry_ptr_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_get_entry_ptr_from_addr() failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_get_entry_ptr_from_addr() */ +#endif /* NDEBUG */ + + +/*------------------------------------------------------------------------- + * Function: H5AC_flush_dependency_exists() + * + * Purpose: Test to see if a flush dependency relationship exists + * between the supplied parent and child. Both parties + * are indicated by addresses so as to avoid the necessity + * of protect / unprotect calls prior to this call. + * + * If either the parent or the child is not in the metadata + * cache, the function sets *fd_exists_ptr to FALSE. + * + * If both are in the cache, the childs list of parents is + * searched for the proposed parent. If the proposed parent + * is found in the childs parent list, the function sets + * *fd_exists_ptr to TRUE. In all other non-error cases, + * the function sets *fd_exists_ptr FALSE. + * + * Return: SUCCEED on success/FAIL on failure. Note that + * *fd_exists_ptr is undefined on failure. + * + * Programmer: John Mainzer + * 9/28/16 + * + *------------------------------------------------------------------------- + */ +#ifndef NDEBUG +herr_t +H5AC_flush_dependency_exists(H5F_t *f, haddr_t parent_addr, haddr_t child_addr, + hbool_t *fd_exists_ptr) +{ + H5C_t *cache_ptr; /* Ptr to cache */ + herr_t ret_value = FAIL; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + + ret_value = H5C_flush_dependency_exists(cache_ptr, parent_addr, child_addr, fd_exists_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_flush_dependency_exists() */ +#endif /* NDEBUG */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC_verify_entry_type() + * + * Purpose: Debugging function that attempts to look up an entry in the + * cache by its file address, and if found, test to see if its + * type field contains the expected value. + * + * If the specified entry is in cache, *in_cache_ptr is set + * to TRUE, and *type_ok_ptr is set to TRUE or FALSE depending + * on whether the entries type field matches the + * expected_type parameter + * + * If the target entry is not in cache, *in_cache_ptr is + * set to FALSE, and *type_ok_ptr is undefined. + * + * Note that this function is only defined if NDEBUG + * is not defined. + * + * This function is just a wrapper that calls the H5C + * version of the function. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: John Mainzer, 5/30/14 + * + * Changes: None. + * + * JRM -- 9/17/16 + * + *------------------------------------------------------------------------- + */ +#ifndef NDEBUG +herr_t +H5AC_verify_entry_type(const H5F_t *f, haddr_t addr, + const H5AC_class_t *expected_type, hbool_t *in_cache_ptr, + hbool_t *type_ok_ptr) +{ + H5C_t * cache_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + + if(H5C_verify_entry_type(cache_ptr, addr, expected_type, in_cache_ptr, type_ok_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_verify_entry_type() failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_verify_entry_type() */ +#endif /* NDEBUG */ + + +/*------------------------------------------------------------------------- + * Function: H5AC_get_serialization_in_progress + * + * Purpose: Return the current value of + * cache_ptr->serialization_in_progress. + * + * Return: Current value of cache_ptr->serialization_in_progress. + * + * Programmer: John Mainzer + * 8/24/15 + * + *------------------------------------------------------------------------- + */ +#ifndef NDEBUG +hbool_t +H5AC_get_serialization_in_progress(H5F_t *f) +{ + H5C_t * cache_ptr; + hbool_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + + /* Set return value */ + ret_value = H5C_get_serialization_in_progress(cache_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_get_serialization_in_progress() */ +#endif /* NDEBUG */ + + +/*------------------------------------------------------------------------- + * * Function: H5AC_cache_is_clean() * * Purpose: Debugging function that verifies that all rings in the diff --git a/src/H5ACpkg.h b/src/H5ACpkg.h index dbbd8a0..816f1d5 100644 --- a/src/H5ACpkg.h +++ b/src/H5ACpkg.h @@ -351,6 +351,12 @@ H5FL_EXTERN(H5AC_aux_t); * this verification. The field is set to NULL when the * callback is not needed. * + * The following field supports the metadata cache image feature. + * + * p0_image_len: unsiged integer containing the length of the metadata cache + * image constructed by MPI process 0. This field should be 0 + * if the value is unknown, or if cache image is not enabled. + * ****************************************************************************/ #ifdef H5_HAVE_PARALLEL @@ -400,6 +406,9 @@ typedef struct H5AC_aux_t void (* sync_point_done)(int num_writes, haddr_t * written_entries_tbl); + + unsigned p0_image_len; + } H5AC_aux_t; /* struct H5AC_aux_t */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 683be2b..9f28b18 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -88,6 +88,7 @@ typedef enum { H5AC_DRVRINFO_ID, /* (26) driver info block (supplements superblock) */ H5AC_EPOCH_MARKER_ID, /* (27) epoch marker - always internal to cache */ H5AC_PROXY_ENTRY_ID, /* (28) cache entry proxy */ + H5AC_PREFETCHED_ENTRY_ID, /* (29) prefetched entry - always internal to cache */ H5AC_NTYPES /* Number of types, must be last */ } H5AC_type_t; @@ -110,14 +111,22 @@ typedef enum { * use the dump_stats parameter to takedown_cache(), or call * H5C_stats() directly. * JRM -- 4/12/15 + * + * Added the H5AC_DUMP_IMAGE_STATS_ON_CLOSE #define, which works much + * the same way as H5AC_DUMP_STATS_ON_CLOSE. However, the set of stats + * displayed is much smaller, and directed purely at the cache image feature. + * + * JRM -- 11/1/15 */ #if H5C_COLLECT_CACHE_STATS #define H5AC_DUMP_STATS_ON_CLOSE 0 +#define H5AC_DUMP_IMAGE_STATS_ON_CLOSE 0 #else /* H5C_COLLECT_CACHE_STATS */ #define H5AC_DUMP_STATS_ON_CLOSE 0 +#define H5AC_DUMP_IMAGE_STATS_ON_CLOSE 0 #endif /* H5C_COLLECT_CACHE_STATS */ @@ -318,7 +327,13 @@ H5_DLLVAR hid_t H5AC_rawdata_dxpl_id; } #endif /* H5_HAVE_PARALLEL */ - +#define H5AC__DEFAULT_CACHE_IMAGE_CONFIG \ +{ \ + /* int32_t version = */ H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, \ + /* hbool_t generate_image = */ FALSE, \ + /* hbool_t save_resize_status = */ FALSE, \ + /* int32_t entry_ageout = */ H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE \ +} /* * Library prototypes. */ @@ -388,12 +403,14 @@ H5_DLLVAR const H5AC_class_t H5AC_SUPERBLOCK[1]; H5_DLLVAR const H5AC_class_t H5AC_DRVRINFO[1]; H5_DLLVAR const H5AC_class_t H5AC_EPOCH_MARKER[1]; H5_DLLVAR const H5AC_class_t H5AC_PROXY_ENTRY[1]; +H5_DLLVAR const H5AC_class_t H5AC_PREFETCHED_ENTRY[1]; /* external function declarations: */ H5_DLL herr_t H5AC_init(void); -H5_DLL herr_t H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr); +H5_DLL herr_t H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, + H5AC_cache_image_config_t * image_config_ptr); H5_DLL herr_t H5AC_get_entry_status(const H5F_t *f, haddr_t addr, unsigned *status_ptr); H5_DLL herr_t H5AC_insert_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, @@ -430,6 +447,11 @@ H5_DLL herr_t H5AC_set_cache_auto_resize_config(H5AC_t *cache_ptr, H5AC_cache_config_t *config_ptr); H5_DLL herr_t H5AC_validate_config(H5AC_cache_config_t *config_ptr); +/* Cache image routines */ +H5_DLL herr_t H5AC_load_cache_image_on_next_protect(H5F_t *f, haddr_t addr, + hsize_t len, hbool_t rw); +H5_DLL herr_t H5AC_validate_cache_image_config(H5AC_cache_image_config_t *config_ptr); + /* Tag & Ring routines */ H5_DLL herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag); H5_DLL herr_t H5AC_flush_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id); @@ -462,6 +484,14 @@ H5_DLL herr_t H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr); H5_DLL herr_t H5AC_stats(const H5F_t *f); H5_DLL herr_t H5AC_dump_cache(const H5F_t *f); #ifndef NDEBUG +H5_DLL herr_t H5AC_get_entry_ptr_from_addr(const H5F_t *f, haddr_t addr, + void **entry_ptr_ptr); +H5_DLL herr_t H5AC_flush_dependency_exists(H5F_t *f, haddr_t parent_addr, + haddr_t child_addr, hbool_t *fd_exists_ptr); +H5_DLL herr_t H5AC_verify_entry_type(const H5F_t *f, haddr_t addr, + const H5AC_class_t *expected_type, hbool_t *in_cache_ptr, + hbool_t *type_ok_ptr); +H5_DLL hbool_t H5AC_get_serialization_in_progress(H5F_t *f); H5_DLL hbool_t H5AC_cache_is_clean(const H5F_t *f, H5AC_ring_t inner_ring); #endif /* NDEBUG */ /* end debugging functions */ diff --git a/src/H5ACpublic.h b/src/H5ACpublic.h index dd16764..5fdb3f4 100644 --- a/src/H5ACpublic.h +++ b/src/H5ACpublic.h @@ -508,6 +508,67 @@ typedef struct H5AC_cache_config_t } H5AC_cache_config_t; +/**************************************************************************** + * + * structure H5AC_cache_image_config_t + * + * H5AC_cache_image_ctl_t is a public structure intended for use in public + * APIs. At least in its initial incarnation, it is a copy of struct + * H5C_cache_image_ctl_t. + * + * The fields of the structure are discussed individually below: + * + * version: Integer field containing the version number of this version + * of the H5C_image_ctl_t structure. Any instance of + * H5C_image_ctl_t passed to the cache must have a known + * version number, or an error will be flagged. + * + * generate_image: Boolean flag indicating whether a cache image should + * be created on file close. + * + * save_resize_status: Boolean flag indicating whether the cache image + * should include the adaptive cache resize configuration and status. + * Note that this field is ignored at present. + * + * entry_ageout: Integer field indicating the maximum number of + * times a prefetched entry can appear in subsequent cache images. + * This field exists to allow the user to avoid the buildup of + * infrequently used entries in long sequences of cache images. + * + * The value of this field must lie in the range + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE (-1) to + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX (100). + * + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE means that no limit + * is imposed on number of times a prefeteched entry can appear + * in subsequent cache images. + * + * A value of 0 prevents prefetched entries from being included + * in cache images. + * + * Positive integers restrict prefetched entries to the specified + * number of appearances. + * + * Note that the number of subsequent cache images that a prefetched + * entry has appeared in is tracked in an 8 bit field. Thus, while + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX can be increased from its + * current value, any value in excess of 255 will be the functional + * equivalent of H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE. + * + ****************************************************************************/ + +#define H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION 1 + +#define H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE -1 +#define H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX 100 + +typedef struct H5AC_cache_image_config_t { + int32_t version; + hbool_t generate_image; + hbool_t save_resize_status; + int32_t entry_ageout; +} H5AC_cache_image_config_t; + #ifdef __cplusplus } #endif @@ -46,9 +46,9 @@ * - Change protect/unprotect to lock/unlock. * * - Flush entries in increasing address order in - * H5C_make_space_in_cache(). + * H5C__make_space_in_cache(). * - * - Also in H5C_make_space_in_cache(), use high and low water marks + * - Also in H5C__make_space_in_cache(), use high and low water marks * to reduce the number of I/O calls. * * - When flushing, attempt to combine contiguous entries to reduce @@ -155,11 +155,6 @@ static void * H5C_load_entry(H5F_t * f, haddr_t addr, void * udata); -static herr_t H5C_make_space_in_cache(H5F_t * f, - hid_t dxpl_id, - size_t space_needed, - hbool_t write_permitted); - static herr_t H5C__mark_flush_dep_dirty(H5C_cache_entry_t * entry); static herr_t H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry); @@ -389,6 +384,7 @@ H5C_create(size_t max_cache_size, cache_ptr->resize_enabled = FALSE; cache_ptr->cache_full = FALSE; cache_ptr->size_decreased = FALSE; + cache_ptr->resize_in_progress = FALSE; (cache_ptr->resize_ctl).version = H5C__CURR_AUTO_SIZE_CTL_VER; (cache_ptr->resize_ctl).rpt_fcn = NULL; @@ -437,9 +433,32 @@ H5C_create(size_t max_cache_size, ((cache_ptr->epoch_markers)[i]).type = H5AC_EPOCH_MARKER; } + /* Initialize cache image generation on file close related fields. + * Initial value of image_ctl must match H5C__DEFAULT_CACHE_IMAGE_CTL + * in H5Cprivate.h. + */ + cache_ptr->image_ctl.version = H5C__CURR_CACHE_IMAGE_CTL_VER; + cache_ptr->image_ctl.generate_image = FALSE; + cache_ptr->image_ctl.save_resize_status = FALSE; + cache_ptr->image_ctl.entry_ageout = -1; + cache_ptr->image_ctl.flags = H5C_CI__ALL_FLAGS; + + cache_ptr->serialization_in_progress= FALSE; + cache_ptr->load_image = FALSE; + cache_ptr->image_loaded = FALSE; + cache_ptr->delete_image = FALSE; + cache_ptr->image_addr = HADDR_UNDEF; + cache_ptr->image_len = 0; + cache_ptr->image_data_len = 0; + cache_ptr->entries_loaded_counter = 0; cache_ptr->entries_inserted_counter = 0; cache_ptr->entries_relocated_counter = 0; + cache_ptr->entry_fd_height_change_counter = 0; + + cache_ptr->num_entries_in_image = 0; + cache_ptr->image_entries = NULL; + cache_ptr->image_buffer = NULL; /* initialize free space manager related fields: */ cache_ptr->rdfsm_settled = FALSE; @@ -456,6 +475,10 @@ H5C_create(size_t max_cache_size, cache_ptr->prefix[0] = '\0'; /* empty string */ +#ifndef NDEBUG + cache_ptr->get_entry_ptr_from_addr_counter = 0; +#endif /* NDEBUG */ + /* Set return value */ ret_value = cache_ptr; @@ -733,6 +756,21 @@ H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id) /* Make certain there aren't any protected entries */ HDassert(cache_ptr->pl_len == 0); + /* If the file is opened and closed without any access to + * any group or dataset, it is possible that the cache image (if + * it exists) has not been read yet. Do this now if required. + */ + if(cache_ptr->load_image) { + cache_ptr->load_image = FALSE; + if(H5C__load_cache_image(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't load cache image") + } /* end if */ + + /* Generate the cache image, if requested */ + if(cache_ptr->image_ctl.generate_image) + if(H5C__prep_image_for_file_close(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "Can't create cache image") + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5C_prep_for_file_close() */ @@ -775,10 +813,20 @@ H5C_dest(H5F_t * f, hid_t dxpl_id) HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); HDassert(cache_ptr->close_warning_received); +#if H5AC_DUMP_IMAGE_STATS_ON_CLOSE + if(H5C_image_stats(cache_ptr, TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't display cache image stats") +#endif /* H5AC_DUMP_IMAGE_STATS_ON_CLOSE */ + /* Flush and invalidate all cache entries */ if(H5C_flush_invalidate_cache(f, dxpl_id, H5C__NO_FLAGS_SET) < 0 ) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache") + /* Generate & write cache image if requested */ + if(cache_ptr->image_ctl.generate_image) + if(H5C__generate_cache_image(f, dxpl_id, cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "Can't generate metadata cache image") + if(cache_ptr->slist_ptr != NULL) { H5SL_close(cache_ptr->slist_ptr); cache_ptr->slist_ptr = NULL; @@ -790,6 +838,12 @@ H5C_dest(H5F_t * f, hid_t dxpl_id) } /* end if */ #ifndef NDEBUG +#if H5C_DO_SANITY_CHECKS + if(cache_ptr->get_entry_ptr_from_addr_counter > 0) + HDfprintf(stdout, "*** %ld calls to H5C_get_entry_ptr_from_add(). ***\n", + cache_ptr->get_entry_ptr_from_addr_counter); +#endif /* H5C_DO_SANITY_CHECKS */ + cache_ptr->magic = 0; #endif /* NDEBUG */ @@ -1173,7 +1227,7 @@ H5C_flush_to_min_clean(H5F_t * f, "cache write is not permitted!?!\n"); } #if 1 /* original code */ - if(H5C_make_space_in_cache(f, dxpl_id, (size_t)0, write_permitted) < 0) + if(H5C__make_space_in_cache(f, dxpl_id, (size_t)0, write_permitted) < 0) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C__make_space_in_cache failed") #else /* modified code -- commented out for now */ if ( cache_ptr->max_cache_size > cache_ptr->index_size ) { @@ -1343,6 +1397,7 @@ H5C_insert_entry(H5F_t * f, HDassert( cache_ptr ); HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); HDassert( type ); + HDassert( type->mem_type == cache_ptr->class_table_ptr[type->id]->mem_type ); HDassert( type->image_len ); HDassert( H5F_addr_defined(addr) ); HDassert( thing ); @@ -1442,6 +1497,22 @@ H5C_insert_entry(H5F_t * f, entry_ptr->aux_next = NULL; entry_ptr->aux_prev = NULL; + /* initialize cache image related fields */ + entry_ptr->include_in_image = FALSE; + entry_ptr->lru_rank = 0; + entry_ptr->image_dirty = FALSE; + entry_ptr->fd_parent_count = 0; + entry_ptr->fd_parent_addrs = NULL; + entry_ptr->fd_child_count = 0; + entry_ptr->fd_dirty_child_count = 0; + entry_ptr->image_fd_height = 0; + entry_ptr->prefetched = FALSE; + entry_ptr->prefetch_type_id = 0; + entry_ptr->age = 0; +#ifndef NDEBUG /* debugging field */ + entry_ptr->serialization_count = 0; +#endif /* NDEBUG */ + #ifdef H5_HAVE_PARALLEL entry_ptr->coll_next = NULL; entry_ptr->coll_prev = NULL; @@ -1487,7 +1558,7 @@ H5C_insert_entry(H5F_t * f, /* Note that space_needed is just the amount of space that * needed to insert the new entry without exceeding the cache - * size limit. The subsequent call to H5C_make_space_in_cache() + * size limit. The subsequent call to H5C__make_space_in_cache() * may evict the entries required to free more or less space * depending on conditions. It MAY be less if the cache is * currently undersized, or more if the cache is oversized. @@ -1510,8 +1581,8 @@ H5C_insert_entry(H5F_t * f, * no point in worrying about the third. */ - if(H5C_make_space_in_cache(f, dxpl_id, space_needed, write_permitted) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, "H5C_make_space_in_cache failed.") + if(H5C__make_space_in_cache(f, dxpl_id, space_needed, write_permitted) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, "H5C__make_space_in_cache failed.") } H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, FAIL) @@ -2342,6 +2413,7 @@ H5C_protect(H5F_t * f, HDassert( cache_ptr ); HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); HDassert( type ); + HDassert( type->mem_type == cache_ptr->class_table_ptr[type->id]->mem_type ); HDassert( H5F_addr_defined(addr) ); #if H5C_DO_EXTREME_SANITY_CHECKS @@ -2351,6 +2423,13 @@ H5C_protect(H5F_t * f, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, NULL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + /* Load the cache image, if requested */ + if(cache_ptr->load_image) { + cache_ptr->load_image = FALSE; + if(H5C__load_cache_image(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't load cache image") + } /* end if */ + read_only = ( (flags & H5C__READ_ONLY_FLAG) != 0 ); flush_last = ( (flags & H5C__FLUSH_LAST_FLAG) != 0 ); @@ -2384,6 +2463,21 @@ H5C_protect(H5F_t * f, if(entry_ptr->ring != ring) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, NULL, "ring type mismatch occured for cache entry\n") + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + + if(entry_ptr->prefetched) { + /* This call removes the prefetched entry from the cache, + * and replaces it with an entry deserialized from the + * image of the prefetched entry. + */ + if(H5C__deserialize_prefetched_entry(f, dxpl_id, cache_ptr, &entry_ptr, type, addr, udata) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "can't deserialize prefetched entry.") + + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(!entry_ptr->prefetched); + HDassert(entry_ptr->addr == addr); + } /* end if */ + /* Check for trying to load the wrong type of entry from an address */ if(entry_ptr->type != type) HGOTO_ERROR(H5E_CACHE, H5E_BADTYPE, NULL, "incorrect cache entry type") @@ -2501,7 +2595,7 @@ H5C_protect(H5F_t * f, empty_space = cache_ptr->max_cache_size - cache_ptr->index_size; /* try to free up if necceary and if evictions are permitted. Note - * that if evictions are enabled, we will call H5C_make_space_in_cache() + * that if evictions are enabled, we will call H5C__make_space_in_cache() * regardless if the min_free_space requirement is not met. */ if ( ( cache_ptr->evictions_enabled ) && @@ -2536,7 +2630,7 @@ H5C_protect(H5F_t * f, /* Note that space_needed is just the amount of space that * needed to insert the new entry without exceeding the cache - * size limit. The subsequent call to H5C_make_space_in_cache() + * size limit. The subsequent call to H5C__make_space_in_cache() * may evict the entries required to free more or less space * depending on conditions. It MAY be less if the cache is * currently undersized, or more if the cache is oversized. @@ -2563,8 +2657,8 @@ H5C_protect(H5F_t * f, * see no point in worrying about the fourth. */ - if(H5C_make_space_in_cache(f, dxpl_id, space_needed, write_permitted) < 0 ) - HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, "H5C_make_space_in_cache failed 1.") + if(H5C__make_space_in_cache(f, dxpl_id, space_needed, write_permitted) < 0 ) + HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, "H5C__make_space_in_cache failed 1.") } /* Insert the entry in the hash table. It can't be dirty yet, so @@ -2684,7 +2778,7 @@ H5C_protect(H5F_t * f, * bring the cache size down to the current maximum cache size. * * Also, if the min_clean_size requirement is not met, we - * should also call H5C_make_space_in_cache() to bring us + * should also call H5C__make_space_in_cache() to bring us * into complience. */ @@ -2701,8 +2795,8 @@ H5C_protect(H5F_t * f, if(cache_ptr->index_size > cache_ptr->max_cache_size) cache_ptr->cache_full = TRUE; - if(H5C_make_space_in_cache(f, dxpl_id, (size_t)0, write_permitted) < 0 ) - HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, "H5C_make_space_in_cache failed 2.") + if(H5C__make_space_in_cache(f, dxpl_id, (size_t)0, write_permitted) < 0 ) + HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, "H5C__make_space_in_cache failed 2.") } } } @@ -3815,6 +3909,7 @@ H5C_create_flush_dependency(void * parent_thing, void * child_thing) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for flush dependency parent list") child_entry->flush_dep_parent_nalloc *= 2; } /* end else */ + cache_ptr->entry_fd_height_change_counter++; } /* end if */ /* Add the dependency to the child's parent array */ @@ -4023,6 +4118,7 @@ H5C__auto_adjust_cache_size(H5F_t * f, { H5C_t * cache_ptr = f->shared->cache; herr_t result; + hbool_t reentrant_call = FALSE; hbool_t inserted_epoch_marker = FALSE; size_t new_max_cache_size = 0; size_t old_max_cache_size = 0; @@ -4042,6 +4138,18 @@ H5C__auto_adjust_cache_size(H5F_t * f, HDassert( (double)0.0f <= (cache_ptr->resize_ctl).min_clean_fraction ); HDassert( (cache_ptr->resize_ctl).min_clean_fraction <= (double)100.0f ); + /* check to see if cache_ptr->resize_in_progress is TRUE. If it, this + * is a re-entrant call via a client callback called in the resize + * process. To avoid an infinite recursion, set reentrant_call to + * TRUE, and goto done. + */ + if(cache_ptr->resize_in_progress) { + reentrant_call = TRUE; + HGOTO_DONE(SUCCEED); + } /* end if */ + + cache_ptr->resize_in_progress = TRUE; + if ( !cache_ptr->resize_enabled ) { HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Auto cache resize disabled.") @@ -4340,6 +4448,12 @@ H5C__auto_adjust_cache_size(H5F_t * f, } done: + /* Sanity checks */ + HDassert(cache_ptr->resize_in_progress); + if(!reentrant_call) + cache_ptr->resize_in_progress = FALSE; + HDassert((!reentrant_call) || (cache_ptr->resize_in_progress)); + FUNC_LEAVE_NOAPI(ret_value) } /* H5C__auto_adjust_cache_size() */ @@ -6136,6 +6250,8 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, hbool_t destroy_entry; /* internal flag */ hbool_t generate_image; /* internal flag */ hbool_t was_dirty; + hbool_t suppress_image_entry_writes = FALSE; + hbool_t suppress_image_entry_frees = FALSE; haddr_t entry_addr = HADDR_UNDEF; herr_t ret_value = SUCCEED; /* Return value */ @@ -6178,6 +6294,28 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, else write_entry = FALSE; + /* if we have received close warning, and we have been instructed to + * generate a metadata cache image, and we have actually constructed + * the entry images, set suppress_image_entry_frees to TRUE. + * + * Set suppress_image_entry_writes to TRUE if indicated by the + * image_ctl flags. + */ + if(cache_ptr->close_warning_received && cache_ptr->image_ctl.generate_image + && cache_ptr->num_entries_in_image > 0 && cache_ptr->image_entries) { + /* Sanity checks */ + HDassert(entry_ptr->image_up_to_date || !(entry_ptr->include_in_image)); + HDassert(entry_ptr->image_ptr || !(entry_ptr->include_in_image)); + HDassert((!clear_only) || !(entry_ptr->include_in_image)); + HDassert((!take_ownership) || !(entry_ptr->include_in_image)); + HDassert((!free_file_space) || !(entry_ptr->include_in_image)); + + suppress_image_entry_frees = TRUE; + + if(cache_ptr->image_ctl.flags & H5C_CI__SUPRESS_ENTRY_WRITES) + suppress_image_entry_writes = TRUE; + } /* end if */ + /* run initial sanity checks */ #if H5C_DO_SANITY_CHECKS if(entry_ptr->in_slist) { @@ -6232,6 +6370,9 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, } /* end if */ if(!(entry_ptr->image_up_to_date)) { + /* Sanity check */ + HDassert(!entry_ptr->prefetched); + /* Generate the entry's image */ if(H5C__generate_image(f, cache_ptr, entry_ptr, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't generate entry's image") @@ -6252,7 +6393,17 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Write when writes are always forbidden!?!?!") #endif /* H5C_DO_SANITY_CHECKS */ - if(((entry_ptr->type->flags) & H5C__CLASS_SKIP_WRITES) == 0) { + /* Write the image to disk unless the write is suppressed. + * + * This happens if both suppress_image_entry_writes and + * entry_ptr->include_in_image are TRUE, or if the + * H5AC__CLASS_SKIP_WRITES is set in the entry's type. This + * flag should only be used in test code + */ + if((!suppress_image_entry_writes || !entry_ptr->include_in_image) + && (((entry_ptr->type->flags) & H5C__CLASS_SKIP_WRITES) == 0)) { + H5FD_mem_t mem_type = H5FD_MEM_DEFAULT; + #ifdef H5_HAVE_PARALLEL if(cache_ptr->coll_write_list) { if(H5SL_insert(cache_ptr->coll_write_list, entry_ptr, &entry_ptr->addr) < 0) @@ -6260,7 +6411,15 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, } /* end if */ else #endif /* H5_HAVE_PARALLEL */ - if(H5F_block_write(f, entry_ptr->type->mem_type, entry_ptr->addr, entry_ptr->size, dxpl_id, entry_ptr->image_ptr) < 0) + + if(entry_ptr->prefetched) { + HDassert(entry_ptr->type->id == H5AC_PREFETCHED_ENTRY_ID); + mem_type = cache_ptr->class_table_ptr[entry_ptr->prefetch_type_id]->mem_type; + } /* end if */ + else + mem_type = entry_ptr->type->mem_type; + + if(H5F_block_write(f, mem_type, entry_ptr->addr, entry_ptr->size, dxpl_id, entry_ptr->image_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't write image to file.") } /* end if */ @@ -6405,10 +6564,29 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, /* Sanity check */ HDassert(0 == entry_ptr->flush_dep_nparents); - /* Start by freeing the buffer for the on disk image */ - if(entry_ptr->image_ptr != NULL) + /* if both suppress_image_entry_frees and entry_ptr->include_in_image + * are true, simply set entry_ptr->image_ptr to NULL, as we have + * another pointer to the buffer in an instance of H5C_image_entry_t + * in cache_ptr->image_entries. + * + * Otherwise, free the buffer if it exists. + */ + if(suppress_image_entry_frees && entry_ptr->include_in_image) + entry_ptr->image_ptr = NULL; + else if(entry_ptr->image_ptr != NULL) entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); + /* If the entry is not a prefetched entry, verify that the flush + * dependency parents addresses array has been transfered. + * + * If the entry is prefetched, the free_isr routine will dispose of + * the flush dependency parents adresses array if necessary. + */ + if(!entry_ptr->prefetched) { + HDassert(0 == entry_ptr->fd_parent_count); + HDassert(NULL == entry_ptr->fd_parent_addrs); + } /* end if */ + /* Check whether we should free the space in the file that * the entry occupies */ @@ -6884,6 +7062,22 @@ H5C_load_entry(H5F_t * f, entry->coll_prev = NULL; #endif /* H5_HAVE_PARALLEL */ + /* initialize cache image related fields */ + entry->include_in_image = FALSE; + entry->lru_rank = 0; + entry->image_dirty = FALSE; + entry->fd_parent_count = 0; + entry->fd_parent_addrs = NULL; + entry->fd_child_count = 0; + entry->fd_dirty_child_count = 0; + entry->image_fd_height = 0; + entry->prefetched = FALSE; + entry->prefetch_type_id = 0; + entry->age = 0; +#ifndef NDEBUG /* debugging field */ + entry->serialization_count = 0; +#endif /* NDEBUG */ + H5C__RESET_CACHE_ENTRY_STATS(entry); ret_value = thing; @@ -6904,7 +7098,7 @@ done: /*------------------------------------------------------------------------- * - * Function: H5C_make_space_in_cache + * Function: H5C__make_space_in_cache * * Purpose: Attempt to evict cache entries until the index_size * is at least needed_space below max_cache_size. @@ -6970,7 +7164,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5C_make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, +H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, hbool_t write_permitted) { H5C_t * cache_ptr = f->shared->cache; @@ -6990,7 +7184,7 @@ H5C_make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, uint32_t num_corked_entries = 0; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_PACKAGE /* Sanity checks */ HDassert(f); @@ -7260,7 +7454,7 @@ H5C_make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_make_space_in_cache() */ +} /* H5C__make_space_in_cache() */ /*------------------------------------------------------------------------- @@ -8083,6 +8277,10 @@ H5C__assert_flush_dep_nocycle(const H5C_cache_entry_t * entry, * are about to delete the entry from the cache (i.e. on a * flush destroy). * + * Note: This routine is very similar to H5C__serialize_single_entry + * and changes to one should probably be reflected in the other. + * Ideally, one should be eliminated. + * * Return: Non-negative on success/Negative on failure * * Programmer: Mohamad Chaarawi @@ -8117,8 +8315,7 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr, /* Check for any flags set in the pre-serialize callback */ if(serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) { /* Check for unexpected flags from serialize callback */ - if(serialize_flags & ~(H5C__SERIALIZE_RESIZED_FLAG | - H5C__SERIALIZE_MOVED_FLAG)) + if(serialize_flags & ~(H5C__SERIALIZE_RESIZED_FLAG | H5C__SERIALIZE_MOVED_FLAG)) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unknown serialize flag(s)") #ifdef H5_HAVE_PARALLEL @@ -8165,9 +8362,8 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr, /* Update statistics for resizing the entry */ H5C__UPDATE_STATS_FOR_ENTRY_SIZE_CHANGE(cache_ptr, entry_ptr, new_len); - /* update the hash table for the size change */ - H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, \ - new_len, entry_ptr, !(entry_ptr->is_dirty)); + /* Update the hash table for the size change */ + H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, new_len, entry_ptr, !(entry_ptr->is_dirty)); /* The entry can't be protected since we are in the process of * flushing it. Thus we must update the replacement policy data @@ -8180,10 +8376,11 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr, * for the flush or flush destroy yet, the entry should * be in the slist. Thus update it for the size change. */ + HDassert(entry_ptr->is_dirty); HDassert(entry_ptr->in_slist); H5C__UPDATE_SLIST_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, new_len); - /* finally, update the entry for its new size */ + /* Finally, update the entry for its new size */ entry_ptr->size = new_len; } /* end if */ @@ -8200,10 +8397,10 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr, H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL); H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE); - /* update the entry for its new address */ + /* Update the entry for its new address */ entry_ptr->addr = new_addr; - /* and then reinsert in the index and slist */ + /* And then reinsert in the index and slist */ H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, FAIL); H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL); } /* end if */ diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index 7efba23..7f4c8e0 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -35,6 +35,7 @@ /* Headers */ /***********/ #include "H5private.h" /* Generic Functions */ +#include "H5ACprivate.h" /* Metadata Cache */ #include "H5Cpkg.h" /* Cache */ #include "H5Eprivate.h" /* Error Handling */ @@ -385,6 +386,7 @@ H5C_stats(H5C_t * cache_ptr, size_t aggregate_max_size = 0; int32_t aggregate_max_pins = 0; double hit_rate; + double prefetch_use_rate; double average_successful_search_depth = 0.0f; double average_failed_search_depth = 0.0f; double average_entries_skipped_per_calls_to_msic = 0.0f; @@ -648,6 +650,38 @@ H5C_stats(H5C_t * cache_ptr, (long long)(cache_ptr->LRU_scan_restarts), (long long)(cache_ptr->index_scan_restarts)); + HDfprintf(stdout, + "%s cache image creations/loads/size = %d / %d / %lld\n", + cache_ptr->prefix, + cache_ptr->images_created, + cache_ptr->images_loaded, + (long long)cache_ptr->last_image_size); + + HDfprintf(stdout, + "%s prefetches / dirty prefetches = %lld / %lld\n", + cache_ptr->prefix, + (long long)(cache_ptr->prefetches), + (long long)(cache_ptr->dirty_prefetches)); + + HDfprintf(stdout, + "%s prefetch hits/flushes/evictions = %lld / %lld / %lld\n", + cache_ptr->prefix, + (long long)(cache_ptr->prefetch_hits), + (long long)(cache_ptr->flushes[H5AC_PREFETCHED_ENTRY_ID]), + (long long)(cache_ptr->evictions[H5AC_PREFETCHED_ENTRY_ID])); + + if(cache_ptr->prefetches > 0) + prefetch_use_rate = + (double)100.0f * ((double)(cache_ptr->prefetch_hits)) / + ((double)(cache_ptr->prefetches)); + else + prefetch_use_rate = 0.0f; + + HDfprintf(stdout, + "%s prefetched entry use rate = %lf\n", + cache_ptr->prefix, + prefetch_use_rate); + #if H5C_COLLECT_CACHE_ENTRY_STATS HDfprintf(stdout, "%s aggregate max / min accesses = %d / %d\n", @@ -869,6 +903,14 @@ H5C_stats__reset(H5C_t H5_ATTR_UNUSED * cache_ptr) cache_ptr->LRU_scan_restarts = 0; cache_ptr->index_scan_restarts = 0; + cache_ptr->images_created = 0; + cache_ptr->images_loaded = 0; + cache_ptr->last_image_size = (size_t)0; + + cache_ptr->prefetches = 0; + cache_ptr->dirty_prefetches = 0; + cache_ptr->prefetch_hits = 0; + #if H5C_COLLECT_CACHE_ENTRY_STATS for(i = 0; i <= cache_ptr->max_type_id; i++) { cache_ptr->max_accesses[i] = 0; @@ -952,6 +994,303 @@ H5C__dump_entry(H5C_t *cache_ptr, const H5C_cache_entry_t *entry_ptr, if(entry_ptr->flush_dep_nchildren) H5C__dump_children(cache_ptr, entry_ptr, FALSE, "Child", indent); } /* end H5C__dump_entry() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_flush_dependency_exists() + * + * Purpose: Test to see if a flush dependency relationship exists + * between the supplied parent and child. Both parties + * are indicated by addresses so as to avoid the necessity + * of protect / unprotect calls prior to this call. + * + * If either the parent or the child is not in the metadata + * cache, the function sets *fd_exists_ptr to FALSE. + * + * If both are in the cache, the childs list of parents is + * searched for the proposed parent. If the proposed parent + * is found in the childs parent list, the function sets + * *fd_exists_ptr to TRUE. In all other non-error cases, + * the function sets *fd_exists_ptr FALSE. + * + * Return: SUCCEED on success/FAIL on failure. Note that + * *fd_exists_ptr is undefined on failure. + * + * Programmer: John Mainzer + * 9/28/16 + * + *------------------------------------------------------------------------- + */ +#ifndef NDEBUG +herr_t +H5C_flush_dependency_exists(H5C_t *cache_ptr, haddr_t parent_addr, haddr_t child_addr, + hbool_t *fd_exists_ptr) +{ + hbool_t fd_exists = FALSE; /* whether flush dependency exists */ + H5C_cache_entry_t * parent_ptr; /* Ptr to parent entry */ + H5C_cache_entry_t * child_ptr; /* Ptr to child entry */ + hbool_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI(NULL) + + /* Sanity checks */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(H5F_addr_defined(parent_addr)); + HDassert(H5F_addr_defined(child_addr)); + HDassert(fd_exists_ptr); + + H5C__SEARCH_INDEX(cache_ptr, parent_addr, parent_ptr, FAIL) + H5C__SEARCH_INDEX(cache_ptr, child_addr, child_ptr, FAIL) + + if(parent_ptr && child_ptr) { + HDassert(parent_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(child_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + + if(child_ptr->flush_dep_nparents > 0) { + unsigned u; /* Local index variable */ + + HDassert(child_ptr->flush_dep_parent); + HDassert(child_ptr->flush_dep_parent_nalloc >= child_ptr->flush_dep_nparents); + + for(u = 0; u < child_ptr->flush_dep_nparents; u++) { + if(child_ptr->flush_dep_parent[u] == parent_ptr) { + fd_exists = TRUE; + HDassert(parent_ptr->flush_dep_nchildren > 0); + break; + } /* end if */ + } /* end for */ + } /* end if */ + } /* end if */ + + *fd_exists_ptr = fd_exists; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_flush_dependency_exists() */ +#endif /* NDEBUG */ + + +/*------------------------------------------------------------------------- + * + * Function: H5C_validate_index_list + * + * Purpose: Debugging function that scans the index list for errors. + * + * If an error is detected, the function generates a + * diagnostic and returns FAIL. If no error is detected, + * the function returns SUCCEED. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: John Mainzer, 9/16/16 + * + *------------------------------------------------------------------------- + */ +#ifndef NDEBUG +herr_t +H5C_validate_index_list(H5C_t *cache_ptr) +{ + H5C_cache_entry_t * entry_ptr = NULL; + uint32_t len = 0; + int32_t index_ring_len[H5C_RING_NTYPES]; + size_t size = 0; + size_t clean_size = 0; + size_t dirty_size = 0; + size_t index_ring_size[H5C_RING_NTYPES]; + size_t clean_index_ring_size[H5C_RING_NTYPES]; + size_t dirty_index_ring_size[H5C_RING_NTYPES]; + int i; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Sanity checks */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + for(i = 0; i < H5C_RING_NTYPES; i++) { + index_ring_len[i] = 0; + index_ring_size[i] = 0; + clean_index_ring_size[i] = 0; + dirty_index_ring_size[i] = 0; + } /* end if */ + + if(((cache_ptr->il_head == NULL) || (cache_ptr->il_tail == NULL)) + && (cache_ptr->il_head != cache_ptr->il_tail)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 1 failed") + + if((cache_ptr->index_len == 1) && ((cache_ptr->il_head != cache_ptr->il_tail) + || (cache_ptr->il_head == NULL) || (cache_ptr->il_head->size != cache_ptr->index_size))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 2 failed") + + if((cache_ptr->index_len >= 1) + && ((cache_ptr->il_head == NULL) + || (cache_ptr->il_head->il_prev != NULL) + || (cache_ptr->il_tail == NULL) + || (cache_ptr->il_tail->il_next != NULL))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 3 failed") + + entry_ptr = cache_ptr->il_head; + while(entry_ptr != NULL) { + if((entry_ptr != cache_ptr->il_head) + && ((entry_ptr->il_prev == NULL) || (entry_ptr->il_prev->il_next != entry_ptr))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 4 failed") + + if((entry_ptr != cache_ptr->il_tail) + && ((entry_ptr->il_next == NULL) || (entry_ptr->il_next->il_prev != entry_ptr))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 5 failed") + + HDassert(entry_ptr->ring > 0); + HDassert(entry_ptr->ring < H5C_RING_NTYPES); + + len++; + index_ring_len[entry_ptr->ring] += 1; + + size += entry_ptr->size; + index_ring_size[entry_ptr->ring] += entry_ptr->size; + + if(entry_ptr->is_dirty) { + dirty_size += entry_ptr->size; + dirty_index_ring_size[entry_ptr->ring] += entry_ptr->size; + } /* end if */ + else { + clean_size += entry_ptr->size; + clean_index_ring_size[entry_ptr->ring] += entry_ptr->size; + } /* end else */ + + entry_ptr = entry_ptr->il_next; + } /* end while */ + + if((cache_ptr->index_len != len) || (cache_ptr->il_len != len) + || (cache_ptr->index_size != size) || (cache_ptr->il_size != size) + || (cache_ptr->clean_index_size != clean_size) + || (cache_ptr->dirty_index_size != dirty_size) + || (clean_size + dirty_size != size)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 6 failed") + + size = 0; + clean_size = 0; + dirty_size = 0; + for(i = 0; i < H5C_RING_NTYPES; i++) { + size += clean_index_ring_size[i] + dirty_index_ring_size[i]; + clean_size += clean_index_ring_size[i]; + dirty_size += dirty_index_ring_size[i]; + } /* end for */ + + if((cache_ptr->index_size != size) + || (cache_ptr->clean_index_size != clean_size) + || (cache_ptr->dirty_index_size != dirty_size)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 7 failed") + +done: + if(ret_value != SUCCEED) + HDassert(0); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_validate_index_list() */ +#endif /* NDEBUG */ + + +/*------------------------------------------------------------------------- + * + * Function: H5C_get_entry_ptr_from_addr() + * + * Purpose: Debugging function that attempts to look up an entry in the + * cache by its file address, and if found, returns a pointer + * to the entry in *entry_ptr_ptr. If the entry is not in the + * cache, *entry_ptr_ptr is set to NULL. + * + * WARNING: This call should be used only in debugging + * routines, and it should be avoided when + * possible. + * + * Further, if we ever multi-thread the cache, + * this routine will have to be either discarded + * or heavily re-worked. + * + * Finally, keep in mind that the entry whose + * pointer is obtained in this fashion may not + * be in a stable state. + * + * Note that this function is only defined if NDEBUG + * is not defined. + * + * As heavy use of this function is almost certainly a + * bad idea, the metadata cache tracks the number of + * successful calls to this function, and (if + * H5C_DO_SANITY_CHECKS is defined) displays any + * non-zero count on cache shutdown. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: John Mainzer, 5/30/14 + * + *------------------------------------------------------------------------- + */ +#ifndef NDEBUG +herr_t +H5C_get_entry_ptr_from_addr(H5C_t *cache_ptr, haddr_t addr, void **entry_ptr_ptr) +{ + H5C_cache_entry_t * entry_ptr = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(H5F_addr_defined(addr)); + HDassert(entry_ptr_ptr); + + H5C__SEARCH_INDEX(cache_ptr, addr, entry_ptr, FAIL) + + if(entry_ptr == NULL) + /* the entry doesn't exist in the cache -- report this + * and quit. + */ + *entry_ptr_ptr = NULL; + else { + *entry_ptr_ptr = entry_ptr; + + /* increment call counter */ + (cache_ptr->get_entry_ptr_from_addr_counter)++; + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_get_entry_ptr_from_addr() */ +#endif /* NDEBUG */ + + +/*------------------------------------------------------------------------- + * Function: H5C_get_serialization_in_progress + * + * Purpose: Return the current value of + * cache_ptr->serialization_in_progress. + * + * Return: Current value of cache_ptr->serialization_in_progress. + * + * Programmer: John Mainzer + * 8/24/15 + * + *------------------------------------------------------------------------- + */ +#ifndef NDEBUG +hbool_t +H5C_get_serialization_in_progress(const H5C_t *cache_ptr) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + FUNC_LEAVE_NOAPI(cache_ptr->serialization_in_progress) +} /* H5C_get_serialization_in_progress() */ +#endif /* NDEBUG */ + + /*------------------------------------------------------------------------- * * Function: H5C_cache_is_clean() @@ -995,3 +1334,69 @@ H5C_cache_is_clean(const H5C_t *cache_ptr, H5C_ring_t inner_ring) } /* H5C_cache_is_clean() */ #endif /* NDEBUG */ + +/*------------------------------------------------------------------------- + * + * Function: H5C_verify_entry_type() + * + * Purpose: Debugging function that attempts to look up an entry in the + * cache by its file address, and if found, test to see if its + * type field contains the expted value. + * + * If the specified entry is in cache, *in_cache_ptr is set + * to TRUE, and *type_ok_ptr is set to TRUE or FALSE depending + * on whether the entries type field matches the expected_type + * parameter. + * + * If the target entry is not in cache, *in_cache_ptr is + * set to FALSE, and *type_ok_ptr is undefined. + * + * Note that this function is only defined if NDEBUG + * is not defined. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: John Mainzer, 5/30/14 + * + *------------------------------------------------------------------------- + */ +#ifndef NDEBUG +herr_t +H5C_verify_entry_type(H5C_t *cache_ptr, haddr_t addr, + const H5C_class_t *expected_type, hbool_t *in_cache_ptr, + hbool_t *type_ok_ptr) +{ + H5C_cache_entry_t * entry_ptr = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(H5F_addr_defined(addr)); + HDassert(expected_type); + HDassert(in_cache_ptr); + HDassert(type_ok_ptr); + + H5C__SEARCH_INDEX(cache_ptr, addr, entry_ptr, FAIL) + + if(entry_ptr == NULL) + /* the entry doesn't exist in the cache -- report this + * and quit. + */ + *in_cache_ptr = FALSE; + else { + *in_cache_ptr = TRUE; + + if(entry_ptr->prefetched) + *type_ok_ptr = (expected_type->id == entry_ptr->prefetch_type_id); + else + *type_ok_ptr = (expected_type == entry_ptr->type); + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_verify_entry_type() */ +#endif /* NDEBUG */ + diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index f237b7b..1656a32 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -682,6 +682,31 @@ if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ #define H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) \ ((cache_ptr)->index_scan_restarts)++; +#define H5C__UPDATE_STATS_FOR_CACHE_IMAGE_CREATE(cache_ptr) \ +{ \ + (cache_ptr)->images_created++; \ +} + +#define H5C__UPDATE_STATS_FOR_CACHE_IMAGE_LOAD(cache_ptr) \ +{ \ + /* make sure image len is still good */ \ + HDassert((cache_ptr)->image_len > 0); \ + (cache_ptr)->images_loaded++; \ + (cache_ptr)->last_image_size = (cache_ptr)->image_len; \ +} + +#define H5C__UPDATE_STATS_FOR_PREFETCH(cache_ptr, dirty) \ +{ \ + (cache_ptr)->prefetches++; \ + if ( dirty ) \ + (cache_ptr)->dirty_prefetches++; \ +} + +#define H5C__UPDATE_STATS_FOR_PREFETCH_HIT(cache_ptr) \ +{ \ + (cache_ptr)->prefetch_hits++; \ +} + #if H5C_COLLECT_CACHE_ENTRY_STATS #define H5C__RESET_CACHE_ENTRY_STATS(entry_ptr) \ @@ -906,6 +931,10 @@ if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ #define H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr) #define H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) #define H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) +#define H5C__UPDATE_STATS_FOR_CACHE_IMAGE_CREATE(cache_ptr) +#define H5C__UPDATE_STATS_FOR_CACHE_IMAGE_LOAD(cache_ptr) +#define H5C__UPDATE_STATS_FOR_PREFETCH(cache_ptr, dirty) +#define H5C__UPDATE_STATS_FOR_PREFETCH_HIT(cache_ptr) #endif /* H5C_COLLECT_CACHE_STATS */ @@ -2219,6 +2248,120 @@ if ( ( (cache_ptr)->index_size != \ /*------------------------------------------------------------------------- * + * Macro: H5C__UPDATE_RP_FOR_INSERT_APPEND + * + * Purpose: Update the replacement policy data structures for an + * insertion of the specified cache entry. + * + * Unlike H5C__UPDATE_RP_FOR_INSERTION below, mark the + * new entry as the LEAST recently used entry, not the + * most recently used. + * + * For now at least, this macro should only be used in + * the reconstruction of the metadata cache from a cache + * image block. + * + * At present, we only support the modified LRU policy, so + * this function deals with that case unconditionally. If + * we ever support other replacement policies, the function + * should switch on the current policy and act accordingly. + * + * Return: N/A + * + * Programmer: John Mainzer, 8/15/15 + * + *------------------------------------------------------------------------- + */ + +#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS + +#define H5C__UPDATE_RP_FOR_INSERT_APPEND(cache_ptr, entry_ptr, fail_val) \ +{ \ + HDassert( (cache_ptr) ); \ + HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ + HDassert( (entry_ptr) ); \ + HDassert( !((entry_ptr)->is_protected) ); \ + HDassert( !((entry_ptr)->is_read_only) ); \ + HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ + HDassert( (entry_ptr)->size > 0 ); \ + \ + if ( (entry_ptr)->is_pinned ) { \ + \ + H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->pel_head_ptr, \ + (cache_ptr)->pel_tail_ptr, \ + (cache_ptr)->pel_len, \ + (cache_ptr)->pel_size, (fail_val)) \ + \ + } else { \ + \ + /* modified LRU specific code */ \ + \ + /* insert the entry at the tail of the LRU list. */ \ + \ + H5C__DLL_APPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ + (cache_ptr)->LRU_tail_ptr, \ + (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_size, (fail_val)) \ + \ + /* insert the entry at the tail of the clean or dirty LRU list as \ + * appropriate. \ + */ \ + \ + if ( entry_ptr->is_dirty ) { \ + H5C__AUX_DLL_APPEND((entry_ptr), (cache_ptr)->dLRU_head_ptr, \ + (cache_ptr)->dLRU_tail_ptr, \ + (cache_ptr)->dLRU_list_len, \ + (cache_ptr)->dLRU_list_size, (fail_val)) \ + } else { \ + H5C__AUX_DLL_APPEND((entry_ptr), (cache_ptr)->cLRU_head_ptr, \ + (cache_ptr)->cLRU_tail_ptr, \ + (cache_ptr)->cLRU_list_len, \ + (cache_ptr)->cLRU_list_size, (fail_val)) \ + } \ + \ + /* End modified LRU specific code. */ \ + } \ +} + +#else /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ + +#define H5C__UPDATE_RP_FOR_INSERT_APPEND(cache_ptr, entry_ptr, fail_val) \ +{ \ + HDassert( (cache_ptr) ); \ + HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ + HDassert( (entry_ptr) ); \ + HDassert( !((entry_ptr)->is_protected) ); \ + HDassert( !((entry_ptr)->is_read_only) ); \ + HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ + HDassert( (entry_ptr)->size > 0 ); \ + \ + if ( (entry_ptr)->is_pinned ) { \ + \ + H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->pel_head_ptr, \ + (cache_ptr)->pel_tail_ptr, \ + (cache_ptr)->pel_len, \ + (cache_ptr)->pel_size, (fail_val)) \ + \ + } else { \ + \ + /* modified LRU specific code */ \ + \ + /* insert the entry at the tail of the LRU list. */ \ + \ + H5C__DLL_APPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ + (cache_ptr)->LRU_tail_ptr, \ + (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_size, (fail_val)) \ + \ + /* End modified LRU specific code. */ \ + } \ +} + +#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ + + +/*------------------------------------------------------------------------- + * * Macro: H5C__UPDATE_RP_FOR_INSERTION * * Purpose: Update the replacement policy data structures for an @@ -3962,7 +4105,22 @@ typedef struct H5C_tag_info_t { * * size_decreased: Boolean flag set to TRUE whenever the maximum cache * size is decreased. The flag triggers a call to - * H5C_make_space_in_cache() on the next call to H5C_protect(). + * H5C__make_space_in_cache() on the next call to H5C_protect(). + * + * resize_in_progress: As the metadata cache has become re-entrant, it is + * possible that a protect may trigger a call to + * H5C__auto_adjust_cache_size(), which may trigger a flush, + * which may trigger a protect, which will result in another + * call to H5C__auto_adjust_cache_size(). + * + * The resize_in_progress boolean flag is used to detect this, + * and to prevent the infinite recursion that would otherwise + * occur. + * + * Note that this issue is not hypothetical -- this field + * was added 12/29/15 to fix a bug exposed in the testing + * of changes to the file driver info superblock extension + * management code needed to support rings. * * resize_ctl: Instance of H5C_auto_size_ctl_t containing configuration * data for automatic cache resizing. @@ -4039,6 +4197,77 @@ typedef struct H5C_tag_info_t { * this field will be reset every automatic resize epoch. * * + * Metadata cache image management related fields. + * + * image_ctl: Instance of H5C_cache_image_ctl_t containing configuration + * data for generation of a cache image on file close. + * + * serialization_in_progress: Boolean field that is set to TRUE iff + * the cache is in the process of being serialized. This + * field is needed to support the H5C_serialization_in_progress() + * call, which is in turn required for sanity checks in some + * cache clients. + * + * load_image: Boolean flag indicating that the metadata cache image + * superblock extension message exists and should be + * read, and the image block read and decoded on the next + * call to H5C_protect(). + * + * image_loaded: Boolean flag indicating that the metadata cache has + * loaded the metadata cache image as directed by the + * MDC cache image superblock extension message. + * + * delete_image: Boolean flag indicating whether the metadata cache image + * superblock message should be deleted and the cache image + * file space freed after they have been read and decoded. + * + * This flag should be set to TRUE iff the file is opened + * R/W and there is a cache image to be read. + * + * image_addr: haddr_t containing the base address of the on disk + * metadata cache image, or HADDR_UNDEF if that value is + * undefined. Note that this field is used both in the + * construction and write, and the read and decode of + * metadata cache image blocks. + * + * image_len: hsize_t containing the size of the on disk metadata cache + * image, or zero if that value is undefined. Note that this + * field is used both in the construction and write, and the + * read and decode of metadata cache image blocks. + * + * image_data_len: size_t containing the number of bytes of data in the + * on disk metadata cache image, or zero if that value is + * undefined. + * + * In most cases, this value is the same as the image_len + * above. It exists to allow for metadata cache image blocks + * that are larger than the actual image. Thus in all + * cases image_data_len <= image_len. + * + * To create the metadata cache image, we must first serialize all the + * entries in the metadata cache. This is done by a scan of the index. + * As entries must be serialized in increasing flush dependency height + * order, we scan the index repeatedly, once for each flush dependency + * height in increasing order. + * + * This operation is complicated by the fact that entries other the the + * target may be inserted, loaded, relocated, or removed from the cache + * (either by eviction or the take ownership flag) as the result of a + * pre_serialize or serialize callback. While entry removals are not + * a problem for the scan of the index, insertions, loads, and relocations + * are. Hence the entries loaded, inserted, and relocated counters + * listed below have been implemented to allow these conditions to be + * detected and dealt with by restarting the scan. + * + * The serialization operation is further complicated by the fact that + * the flush dependency height of a given entry may increase (as the + * result of an entry load or insert) or decrease (as the result of an + * entry removal -- via either eviction or the take ownership flag). The + * entry_fd_height_change_counter field is maintained to allow detection + * of this condition, and a restart of the scan when it occurs. + * + * Note that all these new fields would work just as well as booleans. + * * entries_loaded_counter: Number of entries loaded into the cache * since the last time this field was reset. * @@ -4048,6 +4277,29 @@ typedef struct H5C_tag_info_t { * entries relocated_counter: Number of entries whose base address has * been changed since the last time this field was reset. * + * entry_fd_height_change_counter: Number of entries whose flush dependency + * height has changed since the last time this field was reset. + * + * The following fields are used assemble the cache image prior to + * writing it to disk. + * + * num_entries_in_image: Unsigned integer field containing the number of entries + * to be copied into the metadata cache image. Note that + * this value will be less than the number of entries in + * the cache, and the superblock and its related entries + * are not written to the metadata cache image. + * + * image_entries: Pointer to a dynamically allocated array of instance of + * H5C_image_entry_t of length num_entries_in_image, or NULL + * if that array does not exist. This array is used to + * assemble entry data to be included in the image, and to + * sort them by flush dependency height and LRU rank. + * + * image_buffer: Pointer to the dynamically allocated buffer of length + * image_len in which the metadata cache image is assembled, + * or NULL if that buffer does not exist. + * + * * Free Space Manager Related fields: * * The free space managers must be informed when we are about to close @@ -4261,23 +4513,63 @@ typedef struct H5C_tag_info_t { * max_pel_size: Largest value attained by the pel_size field in the * current epoch. * - * calls_to_msic: Total number of calls to H5C_make_space_in_cache + * calls_to_msic: Total number of calls to H5C__make_space_in_cache * * total_entries_skipped_in_msic: Number of clean entries skipped while - * enforcing the min_clean_fraction in H5C_make_space_in_cache(). + * enforcing the min_clean_fraction in H5C__make_space_in_cache(). * * total_entries_scanned_in_msic: Number of clean entries skipped while - * enforcing the min_clean_fraction in H5C_make_space_in_cache(). + * enforcing the min_clean_fraction in H5C__make_space_in_cache(). * * max_entries_skipped_in_msic: Maximum number of clean entries skipped - * in any one call to H5C_make_space_in_cache(). + * in any one call to H5C__make_space_in_cache(). * * max_entries_scanned_in_msic: Maximum number of entries scanned over - * in any one call to H5C_make_space_in_cache(). + * in any one call to H5C__make_space_in_cache(). * * entries_scanned_to_make_space: Number of entries scanned only when looking * for entries to evict in order to make space in cache. * + * + * The following fields track statistics on cache images. + * + * images_created: Integer field containing the number of cache images + * created since the last time statistics were reset. + * + * At present, this field must always be either 0 or 1. + * Further, since cache images are only created at file + * close, this field should only be set at that time. + * + * images_loaded: Integer field containing the number of cache images + * loaded since the last time statistics were reset. + * + * At present, this field must always be either 0 or 1. + * Further, since cache images are only loaded at the + * time of the first protect or on file close, this value + * should only change on those events. + * + * last_image_size: Size of the most recently loaded metadata cache image + * loaded into the cache, or zero if no image has been + * loaded. + * + * At present, at most one cache image can be loaded into + * the metadata cache for any given file, and this image + * will be loaded either on the first protect, or on file + * close if no entry is protected before then. + * + * + * Fields for tracking prefetched entries. Note that flushes and evictions + * of prefetched entries are tracked in the flushes and evictions arrays + * discused above. + * + * prefetches: Number of prefetched entries that are loaded to the + * cache. + * + * dirty_prefetches: Number of dirty prefetched entries that are loaded + * into the cache. + * + * prefetch_hits: Number of prefetched entries that are actually used. + * * * As entries are now capable of moving, loading, dirtying, and deleting * other entries in their pre_serialize and serialize callbacks, it has @@ -4348,6 +4640,11 @@ typedef struct H5C_tag_info_t { * field is intended to allow marking of output of with * the processes mpi rank. * + * get_entry_ptr_from_addr_counter: Counter used to track the number of + * times the H5C_get_entry_ptr_from_addr() function has been + * called successfully. This field is only defined when + * NDEBUG is not #defined. + * ****************************************************************************/ struct H5C_t { uint32_t magic; @@ -4453,6 +4750,7 @@ struct H5C_t { hbool_t resize_enabled; hbool_t cache_full; hbool_t size_decreased; + hbool_t resize_in_progress; H5C_auto_size_ctl_t resize_ctl; /* Fields for epoch markers used in automatic cache size adjustment */ @@ -4468,9 +4766,23 @@ struct H5C_t { int64_t cache_hits; int64_t cache_accesses; + /* fields supporting generation of a cache image on file close */ + H5C_cache_image_ctl_t image_ctl; + hbool_t serialization_in_progress; + hbool_t load_image; + hbool_t image_loaded; + hbool_t delete_image; + haddr_t image_addr; + hsize_t image_len; + hsize_t image_data_len; int64_t entries_loaded_counter; int64_t entries_inserted_counter; int64_t entries_relocated_counter; + int64_t entry_fd_height_change_counter; + uint32_t num_entries_in_image; + H5C_image_entry_t * image_entries; + void * image_buffer; + /* Free Space Manager Related fields */ hbool_t rdfsm_settled; hbool_t mdfsm_settled; @@ -4538,6 +4850,16 @@ struct H5C_t { int64_t LRU_scan_restarts; int64_t index_scan_restarts; + /* Fields for tracking cache image operations */ + int32_t images_created; + int32_t images_loaded; + hsize_t last_image_size; + + /* Fields for tracking prefetched entries */ + int64_t prefetches; + int64_t dirty_prefetches; + int64_t prefetch_hits; + #if H5C_COLLECT_CACHE_ENTRY_STATS int32_t max_accesses[H5C__MAX_NUM_TYPE_IDS + 1]; int32_t min_accesses[H5C__MAX_NUM_TYPE_IDS + 1]; @@ -4549,6 +4871,10 @@ struct H5C_t { #endif /* H5C_COLLECT_CACHE_STATS */ char prefix[H5C__PREFIX_LEN]; + +#ifndef NDEBUG + int64_t get_entry_ptr_from_addr_counter; +#endif /* NDEBUG */ }; /* Define typedef for tagged cache entry iteration callbacks */ @@ -4563,12 +4889,20 @@ typedef int (*H5C_tag_iter_cb_t)(H5C_cache_entry_t *entry, void *ctx); /******************************/ /* Package Private Prototypes */ /******************************/ +H5_DLL herr_t H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5C__deserialize_prefetched_entry(H5F_t * f, hid_t dxpl_id, + H5C_t * cache_ptr, H5C_cache_entry_t** entry_ptr_ptr, + const H5C_class_t * type, haddr_t addr, void * udata); /* General routines */ H5_DLL herr_t H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, unsigned flags); +H5_DLL herr_t H5C__generate_cache_image(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr); +H5_DLL herr_t H5C__load_cache_image(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5C__mark_flush_dep_serialized(H5C_cache_entry_t * entry_ptr); H5_DLL herr_t H5C__mark_flush_dep_unserialized(H5C_cache_entry_t * entry_ptr); +H5_DLL herr_t H5C__make_space_in_cache(H5F_t * f, hid_t dxpl_id, + size_t space_needed, hbool_t write_permitted); H5_DLL herr_t H5C__flush_marked_entries(H5F_t * f, hid_t dxpl_id); H5_DLL herr_t H5C__iter_tagged_entries(H5C_t *cache, haddr_t tag, hbool_t match_global, H5C_tag_iter_cb_t cb, void *cb_ctx); diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index d7111a8..47573f9 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -41,7 +41,7 @@ /**************************/ /* Cache configuration settings */ -#define H5C__MAX_NUM_TYPE_IDS 29 +#define H5C__MAX_NUM_TYPE_IDS 30 #define H5C__PREFIX_LEN 32 /* This sanity checking constant was picked out of the air. Increase @@ -114,6 +114,7 @@ /* Cache configuration versions */ #define H5C__CURR_AUTO_SIZE_CTL_VER 1 #define H5C__CURR_AUTO_RESIZE_RPT_FCN_VER 1 +#define H5C__CURR_CACHE_IMAGE_CTL_VER 1 /* Default configuration settings */ #define H5C__DEF_AR_UPPER_THRESHHOLD 0.9999f @@ -994,16 +995,16 @@ typedef int H5C_ring_t; * just before the entry is freed. * * This is necessary, as the LRU list can be changed out - * from under H5C_make_space_in_cache() by the serialize + * from under H5C__make_space_in_cache() by the serialize * callback which may change the size of an existing entry, * and/or load a new entry while serializing the target entry. * * This in turn can cause a recursive call to - * H5C_make_space_in_cache() which may either flush or evict + * H5C__make_space_in_cache() which may either flush or evict * the next entry that the first invocation of that function * was about to examine. * - * The magic field allows H5C_make_space_in_cache() to + * The magic field allows H5C__make_space_in_cache() to * detect this case, and re-start its scan from the bottom * of the LRU when this situation occurs. * @@ -1338,6 +1339,187 @@ typedef int H5C_ring_t; * In either case, when there is no previous item, it should * be NULL. * + * Fields supporting the cache image feature: + * + * The following fields are used to store data about the entry which must + * be stored in the cache image block, but which will typically be either + * lost or heavily altered in the process of serializing the cache and + * preparing its contents to be copied into the cache image block. + * + * Some fields are also used in loading the contents of the metadata cache + * image back into the cache, and in managing such entries until they are + * either protected by the library (at which point they become regular + * entries) or are evicted. See discussion of the prefetched field for + * further details. + * + * include_in_image: Boolean flag indicating whether this entry should + * be included in the metadata cache image. This field should + * always be false prior to the H5C_prep_for_file_close() call. + * During that call, it should be set to TRUE for all entries + * that are to be included in the metadata cache image. At + * present, only the superblock, the superblock extension + * object header and its chunks (if any) are omitted from + * the image. + * + * lru_rank: Rank of the entry in the LRU just prior to file close. + * + * Note that the first entry on the LRU has lru_rank 1, + * and that entries not on the LRU at that time will have + * either lru_rank -1 (if pinned) or 0 (if loaded during + * the process of flushing the cache. + * + * image_dirty: Boolean flag indicating whether the entry should be marked + * as dirty in the metadata cache image. The flag is set to + * TRUE iff the entry is dirty when H5C_prep_for_file_close() + * is called. + * + * fd_parent_count: If the entry is a child in one or more flush dependency + * relationships, this field contains the number of flush + * dependency parents. + * + * In all other cases, the field is set to zero. + * + * Note that while this count is initially taken from the + * flush dependency fields above, if the entry is in the + * cache image (i.e. include_in_image is TRUE), any parents + * that are not in the image are removed from this count and + * from the fd_parent_addrs array below. + * + * Finally observe that if the entry is dirty and in the + * cache image, and its parent is dirty and not in the cache + * image, then the entry must be removed from the cache image + * to avoid violating the flush dependency flush ordering. + * + * fd_parent_addrs: If the entry is a child in one or more flush dependency + * relationship when H5C_prep_for_file_close() is called, this + * field must contain a pointer to an array of size + * fd_parent_count containing the on disk addresses of the + * parent. + * + * In all other cases, the field is set to NULL. + * + * Note that while this list of addresses is initially taken + * from the flush dependency fields above, if the entry is in the + * cache image (i.e. include_in_image is TRUE), any parents + * that are not in the image are removed from this list, and + * and from the fd_parent_count above. + * + * Finally observe that if the entry is dirty and in the + * cache image, and its parent is dirty and not in the cache + * image, then the entry must be removed from the cache image + * to avoid violating the flush dependency flush ordering. + * + * fd_child_count: If the entry is a parent in a flush dependency + * relationship, this field contains the number of flush + * dependency children. + * + * In all other cases, the field is set to zero. + * + * Note that while this count is initially taken from the + * flush dependency fields above, if the entry is in the + * cache image (i.e. include_in_image is TRUE), any children + * that are not in the image are removed from this count. + * + * fd_dirty_child_count: If the entry is a parent in a flush dependency + * relationship, this field contains the number of dirty flush + * dependency children. + * + * In all other cases, the field is set to zero. + * + * Note that while this count is initially taken from the + * flush dependency fields above, if the entry is in the + * cache image (i.e. include_in_image is TRUE), any dirty + * children that are not in the image are removed from this + * count. + * + * image_fd_height: Flush dependency height of the entry in the cache image. + * + * The flush dependency height of any entry involved in a + * flush dependency relationship is defined to be the + * longest flush dependency path from that entry to an entry + * with no flush depenency children. + * + * Since the image_fd_height is used to order entries in the + * cache image so that fd parents preceed fd children, for + * purposes of this field, and entry is at flush dependency + * level 0 if it either has no children, or if all of its + * children are not in the cache image. + * + * Note that if a child in a flush dependency relationship is + * dirty and in the cache image, and its parent is dirty and + * not in the cache image, then the child must be excluded + * from the cache image to maintain flush ordering. + * + * prefetched: Boolean flag indicating that the on disk image of the entry + * has been loaded into the cache prior any request for the + * entry by the rest of the library. + * + * As of this writing (8/10/15), this can only happen through + * the load of a cache image block, although other scenarios + * are contemplated for the use of this feature. Note that + * unlike the usual prefetch situation, this means that a + * pre fetched entry can be dirty, and/or can be a party to + * flush dependency relationship(s). This complicates matters + * somewhat. + * + * The essential feature of a pre-fetched entry is that it + * consists only of a buffer containing the on disk image of + * the entry. Thus it must be deserialized before it can + * be passed back to the library on a protect call. This + * task is handled by H5C_deserialized_prefetched_entry(). + * In essence, this routine calls the deserialize callback + * provided in the protect call with the on disk image, + * deletes the prefetched entry from the cache, and replaces + * it with the deserialized entry returned by the deserialize + * callback. + * + * Further, if the prefetched entry is a flush dependency parent, + * all its flush dependency children (which must also be + * pre-fetched entries), must be tranfered to the new cache + * entry returned by the deserailization callback. + * + * Finally, if the prefetched entry is a flush dependency child, + * this flush dependency must be destroyed prior to the + * deserialize call. + * + * In addition to the above special processing on the first + * protect call on a prefetched entry (after which is no longer + * a prefetched entry), prefetched entries also require special + * tretment on flush and evict. + * + * On flush, a dirty prefetched entry must simply be written + * to disk and marked clean without any call to any client + * callback. + * + * On eviction, if a prefetched entry is a flush dependency + * child, that flush dependency relationship must be destroyed + * just prior to the eviction. If the flush dependency code + * is working properly, it should be impossible for any entry + * that is a flush dependency parent to be evicted. + * + * prefetch_type_id: Integer field containing the type ID of the prefetched + * entry. This ID must match the ID of the type provided in any + * protect call on the prefetched entry. + * + * The value of this field is undefined in prefetched is FALSE. + * + * age: Number of times a prefetched entry has appeared in + * subsequent cache images. The field exists to allow + * imposition of a limit on how many times a prefetched + * entry can appear in subsequent cache images without being + * converted to a regular entry. + * + * This field must be zero if prefetched is FALSE. + * + * serialization_count: Integer field used to maintain a count of the + * number of times each entry is serialized during cache + * serialization. While no entry should be serialized more than + * once in any serialization call, throw an assertion if any + * flush depencency parent is serialized more than once during + * a single cache serialization. + * + * This is a debugging field, and thus is maintained only if + * NDEBUG is undefined. * * Fields supporting tagged entries: * @@ -1433,6 +1615,23 @@ typedef struct H5C_cache_entry_t { struct H5C_cache_entry_t *coll_prev; #endif /* H5_HAVE_PARALLEL */ + /* fields supporting cache image */ + hbool_t include_in_image; + int32_t lru_rank; + hbool_t image_dirty; + uint64_t fd_parent_count; + haddr_t *fd_parent_addrs; + uint64_t fd_child_count; + uint64_t fd_dirty_child_count; + uint32_t image_fd_height; + hbool_t prefetched; + int prefetch_type_id; + int32_t age; + +#ifndef NDEBUG /* debugging field */ + int serialization_count; +#endif /* NDEBUG */ + /* fields supporting tag lists */ struct H5C_cache_entry_t *tl_next; struct H5C_cache_entry_t *tl_prev; @@ -1447,6 +1646,168 @@ typedef struct H5C_cache_entry_t { #endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ } H5C_cache_entry_t; + +/**************************************************************************** + * + * structure H5C_image_entry_t + * + * Instances of the H5C_image_entry_t structure are used to store data on + * metadata cache entries used in the construction of the metadata cache + * image block. In essence this structure is a greatly simplified version + * of H5C_cache_entry_t. + * + * The fields of this structure are discussed individually below: + * + * JRM - 8/5/15 + * + * magic: Unsigned 32 bit integer that must always be set to + * H5C__H5C_IMAGE_ENTRY_T_MAGIC when the entry is valid. + * The field must be set to H5C__H5C_IMAGE__ENTRY_T_BAD_MAGIC + * just before the entry is freed. + * + * addr: Base address of the cache entry on disk. + * + * size: Length of the cache entry on disk in bytes. + * + * ring: Instance of H5C_ring_t indicating the flush ordering ring + * to which this entry is assigned. + * + * age: Number of times this prefetech entry has appeared in + * the current sequence of cache images. This field is + * initialized to 0 if the instance of H5C_image_entry_t + * is constructed from a regular entry. + * + * If the instance is constructed from a prefetched entry + * currently residing in the metadata cache, the field is + * set to 1 + the age of the prefetched entry, or to + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX if that sum exceeds + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX. + * + * type_id: Integer field containing the type ID of the entry. + * + * lru_rank: Rank of the entry in the LRU just prior to file close. + * + * Note that the first entry on the LRU has lru_rank 1, + * and that entries not on the LRU at that time will have + * either lru_rank -1 (if pinned) or 0 (if loaded during + * the process of flushing the cache. + * + * is_dirty: Boolean flag indicating whether the contents of the cache + * entry has been modified since the last time it was written + * to disk as a regular piece of metadata. + * + * image_fd_height: Flush dependency height of the entry in the cache image. + * + * The flush dependency height of any entry involved in a + * flush dependency relationship is defined to be the + * longest flush dependency path from that entry to an entry + * with no flush depenency children. + * + * Since the image_fd_height is used to order entries in the + * cache image so that fd parents preceed fd children, for + * purposes of this field, an entry is at flush dependency + * level 0 if it either has no children, or if all of its + * children are not in the cache image. + * + * Note that if a child in a flush dependency relationship is + * dirty and in the cache image, and its parent is dirty and + * not in the cache image, then the child must be excluded + * from the cache image to maintain flush ordering. + * + * fd_parent_count: If the entry is a child in one or more flush dependency + * relationships, this field contains the number of flush + * dependency parents. + * + * In all other cases, the field is set to zero. + * + * Note that while this count is initially taken from the + * flush dependency fields in the associated instance of + * H5C_cache_entry_t, if the entry is in the cache image + * (i.e. include_in_image is TRUE), any parents that are + * not in the image are removed from this count and + * from the fd_parent_addrs array below. + * + * Finally observe that if the entry is dirty and in the + * cache image, and its parent is dirty and not in the cache + * image, then the entry must be removed from the cache image + * to avoid violating the flush dependency flush ordering. + * This should have happened before the construction of + * the instance of H5C_image_entry_t. + * + * fd_parent_addrs: If the entry is a child in one or more flush dependency + * relationship when H5C_prep_for_file_close() is called, this + * field must contain a pointer to an array of size + * fd_parent_count containing the on disk addresses of the + * parents. + * + * In all other cases, the field is set to NULL. + * + * Note that while this list of addresses is initially taken + * from the flush dependency fields in the associated instance of + * H5C_cache_entry_t, if the entry is in the cache image + * (i.e. include_in_image is TRUE), any parents that are not + * in the image are removed from this list, and from the + * fd_parent_count above. + * + * Finally observe that if the entry is dirty and in the + * cache image, and its parent is dirty and not in the cache + * image, then the entry must be removed from the cache image + * to avoid violating the flush dependency flush ordering. + * This should have happened before the construction of + * the instance of H5C_image_entry_t. + * + * fd_child_count: If the entry is a parent in a flush dependency + * relationship, this field contains the number of flush + * dependency children. + * + * In all other cases, the field is set to zero. + * + * Note that while this count is initially taken from the + * flush dependency fields in the associated instance of + * H5C_cache_entry_t, if the entry is in the cache image + * (i.e. include_in_image is TRUE), any children + * that are not in the image are removed from this count. + * + * fd_dirty_child_count: If the entry is a parent in a flush dependency + * relationship, this field contains the number of dirty flush + * dependency children. + * + * In all other cases, the field is set to zero. + * + * Note that while this count is initially taken from the + * flush dependency fields in the associated instance of + * H5C_cache_entry_t, if the entry is in the cache image + * (i.e. include_in_image is TRUE), any dirty children + * that are not in the image are removed from this count. + * + * image_ptr: Pointer to void. When not NULL, this field points to a + * dynamically allocated block of size bytes in which the + * on disk image of the metadata cache entry is stored. + * + * If the entry is dirty, the pre-serialize and serialize + * callbacks must be used to update this image before it is + * written to disk + * + * + ****************************************************************************/ + +typedef struct H5C_image_entry_t { + uint32_t magic; + haddr_t addr; + size_t size; + H5C_ring_t ring; + int32_t age; + int32_t type_id; + int32_t lru_rank; + hbool_t is_dirty; + unsigned image_fd_height; + uint64_t fd_parent_count; + haddr_t *fd_parent_addrs; + uint64_t fd_child_count; + uint64_t fd_dirty_child_count; + void *image_ptr; +} H5C_image_entry_t; + /**************************************************************************** * * structure H5C_auto_size_ctl_t @@ -1736,6 +2097,92 @@ typedef struct H5C_auto_size_ctl_t { double empty_reserve; } H5C_auto_size_ctl_t; +/**************************************************************************** + * + * structure H5C_cache_image_ctl_t + * + * Instances of H5C_image_ctl_t are used to get and set the control + * fields for generation of a metadata cache image on file close. + * + * At present control of construction of a cache image is via a FAPL + * property at file open / create. + * + * The fields of the structure are discussed individually below: + * + * version: Integer field containing the version number of this version + * of the H5C_image_ctl_t structure. Any instance of + * H5C_image_ctl_t passed to the cache must have a known + * version number, or an error will be flagged. + * + * generate_image: Boolean flag indicating whether a cache image should + * be created on file close. + * + * save_resize_status: Boolean flag indicating whether the cache image + * should include the adaptive cache resize configuration and status. + * Note that this field is ignored at present. + * + * entry_ageout: Integer field indicating the maximum number of + * times a prefetched entry can appear in subsequent cache images. + * This field exists to allow the user to avoid the buildup of + * infrequently used entries in long sequences of cache images. + * + * The value of this field must lie in the range + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE (-1) to + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX (100). + * + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE means that no limit + * is imposed on number of times a prefeteched entry can appear + * in subsequent cache images. + * + * A value of 0 prevents prefetched entries from being included + * in cache images. + * + * Positive integers restrict prefetched entries to the specified + * number of appearances. + * + * Note that the number of subsequent cache images that a prefetched + * entry has appeared in is tracked in an 8 bit field. Thus, while + * H5AC__CACHE_IMAGE__ENTRY_AGEOUT__MAX can be increased from its + * current value, any value in excess of 255 will be the functional + * equivalent of H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE. + * + * flags: Unsigned integer containing flags controling which aspects of the + * cache image functinality is actually executed. The primary impetus + * behind this field is to allow developement of tests for partial + * implementations that will require little if any modification to run + * with the full implementation. In normal operation, all flags should + * be set. + * + ****************************************************************************/ + +#define H5C_CI__GEN_MDCI_SBE_MESG ((unsigned)0x0001) +#define H5C_CI__GEN_MDC_IMAGE_BLK ((unsigned)0x0002) +#define H5C_CI__SUPRESS_ENTRY_WRITES ((unsigned)0x0004) +#define H5C_CI__WRITE_CACHE_IMAGE ((unsigned)0x0008) + +/* This #define must set all defined H5C_CI flags. It is + * used in the default value for instances of H5C_cache_image_ctl_t. + * This value will only be modified in test code. + */ +#define H5C_CI__ALL_FLAGS ((unsigned)0x000F) + +#define H5C__DEFAULT_CACHE_IMAGE_CTL \ +{ \ + /* version = */ H5C__CURR_CACHE_IMAGE_CTL_VER, \ + /* generate_image = */ FALSE, \ + /* save_resize_status = */ FALSE, \ + /* entry_ageout = */ H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE, \ + /* flags = */ H5C_CI__ALL_FLAGS \ +} + +typedef struct H5C_cache_image_ctl_t { + int32_t version; + hbool_t generate_image; + hbool_t save_resize_status; + int32_t entry_ageout; + unsigned flags; +} H5C_cache_image_ctl_t; + /***************************************/ /* Library-private Function Prototypes */ /***************************************/ @@ -1770,6 +2217,8 @@ herr_t H5C_verify_tag(int id, haddr_t tag); H5_DLL herr_t H5C_flush_to_min_clean(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5C_get_cache_auto_resize_config(const H5C_t *cache_ptr, H5C_auto_size_ctl_t *config_ptr); +H5_DLL herr_t H5C_get_cache_image_config(const H5C_t * cache_ptr, + H5C_cache_image_ctl_t *config_ptr); H5_DLL herr_t H5C_get_cache_size(H5C_t *cache_ptr, size_t *max_size_ptr, size_t *min_clean_size_ptr, size_t *cur_size_ptr, uint32_t *cur_num_entries_ptr); @@ -1783,8 +2232,11 @@ H5_DLL herr_t H5C_get_evictions_enabled(const H5C_t *cache_ptr, hbool_t *evictio H5_DLL void * H5C_get_aux_ptr(const H5C_t *cache_ptr); H5_DLL FILE *H5C_get_trace_file_ptr(const H5C_t *cache_ptr); H5_DLL FILE *H5C_get_trace_file_ptr_from_entry(const H5C_cache_entry_t *entry_ptr); +H5_DLL herr_t H5C_image_stats(H5C_t * cache_ptr, hbool_t print_header); H5_DLL herr_t H5C_insert_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, haddr_t addr, void *thing, unsigned int flags); +H5_DLL herr_t H5C_load_cache_image_on_next_protect(H5F_t *f, haddr_t addr, + hsize_t len, hbool_t rw); H5_DLL herr_t H5C_mark_entry_dirty(void *thing); H5_DLL herr_t H5C_mark_entry_clean(void *thing); H5_DLL herr_t H5C_mark_entry_unserialized(void *thing); @@ -1799,6 +2251,8 @@ H5_DLL void * H5C_protect(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, H5_DLL herr_t H5C_reset_cache_hit_rate_stats(H5C_t *cache_ptr); H5_DLL herr_t H5C_resize_entry(void *thing, size_t new_size); H5_DLL herr_t H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, H5C_auto_size_ctl_t *config_ptr); +H5_DLL herr_t H5C_set_cache_image_config(const H5F_t *f, H5C_t *cache_ptr, + H5C_cache_image_ctl_t *config_ptr); H5_DLL herr_t H5C_set_evictions_enabled(H5C_t *cache_ptr, hbool_t evictions_enabled); H5_DLL herr_t H5C_set_prefix(H5C_t *cache_ptr, char *prefix); H5_DLL herr_t H5C_set_trace_file_ptr(H5C_t *cache_ptr, FILE *trace_file_ptr); @@ -1810,6 +2264,7 @@ H5_DLL herr_t H5C_unpin_entry(void *thing); H5_DLL herr_t H5C_destroy_flush_dependency(void *parent_thing, void *child_thing); H5_DLL herr_t H5C_unprotect(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *thing, unsigned int flags); +H5_DLL herr_t H5C_validate_cache_image_config(H5C_cache_image_ctl_t * ctl_ptr); H5_DLL herr_t H5C_validate_resize_config(H5C_auto_size_ctl_t *config_ptr, unsigned int tests); H5_DLL herr_t H5C_ignore_tags(H5C_t *cache_ptr); @@ -1819,6 +2274,8 @@ H5_DLL herr_t H5C_cork(H5C_t *cache_ptr, haddr_t obj_addr, unsigned action, hboo H5_DLL herr_t H5C_get_entry_ring(const H5F_t *f, haddr_t addr, H5C_ring_t *ring); H5_DLL herr_t H5C_unsettle_entry_ring(void *thing); H5_DLL herr_t H5C_remove_entry(void *thing); +H5_DLL herr_t H5C_cache_image_status(H5F_t * f, hbool_t *load_ci_ptr, + hbool_t *write_ci_ptr); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id, @@ -1832,7 +2289,16 @@ H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t *f, hid_t dxpl_id, int32_t ce_arra #endif /* H5_HAVE_PARALLEL */ #ifndef NDEBUG /* debugging functions */ +H5_DLL hbool_t H5C_get_serialization_in_progress(const H5C_t *cache_ptr); H5_DLL hbool_t H5C_cache_is_clean(const H5C_t *cache_ptr, H5C_ring_t inner_ring); +H5_DLL herr_t H5C_get_entry_ptr_from_addr(H5C_t *cache_ptr, haddr_t addr, + void **entry_ptr_ptr); +H5_DLL herr_t H5C_flush_dependency_exists(H5C_t *cache_ptr, haddr_t parent_addr, + haddr_t child_addr, hbool_t *fd_exists_ptr); +H5_DLL herr_t H5C_verify_entry_type(H5C_t *cache_ptr, haddr_t addr, + const H5C_class_t *expected_type, hbool_t *in_cache_ptr, + hbool_t *type_ok_ptr); +H5_DLL herr_t H5C_validate_index_list(H5C_t *cache_ptr); #endif /* NDEBUG */ #endif /* !_H5Cprivate_H */ @@ -445,6 +445,8 @@ done: hid_t H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id) { + hbool_t ci_load = FALSE; /* whether MDC ci load requested */ + hbool_t ci_write = FALSE; /* whether MDC CI write requested */ H5F_t *new_file = NULL; /*file struct for new file */ hid_t dxpl_id = H5AC_ind_read_dxpl_id; /*dxpl used by library */ hid_t ret_value; /*return value */ @@ -490,6 +492,14 @@ H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(NULL == (new_file = H5F_open(filename, flags, fcpl_id, fapl_id, dxpl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create file") + /* check to see if both SWMR and cache image are requested. Fail if so */ + if ( H5C_cache_image_status(new_file, &ci_load, &ci_write) < 0 ) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "can't get MDC CI status") + + if ( ( ( ci_load ) || ( ci_write ) ) && + ( flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE) ) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "can't have both SWMR and cache image.") + /* Get an atom for the file */ if((ret_value = H5I_register(H5I_FILE, new_file, TRUE)) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file") @@ -498,9 +508,29 @@ H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id) new_file->file_id = ret_value; done: +#if 0 + /* Quincy: please review this. With the original cleanup code (below) + * my code (above) to disable the combination of SWMR and cache image, + * triggering results in an assertion failure in H5F_close() either + * immediately or on library shutdown depending on the exact location + * of my code. + * + * I noticed that the issue did not appear in H5Fopen(). As that function + * calls H5F_try_close() instead of H5F_close(), I tried copying that + * cleanup code here. It seems to work -- but since I'm not familiar + * with this section of the code, it would be good if you would check me. + * + * Please delete the old version if you buy the fix. + * + * JRM -- 12/29/16 + */ if(ret_value < 0 && new_file) if(H5F_close(new_file) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "problems closing file") +#else + if(ret_value < 0 && new_file && H5F_try_close(new_file, NULL) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "problems closing file") +#endif FUNC_LEAVE_API(ret_value) } /* end H5Fcreate() */ @@ -549,6 +579,8 @@ done: hid_t H5Fopen(const char *filename, unsigned flags, hid_t fapl_id) { + hbool_t ci_load = FALSE; /* whether MDC ci load requested */ + hbool_t ci_write = FALSE; /* whether MDC CI write requested */ H5F_t *new_file = NULL; /*file struct for new file */ hid_t dxpl_id = H5AC_ind_read_dxpl_id; /*dxpl used by library */ hid_t ret_value; /*return value */ @@ -578,6 +610,14 @@ H5Fopen(const char *filename, unsigned flags, hid_t fapl_id) if(NULL == (new_file = H5F_open(filename, flags, H5P_FILE_CREATE_DEFAULT, fapl_id, dxpl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to open file") + /* check to see if both SWMR and cache image are requested. Fail if so */ + if ( H5C_cache_image_status(new_file, &ci_load, &ci_write) < 0 ) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "can't get MDC CI status") + + if ( ( ( ci_load ) || ( ci_write ) ) && + ( flags & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE) ) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "can't have both SWMR and cache image") + /* Get an atom for the file */ if((ret_value = H5I_register(H5I_FILE, new_file, TRUE)) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file handle") @@ -1593,6 +1633,8 @@ done: herr_t H5Fstart_swmr_write(hid_t file_id) { + hbool_t ci_load = FALSE; /* whether MDC ci load requested */ + hbool_t ci_write = FALSE; /* whether MDC CI write requested */ H5F_t *file = NULL; /* File info */ size_t grp_dset_count=0; /* # of open objects: groups & datasets */ size_t nt_attr_count=0; /* # of opened named datatypes + opened attributes */ @@ -1626,6 +1668,13 @@ H5Fstart_swmr_write(hid_t file_id) HDassert(file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS); + /* check to see if cache image is enabled. Fail if so */ + if ( H5C_cache_image_status(file, &ci_load, &ci_write) < 0 ) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "can't get MDC CI status") + + if ( ( ci_load ) || ( ci_write ) ) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "can't have both SWMR and MDC cache image.") + /* Flush data buffers */ if(H5F_flush(file, H5AC_ind_read_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") @@ -1868,7 +1917,48 @@ H5Fget_mdc_logging_status(hid_t file_id, hbool_t *is_enabled, done: FUNC_LEAVE_API(ret_value) -} /* H5Fstop_mdc_logging() */ +} /* H5Fget_mdc_logging_status() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fset_latest_format + * + * Purpose: Enable switching the "latest format" flag while a file is open. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Monday, September 21, 2015 + *------------------------------------------------------------------------- + */ +herr_t +H5Fset_latest_format(hid_t file_id, hbool_t latest_format) +{ + H5F_t *f; /* File */ + unsigned latest_flags; /* Latest format flags for file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ib", file_id, latest_format); + + /* Check args */ + if(NULL == (f = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "not a file ID") + + /* Check if the value is changing */ + latest_flags = H5F_USE_LATEST_FLAGS(f, H5F_LATEST_ALL_FLAGS); + if(latest_format != (H5F_LATEST_ALL_FLAGS == latest_flags)) { + /* Call the flush routine, for this file */ + if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") + + /* Toggle the 'latest format' flag */ + H5F_SET_LATEST_FLAGS(f, latest_format ? H5F_LATEST_ALL_FLAGS : 0); + } /* end if */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Fset_latest_format() */ /*------------------------------------------------------------------------- diff --git a/src/H5Fint.c b/src/H5Fint.c index 8ad97a8..4783a36 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -188,6 +188,8 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) if(H5P_set(new_plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &(f->coll_md_write)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set collective metadata read flag") #endif /* H5_HAVE_PARALLEL */ + if(H5P_set(new_plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set initial metadata cache resize config.") /* Prepare the driver property */ driver_prop.driver_id = f->shared->lf->driver_id; @@ -667,6 +669,8 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t if(H5P_get(plist, H5F_ACS_COLL_MD_WRITE_FLAG_NAME, &(f->coll_md_write)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get collective metadata write flag") #endif /* H5_HAVE_PARALLEL */ + if(H5P_get(plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, &(f->shared->mdc_initCacheImageCfg)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get initial metadata cache resize config") /* Get the VFD values to cache */ f->shared->maxaddr = H5FD_get_maxaddr(lf); @@ -750,7 +754,7 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t * The cache might be created with a different number of elements and * the access property list should be updated to reflect that. */ - if(H5AC_create(f, &(f->shared->mdc_initCacheCfg)) < 0) + if(H5AC_create(f, &(f->shared->mdc_initCacheCfg), &(f->shared->mdc_initCacheImageCfg)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create metadata cache") /* Create the file's "open object" information */ @@ -2671,3 +2675,33 @@ H5F_set_coll_md_read(H5F_t *f, H5P_coll_md_read_flag_t cmr) } /* H5F_set_coll_md_read() */ #endif /* H5_HAVE_PARALLEL */ + +/*------------------------------------------------------------------------- + * Function: H5F_set_latest_flags + * + * Purpose: Set the latest_flags field with a new value. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * 4/26/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_set_latest_flags(H5F_t *f, unsigned flags) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(f); + HDassert(f->shared); + HDassert(0 == ((~flags) & H5F_LATEST_ALL_FLAGS)); + + f->shared->latest_flags = flags; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5F_set_latest_flags() */ + diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index c067f26..93a3978 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -287,6 +287,12 @@ struct H5F_file_t { /* metadata cache. This structure is */ /* fixed at creation time and should */ /* not change thereafter. */ + H5AC_cache_image_config_t + mdc_initCacheImageCfg; /* initial configuration for the */ + /* generate metadata cache image on */ + /* close option. This structure is */ + /* fixed at creation time and should */ + /* not change thereafter. */ hbool_t use_mdc_logging; /* Set when metadata logging is desired */ hbool_t start_mdc_log_on_access; /* set when mdc logging should */ /* begin on file access/create */ @@ -397,7 +403,8 @@ H5_DLL herr_t H5F__super_free(H5F_super_t *sblock); /* Superblock extension related routines */ H5_DLL herr_t H5F_super_ext_open(H5F_t *f, haddr_t ext_addr, H5O_loc_t *ext_ptr); -H5_DLL herr_t H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, unsigned id, void *mesg, hbool_t may_create); +H5_DLL herr_t H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, unsigned id, + void *mesg, hbool_t may_create, unsigned mesg_flags); H5_DLL herr_t H5F_super_ext_remove_msg(H5F_t *f, hid_t dxpl_id, unsigned id); H5_DLL herr_t H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr, hid_t dxpl_id, hbool_t was_created); diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index bcc56c6..7d288fa 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -317,6 +317,7 @@ #define H5F_SET_GRP_BTREE_SHARED(F, RC) (((F)->shared->grp_btree_shared = (RC)) ? SUCCEED : FAIL) #define H5F_USE_TMP_SPACE(F) ((F)->shared->use_tmp_space) #define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->tmp_addr, (ADDR))) +#define H5F_SET_LATEST_FLAGS(F, FL) ((F)->shared->latest_flags = (FL)) #ifdef H5_HAVE_PARALLEL #define H5F_COLL_MD_READ(F) ((F)->coll_md_read) #endif /* H5_HAVE_PARALLEL */ @@ -367,6 +368,7 @@ #define H5F_SET_GRP_BTREE_SHARED(F, RC) (H5F_set_grp_btree_shared((F), (RC))) #define H5F_USE_TMP_SPACE(F) (H5F_use_tmp_space(F)) #define H5F_IS_TMP_ADDR(F, ADDR) (H5F_is_tmp_addr((F), (ADDR))) +#define H5F_SET_LATEST_FLAGS(F, FL) (H5F_set_latest_flags((F), (FL))) #ifdef H5_HAVE_PARALLEL #define H5F_COLL_MD_READ(F) (H5F_coll_md_read(F)) #endif /* H5_HAVE_PARALLEL */ @@ -481,6 +483,7 @@ #define H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME "evict_on_close_flag" /* Whether or not the metadata cache will evict objects on close */ #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */ #define H5F_ACS_COLL_MD_WRITE_FLAG_NAME "collective_metadata_write" /* property indicating whether metadata writes are done collectively or not */ +#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME "mdc_initCacheImageCfg" /* Initial metadata cache image creation configuration */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ @@ -700,6 +703,7 @@ H5_DLL struct H5UC_t *H5F_grp_btree_shared(const H5F_t *f); H5_DLL herr_t H5F_set_grp_btree_shared(H5F_t *f, struct H5UC_t *rc); H5_DLL hbool_t H5F_use_tmp_space(const H5F_t *f); H5_DLL hbool_t H5F_is_tmp_addr(const H5F_t *f, haddr_t addr); +H5_DLL herr_t H5F_set_latest_flags(H5F_t *f, unsigned flags); #ifdef H5_HAVE_PARALLEL H5_DLL H5P_coll_md_read_flag_t H5F_coll_md_read(const H5F_t *f); H5_DLL void H5F_set_coll_md_read(H5F_t *f, H5P_coll_md_read_flag_t flag); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index a79da75..c57a821 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -246,6 +246,7 @@ H5_DLL herr_t H5Fstart_swmr_write(hid_t file_id); H5_DLL ssize_t H5Fget_free_sections(hid_t file_id, H5F_mem_t type, size_t nsects, H5F_sect_info_t *sect_info/*out*/); H5_DLL herr_t H5Fclear_elink_file_cache(hid_t file_id); +H5_DLL herr_t H5Fset_latest_format(hid_t file_id, hbool_t latest_format); H5_DLL herr_t H5Fstart_mdc_logging(hid_t file_id); H5_DLL herr_t H5Fstop_mdc_logging(hid_t file_id); H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id, diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 893ce26..e203c38 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -290,7 +290,7 @@ H5F__update_super_ext_driver_msg(H5F_t *f, hid_t dxpl_id) */ drvinfo.len = driver_size; drvinfo.buf = dbuf; - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE) < 0) + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message") } /* end if driver_size > 0 */ } /* end if !H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO) */ @@ -692,6 +692,34 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) f->shared->fs_addr[u] = fsinfo.fs_addr[u-1]; } /* end if */ + /* Check for the extension having a 'metadata cache image' message */ + if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, dxpl_id)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") + if(status) { + hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0); + H5O_mdci_t mdci_msg; + + /* if the metadata cache image superblock extension message exists, + * read its contents and pass the data on to the metadata cache. + * Given this data, the cache will load and decode the metadata + * cache image block, decoded it and load its contents into the + * the cache on the test protect call. + * + * Further, if the file is opened R/W, the metadata cache will + * delete the metadata cache image superblock extension and free + * the cache image block. Don't do this now as f->shared + * is not fully setup, which complicates matters. + */ + + /* Retrieve the 'metadata cache image message' structure */ + if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, dxpl_id)) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get metadata cache image message") + + /* Indicate to the cache that there's an image to load on first protect call */ + if(H5AC_load_cache_image_on_next_protect(f, mdci_msg.addr, mdci_msg.size, rw) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTLOAD, FAIL, "call to H5AC_load_cache_image_on_next_protect failed"); + } /* end if */ + /* Close superblock extension */ if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") @@ -737,7 +765,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) HDassert(f->shared->sblock == NULL); f->shared->sblock = sblock; #endif /* JRM */ - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE) < 0) + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") #if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ @@ -1365,7 +1393,8 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, unsigned id, void *mesg, hbool_t may_create) +H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, unsigned id, void *mesg, + hbool_t may_create, unsigned mesg_flags) { H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ @@ -1410,7 +1439,7 @@ H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, unsigned id, void *mesg, hbool_ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should not exist") /* Create the message with ID in the superblock extension */ - if(H5O_msg_create(&ext_loc, id, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, mesg, dxpl_id) < 0) + if(H5O_msg_create(&ext_loc, id, (mesg_flags | H5O_MSG_FLAG_DONTSHARE), H5O_UPDATE_TIME, mesg, dxpl_id) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to create the message in object header") } /* end if */ else { @@ -1418,7 +1447,7 @@ H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, unsigned id, void *mesg, hbool_ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should exist") /* Update the message with ID in the superblock extension */ - if(H5O_msg_write(&ext_loc, id, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, mesg, dxpl_id) < 0) + if(H5O_msg_write(&ext_loc, id, (mesg_flags | H5O_MSG_FLAG_DONTSHARE), H5O_UPDATE_TIME, mesg, dxpl_id) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to write the message in object header") } /* end else */ diff --git a/src/H5HFcache.c b/src/H5HFcache.c index 302fe04..8e5ed76 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -116,14 +116,17 @@ static herr_t H5HF__cache_dblock_free_icr(void *thing); /* Debugging Function Prototypes */ #ifndef NDEBUG -static herr_t H5HF__cache_verify_hdr_descendants_clean(H5F_t *f, H5HF_hdr_t *hdr, - hbool_t *clean); -static herr_t H5HF__cache_verify_iblock_descendants_clean(H5F_t *f, - H5HF_indirect_t *iblock, unsigned *iblock_status, hbool_t *clean); -static herr_t H5HF__cache_verify_iblocks_dblocks_clean(H5F_t *f, - H5HF_indirect_t *iblock, hbool_t *clean, hbool_t *has_dblocks); -static herr_t H5HF__cache_verify_descendant_iblocks_clean(H5F_t *f, - H5HF_indirect_t *iblock, hbool_t *clean, hbool_t *has_iblocks); +static herr_t H5HF__cache_verify_hdr_descendants_clean(H5F_t *f, hid_t dxpl_id, + H5HF_hdr_t * hdr, hbool_t *fd_clean, hbool_t *clean); +static herr_t H5HF__cache_verify_iblock_descendants_clean(H5F_t *f, + hid_t dxpl_id, haddr_t fd_parent_addr, H5HF_indirect_t *iblock, + unsigned *iblock_status, hbool_t * fd_clean, hbool_t *clean); +static herr_t H5HF__cache_verify_iblocks_dblocks_clean(H5F_t *f, + haddr_t fd_parent_addr, H5HF_indirect_t *iblock, hbool_t *fd_clean, + hbool_t *clean, hbool_t *has_dblocks); +static herr_t H5HF__cache_verify_descendant_iblocks_clean(H5F_t *f, + hid_t dxpl_id, haddr_t fd_parent_addr, H5HF_indirect_t *iblock, + hbool_t *fd_clean, hbool_t *clean, hbool_t *has_iblocks); #endif /* NDEBUG */ @@ -687,6 +690,7 @@ H5HF__cache_hdr_pre_serialize(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, #ifndef NDEBUG { hbool_t descendants_clean = TRUE; + hbool_t fd_children_clean = TRUE; /* Verify that flush dependencies are working correctly. Do this * by verifying that either: @@ -701,10 +705,22 @@ H5HF__cache_hdr_pre_serialize(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, * constraint is met by default. * * Do this with a call to H5HF__cache_verify_hdr_descendants_clean(). + * + * Note that decendants need not be clean if the pre_serialize call + * is made during a cache serialization instead of an entry or cache + * flush. + * + * Note also that with the recent change in the definition of flush + * dependency, not all decendants need be clean -- only direct flush + * dependency children. + * + * Finally, observe that the H5HF__cache_verify_hdr_descendants_clean() + * call still looks for dirty descendants. At present we do not check + * this value. */ - if(H5HF__cache_verify_hdr_descendants_clean((H5F_t *)f, hdr, &descendants_clean) < 0) + if(H5HF__cache_verify_hdr_descendants_clean((H5F_t *)f, dxpl_id, hdr, &fd_children_clean, &descendants_clean) < 0) HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "can't verify hdr descendants clean.") - HDassert(descendants_clean); + HDassert(fd_children_clean); } #endif /* NDEBUG */ @@ -1176,8 +1192,9 @@ H5HF__cache_iblock_image_len(const void *_thing, size_t *image_len) * and if so, to move it to real file space before the entry is * serialized. * - * In debug compiles, this function also verifies that all children - * of this indirect block are either clean or are not in cache. + * In debug compiles, this function also verifies that all + * immediate flush dependency children of this indirect block + * are either clean or are not in cache. * * Return: Success: SUCCEED * Failure: FAIL @@ -1217,10 +1234,12 @@ H5HF__cache_iblock_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing, #ifndef NDEBUG { hbool_t descendants_clean = TRUE; + hbool_t fd_children_clean = TRUE; unsigned iblock_status = 0; /* verify that flush dependencies are working correctly. Do this - * by verifying that all children of this iblock are clean. + * by verifying that all immediate flush dependency children of this + * iblock are clean. */ if(H5AC_get_entry_status(f, iblock->addr, &iblock_status) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't get iblock status") @@ -1230,9 +1249,9 @@ H5HF__cache_iblock_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing, * there is no need to check to see if it is pinned or protected, or to * protect it if it is not. */ - if(H5HF__cache_verify_iblock_descendants_clean((H5F_t *)f, iblock, &iblock_status, &descendants_clean) < 0) + if(H5HF__cache_verify_iblock_descendants_clean((H5F_t *)f, dxpl_id, iblock->addr, iblock, &iblock_status, &fd_children_clean, &descendants_clean) < 0) HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "can't verify descendants clean.") - HDassert(descendants_clean); + HDassert(fd_children_clean); } #endif /* NDEBUG */ @@ -2585,6 +2604,54 @@ done: * instance of H5HF_hdr_t are clean. Set *clean to * TRUE if this is the case, and to FALSE otherwise. * + * Update -- 8/24/15 + * + * With the advent of the metadata cache image feature, it is + * possible for the pre-serialize and serialize calls to be + * invoked outside of a flush. While this serialization + * observes flush dependencies for the order of serialization, + * the entries are not written to disk, and hence dirty entries + * remain dirty. + * + * To address this, updated the sanity checks in this function + * to treat entries whose images are up to date as clean if + * a cache serialization is in progress. + * + * Update -- 9/29/16 + * + * The implementation of flush dependencies has been changed. + * Prior to this change, a flush dependency parent could be + * flushed if and only if all its flush dependency decendants + * were clean. In the new definition, a flush dependency + * parent can be flushed if all its immediate flush dependency + * children are clean, regardless of any other dirty + * decendants. + * + * Further, metadata cache entries are now allowed to have + * multiple flush dependency parents. + * + * This means that the fractal heap is no longer ncessarily + * flushed from the bottom up. + * + * For example, it is now possible for a dirty fractal heap + * header to be flushed before a dirty dblock, as long as the + * there in an interviening iblock, and the header has no + * dirty immediate flush dependency children. + * + * Also, I gather that under some circumstances, a dblock + * will be direct a flush dependency child both of the iblock + * that points to it, and of the fractal heap header. + * + * As a result of these changes, the functionality of these + * sanity checking routines has been modified significantly. + * Instead of scanning the fractal heap from a starting point + * down, and verifying that there were no dirty entries, the + * functions now scan downward from the starting point and + * verify that there are no dirty flush dependency children + * of the specified flush dependency parent. In passing, + * they also walk the data structure, and verify it. + * + * * Return: Non-negative on success/Negative on failure * * Programmer: John Mainzer @@ -2594,9 +2661,10 @@ done: */ #ifndef NDEBUG static herr_t -H5HF__cache_verify_hdr_descendants_clean(H5F_t *f, H5HF_hdr_t *hdr, - hbool_t *clean) +H5HF__cache_verify_hdr_descendants_clean(H5F_t *f, hid_t dxpl_id, + H5HF_hdr_t *hdr, hbool_t *fd_clean, hbool_t *clean) { + hbool_t fd_exists = FALSE; /* whether flush dependency exists. */ haddr_t hdr_addr; /* Address of header */ unsigned hdr_status = 0; /* Header cache entry status */ herr_t ret_value = SUCCEED; /* Return value */ @@ -2608,6 +2676,7 @@ H5HF__cache_verify_hdr_descendants_clean(H5F_t *f, H5HF_hdr_t *hdr, HDassert(hdr); HDassert(hdr->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(hdr->cache_info.type == H5AC_FHEAP_HDR); + HDassert(fd_clean); HDassert(clean); hdr_addr = hdr->cache_info.addr; HDassert(hdr_addr == hdr->heap_addr); @@ -2672,15 +2741,165 @@ H5HF__cache_verify_hdr_descendants_clean(H5F_t *f, H5HF_hdr_t *hdr, root_iblock_in_cache = ( (root_iblock_status & H5AC_ES__IN_CACHE) != 0); HDassert(root_iblock_in_cache || (root_iblock == NULL)); - if(!root_iblock_in_cache) /* we are done */ - *clean = TRUE; - else if(root_iblock_status & H5AC_ES__IS_DIRTY) - *clean = FALSE; + if(!root_iblock_in_cache) { /* we are done */ + *clean = TRUE; + *fd_clean = TRUE; + } /* end if */ + else if((root_iblock_status & H5AC_ES__IS_DIRTY) && + (((root_iblock_status & H5AC_ES__IMAGE_IS_UP_TO_DATE) == 0) || + (!H5AC_get_serialization_in_progress(f)))) { + *clean = FALSE; + + /* verify that a flush dependency exists between the header and + * the root inode. + */ + if(H5AC_flush_dependency_exists(f, hdr->heap_addr, root_iblock_addr, &fd_exists) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't check flush dependency") + HDassert(fd_exists); + + *fd_clean = FALSE; + } /* end else-if */ + else { /* must examine children */ + hbool_t unprotect_root_iblock = FALSE; + + /* At this point, the root iblock may be pinned, protected, + * both, or neither, and we may or may not have a pointer + * to root iblock in memory. + * + * Before we call H5HF__cache_verify_iblock_descendants_clean(), + * we must ensure that the root iblock is either pinned or + * protected or both, and that we have a pointer to it. + * Do this as follows: + */ + if(root_iblock == NULL) { /* we don't have ptr to root iblock */ + if(0 == (root_iblock_status & H5AC_ES__IS_PROTECTED)) { + /* just protect the root iblock -- this will give us + * the pointer we need to proceed, and ensure that + * it is locked into the metadata cache for the + * duration. + * + * Note that the udata is only used in the load callback. + * While the fractal heap makes heavy use of the udata + * in this case, since we know that the entry is in cache, + * we can pass NULL udata. + * + * The tag specified in the dxpl we received + * as a parameter (via dxpl_id) may not be correct. + * Grab the (hopefully) correct tag from the header, + * and load it into the dxpl via the H5_BEGIN_TAG and + * H5_END_TAG macros. Note that any error bracked by + * these macros must be reported with HGOTO_ERROR_TAG. + */ + H5_BEGIN_TAG(dxpl_id, hdr->heap_addr, FAIL) + + if(NULL == (root_iblock = (H5HF_indirect_t *)H5AC_protect(f, dxpl_id, H5AC_FHEAP_IBLOCK, root_iblock_addr, NULL, H5AC__READ_ONLY_FLAG))) + HGOTO_ERROR_TAG(H5E_HEAP, H5E_CANTPROTECT, FAIL, "H5AC_protect() faild.") + + H5_END_TAG(FAIL) + + unprotect_root_iblock = TRUE; + } /* end if */ + else { + /* the root iblock is protected, and we have no + * legitimate way of getting a pointer to it. + * + * We square this circle by using the + * H5AC_get_entry_ptr_from_addr() to get the needed + * pointer. + * + * WARNING: This call should be used only in debugging + * routines, and it should be avoided there when + * possible. + * + * Further, if we ever multi-thread the cache, + * this routine will have to be either discarded + * or heavily re-worked. + * + * Finally, keep in mind that the entry whose + * pointer is obtained in this fashion may not + * be in a stable state. + * + * Assuming that the flush dependency code is working + * as it should, the only reason for the root iblock to + * be unpinned is if none of its children are in cache. + * This unfortunately means that if it is protected and + * not pinned, the fractal heap is in the process of loading + * or inserting one of its children. The obvious + * implication is that there is a significant chance that + * the root iblock is in an unstable state. + * + * All this suggests that using + * H5AC_get_entry_ptr_from_addr() to obtain the pointer + * to the protected root iblock is questionable here. + * However, since this is test/debugging code, I expect + * that we will use this approach until it causes problems, + * or we think of a better way. + */ + if(H5AC_get_entry_ptr_from_addr(f, root_iblock_addr, (void **)(&root_iblock)) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "H5AC_get_entry_ptr_from_addr() failed.") + HDassert(root_iblock); + } /* end else */ + } /* end if */ + else { /* root_iblock != NULL */ + /* we have the pointer to the root iblock. Protect it + * if it is neither pinned nor protected -- otherwise we + * are ready to go. + */ + H5HF_indirect_t * iblock = NULL; + + if(((root_iblock_status & H5AC_ES__IS_PINNED) == 0) && + ((root_iblock_status & H5AC_ES__IS_PROTECTED) == 0)) { + /* the root iblock is neither pinned nor protected -- hence + * we must protect it before we proceed + * + * Note that the udata is only used in the load callback. + * While the fractal heap makes heavy use of the udata + * in this case, since we know that the entry is in cache, + * we can pass NULL udata. + * + * The tag associated specified in the dxpl we received + * as a parameter (via dxpl_id) may not be correct. + * Grab the (hopefully) correct tag from the header, + * and load it into the dxpl via the H5_BEGIN_TAG and + * H5_END_TAG macros. Note that any error bracked by + * these macros must be reported with HGOTO_ERROR_TAG. + */ + H5_BEGIN_TAG(dxpl_id, hdr->heap_addr, FAIL) + + if(NULL == (iblock = (H5HF_indirect_t *)H5AC_protect(f, dxpl_id, H5AC_FHEAP_IBLOCK, root_iblock_addr, NULL, H5AC__READ_ONLY_FLAG))) + HGOTO_ERROR_TAG(H5E_HEAP, H5E_CANTPROTECT, FAIL, "H5AC_protect() faild.") + + H5_END_TAG(FAIL) + + unprotect_root_iblock = TRUE; + HDassert(iblock == root_iblock); + } /* end if */ + } /* end else */ + + /* at this point, one way or another, the root iblock is locked + * in memory for the duration of the call. Do some sanity checks, + * and then call H5HF__cache_verify_iblock_descendants_clean(). + */ + HDassert(root_iblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(root_iblock->cache_info.type == H5AC_FHEAP_IBLOCK); + + if(H5HF__cache_verify_iblock_descendants_clean(f, dxpl_id, hdr->heap_addr, root_iblock, &root_iblock_status, fd_clean, clean) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "can't verify root iblock & descendants clean.") + + /* Unprotect the root indirect block if required */ + if(unprotect_root_iblock) { + HDassert(root_iblock); + if(H5AC_unprotect(f, dxpl_id, H5AC_FHEAP_IBLOCK, root_iblock_addr, root_iblock, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTUNPROTECT, FAIL, "H5AC_unprotect() faild.") + } /* end if */ + } /* end else */ } /* end if */ else if((hdr->man_dtable.curr_root_rows == 0) && (HADDR_UNDEF != hdr->man_dtable.table_addr)) { haddr_t root_dblock_addr; unsigned root_dblock_status = 0; + hbool_t in_cache; + hbool_t type_ok; /* this is scenario 2 -- we have a root dblock */ root_dblock_addr = hdr->man_dtable.table_addr; @@ -2688,25 +2907,48 @@ H5HF__cache_verify_hdr_descendants_clean(H5F_t *f, H5HF_hdr_t *hdr, HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't get root dblock status") if(root_dblock_status & H5AC_ES__IN_CACHE) { + if(H5AC_verify_entry_type(f, root_dblock_addr, &H5AC_FHEAP_DBLOCK[0], &in_cache, &type_ok) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't check dblock type") + HDassert(in_cache); + if(!type_ok) + HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "root dblock addr doesn't refer to a dblock?!?") + /* If a root dblock is in cache, it must have a flush - * dependency relationship with the header. + * dependency relationship with the header, and it + * may not be the parent in any flush dependency + * relationship. + * + * We don't test this fully, but we will verify that + * the root iblock is a child in a flush dependency + * relationship with the header. */ - if(0 == (root_dblock_status & H5AC_ES__IS_FLUSH_DEP_CHILD)) - HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "root dblock in cache and not a flush dep child.") + if(H5AC_flush_dependency_exists(f, hdr->heap_addr, root_dblock_addr, &fd_exists) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't check flush dependency") + if(!fd_exists) + HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "root dblock is not a flush dep parent of header.") + if(0 != (root_dblock_status & H5AC_ES__IS_FLUSH_DEP_PARENT)) HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "root dblock in cache and is a flush dep parent.") - if(root_dblock_status & H5AC_ES__IS_DIRTY) - *clean = FALSE; - } /* end if */ - else /* root dblock not in cache */ - *clean = TRUE; + *clean = !((root_dblock_status & H5AC_ES__IS_DIRTY) && + (((root_dblock_status & + H5AC_ES__IMAGE_IS_UP_TO_DATE) == 0) || + (!H5AC_get_serialization_in_progress(f)))); + + *fd_clean = *clean; + } /* end if */ + else { /* root dblock not in cache */ + *fd_clean = TRUE; + *clean = TRUE; + } /* end else */ } /* end else-if */ - else - /* this is scenario 3 -- the fractal heap is empty, and we - * have nothing to do. - */ - *clean = TRUE; + else { + /* this is scenario 3 -- the fractal heap is empty, and we + * have nothing to do. + */ + *fd_clean = TRUE; + *clean = TRUE; + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -2741,6 +2983,40 @@ done: * H5HF__cache_verify_descendant_iblocks_clean() are * recursive co-routines. * + * Update -- 9/29/16 + * + * The implementation of flush dependencies has been changed. + * Prior to this change, a flush dependency parent could be + * flushed if and only if all its flush dependency decendants + * were clean. In the new definition, a flush dependency + * parent can be flushed if all its immediate flush dependency + * children are clean, regardless of any other dirty + * decendants. + * + * Further, metadata cache entries are now allowed to have + * multiple flush dependency parents. + * + * This means that the fractal heap is no longer ncessarily + * flushed from the bottom up. + * + * For example, it is now possible for a dirty fractal heap + * header to be flushed before a dirty dblock, as long as the + * there in an interviening iblock, and the header has no + * dirty immediate flush dependency children. + * + * Also, I gather that under some circumstances, a dblock + * will be direct a flush dependency child both of the iblock + * that points to it, and of the fractal heap header. + * + * As a result of these changes, the functionality of these + * sanity checking routines has been modified significantly. + * Instead of scanning the fractal heap from a starting point + * down, and verifying that there were no dirty entries, the + * functions now scan downward from the starting point and + * verify that there are no dirty flush dependency children + * of the specified flush dependency parent. In passing, + * they also walk the data structure, and verify it. + * * Return: Non-negative on success/Negative on failure * * Programmer: John Mainzer @@ -2750,8 +3026,9 @@ done: */ #ifndef NDEBUG static herr_t -H5HF__cache_verify_iblock_descendants_clean(H5F_t *f, H5HF_indirect_t *iblock, - unsigned *iblock_status, hbool_t *clean) +H5HF__cache_verify_iblock_descendants_clean(H5F_t *f, hid_t dxpl_id, + haddr_t fd_parent_addr, H5HF_indirect_t *iblock, unsigned *iblock_status, + hbool_t * fd_clean, hbool_t *clean) { hbool_t has_dblocks = FALSE; hbool_t has_iblocks = FALSE; @@ -2761,17 +3038,19 @@ H5HF__cache_verify_iblock_descendants_clean(H5F_t *f, H5HF_indirect_t *iblock, /* Sanity checks */ HDassert(f); + HDassert(H5F_addr_defined(fd_parent_addr)); HDassert(iblock); HDassert(iblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(iblock->cache_info.type == H5AC_FHEAP_IBLOCK); HDassert(iblock_status); - HDassert(clean); - HDassert(*clean); + HDassert(fd_clean); + HDassert(*fd_clean); + HDassert(clean); /* note that *clean need not be TRUE */ - if((*clean) && H5HF__cache_verify_iblocks_dblocks_clean(f, iblock, clean, &has_dblocks) < 0) + if((*fd_clean) && H5HF__cache_verify_iblocks_dblocks_clean(f, fd_parent_addr, iblock, fd_clean, clean, &has_dblocks) < 0) HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "can't verify dblocks clean.") - if((*clean) && H5HF__cache_verify_descendant_iblocks_clean(f, iblock, clean, &has_iblocks) < 0) + if((*fd_clean) && H5HF__cache_verify_descendant_iblocks_clean(f, dxpl_id, fd_parent_addr, iblock, fd_clean, clean, &has_iblocks) < 0) HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "can't verify iblocks clean.") /* verify that flush dependency setup is plausible */ @@ -2808,6 +3087,53 @@ done: * during the call. Caller must ensure that this is * the case before the call. * + * Update -- 8/24/15 + * + * With the advent of the metadata cache image feature, it is + * possible for the pre-serialize and serialize calls to be + * invoked outside of a flush. While this serialization + * observes flush dependencies for the order of serialization, + * the entries are not written to disk, and hence dirty entries + * remain dirty. + * + * To address this, updated the sanity checks in this function + * to treat entries whose images are up to date as clean if + * a cache serialization is in progress. + * + * Update -- 9/29/16 + * + * The implementation of flush dependencies has been changed. + * Prior to this change, a flush dependency parent could be + * flushed if and only if all its flush dependency decendants + * were clean. In the new definition, a flush dependency + * parent can be flushed if all its immediate flush dependency + * children are clean, regardless of any other dirty + * decendants. + * + * Further, metadata cache entries are now allowed to have + * multiple flush dependency parents. + * + * This means that the fractal heap is no longer ncessarily + * flushed from the bottom up. + * + * For example, it is now possible for a dirty fractal heap + * header to be flushed before a dirty dblock, as long as the + * there in an interviening iblock, and the header has no + * dirty immediate flush dependency children. + * + * Also, I gather that under some circumstances, a dblock + * will be direct a flush dependency child both of the iblock + * that points to it, and of the fractal heap header. + * + * As a result of these changes, the functionality of these + * sanity checking routines has been modified significantly. + * Instead of scanning the fractal heap from a starting point + * down, and verifying that there were no dirty entries, the + * functions now scan downward from the starting point and + * verify that there are no dirty flush dependency children + * of the specified flush dependency parent. In passing, + * they also walk the data structure, and verify it. + * * Return: Non-negative on success/Negative on failure * * Programmer: John Mainzer @@ -2817,53 +3143,82 @@ done: */ #ifndef NDEBUG static herr_t -H5HF__cache_verify_iblocks_dblocks_clean(H5F_t *f, H5HF_indirect_t *iblock, - hbool_t *clean, hbool_t *has_dblocks) +H5HF__cache_verify_iblocks_dblocks_clean(H5F_t *f, haddr_t fd_parent_addr, + H5HF_indirect_t *iblock, hbool_t *fd_clean, hbool_t *clean, + hbool_t *has_dblocks) { unsigned num_direct_rows; unsigned max_dblock_index; unsigned i; + haddr_t iblock_addr; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Sanity checks */ HDassert(f); + HDassert(H5F_addr_defined(fd_parent_addr)); HDassert(iblock); HDassert(iblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(iblock->cache_info.type == H5AC_FHEAP_IBLOCK); - HDassert(clean); - HDassert(*clean); + HDassert(fd_clean); + HDassert(*fd_clean); + HDassert(clean); /* note that *clean need not be true */ HDassert(has_dblocks); i = 0; num_direct_rows = MIN(iblock->nrows, iblock->hdr->man_dtable.max_direct_rows); HDassert(num_direct_rows <= iblock->nrows); max_dblock_index = (num_direct_rows * iblock->hdr->man_dtable.cparam.width) - 1; - while((*clean) && (i <= max_dblock_index)) { + iblock_addr = iblock->addr; + HDassert(H5F_addr_defined(iblock_addr)); + + while((*fd_clean) && (i <= max_dblock_index)) { haddr_t dblock_addr; dblock_addr = iblock->ents[i].addr; if(H5F_addr_defined(dblock_addr)) { - unsigned dblock_status = 0; + hbool_t in_cache; + hbool_t type_ok; + + if(H5AC_verify_entry_type(f, dblock_addr, &H5AC_FHEAP_DBLOCK[0], &in_cache, &type_ok) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't check dblock type") + + if(in_cache) { /* dblock is in cache */ + hbool_t fd_exists; + unsigned dblock_status = 0; + + if(!type_ok) + HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "dblock addr doesn't refer to a dblock?!?") + + if(H5AC_get_entry_status(f, dblock_addr, &dblock_status) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't get dblock status") + + HDassert(dblock_status & H5AC_ES__IN_CACHE); - if(H5AC_get_entry_status(f, dblock_addr, &dblock_status) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't get dblock status") - if(dblock_status & H5AC_ES__IN_CACHE) { *has_dblocks = TRUE; - if(dblock_status & H5AC_ES__IS_DIRTY) + if((dblock_status & H5AC_ES__IS_DIRTY) && + (((dblock_status & H5AC_ES__IMAGE_IS_UP_TO_DATE) == 0) || + (!H5AC_get_serialization_in_progress(f)))) { *clean = FALSE; + + if(H5AC_flush_dependency_exists(f, fd_parent_addr, dblock_addr, &fd_exists) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't check flush dependency") + + if(fd_exists) + *fd_clean = FALSE; + } /* end if */ - /* If a child dblock is in cache, it must have a flush - * dependency relationship with this iblock, and it - * may not be the parent in any flush dependency - * relationship. + /* If a child dblock is in cache, it must have a flush + * dependency relationship with this iblock. Test this + * here. */ - if(0 == (dblock_status & H5AC_ES__IS_FLUSH_DEP_CHILD)) - HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "dblock in cache and not a flush dep child.") - if(0 != (dblock_status & H5AC_ES__IS_FLUSH_DEP_PARENT)) - HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "dblock in cache and is a flush dep parent.") + if(H5AC_flush_dependency_exists(f, iblock_addr, dblock_addr, &fd_exists) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't check flush dependency") + + if(!fd_exists) + HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "dblock in cache and not a flush dep child of iblock.") } /* end if */ } /* end if */ @@ -2896,6 +3251,54 @@ done: * during the call. Caller must ensure that this is * the case before the call. * + * Update -- 8/24/15 + * + * With the advent of the metadata cache image feature, it is + * possible for the pre-serialize and serialize calls to be + * invoked outside of a flush. While this serialization + * observes flush dependencies for the order of serialization, + * the entries are not written to disk, and hence dirty entries + * remain dirty. + * + * To address this, updated the sanity checks in this function + * to treat entries whose images are up to date as clean if + * a cache serialization is in progress. + * + * Update -- 9/29/16 + * + * The implementation of flush dependencies has been changed. + * Prior to this change, a flush dependency parent could be + * flushed if and only if all its flush dependency decendants + * were clean. In the new definition, a flush dependency + * parent can be flushed if all its immediate flush dependency + * children are clean, regardless of any other dirty + * decendants. + * + * Further, metadata cache entries are now allowed to have + * multiple flush dependency parents. + * + * This means that the fractal heap is no longer ncessarily + * flushed from the bottom up. + * + * For example, it is now possible for a dirty fractal heap + * header to be flushed before a dirty dblock, as long as the + * there in an interviening iblock, and the header has no + * dirty immediate flush dependency children. + * + * Also, I gather that under some circumstances, a dblock + * will be direct a flush dependency child both of the iblock + * that points to it, and of the fractal heap header. + * + * As a result of these changes, the functionality of these + * sanity checking routines has been modified significantly. + * Instead of scanning the fractal heap from a starting point + * down, and verifying that there were no dirty entries, the + * functions now scan downward from the starting point and + * verify that there are no dirty flush dependency children + * of the specified flush dependency parent. In passing, + * they also walk the data structure, and verify it. + * + * * Return: Non-negative on success/Negative on failure * * Programmer: John Mainzer @@ -2905,33 +3308,38 @@ done: */ #ifndef NDEBUG static herr_t -H5HF__cache_verify_descendant_iblocks_clean(H5F_t *f, H5HF_indirect_t *iblock, +H5HF__cache_verify_descendant_iblocks_clean(H5F_t *f, hid_t dxpl_id, + haddr_t fd_parent_addr, H5HF_indirect_t *iblock, hbool_t *fd_clean, hbool_t *clean, hbool_t *has_iblocks) { unsigned first_iblock_index; unsigned last_iblock_index; unsigned num_direct_rows; unsigned i; + haddr_t iblock_addr; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Sanity checks */ HDassert(f); + HDassert(H5F_addr_defined(fd_parent_addr)); HDassert(iblock); HDassert(iblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(iblock->cache_info.type == H5AC_FHEAP_IBLOCK); - HDassert(clean); - HDassert(*clean); + HDassert(fd_clean); + HDassert(*fd_clean); + HDassert(clean); /* note that *clean need not be true */ HDassert(has_iblocks); num_direct_rows = MIN(iblock->nrows, iblock->hdr->man_dtable.max_direct_rows); HDassert(num_direct_rows <= iblock->nrows); + iblock_addr = iblock->addr; first_iblock_index = num_direct_rows * iblock->hdr->man_dtable.cparam.width; last_iblock_index = (iblock->nrows * iblock->hdr->man_dtable.cparam.width) - 1; i = first_iblock_index; - while((*clean) && (i <= last_iblock_index)) { + while((*fd_clean) && (i <= last_iblock_index)) { haddr_t child_iblock_addr = iblock->ents[i].addr; if(H5F_addr_defined(child_iblock_addr)) { @@ -2941,9 +3349,157 @@ H5HF__cache_verify_descendant_iblocks_clean(H5F_t *f, H5HF_indirect_t *iblock, HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't get iblock status") if(child_iblock_status & H5AC_ES__IN_CACHE) { + hbool_t fd_exists; + *has_iblocks = TRUE; - if(child_iblock_status & H5AC_ES__IS_DIRTY) - *clean = FALSE; + + if((child_iblock_status & H5AC_ES__IS_DIRTY) && + (((child_iblock_status & H5AC_ES__IMAGE_IS_UP_TO_DATE) == 0) || + (!H5AC_get_serialization_in_progress(f)))) { + + *clean = FALSE; + + if(H5AC_flush_dependency_exists(f, fd_parent_addr, child_iblock_addr, &fd_exists) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't check flush dependency") + + if(fd_exists) + *fd_clean = FALSE; + } /* end if */ + + /* if the child iblock is in cache and *fd_clean is TRUE, + * we must continue to explore down the fractal heap tree + * structure to verify that all descendant blocks that are + * flush dependency children of the entry at parent_addr are + * either clean, or not in the metadata cache. We do this + * with a recursive call to + * H5HF__cache_verify_iblock_descendants_clean(). + * However, we can't make this call unless the child iblock + * is somehow locked into the cache -- typically via either + * pinning or protecting. + * + * If the child iblock is pinned, we can look up its pointer + * on the current iblock's pinned child iblock list, and + * and use that pointer in the recursive call. + * + * If the entry is unprotected and unpinned, we simply + * protect it. + * + * If, however, the the child iblock is already protected, + * but not pinned, we have a bit of a problem, as we have + * no legitimate way of looking up its pointer in memory. + * + * To solve this problem, I have added a new metadata cache + * call to obtain the pointer. + * + * WARNING: This call should be used only in debugging + * routines, and it should be avoided there when + * possible. + * + * Further, if we ever multi-thread the cache, + * this routine will have to be either discarded + * or heavily re-worked. + * + * Finally, keep in mind that the entry whose + * pointer is obtained in this fashion may not + * be in a stable state. + * + * Assuming that the flush dependency code is working + * as it should, the only reason for the child entry to + * be unpinned is if none of its children are in cache. + * This unfortunately means that if it is protected and + * not pinned, the fractal heap is in the process of loading + * or inserting one of its children. The obvious implication + * is that there is a significant chance that the child + * iblock is in an unstable state. + * + * All this suggests that using the new call to obtain the + * pointer to the protected child iblock is questionable + * here. However, since this is test/debugging code, I + * expect that we will use this approach until it causes + * problems, or we think of a better way. + */ + if(*fd_clean) { + H5HF_indirect_t *child_iblock = NULL; + hbool_t unprotect_child_iblock = FALSE; + + if(0 == (child_iblock_status & H5AC_ES__IS_PINNED)) { + /* child iblock is not pinned */ + if(0 == (child_iblock_status & H5AC_ES__IS_PROTECTED)) { + /* child iblock is unprotected, and unpinned */ + /* protect it. Note that the udata is only */ + /* used in the load callback. While the */ + /* fractal heap makes heavy use of the udata */ + /* in this case, since we know that the */ + /* entry is in cache, we can pass NULL udata */ + /* */ + /* The tag associated specified in the dxpl */ + /* we received as a parameter (via dxpl_id) */ + /* may not be correct. */ + /* */ + /* Grab the (hopefully) correct tag from the */ + /* parent iblock, and load it into the dxpl */ + /* via the H5_BEGIN_TAG and H5_END_TAG */ + /* macros. Note that any error bracked by */ + /* these macros must be reported with */ + /* HGOTO_ERROR_TAG. */ + + H5_BEGIN_TAG(dxpl_id, iblock->hdr->heap_addr, FAIL) + + if(NULL == (child_iblock = (H5HF_indirect_t *) H5AC_protect(f, dxpl_id, H5AC_FHEAP_IBLOCK, child_iblock_addr, NULL, H5AC__READ_ONLY_FLAG))) + HGOTO_ERROR_TAG(H5E_HEAP, H5E_CANTPROTECT, FAIL, "H5AC_protect() faild.") + + H5_END_TAG(FAIL) + + unprotect_child_iblock = TRUE; + } /* end if */ + else { + /* child iblock is protected -- use */ + /* H5AC_get_entry_ptr_from_addr() to get a */ + /* pointer to the entry. This is very slimy -- */ + /* come up with a better solution. */ + if(H5AC_get_entry_ptr_from_addr(f, child_iblock_addr, (void **)(&child_iblock)) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "H5AC_get_entry_ptr_from_addr() faild.") + HDassert(child_iblock); + } /* end else */ + } /* end if */ + else { + /* child iblock is pinned -- look it up in the */ + /* parent iblocks child_iblocks array. */ + HDassert(iblock->child_iblocks); + child_iblock = iblock->child_iblocks[i - first_iblock_index]; + } /* end else */ + + /* At this point, one way or another we should have + * a pointer to the child iblock. Verify that we + * that we have the correct one. + */ + HDassert(child_iblock); + HDassert(child_iblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(child_iblock->cache_info.type == H5AC_FHEAP_IBLOCK); + HDassert(child_iblock->addr == child_iblock_addr); + + /* now make the recursive call */ + if(H5HF__cache_verify_iblock_descendants_clean(f, dxpl_id, fd_parent_addr, child_iblock, &child_iblock_status, fd_clean, clean) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "can't verify child iblock clean.") + + /* if iblock_addr != fd_parent_addr, verify that a flush + * dependency relationship exists between iblock and + * the child iblock. + */ + if(fd_parent_addr != iblock_addr) { + if(H5AC_flush_dependency_exists(f, iblock_addr, child_iblock_addr, &fd_exists) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't check flush dependency") + + if(!fd_exists) + HGOTO_ERROR(H5E_HEAP, H5E_SYSTEM, FAIL, "iblock is not a flush dep parent of child_iblock.") + } /* end if */ + + /* if we protected the child iblock, unprotect it now */ + if(unprotect_child_iblock) { + if(H5AC_unprotect(f, dxpl_id, H5AC_FHEAP_IBLOCK, child_iblock_addr, child_iblock, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTUNPROTECT, FAIL, "H5AC_unprotect() faild.") + } /* end if */ + } /* end if */ } /* end if */ } /* end if */ @@ -1489,7 +1489,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) fsinfo.fs_addr[type-1] = HADDR_UNDEF; fsinfo.strategy = f->shared->fs_strategy; fsinfo.threshold = f->shared->fs_threshold; - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE) < 0) + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") @@ -2113,7 +2113,7 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); fsinfo.threshold = f->shared->fs_threshold; /* Write the free space manager message -- message must already exist */ - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE) < 0) + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") /* Final close of free-space managers */ @@ -130,11 +130,12 @@ const H5O_msg_class_t *const H5O_msg_class_g[] = { H5O_MSG_AINFO, /*0x0015 Attribute information */ H5O_MSG_REFCOUNT, /*0x0016 Object's ref. count */ H5O_MSG_FSINFO, /*0x0017 Free-space manager info */ - H5O_MSG_UNKNOWN, /*0x0018 Placeholder for unknown message */ + H5O_MSG_MDCI, /*0x0018 Metadata cache image */ + H5O_MSG_UNKNOWN, /*0x0019 Placeholder for unknown message */ #ifdef H5O_ENABLE_BOGUS - H5O_MSG_BOGUS_INVALID, /*0x0019 "Bogus invalid" (for testing) */ + H5O_MSG_BOGUS_INVALID, /*0x001A "Bogus invalid" (for testing) */ #else /* H5O_ENABLE_BOGUS */ - NULL, /*0x0019 "Bogus invalid" (for testing) */ + NULL, /*0x001A "Bogus invalid" (for testing) */ #endif /* H5O_ENABLE_BOGUS */ }; diff --git a/src/H5Opkg.h b/src/H5Opkg.h index c2866f7..e1fa0be 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -31,7 +31,7 @@ #define H5O_NMESGS 8 /*initial number of messages */ #define H5O_NCHUNKS 2 /*initial number of chunks */ #define H5O_MIN_SIZE 22 /* Min. obj header data size (must be big enough for a message prefix and a continuation message) */ -#define H5O_MSG_TYPES 26 /* # of types of messages */ +#define H5O_MSG_TYPES 27 /* # of types of messages */ #define H5O_MAX_CRT_ORDER_IDX 65535 /* Max. creation order index value */ /* Versions of object header structure */ @@ -538,7 +538,10 @@ H5_DLLVAR const H5O_msg_class_t H5O_MSG_REFCOUNT[1]; /* Free-space Manager Info message. (0x0017) */ H5_DLLVAR const H5O_msg_class_t H5O_MSG_FSINFO[1]; -/* Placeholder for unknown message. (0x0018) */ +/* Metadata Cache Image message. (0x0018) */ +H5_DLLVAR const H5O_msg_class_t H5O_MSG_MDCI[1]; + +/* Placeholder for unknown message. (0x0019) */ H5_DLLVAR const H5O_msg_class_t H5O_MSG_UNKNOWN[1]; diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index f6df874..71c512c 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -69,6 +69,7 @@ typedef struct H5O_t H5O_t; #define H5O_FIRST (-2) /* Operate on first message of type */ /* Flags needed when encoding messages */ +#define H5O_MSG_NO_FLAGS_SET 0x00u #define H5O_MSG_FLAG_CONSTANT 0x01u #define H5O_MSG_FLAG_SHARED 0x02u #define H5O_MSG_FLAG_DONTSHARE 0x04u @@ -204,7 +205,8 @@ typedef struct H5O_copy_t { #define H5O_AINFO_ID 0x0015 /* Attribute info message. */ #define H5O_REFCOUNT_ID 0x0016 /* Reference count message. */ #define H5O_FSINFO_ID 0x0017 /* File space info message. */ -#define H5O_UNKNOWN_ID 0x0018 /* Placeholder message ID for unknown message. */ +#define H5O_MDCI_MSG_ID 0x0018 /* Metadata Cache Image Message */ +#define H5O_UNKNOWN_ID 0x0019 /* Placeholder message ID for unknown message. */ /* (this should never exist in a file) */ /* * Note: Must increment H5O_MSG_TYPES in H5Opkg.h and update H5O_msg_class_g @@ -214,7 +216,7 @@ typedef struct H5O_copy_t { * * (this should never exist in a file) */ -#define H5O_BOGUS_INVALID_ID 0x0019 /* "Bogus invalid" Message. */ +#define H5O_BOGUS_INVALID_ID 0x001A /* "Bogus invalid" Message. */ /* Shared object message types. * Shared objects can be committed, in which case the shared message contains @@ -794,6 +796,16 @@ typedef struct H5O_fsinfo_t { haddr_t fs_addr[H5FD_MEM_NTYPES-1]; /* Addresses of free space managers */ } H5O_fsinfo_t; +/* + * Metadata Cache Image Message. + * Contains base address and length of the metadata cache image. + * (Data structure in memory) + */ +typedef struct H5O_mdci_t { + haddr_t addr; /* address of MDC image block */ + hsize_t size; /* size of MDC image block */ +} H5O_mdci_t; + /* Typedef for "application" iteration operations */ typedef herr_t (*H5O_operator_t)(const void *mesg/*in*/, unsigned idx, void *operator_data/*in,out*/); diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 91a0e03..7ea0fe0 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -225,6 +225,13 @@ #define H5F_ACS_COLL_MD_WRITE_FLAG_ENC H5P__encode_hbool_t #define H5F_ACS_COLL_MD_WRITE_FLAG_DEC H5P__decode_hbool_t #endif /* H5_HAVE_PARALLEL */ +/* Definitions for the initial metadata cache image configuration */ +#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_SIZE sizeof(H5AC_cache_image_config_t) +#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEF H5AC__DEFAULT_CACHE_IMAGE_CONFIG +#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_ENC H5P__facc_cache_image_config_enc +#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEC H5P__facc_cache_image_config_dec +#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP H5P__facc_cache_image_config_cmp + /******************/ /* Local Typedefs */ @@ -279,6 +286,11 @@ static herr_t H5P_facc_mdc_log_location_copy(const char *name, size_t size, void static int H5P_facc_mdc_log_location_cmp(const void *value1, const void *value2, size_t size); static herr_t H5P_facc_mdc_log_location_close(const char *name, size_t size, void *value); +/* Metadata cache image property callbacks */ +static int H5P__facc_cache_image_config_cmp(const void *_config1, const void *_config2, size_t H5_ATTR_UNUSED size); +static herr_t H5P__facc_cache_image_config_enc(const void *value, void **_pp, size_t *size); +static herr_t H5P__facc_cache_image_config_dec(const void **_pp, void *_value); + /*********************/ /* Package Variables */ @@ -346,6 +358,7 @@ static const hbool_t H5F_def_evict_on_close_flag_g = H5F_ACS_EVICT_ON_CLOSE_FLAG static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_MD_READ_FLAG_DEF; /* Default setting for the collective metedata read flag */ static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF; /* Default setting for the collective metedata write flag */ #endif /* H5_HAVE_PARALLEL */ +static const H5AC_cache_image_config_t H5F_def_mdc_initCacheImageCfg_g = H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEF; /* Default metadata cache image settings */ /*------------------------------------------------------------------------- @@ -555,6 +568,12 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") #endif /* H5_HAVE_PARALLEL */ + /* Register the initial metadata cache image configuration */ + if(H5P_register_real(pclass, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_SIZE, &H5F_def_mdc_initCacheImageCfg_g, + NULL, NULL, NULL, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_ENC, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEC, + NULL, NULL, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -1543,6 +1562,101 @@ done: /*------------------------------------------------------------------------- + * Function: H5Pset_mdc_image_config + * + * Purpose: Set the initial metadata cache image configuration in the + * target FAPL. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: J. Mainzer + * Thursday, June 25, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", plist_id, config_ptr); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* validate the new configuration */ + if(H5AC_validate_cache_image_config(config_ptr) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid metadata cache image configuration") + + /* set the modified metadata cache image config */ + + /* If we ever support multiple versions of H5AC_cache_image_config_t, we + * will have to test the version and do translation here. + */ + + if(H5P_set(plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, config_ptr) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set metadata cache image initial config") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_mdc_image_config() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_mdc_image_config + * + * Purpose: Retrieve the metadata cache initial image configuration + * from the target FAPL. + * + * Observe that the function will fail if config_ptr is + * NULL, or if config_ptr->version specifies an unknown + * version of H5AC_cache_image_config_t. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: J. Mainzer + * Friday, June 26, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", plist_id, config_ptr); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* validate the config_ptr */ + if(config_ptr == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL config_ptr on entry.") + + if(config_ptr->version != H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Unknown image config version.") + + /* If we ever support multiple versions of H5AC_cache_config_t, we + * will have to get the cannonical version here, and then translate + * to the version of the structure supplied. + */ + + /* Get the current initial metadata cache resize configuration */ + if(H5P_get(plist, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME, config_ptr) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get metadata cache initial image config") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_mdc_image_config() */ + + +/*------------------------------------------------------------------------- * Function: H5Pset_mdc_config * * Purpose: Set the initial metadata cache resize configuration in the @@ -2754,6 +2868,147 @@ done: /*------------------------------------------------------------------------- + * Function: H5P__facc_cache_image_config_cmp + * + * Purpose: Compare two cache image configurations. + * + * Return: positive if VALUE1 is greater than VALUE2, negative if VALUE2 is + * greater than VALUE1 and zero if VALUE1 and VALUE2 are equal. + * + * Programmer: John Mainzer + * June 26, 2015 + * + *------------------------------------------------------------------------- + */ +static int +H5P__facc_cache_image_config_cmp(const void *_config1, const void *_config2, size_t H5_ATTR_UNUSED size) +{ + const H5AC_cache_image_config_t *config1 = (const H5AC_cache_image_config_t *)_config1; /* Create local aliases for values */ + const H5AC_cache_image_config_t *config2 = (const H5AC_cache_image_config_t *)_config2; /* Create local aliases for values */ + int ret_value = 0; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check for a property being set */ + if(config1 == NULL && config2 != NULL) HGOTO_DONE(-1); + if(config1 != NULL && config2 == NULL) HGOTO_DONE(1); + + if(config1->version < config2->version) HGOTO_DONE(-1); + if(config1->version > config2->version) HGOTO_DONE(1); + + if(config1->generate_image < config2->generate_image) HGOTO_DONE(-1); + if(config1->generate_image > config2->generate_image) HGOTO_DONE(1); + + if(config1->save_resize_status < config2->save_resize_status) HGOTO_DONE(-1); + if(config1->save_resize_status > config2->save_resize_status) HGOTO_DONE(1); + + if(config1->entry_ageout < config2->entry_ageout) HGOTO_DONE(-1); + if(config1->entry_ageout > config2->entry_ageout) HGOTO_DONE(1); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_cache_image_config_cmp() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_cache_image_config_enc + * + * Purpose: Callback routine which is called whenever the default + * cache image config property in the file creation + * property list is encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: John Mainzer + * June 26, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_cache_image_config_enc(const void *value, void **_pp, size_t *size) +{ + const H5AC_cache_image_config_t *config = (const H5AC_cache_image_config_t *)value; /* Create local aliases for value */ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(value); + + if(NULL != *pp) { + /* Encode type sizes (as a safety check) */ + *(*pp)++ = (uint8_t)sizeof(unsigned); + + INT32ENCODE(*pp, (int32_t)config->version); + + H5_ENCODE_UNSIGNED(*pp, config->generate_image); + + H5_ENCODE_UNSIGNED(*pp, config->save_resize_status); + + INT32ENCODE(*pp, (int32_t)config->entry_ageout); + } /* end if */ + + /* Compute encoded size of fixed-size values */ + *size += (1 + (2 * sizeof(unsigned)) + (2 * sizeof(int32_t))); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__facc_cache_image_config_enc() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__facc_cache_image_config_dec + * + * Purpose: Callback routine which is called whenever the default + * cache image config property in the file creation property + * list is decoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: John Mainzer + * June 26, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_cache_image_config_dec(const void **_pp, void *_value) +{ + H5AC_cache_image_config_t *config = (H5AC_cache_image_config_t *)_value; + const uint8_t **pp = (const uint8_t **)_pp; + unsigned enc_size; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(config); + HDcompile_assert(sizeof(size_t) <= sizeof(uint64_t)); + + /* Set property to default value */ + HDmemcpy(config, &H5F_def_mdc_initCacheImageCfg_g, sizeof(H5AC_cache_image_config_t)); + + /* Decode type sizes */ + enc_size = *(*pp)++; + if(enc_size != sizeof(unsigned)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "unsigned value can't be decoded") + + INT32DECODE(*pp, config->version); + + H5_DECODE_UNSIGNED(*pp, config->generate_image); + + H5_DECODE_UNSIGNED(*pp, config->save_resize_status); + + INT32DECODE(*pp, config->entry_ageout); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_cache_image_config_dec() */ + + +/*------------------------------------------------------------------------- * Function: H5P__facc_file_image_info_set * * Purpose: Copies a file image property when it's set for a property list diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index c736d7b..5aa8301 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -365,6 +365,8 @@ H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collectiv H5_DLL herr_t H5Pset_coll_metadata_write(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective); #endif /* H5_HAVE_PARALLEL */ +H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr); +H5_DLL herr_t H5Pget_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr /*out*/); /* Dataset creation property list (DCPL) routines */ H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout); diff --git a/src/Makefile.am b/src/Makefile.am index 939e151..69b54b4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -46,7 +46,8 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5B.c H5Bcache.c H5Bdbg.c \ H5B2.c H5B2cache.c H5B2dbg.c H5B2hdr.c H5B2int.c H5B2internal.c \ H5B2leaf.c H5B2stat.c H5B2test.c \ - H5C.c H5Cdbg.c H5Cepoch.c H5Clog.c H5Cquery.c H5Ctag.c H5Ctest.c \ + H5C.c H5Cdbg.c H5Cepoch.c H5Cimage.c H5Clog.c H5Cprefetched.c \ + H5Cquery.c H5Ctag.c H5Ctest.c \ H5CS.c \ H5D.c H5Dbtree.c H5Dbtree2.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ H5Ddeprec.c H5Dearray.c H5Defl.c H5Dfarray.c H5Dfill.c H5Dint.c \ @@ -81,7 +82,8 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5MF.c H5MFaggr.c H5MFdbg.c H5MFsection.c \ H5MM.c H5MP.c H5MPtest.c \ H5O.c H5Oainfo.c H5Oalloc.c H5Oattr.c \ - H5Oattribute.c H5Obogus.c H5Obtreek.c H5Ocache.c H5Ochunk.c \ + H5Oattribute.c H5Obogus.c H5Obtreek.c H5Ocache.c H5Ocache_image.c \ + H5Ochunk.c \ H5Ocont.c H5Ocopy.c H5Odbg.c H5Odrvinfo.c H5Odtype.c H5Oefl.c \ H5Ofill.c H5Oflush.c H5Ofsinfo.c H5Oginfo.c \ H5Olayout.c \ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bd73c84..fb9a647 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -269,6 +269,28 @@ endforeach () ############################################################################## ############################################################################## +#-- Adding test for cache_image +add_executable (cache_image + ${HDF5_TEST_SOURCE_DIR}/cache_image.c + ${HDF5_TEST_SOURCE_DIR}/cache_common.c + ${HDF5_TEST_SOURCE_DIR}/genall5.c +) +TARGET_NAMING (cache_image STATIC) +TARGET_C_PROPERTIES (cache_image STATIC " " " ") +target_link_libraries (cache_image ${HDF5_LIB_TARGET} ${HDF5_TEST_LIB_TARGET}) +set_target_properties (cache_image PROPERTIES FOLDER test) +if (BUILD_SHARED_LIBS) + add_executable (cache_image-shared + ${HDF5_TEST_SOURCE_DIR}/cache_image.c + ${HDF5_TEST_SOURCE_DIR}/cache_common.c + ${HDF5_TEST_SOURCE_DIR}/genall5.c + ) + TARGET_NAMING (cache_image-shared SHARED) + TARGET_C_PROPERTIES (cache_image-shared SHARED " " " ") + target_link_libraries (cache_image-shared ${HDF5_TEST_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) + set_target_properties (cache_image-shared PROPERTIES FOLDER test) +endif (BUILD_SHARED_LIBS) + #-- Adding test for hyperslab add_executable (hyperslab ${HDF5_TEST_SOURCE_DIR}/hyperslab.c) TARGET_NAMING (hyperslab STATIC) diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index 20f43bc..05a9be0 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -729,6 +729,22 @@ if (NOT CYGWIN) set_tests_properties (H5TEST-cache PROPERTIES TIMEOUT 1800) endif () +#-- Adding test for cache_image +add_test ( + NAME H5TEST-clear-cache_image-objects + COMMAND ${CMAKE_COMMAND} + -E remove + cache_image_test.h5 + WORKING_DIRECTORY + ${HDF5_TEST_BINARY_DIR}/H5TEST +) +add_test (NAME H5TEST-cache_image COMMAND $<TARGET_FILE:cache_image>) +set_tests_properties (H5TEST-cache_image PROPERTIES + DEPENDS H5TEST-clear-cache_image-objects + ENVIRONMENT "srcdir=${HDF5_TEST_BINARY_DIR}/H5TEST" + WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST +) + #-- Adding test for err_compat if (HDF5_ENABLE_DEPRECATED_SYMBOLS) add_test (NAME H5TEST-clear-err_compat-objects diff --git a/test/Makefile.am b/test/Makefile.am index 50b2ee2..a26f355 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -44,7 +44,8 @@ check_SCRIPTS = $(TEST_SCRIPT) # executed, generally most specific tests to least specific tests. # As an exception, long-running tests should occur earlier in the list. # This gives them more time to run when tests are executing in parallel. -TEST_PROG= testhdf5 cache cache_api cache_tagging lheap ohdr stab gheap \ +TEST_PROG= testhdf5 \ + cache cache_api cache_image cache_tagging lheap ohdr stab gheap \ evict_on_close farray earray btree2 fheap \ pool accum hyperslab istore bittests dt_arith \ dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ @@ -114,6 +115,7 @@ LDADD=libh5test.la $(LIBHDF5) # List the source files for tests that have more than one ttsafe_SOURCES=ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \ ttsafe_acreate.c +cache_image_SOURCES=cache_image.c genall5.c VFD_LIST = sec2 stdio core core_paged split multi family if DIRECT_VFD_CONDITIONAL @@ -174,7 +176,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse flushrefresh_VERIFICATION_CHECKPOINT1 flushrefresh_VERIFICATION_CHECKPOINT2 \ flushrefresh_VERIFICATION_DONE accum_swmr_big.h5 ohdr_swmr.h5 \ cache_logging.h5 cache_logging.out \ - swmr[0-2].h5 tbogus.h5.copy + swmr[0-2].h5 tbogus.h5.copy cache_image_test.h5 # Sources for testhdf5 executable testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \ tgenprop.c th5o.c th5s.c tcoords.c theap.c tid.c titerate.c tmeta.c tmisc.c \ diff --git a/test/cache.c b/test/cache.c index 5c95b77..9ca48e7 100644 --- a/test/cache.c +++ b/test/cache.c @@ -34580,7 +34580,7 @@ cedds__expunge_dirty_entry_in_flush_test(H5F_t * file_ptr) /*------------------------------------------------------------------------- * Function: cedds__H5C_make_space_in_cache() * - * Purpose: Verify that H5C_make_space_in_cache() can handle the + * Purpose: Verify that H5C__make_space_in_cache() can handle the * removal from the cache of the next item in its reverse scan * of the LRU list. * @@ -34590,7 +34590,7 @@ cedds__expunge_dirty_entry_in_flush_test(H5F_t * file_ptr) * load an additional entry, triggering the flush of the last * item, and thereby the deletion of the second to last item. * - * H5C_make_space_in_cache() should detect this deletion, and + * H5C__make_space_in_cache() should detect this deletion, and * restart its scan of the LRU from the tail, instead of * examining the now deleted next item up on the LRU. * @@ -34668,7 +34668,7 @@ cedds__H5C_make_space_in_cache(H5F_t * file_ptr) if(cache_ptr == NULL) { pass = FALSE; - failure_mssg = "cache_ptr NULL on entry to cedds for H5C_make_space_in_cache() test."; + failure_mssg = "cache_ptr NULL on entry to cedds for H5C__make_space_in_cache() test."; } else if((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0)) { @@ -34808,7 +34808,7 @@ cedds__H5C_make_space_in_cache(H5F_t * file_ptr) * and HET 0, 2, and 3 will be evicted to make room for the new * monster entry (MET, 31). * - * Verify this. If H5C_make_space_in_cache() chokes, failure will + * Verify this. If H5C__make_space_in_cache() chokes, failure will * be detected in protect_entry(). Thus end the "if(pass)" clause * there so the error message will not be overwritten. */ @@ -34971,7 +34971,7 @@ cedds__H5C_make_space_in_cache(H5F_t * file_ptr) * access the first item in the LRU repeatedly until the * item, and thereby the deletion of the second to last item. * - * H5C_make_space_in_cache() should detect this deletion, and + * H5C__make_space_in_cache() should detect this deletion, and * restart its scan of the LRU from the tail, instead of * examining the now deleted next item up on the LRU. * diff --git a/test/enc_dec_plist.c b/test/enc_dec_plist.c index 4f0147e..5179e8e 100644 --- a/test/enc_dec_plist.c +++ b/test/enc_dec_plist.c @@ -117,6 +117,11 @@ main(void) 0.2f, (256 * 2048), H5AC__DEFAULT_METADATA_WRITE_STRATEGY}; + H5AC_cache_image_config_t my_cache_image_config = { + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, + TRUE, + FALSE, + -1}; if(VERBOSE_MED) printf("Encode/Decode DCPLs\n"); @@ -455,6 +460,8 @@ main(void) FAIL_STACK_ERROR if((H5Pset_mdc_config(fapl, &my_cache_config)) < 0) FAIL_STACK_ERROR + if((H5Pset_mdc_image_config(fapl, &my_cache_image_config)) < 0) + FAIL_STACK_ERROR if((H5Pset_core_write_tracking(fapl, TRUE, 1024 * 1024)) < 0) FAIL_STACK_ERROR diff --git a/test/gen_plist.c b/test/gen_plist.c index c617ad0..8cb6c00 100644 --- a/test/gen_plist.c +++ b/test/gen_plist.c @@ -81,6 +81,12 @@ main(void) 0.2f, (256 * 2048), H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY}; + H5AC_cache_image_config_t my_cache_image_config = { + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, + TRUE, + FALSE, + -1}; + /* check endianess */ { @@ -356,6 +362,9 @@ main(void) assert(ret > 0); if((ret = H5Pset_mdc_config(fapl1, &my_cache_config)) < 0) assert(ret > 0); + if((ret = H5Pset_mdc_image_config(fapl1, &my_cache_image_config)) < 0) + assert(ret > 0); + if((ret = H5Pset_core_write_tracking(fapl1, TRUE, (size_t)(1024 * 1024))) < 0) assert(ret > 0); diff --git a/test/testfiles/plist_files/dapl_32be b/test/testfiles/plist_files/dapl_32be Binary files differindex 4df4e7f..4dedda2 100644 --- a/test/testfiles/plist_files/dapl_32be +++ b/test/testfiles/plist_files/dapl_32be diff --git a/test/testfiles/plist_files/dapl_32le b/test/testfiles/plist_files/dapl_32le Binary files differindex 4df4e7f..4dedda2 100644 --- a/test/testfiles/plist_files/dapl_32le +++ b/test/testfiles/plist_files/dapl_32le diff --git a/test/testfiles/plist_files/dapl_64be b/test/testfiles/plist_files/dapl_64be Binary files differindex 4df4e7f..4dedda2 100644 --- a/test/testfiles/plist_files/dapl_64be +++ b/test/testfiles/plist_files/dapl_64be diff --git a/test/testfiles/plist_files/dapl_64le b/test/testfiles/plist_files/dapl_64le Binary files differindex 4df4e7f..4dedda2 100644 --- a/test/testfiles/plist_files/dapl_64le +++ b/test/testfiles/plist_files/dapl_64le diff --git a/test/testfiles/plist_files/def_dapl_32be b/test/testfiles/plist_files/def_dapl_32be Binary files differindex c9b7ea9..3df7289 100644 --- a/test/testfiles/plist_files/def_dapl_32be +++ b/test/testfiles/plist_files/def_dapl_32be diff --git a/test/testfiles/plist_files/def_dapl_32le b/test/testfiles/plist_files/def_dapl_32le Binary files differindex c9b7ea9..3df7289 100644 --- a/test/testfiles/plist_files/def_dapl_32le +++ b/test/testfiles/plist_files/def_dapl_32le diff --git a/test/testfiles/plist_files/def_dapl_64be b/test/testfiles/plist_files/def_dapl_64be Binary files differindex c9b7ea9..3df7289 100644 --- a/test/testfiles/plist_files/def_dapl_64be +++ b/test/testfiles/plist_files/def_dapl_64be diff --git a/test/testfiles/plist_files/def_dapl_64le b/test/testfiles/plist_files/def_dapl_64le Binary files differindex c9b7ea9..3df7289 100644 --- a/test/testfiles/plist_files/def_dapl_64le +++ b/test/testfiles/plist_files/def_dapl_64le diff --git a/test/testfiles/plist_files/def_dxpl_32be b/test/testfiles/plist_files/def_dxpl_32be Binary files differindex b13f456..3b77a32 100644 --- a/test/testfiles/plist_files/def_dxpl_32be +++ b/test/testfiles/plist_files/def_dxpl_32be diff --git a/test/testfiles/plist_files/def_dxpl_32le b/test/testfiles/plist_files/def_dxpl_32le Binary files differindex b13f456..3b77a32 100644 --- a/test/testfiles/plist_files/def_dxpl_32le +++ b/test/testfiles/plist_files/def_dxpl_32le diff --git a/test/testfiles/plist_files/def_dxpl_64be b/test/testfiles/plist_files/def_dxpl_64be Binary files differindex b13f456..3b77a32 100644 --- a/test/testfiles/plist_files/def_dxpl_64be +++ b/test/testfiles/plist_files/def_dxpl_64be diff --git a/test/testfiles/plist_files/def_dxpl_64le b/test/testfiles/plist_files/def_dxpl_64le Binary files differindex b13f456..3b77a32 100644 --- a/test/testfiles/plist_files/def_dxpl_64le +++ b/test/testfiles/plist_files/def_dxpl_64le diff --git a/test/testfiles/plist_files/def_fapl_32be b/test/testfiles/plist_files/def_fapl_32be Binary files differindex 6b6baee..3b35501 100644 --- a/test/testfiles/plist_files/def_fapl_32be +++ b/test/testfiles/plist_files/def_fapl_32be diff --git a/test/testfiles/plist_files/def_fapl_32le b/test/testfiles/plist_files/def_fapl_32le Binary files differindex 6b6baee..3b35501 100644 --- a/test/testfiles/plist_files/def_fapl_32le +++ b/test/testfiles/plist_files/def_fapl_32le diff --git a/test/testfiles/plist_files/def_fapl_64be b/test/testfiles/plist_files/def_fapl_64be Binary files differindex 6b6baee..3b35501 100644 --- a/test/testfiles/plist_files/def_fapl_64be +++ b/test/testfiles/plist_files/def_fapl_64be diff --git a/test/testfiles/plist_files/def_fapl_64le b/test/testfiles/plist_files/def_fapl_64le Binary files differindex 6b6baee..3b35501 100644 --- a/test/testfiles/plist_files/def_fapl_64le +++ b/test/testfiles/plist_files/def_fapl_64le diff --git a/test/testfiles/plist_files/dxpl_32be b/test/testfiles/plist_files/dxpl_32be Binary files differindex 5ff2ea0..22fbdc8 100644 --- a/test/testfiles/plist_files/dxpl_32be +++ b/test/testfiles/plist_files/dxpl_32be diff --git a/test/testfiles/plist_files/dxpl_32le b/test/testfiles/plist_files/dxpl_32le Binary files differindex 5ff2ea0..22fbdc8 100644 --- a/test/testfiles/plist_files/dxpl_32le +++ b/test/testfiles/plist_files/dxpl_32le diff --git a/test/testfiles/plist_files/dxpl_64be b/test/testfiles/plist_files/dxpl_64be Binary files differindex 5ff2ea0..22fbdc8 100644 --- a/test/testfiles/plist_files/dxpl_64be +++ b/test/testfiles/plist_files/dxpl_64be diff --git a/test/testfiles/plist_files/dxpl_64le b/test/testfiles/plist_files/dxpl_64le Binary files differindex 5ff2ea0..22fbdc8 100644 --- a/test/testfiles/plist_files/dxpl_64le +++ b/test/testfiles/plist_files/dxpl_64le diff --git a/test/testfiles/plist_files/fapl_32be b/test/testfiles/plist_files/fapl_32be Binary files differindex 65e2070..43e5e67 100644 --- a/test/testfiles/plist_files/fapl_32be +++ b/test/testfiles/plist_files/fapl_32be diff --git a/test/testfiles/plist_files/fapl_32le b/test/testfiles/plist_files/fapl_32le Binary files differindex 65e2070..43e5e67 100644 --- a/test/testfiles/plist_files/fapl_32le +++ b/test/testfiles/plist_files/fapl_32le diff --git a/test/testfiles/plist_files/fapl_64be b/test/testfiles/plist_files/fapl_64be Binary files differindex 65e2070..43e5e67 100644 --- a/test/testfiles/plist_files/fapl_64be +++ b/test/testfiles/plist_files/fapl_64be diff --git a/test/testfiles/plist_files/fapl_64le b/test/testfiles/plist_files/fapl_64le Binary files differindex 65e2070..43e5e67 100644 --- a/test/testfiles/plist_files/fapl_64le +++ b/test/testfiles/plist_files/fapl_64le diff --git a/test/testfiles/plist_files/fcpl_32be b/test/testfiles/plist_files/fcpl_32be Binary files differindex ffa5242..3ce8bf4 100644 --- a/test/testfiles/plist_files/fcpl_32be +++ b/test/testfiles/plist_files/fcpl_32be diff --git a/test/testfiles/plist_files/fcpl_32le b/test/testfiles/plist_files/fcpl_32le Binary files differindex ffa5242..3ce8bf4 100644 --- a/test/testfiles/plist_files/fcpl_32le +++ b/test/testfiles/plist_files/fcpl_32le diff --git a/test/testfiles/plist_files/fcpl_64be b/test/testfiles/plist_files/fcpl_64be Binary files differindex ffa5242..3ce8bf4 100644 --- a/test/testfiles/plist_files/fcpl_64be +++ b/test/testfiles/plist_files/fcpl_64be diff --git a/test/testfiles/plist_files/fcpl_64le b/test/testfiles/plist_files/fcpl_64le Binary files differindex ffa5242..3ce8bf4 100644 --- a/test/testfiles/plist_files/fcpl_64le +++ b/test/testfiles/plist_files/fcpl_64le diff --git a/test/testfiles/plist_files/lapl_32be b/test/testfiles/plist_files/lapl_32be Binary files differindex e9f43e2..f3e9865 100644 --- a/test/testfiles/plist_files/lapl_32be +++ b/test/testfiles/plist_files/lapl_32be diff --git a/test/testfiles/plist_files/lapl_32le b/test/testfiles/plist_files/lapl_32le Binary files differindex e9f43e2..f3e9865 100644 --- a/test/testfiles/plist_files/lapl_32le +++ b/test/testfiles/plist_files/lapl_32le diff --git a/test/testfiles/plist_files/lapl_64be b/test/testfiles/plist_files/lapl_64be Binary files differindex e9f43e2..f3e9865 100644 --- a/test/testfiles/plist_files/lapl_64be +++ b/test/testfiles/plist_files/lapl_64be diff --git a/test/testfiles/plist_files/lapl_64le b/test/testfiles/plist_files/lapl_64le Binary files differindex e9f43e2..f3e9865 100644 --- a/test/testfiles/plist_files/lapl_64le +++ b/test/testfiles/plist_files/lapl_64le |