diff options
Diffstat (limited to 'src/H5Cprivate.h')
| -rw-r--r-- | src/H5Cprivate.h | 355 |
1 files changed, 343 insertions, 12 deletions
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 7ae6b58..20d67f2 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -85,6 +85,11 @@ #endif /* H5_HAVE_PARALLEL */ +/* Typedef for the main structure for the cache (defined in H5Cpkg.h) */ + +typedef struct H5C_t H5C_t; + + /* * Class methods pertaining to caching. Each type of cached object will * have a constant variable with permanent life-span that describes how @@ -144,6 +149,18 @@ typedef herr_t (*H5C_write_permitted_func_t)(H5F_t *f, hid_t dxpl_id, hbool_t * write_permitted_ptr); +/* Upper and lower limits on cache size. These limits are picked + * out of a hat -- you should be able to change them as necessary. + * + * However, if you need a very big cache, you should also increase the + * size of the hash table (H5C__HASH_TABLE_LEN in H5C.c). The current + * upper bound on cache size is rather large for the current hash table + * size. + */ + +#define H5C__MAX_MAX_CACHE_SIZE ((size_t)(128 * 1024 * 1024)) +#define H5C__MIN_MAX_CACHE_SIZE ((size_t)(1024)) + /* Default max cache size and min clean size are give here to make * them generally accessable. @@ -177,6 +194,10 @@ typedef herr_t (*H5C_write_permitted_func_t)(H5F_t *f, * The entries should never overlap, and when we do writebacks, * we will want to writeback adjacent entries where possible. * + * NB: At present, entries need not be contiguous on disk. Until + * we fix this, we can't do much with writing back adjacent + * entries. + * * type: Pointer to the instance of H5C_class_t containing pointers * to the methods for cache entries of the current type. This * field should be NULL when the instance of H5C_cache_entry_t @@ -343,9 +364,293 @@ typedef struct H5C_cache_entry_t } H5C_cache_entry_t; -/* Typedef for the main structure for the cache (defined in H5C.c) */ +/**************************************************************************** + * + * structure H5C_auto_size_ctl_t + * + * Instances of H5C_auto_size_ctl_t are used to get and set the control + * fields for automatic cache re-sizing. + * + * The fields of the structure are discussed individually below: + * + * version: Integer field containing the version number of this version + * of the H5C_auto_size_ctl_t structure. Any instance of + * H5C_auto_size_ctl_t passed to the cache must have a known + * version number, or an error will be flagged. + * + * report_fcn: Pointer to the function that is to be called to report + * activities each time the auto cache resize code is executed. If the + * field is NULL, no call is made. + * + * If the field is not NULL, it must contain the address of a function + * of type H5C_auto_resize_report_fcn. + * + * set_initial_size: Boolean flag indicating whether the size of the + * initial size of the cache is to be set to the value given in + * the initial_size field. If set_initial_size is FALSE, the + * initial_size field is ignored. + * + * initial_size: If enabled, this field contain the size the cache is + * to be set to upon receipt of this structure. Needless to say, + * initial_size must lie in the closed interval [min_size, max_size]. + * + * min_clean_fraction: double in the range 0 to 1 indicating the fraction + * of the cache that is to be kept clean. This field is only used + * in parallel mode. Typical values are 0.1 to 0.5. + * + * max_size: Maximum size to which the cache can be adjusted. The + * supplied value must fall in the closed interval + * [MIN_MAX_CACHE_SIZE, MAX_MAX_CACHE_SIZE]. Also, max_size must + * be greater than or equal to min_size. + * + * min_size: Minimum size to which the cache can be adjusted. The + * supplied value must fall in the closed interval + * [MIN_MAX_CACHE_SIZE, MAX_MAX_CACHE_SIZE]. Also, min_size must + * be less than or equal to max_size. + * + * epoch_length: Number of accesses on the cache over which to collect + * hit rate stats before running the automatic cache resize code, + * if it is enabled. + * + * At the end of an epoch, we discard prior hit rate data and start + * collecting afresh. The epoch_length must lie in the closed + * interval [H5C__MIN_AR_EPOCH_LENGTH, H5C__MAX_AR_EPOCH_LENGTH]. + * + * + * Cache size increase control fields: + * + * incr_mode: Instance of the H5C_cache_incr_mode enumerated type whose + * value indicates how we determine whether the cache size should be + * increased. At present there are two possible values: + * + * H5C_incr__off: Don't attempt to increase the size of the cache + * automatically. + * + * When this increment mode is selected, the remaining fields + * in the cache size increase section ar ignored. + * + * H5C_incr__threshold: Attempt to increase the size of the cache + * whenever the average hit rate over the last epoch drops + * below the value supplied in the lower_hr_threshold + * field. + * + * Note that this attempt will fail if the cache is already + * at its maximum size, or if the cache is not already using + * all available space. + * + * lower_hr_threshold: Lower hit rate threshold. If the increment mode + * (incr_mode) is H5C_incr__threshold and the hit rate drops below the + * value supplied in this field in an epoch, increment the cache size by + * size_increment. Note that cache size may not be incremented above + * max_size, and that the increment may be further restricted by the + * max_increment field if it is enabled. + * + * When enabled, this field must contain a value in the range [0.0, 1.0]. + * Depending on the incr_mode selected, it may also have to be less than + * upper_hr_threshold. + * + * increment: Double containing the multiplier used to derive the new + * cache size from the old if a cache size increment is triggered. + * The increment must be greater than 1.0, and should not exceed 2.0. + * + * The new cache size is obtained my multiplying the current max cache + * size by the increment, and then clamping to max_size and to stay + * within the max_increment as necessary. + * + * apply_max_increment: Boolean flag indicating whether the max_increment + * field should be used to limit the maximum cache size increment. + * + * max_increment: If enabled by the apply_max_increment field described + * above, this field contains the maximum number of bytes by which the + * cache size can be increased in a single re-size. + * + * + * Cache size decrease control fields: + * + * decr_mode: Instance of the H5C_cache_decr_mode enumerated type whose + * value indicates how we determine whether the cache size should be + * decreased. At present there are four possibilities. + * + * H5C_decr__off: Don't attempt to decrease the size of the cache + * automatically. + * + * When this increment mode is selected, the remaining fields + * in the cache size decrease section ar ignored. + * + * H5C_decr__threshold: Attempt to decrease the size of the cache + * whenever the average hit rate over the last epoch rises + * above the value supplied in the upper_hr_threshold + * field. + * + * H5C_decr__age_out: At the end of each epoch, search the cache for + * entries that have not been accessed for at least the number + * of epochs specified in the epochs_before_eviction field, and + * evict these entries. Conceptually, the maximum cache size + * is then decreased to match the new actual cache size. However, + * this reduction may be modified by the min_size, the + * max_decrement, and/or the empty_reserve. + * + * H5C_decr__age_out_with_threshold: Same as age_out, but we only + * attempt to reduce the cache size when the hit rate observed + * over the last epoch exceeds the value provided in the + * upper_hr_threshold field. + * + * upper_hr_threshold: Upper hit rate threshold. The use of this field + * varies according to the current decr_mode: + * + * H5C_decr__off or H5C_decr__age_out: The value of this field is + * ignored. + * + * H5C_decr__threshold: If the hit rate exceeds this threshold in any + * epoch, attempt todecrement the cache size by size_decrement. + * + * Note that cache size may not be decremented below min_size. + * + * Note also that if the upper_threshold is 1.0, the cache size + * will never be reduced. + * + * H5C_decr__age_out_with_threshold: If the hit rate exceeds this + * threshold in any epoch, attempt to reduce the cache size + * by evicting entries that have not been accessed for more + * than the specified number of epochs. + * + * decrement: This field is only used when the decr_mode is + * H5C_decr__threshold. + * + * The field is a double containing the multiplier used to derive the + * new cache size from the old if a cache size decrement is triggered. + * The decrement must be in the range 0.0 (in which case the cache will + * try to contract to its minimum size) to 1.0 (in which case the + * cache will never shrink). + * + * apply_max_decrement: Boolean flag used to determine whether decrements + * in cache size are to be limited by the max_decrement field. + * + * max_decrement: Maximum number of bytes by which the cache size can be + * decreased in a single re-size. Note that decrements may also be + * restricted by the min_size of the cache, and (in age out modes) by + * the empty_reserve field. + * + * epochs_before_eviction: Integer field used in H5C_decr__age_out and + * H5C_decr__age_out_with_threshold decrement modes. + * + * This field contains the number of epochs an entry must remain + * unaccessed before it is evicted in an attempt to reduce the + * cache size. If applicable, this field must lie in the range + * [1, H5C__MAX_EPOCH_MARKERS]. + * + * apply_empty_reserve: Boolean field controlling whether the empty_reserve + * field is to be used in computing the new cache size when the + * decr_mode is H5C_decr__age_out or H5C_decr__age_out_with_threshold. + * + * empty_reserve: To avoid a constant racheting down of cache size by small + * amounts in the H5C_decr__age_out and H5C_decr__age_out_with_threshold + * modes, this field allows one to require that any cache size + * reductions leave the specified fraction of unused space in the cache. + * + * The value of this field must be in the range [0.0, 1.0]. I would + * expect typical values to be in the range of 0.01 to 0.1. + * + ****************************************************************************/ + +#define H5C__CURR_AUTO_SIZE_CTL_VER 1 +#define H5C__CURR_AUTO_RESIZE_RPT_FCN_VER 1 + +#define H5C__DEF_AR_UPPER_THRESHHOLD 0.9999 +#define H5C__DEF_AR_LOWER_THRESHHOLD 0.9 +#define H5C__DEF_AR_MAX_SIZE ((size_t)(16 * 1024 * 1024)) +#define H5C__DEF_AR_INIT_SIZE ((size_t)( 1 * 1024 * 1024)) +#define H5C__DEF_AR_MIN_SIZE ((size_t)( 1 * 1024 * 1024)) +#define H5C__DEF_AR_MIN_CLEAN_FRAC 0.5 +#define H5C__DEF_AR_INCREMENT 2.0 +#define H5C__DEF_AR_MAX_INCREMENT ((size_t)( 2 * 1024 * 1024)) +#define H5C__DEF_AR_DECREMENT 0.9 +#define H5C__DEF_AR_MAX_DECREMENT ((size_t)( 1 * 1024 * 1024)) +#define H5C__DEF_AR_EPCHS_B4_EVICT 3 +#define H5C__DEF_AR_EMPTY_RESERVE 0.05 +#define H5C__MIN_AR_EPOCH_LENGTH 100 +#define H5C__DEF_AR_EPOCH_LENGTH 50000 +#define H5C__MAX_AR_EPOCH_LENGTH 1000000 + +enum H5C_resize_status +{ + in_spec, + increase, + decrease, + at_max_size, + at_min_size, + increase_disabled, + decrease_disabled, + not_full +}; /* enum H5C_resize_conditions */ + +enum H5C_cache_incr_mode +{ + H5C_incr__off, + H5C_incr__threshold +}; + +enum H5C_cache_decr_mode +{ + H5C_decr__off, + H5C_decr__threshold, + H5C_decr__age_out, + H5C_decr__age_out_with_threshold +}; + +typedef void (*H5C_auto_resize_rpt_fcn)(H5C_t * cache_ptr, + int32_t version, + double hit_rate, + enum H5C_resize_status status, + size_t old_max_cache_size, + size_t new_max_cache_size, + size_t old_min_clean_size, + size_t new_min_clean_size); +typedef struct H5C_auto_size_ctl_t +{ + /* general configuration fields: */ + int32_t version; + H5C_auto_resize_rpt_fcn rpt_fcn; + + hbool_t set_initial_size; + size_t initial_size; + + double min_clean_fraction; + + size_t max_size; + size_t min_size; + + int64_t epoch_length; + + + /* size increase control fields: */ + enum H5C_cache_incr_mode incr_mode; + + double lower_hr_threshold; + + double increment; + + hbool_t apply_max_increment; + size_t max_increment; + + + /* size decrease control fields: */ + enum H5C_cache_decr_mode decr_mode; + + double upper_hr_threshold; + + double decrement; + + hbool_t apply_max_decrement; + size_t max_decrement; + + int32_t epochs_before_eviction; + + hbool_t apply_empty_reserve; + double empty_reserve; + +} H5C_auto_size_ctl_t; -typedef struct H5C_t H5C_t; /* * Library prototypes. @@ -356,6 +661,15 @@ H5_DLL H5C_t * H5C_create(size_t max_cache_size, const char * (* type_name_table_ptr), H5C_write_permitted_func_t check_write_permitted); +H5_DLL void H5C_def_auto_resize_rpt_fcn(H5C_t * cache_ptr, + int32_t version, + double hit_rate, + enum H5C_resize_status status, + size_t old_max_cache_size, + size_t new_max_cache_size, + size_t old_min_clean_size, + size_t new_min_clean_size); + H5_DLL herr_t H5C_dest(H5F_t * f, hid_t primary_dxpl_id, hid_t secondary_dxpl_id, @@ -369,6 +683,18 @@ H5_DLL herr_t H5C_flush_cache(H5F_t * f, H5C_t * cache_ptr, unsigned flags); +H5_DLL herr_t H5C_get_cache_auto_resize_config(H5C_t * cache_ptr, + H5C_auto_size_ctl_t *config_ptr); + +H5_DLL herr_t H5C_get_cache_size(H5C_t * cache_ptr, + size_t * max_size_ptr, + size_t * min_clean_size_ptr, + size_t * cur_size_ptr, + int32_t * cur_num_entries_ptr); + +H5_DLL herr_t H5C_get_cache_hit_rate(H5C_t * cache_ptr, + double * hit_rate_ptr); + H5_DLL herr_t H5C_insert_entry(H5F_t * f, hid_t primary_dxpl_id, hid_t secondary_dxpl_id, @@ -392,6 +718,21 @@ H5_DLL void * H5C_protect(H5F_t * f, const void * udata1, void * udata2); +H5_DLL herr_t H5C_reset_cache_hit_rate_stats(H5C_t * cache_ptr); + +H5_DLL herr_t H5C_set_cache_auto_resize_config(H5C_t * cache_ptr, + H5C_auto_size_ctl_t *config_ptr); + +H5_DLL herr_t H5C_set_skip_flags(H5C_t * cache_ptr, + hbool_t skip_file_checks, + hbool_t skip_dxpl_id_checks); + +H5_DLL herr_t H5C_stats(H5C_t * cache_ptr, + const char * cache_name, + hbool_t display_detailed_stats); + +H5_DLL void H5C_stats__reset(H5C_t * cache_ptr); + H5_DLL herr_t H5C_unprotect(H5F_t * f, hid_t primary_dxpl_id, hid_t secondary_dxpl_id, @@ -401,15 +742,5 @@ H5_DLL herr_t H5C_unprotect(H5F_t * f, void * thing, hbool_t deleted); -H5_DLL herr_t H5C_stats(H5C_t * cache_ptr, - const char * cache_name, - hbool_t display_detailed_stats); - -H5_DLL void H5C_stats__reset(H5C_t * cache_ptr); - -H5_DLL herr_t H5C_set_skip_flags(H5C_t * cache_ptr, - hbool_t skip_file_checks, - hbool_t skip_dxpl_id_checks); - #endif /* !_H5Cprivate_H */ |