diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/H5AC.c | 23 | ||||
| -rw-r--r-- | src/H5AC2.c | 27 | ||||
| -rw-r--r-- | src/H5AC2private.h | 4 | ||||
| -rw-r--r-- | src/H5AC2public.h | 60 | ||||
| -rw-r--r-- | src/H5ACprivate.h | 4 | ||||
| -rw-r--r-- | src/H5ACpublic.h | 61 | ||||
| -rw-r--r-- | src/H5C.c | 993 | ||||
| -rw-r--r-- | src/H5C2.c | 599 | ||||
| -rw-r--r-- | src/H5C2journal.c | 519 | ||||
| -rw-r--r-- | src/H5C2pkg.h | 14 | ||||
| -rw-r--r-- | src/H5C2private.h | 104 | ||||
| -rw-r--r-- | src/H5C2public.h | 7 | ||||
| -rw-r--r-- | src/H5Cpkg.h | 22 | ||||
| -rw-r--r-- | src/H5Cprivate.h | 130 | ||||
| -rw-r--r-- | src/H5Cpublic.h | 6 | ||||
| -rw-r--r-- | src/H5Fpkg.h | 5 | ||||
| -rw-r--r-- | src/H5Fsuper.c | 105 | ||||
| -rw-r--r-- | src/H5O.c | 15 | ||||
| -rw-r--r-- | src/H5Opkg.h | 7 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 59 | ||||
| -rwxr-xr-x | src/Makefile.am | 2 | ||||
| -rw-r--r-- | src/Makefile.in | 78 |
22 files changed, 2664 insertions, 180 deletions
@@ -2546,6 +2546,10 @@ done: * switches for functions that are only tenuously related * to auto resize configuration. * + * JRM - 1/2/08 + * Added support for the new flash cache increment related + * fields. + * *------------------------------------------------------------------------- */ @@ -2625,6 +2629,9 @@ H5AC_get_cache_auto_resize_config(H5AC_t * cache_ptr, config_ptr->max_increment = internal_config.max_increment; config_ptr->decr_mode = internal_config.decr_mode; config_ptr->upper_hr_threshold = internal_config.upper_hr_threshold; + config_ptr->flash_incr_mode = internal_config.flash_incr_mode; + config_ptr->flash_multiple = internal_config.flash_multiple; + config_ptr->flash_threshold = internal_config.flash_threshold; config_ptr->decrement = internal_config.decrement; config_ptr->apply_max_decrement = internal_config.apply_max_decrement; config_ptr->max_decrement = internal_config.max_decrement; @@ -2816,6 +2823,10 @@ done: * switches for functions that are only tenuously related * to auto resize configuration. * + * John Mainzer -- 1/3/07 + * Updated trace file code to record the new flash cache + * size increase related fields. + * *------------------------------------------------------------------------- */ @@ -2969,7 +2980,7 @@ done: ( trace_file_ptr != NULL ) ) { HDfprintf(trace_file_ptr, - "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %d %d %f %f %d %d %d %d %f %d %d\n", + "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %f %f %d %d %d %f %f %d %d %d %d %f %d %d\n", "H5AC_set_cache_auto_resize_config", trace_config.version, (int)(trace_config.rpt_fcn_enabled), @@ -2986,6 +2997,9 @@ done: (int)(trace_config.incr_mode), trace_config.lower_hr_threshold, trace_config.increment, + (int)(trace_config.flash_incr_mode), + trace_config.flash_multiple, + trace_config.flash_threshold, (int)(trace_config.apply_max_increment), (int)(trace_config.max_increment), (int)(trace_config.decr_mode), @@ -3618,7 +3632,9 @@ done: * * Modifications: * - * None. + * Updated function for flash cache increment fields. + * + * JRM -- 1/2/08 * *------------------------------------------------------------------------- */ @@ -3661,6 +3677,9 @@ H5AC_ext_config_2_int_config(H5AC_cache_config_t * ext_conf_ptr, int_conf_ptr->increment = ext_conf_ptr->increment; int_conf_ptr->apply_max_increment = ext_conf_ptr->apply_max_increment; int_conf_ptr->max_increment = ext_conf_ptr->max_increment; + int_conf_ptr->flash_incr_mode = ext_conf_ptr->flash_incr_mode; + int_conf_ptr->flash_multiple = ext_conf_ptr->flash_multiple; + int_conf_ptr->flash_threshold = ext_conf_ptr->flash_threshold; int_conf_ptr->decr_mode = ext_conf_ptr->decr_mode; int_conf_ptr->upper_hr_threshold = ext_conf_ptr->upper_hr_threshold; diff --git a/src/H5AC2.c b/src/H5AC2.c index a053b36..49e4934 100644 --- a/src/H5AC2.c +++ b/src/H5AC2.c @@ -2587,6 +2587,10 @@ done: * Modified code in support of revised cache API needed * to permit journaling. * + * JRM - 1/2/08 + * Added support for the new flash cache increment related + * fields. + * *------------------------------------------------------------------------- */ @@ -2665,6 +2669,9 @@ H5AC2_get_cache_auto_resize_config(H5AC2_t * cache_ptr, config_ptr->increment = internal_config.increment; config_ptr->apply_max_increment = internal_config.apply_max_increment; config_ptr->max_increment = internal_config.max_increment; + config_ptr->flash_incr_mode = internal_config.flash_incr_mode; + config_ptr->flash_multiple = internal_config.flash_multiple; + config_ptr->flash_threshold = internal_config.flash_threshold; config_ptr->decr_mode = internal_config.decr_mode; config_ptr->upper_hr_threshold = internal_config.upper_hr_threshold; config_ptr->decrement = internal_config.decrement; @@ -2868,6 +2875,10 @@ done: * Modified code in support of revised cache API needed * to permit journaling. * + * John Mainzer -- 1/3/07 + * Updated trace file code to record the new flash cache + * size increase related fields. + * *------------------------------------------------------------------------- */ @@ -3021,7 +3032,7 @@ done: ( trace_file_ptr != NULL ) ) { HDfprintf(trace_file_ptr, - "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %d %d %f %f %d %d %d %d %f %d %d\n", + "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %d %d %f %f %d %f %f %d %d %d %d %f %d %d\n", "H5AC2_set_cache_auto_resize_config", trace_config.version, (int)(trace_config.rpt_fcn_enabled), @@ -3040,6 +3051,9 @@ done: trace_config.increment, (int)(trace_config.apply_max_increment), (int)(trace_config.max_increment), + (int)(trace_config.flash_incr_mode), + trace_config.flash_multiple, + trace_config.flash_threshold, (int)(trace_config.decr_mode), trace_config.upper_hr_threshold, trace_config.decrement, @@ -3379,7 +3393,7 @@ H5AC2_open_trace_file(H5AC2_t * cache_ptr, HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "trace file open failed.") } - HDfprintf(file_ptr, "### HDF5 metadata cache trace file version 1 ###\n"); + HDfprintf(file_ptr, "### HDF5 metadata cache trace file version 2 ###\n"); if ( H5C2_set_trace_file_ptr(cache_ptr, file_ptr) < 0 ) { @@ -3678,7 +3692,10 @@ done: * * Modifications: * - * None. + * Updated function for flash cache increment fields. + * + * JRM -- 1/2/08 + * * *------------------------------------------------------------------------- */ @@ -3721,6 +3738,10 @@ H5AC2_ext_config_2_int_config(H5AC2_cache_config_t * ext_conf_ptr, int_conf_ptr->increment = ext_conf_ptr->increment; int_conf_ptr->apply_max_increment = ext_conf_ptr->apply_max_increment; int_conf_ptr->max_increment = ext_conf_ptr->max_increment; + int_conf_ptr->flash_incr_mode = ext_conf_ptr->flash_incr_mode; + int_conf_ptr->flash_multiple = ext_conf_ptr->flash_multiple; + int_conf_ptr->flash_threshold = ext_conf_ptr->flash_threshold; + int_conf_ptr->decr_mode = ext_conf_ptr->decr_mode; int_conf_ptr->upper_hr_threshold = ext_conf_ptr->upper_hr_threshold; diff --git a/src/H5AC2private.h b/src/H5AC2private.h index de76b40..ca2522f 100644 --- a/src/H5AC2private.h +++ b/src/H5AC2private.h @@ -202,6 +202,10 @@ extern hid_t H5AC2_ind_dxpl_id; /* double increment = */ 2.0, \ /* hbool_t apply_max_increment = */ TRUE, \ /* size_t max_increment = */ (4 * 1024 * 1024), \ + /* enum H5C2_cache_flash_incr_mode */ \ + /* flash_incr_mode = */ H5C2_flash_incr__add_space, \ + /* double flash_multiple = */ 1.0, \ + /* double flash_threshold = */ 0.25, \ /* enum H5C2_cache_decr_mode decr_mode = */ \ H5C2_decr__age_out_with_threshold, \ /* double upper_hr_threshold = */ 0.999, \ diff --git a/src/H5AC2public.h b/src/H5AC2public.h index 3c88ef1..a16ad2a 100644 --- a/src/H5AC2public.h +++ b/src/H5AC2public.h @@ -15,7 +15,7 @@ /*------------------------------------------------------------------------- * - * Created: H5ACpublic.h + * Created: H5AC2public.h * Jul 10 1997 * Robb Matzke <matzke@llnl.gov> * @@ -208,6 +208,60 @@ extern "C" { * above, this field contains the maximum number of bytes by which the * cache size can be increased in a single re-size. * + * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated + * type whose value indicates whether and by which algorithm we should + * make flash increases in the size of the cache to accomodate insertion + * of large entries and large increases in the size of a single entry. + * + * The addition of the flash increment mode was occasioned by performance + * problems that appear when a local heap is increased to a size in excess + * of the current cache size. While the existing re-size code dealt with + * this eventually, performance was very bad for the remainder of the + * epoch. + * + * At present, there are two possible values for the flash_incr_mode: + * + * H5C_flash_incr__off: Don't perform flash increases in the size of + * the cache. + * + * H5C_flash_incr__add_space: Let x be either the size of a newly + * newly inserted entry, or the number of bytes by which the + * size of an existing entry has been increased. + * + * If + * x > flash_threshold * current max cache size, + * + * increase the current maximum cache size by x * flash_multiple + * less any free space in the cache, and star a new epoch. For + * now at least, pay no attention to the maximum increment. + * + * In both of the above cases, the flash increment pays no attention to + * the maximum increment (at least in this first incarnation), but DOES + * stay within max_size. + * + * With a little thought, it should be obvious that the above flash + * cache size increase algorithm is not sufficient for all circumstances + * -- for example, suppose the user round robins through + * (1/flash_threshold) +1 groups, adding one data set to each on each + * pass. Then all will increase in size at about the same time, requiring + * the max cache size to at least double to maintain acceptable + * performance, however the above flash increment algorithm will not be + * triggered. + * + * Hopefully, the add space algorithms detailed above will be sufficient + * for the performance problems encountered to date. However, we should + * expect to revisit the issue. + * + * flash_multiple: Double containing the multiple described above in the + * H5C_flash_incr__add_space section of the discussion of the + * flash_incr_mode section. This field is ignored unless flash_incr_mode + * is H5C_flash_incr__add_space. + * + * flash_threshold: Double containing the factor by which current max cache + * size is multiplied to obtain the size threshold for the add_space flash + * increment algorithm. The field is ignored unless flash_incr_mode is + * H5C_flash_incr__add_space. + * * * Cache size decrease control fields: * @@ -364,6 +418,10 @@ typedef struct H5AC2_cache_config_t hbool_t apply_max_increment; size_t max_increment; + enum H5C2_cache_flash_incr_mode flash_incr_mode; + double flash_multiple; + double flash_threshold; + /* size decrease control fields: */ enum H5C2_cache_decr_mode decr_mode; diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 628a269..bbbcf11 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -213,6 +213,10 @@ extern hid_t H5AC_ind_dxpl_id; /* double increment = */ 2.0, \ /* hbool_t apply_max_increment = */ TRUE, \ /* size_t max_increment = */ (4 * 1024 * 1024), \ + /* enum H5C_cache_flash_incr_mode */ \ + /* flash_incr_mode = */ H5C_flash_incr__add_space, \ + /* double flash_multiple = */ 1.0, \ + /* double flash_threshold = */ 0.25, \ /* enum H5C_cache_decr_mode decr_mode = */ H5C_decr__age_out_with_threshold,\ /* double upper_hr_threshold = */ 0.999, \ /* double decrement = */ 0.9, \ diff --git a/src/H5ACpublic.h b/src/H5ACpublic.h index 0ddef92..0e75117 100644 --- a/src/H5ACpublic.h +++ b/src/H5ACpublic.h @@ -15,7 +15,7 @@ /*------------------------------------------------------------------------- * - * Created: H5ACproto.h + * Created: H5ACpublic.h * Jul 10 1997 * Robb Matzke <matzke@llnl.gov> * @@ -208,6 +208,61 @@ extern "C" { * above, this field contains the maximum number of bytes by which the * cache size can be increased in a single re-size. * + * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated + * type whose value indicates whether and by which algorithm we should + * make flash increases in the size of the cache to accomodate insertion + * of large entries and large increases in the size of a single entry. + * + * The addition of the flash increment mode was occasioned by performance + * problems that appear when a local heap is increased to a size in excess + * of the current cache size. While the existing re-size code dealt with + * this eventually, performance was very bad for the remainder of the + * epoch. + * + * At present, there are two possible values for the flash_incr_mode: + * + * H5C_flash_incr__off: Don't perform flash increases in the size of + * the cache. + * + * H5C_flash_incr__add_space: Let x be either the size of a newly + * newly inserted entry, or the number of bytes by which the + * size of an existing entry has been increased. + * + * If + * x > flash_threshold * current max cache size, + * + * increase the current maximum cache size by x * flash_multiple + * less any free space in the cache, and star a new epoch. For + * now at least, pay no attention to the maximum increment. + * + * In both of the above cases, the flash increment pays no attention to + * the maximum increment (at least in this first incarnation), but DOES + * stay within max_size. + * + * With a little thought, it should be obvious that the above flash + * cache size increase algorithm is not sufficient for all circumstances -- + * for example, suppose the user round robins through + * (1/flash_threshold) +1 groups, adding one data set to each on each + * pass. Then all will increase in size at about the same time, requiring + * the max cache size to at least double to maintain acceptable + * performance, however the above flash increment algorithm will not be + * triggered. + * + * Hopefully, the add space algorithms detailed above will be sufficient + * for the performance problems encountered to date. However, we should + * expect to revisit the issue. + * + * flash_multiple: Double containing the multiple described above in the + * H5C_flash_incr__add_space section of the discussion of the + * flash_incr_mode section. This field is ignored unless flash_incr_mode + * is H5C_flash_incr__add_space. + * + * flash_threshold: Double containing the factor by which current max cache size + * is multiplied to obtain the size threshold for the add_space flash + * increment algorithm. The field is ignored unless flash_incr_mode is + * H5C_flash_incr__add_space. + * + * * * Cache size decrease control fields: * @@ -364,6 +419,10 @@ typedef struct H5AC_cache_config_t hbool_t apply_max_increment; size_t max_increment; + enum H5C_cache_flash_incr_mode flash_incr_mode; + double flash_multiple; + double flash_threshold; + /* size decrease control fields: */ enum H5C_cache_decr_mode decr_mode; @@ -2596,6 +2596,10 @@ static herr_t H5C__autoadjust__ageout__remove_all_markers(H5C_t * cache_ptr); static herr_t H5C__autoadjust__ageout__remove_excess_markers(H5C_t * cache_ptr); +static herr_t H5C__flash_increase_cache_size(H5C_t * cache_ptr, + size_t old_entry_size, + size_t new_entry_size); + static herr_t H5C_flush_single_entry(H5F_t * f, hid_t primary_dxpl_id, hid_t secondary_dxpl_id, @@ -2836,8 +2840,12 @@ done: * ro_ref_count fields. * * JRM -- 7/27/07 -* Added initialization for the new evictions_enabled -* field of H5C_t. + * Added initialization for the new evictions_enabled + * field of H5C_t. + * + * JRM -- 12/31/07 + * Added initialization for the new flash cache size increase + * related fields of H5C_t. * *------------------------------------------------------------------------- */ @@ -2954,6 +2962,8 @@ H5C_create(size_t max_cache_size, cache_ptr->dLRU_tail_ptr = NULL; cache_ptr->size_increase_possible = FALSE; + cache_ptr->flash_size_increase_possible = FALSE; + cache_ptr->flash_size_increase_threshold = 0; cache_ptr->size_decrease_possible = FALSE; cache_ptr->resize_enabled = FALSE; cache_ptr->cache_full = FALSE; @@ -2974,6 +2984,11 @@ H5C_create(size_t max_cache_size, (cache_ptr->resize_ctl).apply_max_increment = TRUE; (cache_ptr->resize_ctl).max_increment = H5C__DEF_AR_MAX_INCREMENT; + (cache_ptr->resize_ctl).flash_incr_mode = H5C_flash_incr__off; + (cache_ptr->resize_ctl).flash_multiple = 1.0; + (cache_ptr->resize_ctl).flash_threshold = 0.25; + + (cache_ptr->resize_ctl).decr_mode = H5C_decr__off; (cache_ptr->resize_ctl).upper_hr_threshold = H5C__DEF_AR_UPPER_THRESHHOLD; (cache_ptr->resize_ctl).decrement = H5C__DEF_AR_DECREMENT; @@ -2993,7 +3008,10 @@ H5C_create(size_t max_cache_size, for ( i = 0; i < H5C__MAX_EPOCH_MARKERS; i++ ) { (cache_ptr->epoch_marker_active)[i] = FALSE; - +#ifndef NDEBUG + ((cache_ptr->epoch_markers)[i]).magic = + H5C__H5C_CACHE_ENTRY_T_MAGIC; +#endif /* NDEBUG */ ((cache_ptr->epoch_markers)[i]).addr = (haddr_t)i; ((cache_ptr->epoch_markers)[i]).size = (size_t)0; ((cache_ptr->epoch_markers)[i]).type = &epoch_marker_class; @@ -3079,6 +3097,9 @@ done: * Updated function for display the new prefix field of * H5C_t in output. * + * JRM 12/31/07 + * Updated function to handle flash size increases. + * *------------------------------------------------------------------------- */ void @@ -3125,6 +3146,24 @@ H5C_def_auto_resize_rpt_fcn(H5C_t * cache_ptr, new_min_clean_size); break; + case flash_increase: + HDassert( old_max_cache_size < new_max_cache_size ); + + HDfprintf(stdout, + "%sflash cache resize(%d) -- size threshold = %Zu.\n", + cache_ptr->prefix, + (int)((cache_ptr->resize_ctl).flash_incr_mode), + cache_ptr->flash_size_increase_threshold); + + HDfprintf(stdout, + "%s cache size increased from (%Zu/%Zu) to (%Zu/%Zu).\n", + cache_ptr->prefix, + old_max_cache_size, + old_min_clean_size, + new_max_cache_size, + new_min_clean_size); + break; + case decrease: HDassert( old_max_cache_size > new_max_cache_size ); @@ -3537,6 +3576,21 @@ done: * callbacks. As a result, we may have to make multiple * passes through the skip list before the cache is flushed. * + * JRM -- 10/13/07 + * Added code to detect and manage the case in which a + * flush callback changes the s-list out from under + * the function. The only way I can think of in which this + * can happen is if a flush function loads an entry + * into the cache that isn't there already. Quincey tells + * me that this will never happen, but I'm not sure I + * believe him. + * + * Note that this is a pretty bad scenario if it ever + * happens. The code I have added should allow us to + * handle the situation under all but the worst conditions, + * but one can argue that I should just scream and die if I + * ever detect the condidtion. + * *------------------------------------------------------------------------- */ herr_t @@ -3558,6 +3612,7 @@ H5C_flush_cache(H5F_t * f, int32_t protected_entries = 0; H5SL_node_t * node_ptr = NULL; H5C_cache_entry_t * entry_ptr = NULL; + H5C_cache_entry_t * next_entry_ptr = NULL; #if H5C_DO_SANITY_CHECKS int64_t flushed_entries_count; size_t flushed_entries_size; @@ -3620,6 +3675,28 @@ H5C_flush_cache(H5F_t * f, { flushed_entries_last_pass = FALSE; node_ptr = H5SL_first(cache_ptr->slist_ptr); + + if ( node_ptr != NULL ) { + + next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + + if ( next_entry_ptr == NULL ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "next_entry_ptr == NULL 1 ?!?!"); + } +#ifndef NDEBUG + HDassert( next_entry_ptr->magic == + H5C__H5C_CACHE_ENTRY_T_MAGIC ); +#endif /* NDEBUG */ + HDassert( next_entry_ptr->is_dirty ); + HDassert( next_entry_ptr->in_slist ); + + } else { + + next_entry_ptr = NULL; + + } HDassert( node_ptr != NULL ); @@ -3667,13 +3744,76 @@ H5C_flush_cache(H5F_t * f, while ( node_ptr != NULL ) { - entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + entry_ptr = next_entry_ptr; + + /* With the advent of the fractal heap, it is possible + * that the flush callback will dirty and/or resize + * other entries in the cache. In particular, while + * Quincey has promised me that this will never happen, + * it is possible that the flush callback for an + * entry may protect an entry that is not in the cache, + * perhaps causing the cache to flush and possibly + * evict the entry associated with node_ptr to make + * space for the new entry. + * + * Thus we do a bit of extra sanity checking on entry_ptr, + * and break out of this scan of the skip list if we + * detect minor problems. We have a bit of leaway on the + * number of passes though the skip list, so this shouldn't + * be an issue in the flush in and of itself, as it should + * be all but impossible for this to happen more than once + * in any flush. + * + * Observe that that breaking out of the scan early + * shouldn't break the sanity checks just after the end + * of this while loop. + * + * If an entry has merely been marked clean and removed from + * the s-list, we simply break out of the scan. + * + * If the entry has been evicted, we flag an error and + * exit. + */ +#ifndef NDEBUG + if ( entry_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "entry_ptr->magic invalid ?!?!"); + + } else +#endif /* NDEBUG */ + if ( ( ! entry_ptr->is_dirty ) || + ( ! entry_ptr->in_slist ) ) { + + /* the s-list has been modified out from under us. + * set node_ptr to NULL and break out of the loop. + */ + node_ptr = NULL; + break; + } /* increment node pointer now, before we delete its target * from the slist. */ node_ptr = H5SL_next(node_ptr); + if ( node_ptr != NULL ) { + next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + + if ( next_entry_ptr == NULL ) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "next_entry_ptr == NULL 2 ?!?!"); + } +#ifndef NDEBUG + HDassert( next_entry_ptr->magic == + H5C__H5C_CACHE_ENTRY_T_MAGIC ); +#endif /* NDEBUG */ + HDassert( next_entry_ptr->is_dirty ); + HDassert( next_entry_ptr->in_slist ); + } else { + next_entry_ptr = NULL; + } + HDassert( entry_ptr != NULL ); HDassert( entry_ptr->in_slist ); @@ -3689,6 +3829,40 @@ H5C_flush_cache(H5F_t * f, tried_to_flush_protected_entry = TRUE; protected_entries++; + } else if ( entry_ptr->is_pinned ) { + /* Test to see if we are can flush the entry now. + * If we can, go ahead and flush. Note that we + * aren't trying to do a destroy here, so that + * is not an issue. + */ + if ( TRUE ) { /* When we get to multithreaded cache, + * we will need either locking code, + * and/or a test to see if the entry + * is in flushable condition here. + */ +#if H5C_DO_SANITY_CHECKS + flushed_entries_count++; + flushed_entries_size += entry_ptr->size; +#endif /* H5C_DO_SANITY_CHECKS */ + status = H5C_flush_single_entry(f, + primary_dxpl_id, + secondary_dxpl_id, + cache_ptr, + NULL, + entry_ptr->addr, + flags, + &first_flush, + FALSE); + if ( status < 0 ) { + + /* This shouldn't happen -- if it does, we are + * toast so just scream and die. + */ + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ + "dirty pinned entry flush failed.") + } + flushed_entries_last_pass = TRUE; + } } else { #if H5C_DO_SANITY_CHECKS flushed_entries_count++; @@ -4434,6 +4608,9 @@ done: * Added code to disable evictions when the new * evictions_enabled field is FALSE. * + * JRM -- 12/31/07 + * Added code supporting flash cache size increases. + * *------------------------------------------------------------------------- */ @@ -4447,6 +4624,7 @@ H5C_insert_entry(H5F_t * f, void * thing, unsigned int flags) { + /* const char * fcn_name = "H5C_insert_entry()"; */ herr_t result; herr_t ret_value = SUCCEED; /* Return value */ hbool_t first_flush = TRUE; @@ -4486,7 +4664,9 @@ H5C_insert_entry(H5F_t * f, insert_pinned = ( (flags & H5C__PIN_ENTRY_FLAG) != 0 ); entry_ptr = (H5C_cache_entry_t *)thing; - +#ifndef NDEBUG + entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC; +#endif /* NDEBUG */ entry_ptr->addr = addr; entry_ptr->type = type; @@ -4524,6 +4704,18 @@ H5C_insert_entry(H5F_t * f, H5C__RESET_CACHE_ENTRY_STATS(entry_ptr) + if ( ( cache_ptr->flash_size_increase_possible ) && + ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) { + + result = H5C__flash_increase_cache_size(cache_ptr, 0, entry_ptr->size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, \ + "H5C__flash_increase_cache_size failed.") + } + } + if ( ( cache_ptr->evictions_enabled ) && ( (cache_ptr->index_size + entry_ptr->size) > cache_ptr->max_cache_size ) ) { @@ -4999,7 +5191,9 @@ done: * * Modifications: * - * None + * Added code to do a flash cache size increase if + * appropriate. + * JRM -- 1/11/08 * *------------------------------------------------------------------------- */ @@ -5010,6 +5204,8 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr, size_t new_size) { herr_t ret_value = SUCCEED; /* Return value */ + herr_t result; + size_t size_increase; H5C_cache_entry_t * entry_ptr; FUNC_ENTER_NOAPI(H5C_mark_pinned_entry_dirty, FAIL) @@ -5039,6 +5235,29 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr, /* update for change in entry size if necessary */ if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) { + /* do a flash cache size increase if appropriate */ + if ( cache_ptr->flash_size_increase_possible ) { + + if ( new_size > entry_ptr->size ) { + + size_increase = new_size - entry_ptr->size; + + if ( size_increase >= + cache_ptr->flash_size_increase_threshold ) { + + result = H5C__flash_increase_cache_size(cache_ptr, + entry_ptr->size, + new_size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ + "H5C__flash_increase_cache_size failed.") + } + } + } + } + /* update the protected entry list */ H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \ (cache_ptr->pel_size), \ @@ -5371,7 +5590,9 @@ done: * * Modifications: * - * None + * Added code to apply a flash cache size increment if + * appropriate. + * JRM -- 1/11/08 * *------------------------------------------------------------------------- */ @@ -5380,8 +5601,11 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr, void * thing, size_t new_size) { + /* const char * fcn_name = "H5C_resize_pinned_entry()"; */ herr_t ret_value = SUCCEED; /* Return value */ + herr_t result; H5C_cache_entry_t * entry_ptr; + size_t size_increase; FUNC_ENTER_NOAPI(H5C_resize_pinned_entry, FAIL) @@ -5417,6 +5641,29 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr, /* update for change in entry size if necessary */ if ( entry_ptr->size != new_size ) { + /* do a flash cache size increase if appropriate */ + if ( cache_ptr->flash_size_increase_possible ) { + + if ( new_size > entry_ptr->size ) { + + size_increase = new_size - entry_ptr->size; + + if ( size_increase >= + cache_ptr->flash_size_increase_threshold ) { + + result = H5C__flash_increase_cache_size(cache_ptr, + entry_ptr->size, + new_size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ + "H5C__flash_increase_cache_size failed.") + } + } + } + } + /* update the protected entry list */ H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \ (cache_ptr->pel_size), \ @@ -5604,9 +5851,13 @@ done: * of cache entries. * * JRM -- 7/27/07 - * Added code supporting the new evictions_enabled fieled + * Added code supporting the new evictions_enabled field * in H5C_t. * + * JRM -- 1/3/08 + * Added to do a flash cache size increase if appropriate + * when a large entry is loaded. + * *------------------------------------------------------------------------- */ @@ -5621,6 +5872,7 @@ H5C_protect(H5F_t * f, void * udata2, unsigned flags) { + /* const char * fcn_name = "H5C_protect()"; */ hbool_t hit; hbool_t first_flush; hbool_t have_write_permitted = FALSE; @@ -5678,7 +5930,25 @@ H5C_protect(H5F_t * f, entry_ptr = (H5C_cache_entry_t *)thing; - /* try to free up some space if necessary and if evictions are permitted */ + /* If the entry is very large, and we are configured to allow it, + * we may wish to perform a flash cache size increase. + */ + if ( ( cache_ptr->flash_size_increase_possible ) && + ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) { + + result = H5C__flash_increase_cache_size(cache_ptr, 0, + entry_ptr->size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, \ + "H5C__flash_increase_cache_size failed.") + } + } + + /* try to free up some space if necessary and if evictions are + * permitted + */ if ( ( cache_ptr->evictions_enabled ) && ( (cache_ptr->index_size + entry_ptr->size) > cache_ptr->max_cache_size ) ) { @@ -5978,6 +6248,10 @@ done: * if the new configuration forces an immediate reduction * in cache size. * + * JRM -- 12/31/07 + * Added code supporting the new flash cache size increase + * code. + * *------------------------------------------------------------------------- */ @@ -5985,6 +6259,7 @@ herr_t H5C_set_cache_auto_resize_config(H5C_t * cache_ptr, H5C_auto_size_ctl_t *config_ptr) { + /* const char *fcn_name = "H5C_set_cache_auto_resize_config()"; */ herr_t ret_value = SUCCEED; /* Return value */ herr_t result; size_t new_max_cache_size; @@ -6039,8 +6314,10 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr, "conflicting threshold fields in new config.") } - cache_ptr->size_increase_possible = TRUE; /* will set to FALSE if needed */ - cache_ptr->size_decrease_possible = TRUE; /* will set to FALSE if needed */ + /* will set the increase possible fields to FALSE later if needed */ + cache_ptr->size_increase_possible = TRUE; + cache_ptr->flash_size_increase_possible = TRUE; + cache_ptr->size_decrease_possible = TRUE; switch ( config_ptr->incr_mode ) { @@ -6062,6 +6339,11 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown incr_mode?!?!?.") } + /* logically, this is were configuration for flash cache size increases + * should go. However, this configuration depends on max_cache_size, so + * we wait until the end of the function, when this field is set. + */ + switch ( config_ptr->decr_mode ) { case H5C_decr__off: @@ -6106,9 +6388,13 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr, if ( config_ptr->max_size == config_ptr->min_size ) { cache_ptr->size_increase_possible = FALSE; + cache_ptr->flash_size_increase_possible = FALSE; cache_ptr->size_decrease_possible = FALSE; } + /* flash_size_increase_possible is intentionally omitted from the + * following: + */ cache_ptr->resize_enabled = cache_ptr->size_increase_possible || cache_ptr->size_decrease_possible; @@ -6196,6 +6482,37 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr, } } + /* configure flash size increase facility. We wait until the + * end of the function, as we need the max_cache_size set before + * we start to keep things simple. + * + * If we haven't already ruled out flash cache size increases above, + * go ahead and configure it. + */ + + if ( cache_ptr->flash_size_increase_possible ) { + + switch ( config_ptr->flash_incr_mode ) + { + case H5C_flash_incr__off: + cache_ptr->flash_size_increase_possible = FALSE; + break; + + case H5C_flash_incr__add_space: + cache_ptr->flash_size_increase_possible = TRUE; + cache_ptr->flash_size_increase_threshold = + (size_t) + (((double)(cache_ptr->max_cache_size)) * + ((cache_ptr->resize_ctl).flash_threshold)); + break; + + default: /* should be unreachable */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Unknown flash_incr_mode?!?!?.") + break; + } + } + done: FUNC_LEAVE_NOAPI(ret_value) @@ -7095,6 +7412,9 @@ done: * Also added sanity checks using the new is_read_only and * ro_ref_count parameters. * + * JRM -- 12/31/07 + * Modified funtion to support flash cache resizes. + * *------------------------------------------------------------------------- */ herr_t @@ -7108,6 +7428,7 @@ H5C_unprotect(H5F_t * f, unsigned int flags, size_t new_size) { + /* const char * fcn_name = "H5C_unprotect()"; */ hbool_t deleted; hbool_t dirtied; hbool_t set_flush_marker; @@ -7118,6 +7439,8 @@ H5C_unprotect(H5F_t * f, hbool_t clear_entry = FALSE; #endif /* H5_HAVE_PARALLEL */ herr_t ret_value = SUCCEED; /* Return value */ + herr_t result; + size_t size_increase = 0; H5C_cache_entry_t * entry_ptr; H5C_cache_entry_t * test_entry_ptr; @@ -7265,6 +7588,29 @@ H5C_unprotect(H5F_t * f, /* update for change in entry size if necessary */ if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) { + /* do a flash cache size increase if appropriate */ + if ( cache_ptr->flash_size_increase_possible ) { + + if ( new_size > entry_ptr->size ) { + + size_increase = new_size - entry_ptr->size; + + if ( size_increase >= + cache_ptr->flash_size_increase_threshold ) { + + result = H5C__flash_increase_cache_size(cache_ptr, + entry_ptr->size, + new_size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ + "H5C__flash_increase_cache_size failed.") + } + } + } + } + /* update the protected list */ H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pl_len), \ (cache_ptr->pl_size), \ @@ -7458,7 +7804,9 @@ done: * * Modifications: * - * None. + * Added validation for the flash increment fields. + * + * JRM -- 12/31/07 * *------------------------------------------------------------------------- */ @@ -7560,7 +7908,7 @@ H5C_validate_resize_config(H5C_auto_size_ctl_t * config_ptr, ( config_ptr->apply_max_increment != FALSE ) ) { HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \ - "apply_max_increment must be either TRUE or FALSE"); + "apply_max_increment must be either TRUE or FALSE"); } /* no need to check max_increment, as it is a size_t, @@ -7568,6 +7916,33 @@ H5C_validate_resize_config(H5C_auto_size_ctl_t * config_ptr, */ } /* H5C_incr__threshold */ + switch ( config_ptr->flash_incr_mode ) + { + case H5C_flash_incr__off: + /* nothing to do here */ + break; + + case H5C_flash_incr__add_space: + if ( ( config_ptr->flash_multiple < 0.1 ) || + ( config_ptr->flash_multiple > 10.0 ) ) { + + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \ + "flash_multiple must be in the range [0.1, 10.0]"); + } + + if ( ( config_ptr->flash_threshold < 0.1 ) || + ( config_ptr->flash_threshold > 1.0 ) ) { + + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \ + "flash_threshold must be in the range [0.1, 1.0]"); + } + break; + + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \ + "Invalid flash_incr_mode"); + break; + } } /* H5C_RESIZE_CFG__VALIDATE_INCREMENT */ @@ -7707,6 +8082,9 @@ done: * reduction, and to adjust to changes in the * H5C_auto_size_ctl_t structure. * + * JRM -- 1/5/08 + * Added support for flash cache size increases. + * *------------------------------------------------------------------------- */ @@ -7990,6 +8368,30 @@ H5C__auto_adjust_cache_size(H5C_t * cache_ptr, cache_ptr->size_decreased = TRUE; } + + /* update flash cache size increase fields as appropriate */ + if ( cache_ptr->flash_size_increase_possible ) { + + switch ( (cache_ptr->resize_ctl).flash_incr_mode ) + { + case H5C_flash_incr__off: + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "flash_size_increase_possible but H5C_flash_incr__off?!") + break; + + case H5C_flash_incr__add_space: + cache_ptr->flash_size_increase_threshold = + (size_t) + (((double)(cache_ptr->max_cache_size)) * + ((cache_ptr->resize_ctl).flash_threshold)); + break; + + default: /* should be unreachable */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Unknown flash_incr_mode?!?!?.") + break; + } + } } if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) { @@ -8308,7 +8710,20 @@ done: * * Modifications: * - * None. + * JRM -- 10/13/07 + * Added code to detect and manage the case in which a + * flush callback changes the LRU-list out from under + * the function. The only way I can think of in which this + * can happen is if a flush function loads an entry + * into the cache that isn't there already. Quincey tells + * me that this will never happen, but I'm not sure I + * believe him. + * + * Note that this is a pretty bad scenario if it ever + * happens. The code I have added should allow us to + * handle the situation under all but the worst conditions, + * but one can argue that I should just scream and die if I + * ever detect the condidtion. * *------------------------------------------------------------------------- */ @@ -8325,7 +8740,9 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, herr_t result; size_t eviction_size_limit; size_t bytes_evicted = 0; + hbool_t prev_is_dirty = FALSE; H5C_cache_entry_t * entry_ptr; + H5C_cache_entry_t * next_ptr; H5C_cache_entry_t * prev_ptr; FUNC_ENTER_NOAPI_NOINIT(H5C__autoadjust__ageout__evict_aged_out_entries) @@ -8358,8 +8775,14 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, { HDassert( ! (entry_ptr->is_protected) ); + next_ptr = entry_ptr->next; prev_ptr = entry_ptr->prev; + if ( prev_ptr != NULL ) { + + prev_is_dirty = prev_ptr->is_dirty; + } + if ( entry_ptr->is_dirty ) { result = H5C_flush_single_entry(f, @@ -8392,8 +8815,41 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, "unable to flush entry") } - entry_ptr = prev_ptr; + if ( prev_ptr != NULL ) { +#ifndef NDEBUG + if ( prev_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) { + + /* something horrible has happened to *prev_ptr -- + * scream and die. + */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "*prev_ptr corrupt") + + } else +#endif /* NDEBUG */ + if ( ( prev_ptr->is_dirty != prev_is_dirty ) + || + ( prev_ptr->next != next_ptr ) + || + ( prev_ptr->is_protected ) + || + ( prev_ptr->is_pinned ) ) { + + /* something has happened to the LRU -- start over + * from the tail. + */ + entry_ptr = cache_ptr->LRU_tail_ptr; + + } else { + + entry_ptr = prev_ptr; + } + } else { + + entry_ptr = NULL; + + } } /* end while */ /* for now at least, don't bother to maintain the minimum clean size, @@ -8741,6 +9197,173 @@ done: } /* H5C__autoadjust__ageout__remove_excess_markers() */ +/*------------------------------------------------------------------------- + * + * Function: H5C__flash_increase_cache_size + * + * Purpose: If there is not at least new_entry_size - old_entry_size + * bytes of free space in the cache and the current + * max_cache_size is less than (cache_ptr->resize_ctl).max_size, + * perform a flash increase in the cache size and then reset + * the full cache hit rate statistics, and exit. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 12/31/07 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static herr_t +H5C__flash_increase_cache_size(H5C_t * cache_ptr, + size_t old_entry_size, + size_t new_entry_size) +{ + /* const char * fcn_name = "H5C__flash_increase_cache_size()";*/ + herr_t ret_value = SUCCEED; /* Return value */ + size_t new_max_cache_size = 0; + size_t old_max_cache_size = 0; + size_t new_min_clean_size = 0; + size_t old_min_clean_size = 0; + size_t space_needed; + enum H5C_resize_status status = flash_increase; /* may change */ + double hit_rate; + + FUNC_ENTER_NOAPI_NOINIT(H5C__flash_increase_cache_size) + HDassert( cache_ptr ); + HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); + HDassert( cache_ptr->flash_size_increase_possible ); + HDassert( new_entry_size > cache_ptr->flash_size_increase_threshold ); + HDassert( old_entry_size < new_entry_size ); + + if ( old_entry_size >= new_entry_size ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "old_entry_size >= new_entry_size") + } + + space_needed = new_entry_size - old_entry_size; + + if ( ( (cache_ptr->index_size + space_needed) > + cache_ptr->max_cache_size ) && + ( cache_ptr->max_cache_size < (cache_ptr->resize_ctl).max_size ) ) { + + /* we have work to do */ + + switch ( (cache_ptr->resize_ctl).flash_incr_mode ) + { + case H5C_flash_incr__off: + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "flash_size_increase_possible but H5C_flash_incr__off?!") + break; + + case H5C_flash_incr__add_space: + if ( cache_ptr->index_size < cache_ptr->max_cache_size ) { + + HDassert( (cache_ptr->max_cache_size - cache_ptr->index_size) + < space_needed ); + space_needed -= cache_ptr->max_cache_size - cache_ptr->index_size; + } + space_needed = + (size_t)(((double)space_needed) * + (cache_ptr->resize_ctl).flash_multiple); + + new_max_cache_size = cache_ptr->max_cache_size + space_needed; + + break; + + default: /* should be unreachable */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Unknown flash_incr_mode?!?!?.") + break; + } + + if ( new_max_cache_size > (cache_ptr->resize_ctl).max_size ) { + + new_max_cache_size = (cache_ptr->resize_ctl).max_size; + } + + HDassert( new_max_cache_size > cache_ptr->max_cache_size ); + + new_min_clean_size = (size_t) + ((double)new_max_cache_size * + ((cache_ptr->resize_ctl).min_clean_fraction)); + + HDassert( new_min_clean_size <= new_max_cache_size ); + + old_max_cache_size = cache_ptr->max_cache_size; + old_min_clean_size = cache_ptr->min_clean_size; + + cache_ptr->max_cache_size = new_max_cache_size; + cache_ptr->min_clean_size = new_min_clean_size; + + /* update flash cache size increase fields as appropriate */ + HDassert ( cache_ptr->flash_size_increase_possible ); + + switch ( (cache_ptr->resize_ctl).flash_incr_mode ) + { + case H5C_flash_incr__off: + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "flash_size_increase_possible but H5C_flash_incr__off?!") + break; + + case H5C_flash_incr__add_space: + cache_ptr->flash_size_increase_threshold = + (size_t) + (((double)(cache_ptr->max_cache_size)) * + ((cache_ptr->resize_ctl).flash_threshold)); + break; + + default: /* should be unreachable */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Unknown flash_incr_mode?!?!?.") + break; + } + + /* note that we don't cycle the epoch markers. We can + * argue either way as to whether we should, but for now + * we don't. + */ + + if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) { + + /* get the hit rate for the reporting function. Should still + * be good as we havent reset the hit rate statistics. + */ + if ( H5C_get_cache_hit_rate(cache_ptr, &hit_rate) != SUCCEED ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't get hit rate.") + } + + (*((cache_ptr->resize_ctl).rpt_fcn)) + (cache_ptr, + H5C__CURR_AUTO_RESIZE_RPT_FCN_VER, + hit_rate, + status, + old_max_cache_size, + new_max_cache_size, + old_min_clean_size, + new_min_clean_size); + } + + if ( H5C_reset_cache_hit_rate_stats(cache_ptr) != SUCCEED ) { + + /* this should be impossible... */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "H5C_reset_cache_hit_rate_stats failed.") + } + } + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5C__flash_increase_cache_size() */ + /*------------------------------------------------------------------------- * Function: H5C_flush_invalidate_cache @@ -8779,12 +9402,28 @@ done: * * Modifications: * - * To support the fractal heap, the cache must now deal with - * entries being dirtied, resized, and/or renamed inside - * flush callbacks. Updated function to support this. + * To support the fractal heap, the cache must now deal with + * entries being dirtied, resized, and/or renamed inside + * flush callbacks. Updated function to support this. * * -- JRM 8/27/06 * + * Added code to detect and manage the case in which a + * flush callback changes the s-list out from under + * the function. The only way I can think of in which this + * can happen is if a flush function loads an entry + * into the cache that isn't there already. Quincey tells + * me that this will never happen, but I'm not sure I + * believe him. + * + * Note that this is a pretty bad scenario if it ever + * happens. The code I have added should allow us to + * handle the situation under all but the worst conditions, + * but one can argue that I should just scream and die if I + * ever detect the condidtion. + * + * -- JRM 10/13/07 + * *------------------------------------------------------------------------- */ herr_t @@ -8895,6 +9534,23 @@ H5C_flush_invalidate_cache(H5F_t * f, node_ptr = H5SL_first(cache_ptr->slist_ptr); + if ( node_ptr == NULL ) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "slist_len != 0 && node_ptr == NULL"); + } + + next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + + if ( next_entry_ptr == NULL ) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "next_entry_ptr == NULL 1 ?!?!"); + } +#ifndef NDEBUG + HDassert( next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC ); +#endif /* NDEBUG */ + HDassert( next_entry_ptr->is_dirty ); + HDassert( next_entry_ptr->in_slist ); + } #if H5C_DO_SANITY_CHECKS /* Depending on circumstances, H5C_flush_single_entry() will @@ -8927,23 +9583,88 @@ H5C_flush_invalidate_cache(H5F_t * f, while ( node_ptr != NULL ) { - /* Note that we now remove nodes from the slist as we flush - * the associated entries, instead of leaving them there - * until we are done, and then destroying all nodes in - * the slist. + entry_ptr = next_entry_ptr; + + /* With the advent of the fractal heap, it is possible + * that the flush callback will dirty and/or resize + * other entries in the cache. In particular, while + * Quincey has promised me that this will never happen, + * it is possible that the flush callback for an + * entry may protect an entry that is not in the cache, + * perhaps causing the cache to flush and possibly + * evict the entry associated with node_ptr to make + * space for the new entry. * - * While this optimization used to be easy, with the possibility - * of new entries being added to the slist in the midst of the - * flush, we must keep the slist in cannonical form at all - * times. + * Thus we do a bit of extra sanity checking on entry_ptr, + * and break out of this scan of the skip list if we + * detect major problems. We have a bit of leaway on the + * number of passes though the skip list, so this shouldn't + * be an issue in the flush in and of itself, as it should + * be all but impossible for this to happen more than once + * in any flush. + * + * Observe that that breaking out of the scan early + * shouldn't break the sanity checks just after the end + * of this while loop. + * + * If an entry has merely been marked clean and removed from + * the s-list, we simply break out of the scan. + * + * If the entry has been evicted, we flag an error and + * exit. */ +#ifndef NDEBUG + if ( entry_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "entry_ptr->magic is invalid ?!?!"); + + } else +#endif /* NDEBUG */ + if ( ( ! entry_ptr->is_dirty ) || + ( ! entry_ptr->in_slist ) ) { - entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + /* the s-list has been modified out from under us. + * break out of the loop. + */ + break; + } /* increment node pointer now, before we delete its target * from the slist. */ + node_ptr = H5SL_next(node_ptr); + if ( node_ptr != NULL ) { + + next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + + if ( next_entry_ptr == NULL ) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "next_entry_ptr == NULL 2 ?!?!"); + } +#ifndef NDEBUG + HDassert( next_entry_ptr->magic == + H5C__H5C_CACHE_ENTRY_T_MAGIC ); +#endif /* NDEBUG */ + HDassert( next_entry_ptr->is_dirty ); + HDassert( next_entry_ptr->in_slist ); + + } else { + + next_entry_ptr = NULL; + } + + /* Note that we now remove nodes from the slist as we flush + * the associated entries, instead of leaving them there + * until we are done, and then destroying all nodes in + * the slist. + * + * While this optimization used to be easy, with the possibility + * of new entries being added to the slist in the midst of the + * flush, we must keep the slist in cannonical form at all + * times. + */ HDassert( entry_ptr != NULL ); HDassert( entry_ptr->in_slist ); @@ -9030,12 +9751,19 @@ H5C_flush_invalidate_cache(H5F_t * f, /* It is possible that entries were added to the slist during * the scan, either before or after scan pointer. The following * asserts take this into account. - */ + * + * Don't bother with the sanity checks if node_ptr != NULL, as + * in this case we broke out of the loop because it got changed + * out from under us. + */ + + if ( node_ptr == NULL ) { - HDassert( (actual_slist_len + cache_ptr->slist_len) == - (initial_slist_len + cache_ptr->slist_len_increase) ); - HDassert( (actual_slist_size + cache_ptr->slist_size) == - (initial_slist_size + cache_ptr->slist_size_increase) ); + HDassert( (actual_slist_len + cache_ptr->slist_len) == + (initial_slist_len + cache_ptr->slist_len_increase) ); + HDassert( (actual_slist_size + cache_ptr->slist_size) == + (initial_slist_size + cache_ptr->slist_size_increase) ); + } #endif /* H5C_DO_SANITY_CHECKS */ /* Since we are doing a destroy, we must make a pass through @@ -9056,8 +9784,13 @@ H5C_flush_invalidate_cache(H5F_t * f, while ( next_entry_ptr != NULL ) { entry_ptr = next_entry_ptr; - next_entry_ptr = entry_ptr->ht_next; + next_entry_ptr = entry_ptr->ht_next; +#ifndef NDEBUG + HDassert ( ( next_entry_ptr == NULL ) || + ( next_entry_ptr->magic == + H5C__H5C_CACHE_ENTRY_T_MAGIC ) ); +#endif /* NDEBUG */ if ( entry_ptr->is_protected ) { /* we have major problems -- but lets flush and destroy @@ -9099,6 +9832,28 @@ H5C_flush_invalidate_cache(H5F_t * f, * of pinned entries from pass to pass. If it stops * shrinking before it hits zero, we scream and die. */ + /* if the flush function on the entry we last evicted + * loaded an entry into cache (as Quincey has promised me + * it never will), and if the cache was full, it is + * possible that *next_entry_ptr was flushed or evicted. + * + * Test to see if this happened here. Note that if this + * test is triggred, we are accessing a deallocated piece + * of dynamically allocated memory, so we just scream and + * die. + */ +#ifndef NDEBUG + if ( ( next_entry_ptr != NULL ) && + ( next_entry_ptr->magic != + H5C__H5C_CACHE_ENTRY_T_MAGIC ) ) { + + /* Something horrible has happened to + * *next_entry_ptr -- scream and die. + */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "next_entry_ptr->magic is invalid?!?!?.") + } +#endif /* NDEBUG */ } /* end while loop scanning hash table bin */ } /* end for loop scanning hash table */ @@ -9541,6 +10296,17 @@ H5C_flush_single_entry(H5F_t * f, /* Clear the dirty flag only, if requested */ if ( clear_only ) { +#ifndef NDEBUG + if ( destroy ) { + /* we are about to call the clear callback with the + * destroy flag set -- this will result in *entry_ptr + * being freed. Set the magic field to bad magic + * so we can detect a freed cache entry if we see + * one. + */ + entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; + } +#endif /* NDEBUG */ /* Call the callback routine to clear all dirty flags for object */ if ( (entry_ptr->type->clear)(f, entry_ptr, destroy) < 0 ) { @@ -9558,6 +10324,18 @@ H5C_flush_single_entry(H5F_t * f, } #endif /* H5C_DO_SANITY_CHECKS */ +#ifndef NDEBUG + if ( destroy ) { + /* we are about to call the flush callback with the + * destroy flag set -- this will result in *entry_ptr + * being freed. Set the magic field to bad magic + * so we can detect a freed cache entry if we see + * one. + */ + entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; + } +#endif /* NDEBUG */ + /* Only block for all the processes on the first piece of metadata */ @@ -9816,7 +10594,9 @@ H5C_load_entry(H5F_t * f, */ HDassert( ( entry_ptr->is_dirty == FALSE ) || ( type->id == 4 ) ); - +#ifndef NDEBUG + entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC; +#endif /* NDEBUG */ entry_ptr->addr = addr; entry_ptr->type = type; entry_ptr->is_protected = FALSE; @@ -9914,6 +10694,21 @@ done: * Added sanity checks using the new is_read_only and * ro_ref_count fields. * + * JRM -- 10/13/07 + * Added code to detect and manage the case in which a + * flush callback changes the LRU-list out from under + * the function. The only way I can think of in which this + * can happen is if a flush function loads an entry + * into the cache that isn't there already. Quincey tells + * me that this will never happen, but I'm not sure I + * believe him. + * + * Note that this is a pretty bad scenario if it ever + * happens. The code I have added should allow us to + * handle the situation under all but the worst conditions, + * but one can argue that I should just scream and die if I + * ever detect the condidtion. + * *------------------------------------------------------------------------- */ @@ -9933,7 +10728,10 @@ H5C_make_space_in_cache(H5F_t * f, #if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS size_t empty_space; #endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ + hbool_t prev_is_dirty = FALSE; + hbool_t entry_is_epoch_maker = FALSE; H5C_cache_entry_t * entry_ptr; + H5C_cache_entry_t * next_ptr; H5C_cache_entry_t * prev_ptr; FUNC_ENTER_NOAPI_NOINIT(H5C_make_space_in_cache) @@ -9962,10 +10760,18 @@ H5C_make_space_in_cache(H5F_t * f, HDassert( ! (entry_ptr->is_read_only) ); HDassert( (entry_ptr->ro_ref_count) == 0 ); - prev_ptr = entry_ptr->prev; + next_ptr = entry_ptr->next; + prev_ptr = entry_ptr->prev; + + if ( prev_ptr != NULL ) { + + prev_is_dirty = prev_ptr->is_dirty; + } if ( (entry_ptr->type)->id != H5C__EPOCH_MARKER_TYPE ) { + entry_is_epoch_maker = FALSE; + if ( entry_ptr->is_dirty ) { result = H5C_flush_single_entry(f, @@ -9994,6 +10800,7 @@ H5C_make_space_in_cache(H5F_t * f, /* Skip epoch markers. Set result to SUCCEED to avoid * triggering the error code below. */ + entry_is_epoch_maker = TRUE; result = SUCCEED; } @@ -10003,11 +10810,54 @@ H5C_make_space_in_cache(H5F_t * f, "unable to flush entry") } - entry_ptr = prev_ptr; - } + if ( prev_ptr != NULL ) { +#ifndef NDEBUG + if ( prev_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) { + + /* something horrible has happened to *prev_ptr -- + * scream and die. + */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "*prev_ptr corrupt 1") + + } +#endif /* NDEBUG */ + if ( entry_is_epoch_maker ) { + + entry_ptr = prev_ptr; + + } else if ( ( prev_ptr->is_dirty != prev_is_dirty ) + || + ( prev_ptr->next != next_ptr ) + || + ( prev_ptr->is_protected ) + || + ( prev_ptr->is_pinned ) ) { + + /* something has happened to the LRU -- start over + * from the tail. + */ + + entry_ptr = cache_ptr->LRU_tail_ptr; + + } else { + + entry_ptr = prev_ptr; + + } + } else { + + entry_ptr = NULL; + + } + + entries_examined++; + + } #if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS + entries_examined = 0; initial_list_len = cache_ptr->dLRU_list_len; entry_ptr = cache_ptr->dLRU_tail_ptr; @@ -10034,6 +10884,13 @@ H5C_make_space_in_cache(H5F_t * f, prev_ptr = entry_ptr->aux_prev; + next_ptr = entry_ptr->aux_next; + + if ( prev_ptr != NULL ) { + + HDassert( prev_ptr->is_dirty ); + } + result = H5C_flush_single_entry(f, primary_dxpl_id, secondary_dxpl_id, @@ -10050,7 +10907,66 @@ H5C_make_space_in_cache(H5F_t * f, "unable to flush entry") } - entry_ptr = prev_ptr; + if ( prev_ptr != NULL ) { +#ifndef NDEBUG + if (prev_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC) { + + /* something horrible has happened to *prev_ptr -- + * scream and die. + */ + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "*prev_ptr corrupt 2") + + } else +#endif /* #ifndef NDEBUG */ + if ( ( ! ( prev_ptr->is_dirty ) ) + || + ( prev_ptr->aux_next != next_ptr ) + || + ( prev_ptr->is_protected ) + || + ( prev_ptr->is_pinned ) ) { + + /* something has happened to the dirty LRU -- start over + * from the tail. + */ + +#if 0 /* This debuging code may be useful in the future -- keep it for now. */ + if ( ! ( prev_ptr->is_dirty ) ) { + HDfprintf(stdout, "%s: ! prev_ptr->is_dirty\n", + fcn_name); + } + if ( prev_ptr->aux_next != next_ptr ) { + HDfprintf(stdout, "%s: prev_ptr->next != next_ptr\n", + fcn_name); + } + if ( prev_ptr->is_protected ) { + HDfprintf(stdout, "%s: prev_ptr->is_protected\n", + fcn_name); + } + if ( prev_ptr->is_pinned ) { + HDfprintf(stdout, "%s:prev_ptr->is_pinned\n", + fcn_name); + } + + HDfprintf(stdout, "%s: re-starting scan of dirty list\n", + fcn_name); +#endif /* JRM */ + entry_ptr = cache_ptr->dLRU_tail_ptr; + + } else { + + entry_ptr = prev_ptr; + + } + } else { + + entry_ptr = NULL; + + } + + entries_examined++; } #endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ @@ -10096,6 +11012,7 @@ H5C_make_space_in_cache(H5F_t * f, } entry_ptr = prev_ptr; + entries_examined++; } } @@ -2588,6 +2588,10 @@ static herr_t H5C2__autoadjust__ageout__remove_all_markers(H5C2_t * cache_ptr); static herr_t H5C2__autoadjust__ageout__remove_excess_markers(H5C2_t * cache_ptr); +static herr_t H5C2__flash_increase_cache_size(H5C2_t * cache_ptr, + size_t old_entry_size, + size_t new_entry_size); + static herr_t H5C2_flush_single_entry(H5F_t * f, hid_t dxpl_id, H5C2_t * cache_ptr, @@ -2850,6 +2854,11 @@ done: * Added initialization for the new is_read_only and * ro_ref_count fields. * + * JRM -- 12/31/07 + * Added initialization for the new flash cache size increase + * related fields of H5C2_t. + * + * *------------------------------------------------------------------------- */ @@ -2969,6 +2978,8 @@ H5C2_create(const H5F_t * f, cache_ptr->dLRU_tail_ptr = NULL; cache_ptr->size_increase_possible = FALSE; + cache_ptr->flash_size_increase_possible = FALSE; + cache_ptr->flash_size_increase_threshold = 0; cache_ptr->size_decrease_possible = FALSE; cache_ptr->resize_enabled = FALSE; cache_ptr->cache_full = FALSE; @@ -2989,6 +3000,10 @@ H5C2_create(const H5F_t * f, (cache_ptr->resize_ctl).apply_max_increment = TRUE; (cache_ptr->resize_ctl).max_increment = H5C2__DEF_AR_MAX_INCREMENT; + (cache_ptr->resize_ctl).flash_incr_mode = H5C2_flash_incr__off; + (cache_ptr->resize_ctl).flash_multiple = 1.0; + (cache_ptr->resize_ctl).flash_threshold = 0.25; + (cache_ptr->resize_ctl).decr_mode = H5C2_decr__off; (cache_ptr->resize_ctl).upper_hr_threshold = H5C2__DEF_AR_UPPER_THRESHHOLD; (cache_ptr->resize_ctl).decrement = H5C2__DEF_AR_DECREMENT; @@ -3008,9 +3023,10 @@ H5C2_create(const H5F_t * f, for ( i = 0; i < H5C2__MAX_EPOCH_MARKERS; i++ ) { (cache_ptr->epoch_marker_active)[i] = FALSE; - +#ifndef NDEBUG ((cache_ptr->epoch_markers)[i]).magic = H5C2__H5C2_CACHE_ENTRY_T_MAGIC; +#endif /* NDEBUG */ ((cache_ptr->epoch_markers)[i]).addr = (haddr_t)i; ((cache_ptr->epoch_markers)[i]).size = (size_t)0; ((cache_ptr->epoch_markers)[i]).type = &epoch_marker_class_2; @@ -3096,6 +3112,9 @@ done: * Updated function for display the new prefix field of * H5C2_t in output. * + * JRM 12/31/07 + * Updated function to handle flash size increases. + * *------------------------------------------------------------------------- */ void @@ -3142,6 +3161,25 @@ H5C2_def_auto_resize_rpt_fcn(H5C2_t * cache_ptr, new_min_clean_size); break; + case flash_increase2: + HDassert( old_max_cache_size < new_max_cache_size ); + + HDfprintf(stdout, + "%sflash cache resize(%d) -- size threshold = %Zu.\n", + cache_ptr->prefix, + (int)((cache_ptr->resize_ctl).flash_incr_mode), + cache_ptr->flash_size_increase_threshold); + + HDfprintf(stdout, + "%s cache size increased from (%Zu/%Zu) to (%Zu/%Zu).\n", + cache_ptr->prefix, + old_max_cache_size, + old_min_clean_size, + new_max_cache_size, + new_min_clean_size); + break; + + case decrease2: HDassert( old_max_cache_size > new_max_cache_size ); @@ -3571,6 +3609,21 @@ done: * handle the situation, but one can argue that I should * just scream and die if I ever detect the condidtion. * + * JRM -- 10/13/07 + * Added code to detect and manage the case in which a + * flush callback changes the s-list out from under + * the function. The only way I can think of in which this + * can happen is if a flush function loads an entry + * into the cache that isn't there already. Quincey tells + * me that this will never happen, but I'm not sure I + * believe him. + * + * Note that this is a pretty bad scenario if it ever + * happens. The code I have added should allow us to + * handle the situation under all but the worst conditions, + * but one can argue that I should just scream and die if I + * ever detect the condidtion. + * *------------------------------------------------------------------------- */ herr_t @@ -3651,19 +3704,27 @@ H5C2_flush_cache(H5C2_t * cache_ptr, { flushed_entries_last_pass = FALSE; node_ptr = H5SL_first(cache_ptr->slist_ptr); + if ( node_ptr != NULL ) { + next_entry_ptr = (H5C2_cache_entry_t *)H5SL_item(node_ptr); if ( next_entry_ptr == NULL ) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "next_entry_ptr == NULL 1 ?!?!"); } +#ifndef NDEBUG HDassert( next_entry_ptr->magic == H5C2__H5C2_CACHE_ENTRY_T_MAGIC ); +#endif /* NDEBUG */ HDassert( next_entry_ptr->is_dirty ); HDassert( next_entry_ptr->in_slist ); + } else { + next_entry_ptr = NULL; + } HDassert( node_ptr != NULL ); @@ -3742,14 +3803,16 @@ H5C2_flush_cache(H5C2_t * cache_ptr, * If the entry has been evicted, we flag an error and * exit. */ - +#ifndef NDEBUG if ( entry_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) { HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "entry_ptr->magic invalid ?!?!"); - } else if ( ( ! entry_ptr->is_dirty ) || - ( ! entry_ptr->in_slist ) ) { + } else +#endif /* NDEBUG */ + if ( ( ! entry_ptr->is_dirty ) || + ( ! entry_ptr->in_slist ) ) { /* the s-list has been modified out from under us. * set node_ptr to NULL and break out of the loop. @@ -3762,15 +3825,20 @@ H5C2_flush_cache(H5C2_t * cache_ptr, * from the slist. */ node_ptr = H5SL_next(node_ptr); + if ( node_ptr != NULL ) { + next_entry_ptr = (H5C2_cache_entry_t *)H5SL_item(node_ptr); if ( next_entry_ptr == NULL ) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "next_entry_ptr == NULL 2 ?!?!"); } +#ifndef NDEBUG HDassert( next_entry_ptr->magic == H5C2__H5C2_CACHE_ENTRY_T_MAGIC ); +#endif /* NDEBUG */ HDassert( next_entry_ptr->is_dirty ); HDassert( next_entry_ptr->in_slist ); } else { @@ -3823,6 +3891,7 @@ H5C2_flush_cache(H5C2_t * cache_ptr, HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ "dirty pinned entry flush failed.") } + flushed_entries_last_pass = TRUE; } } else { #if H5C2_DO_SANITY_CHECKS @@ -4580,6 +4649,9 @@ done: * JRM -- 10/12/07 * Added initialization for the new magic field. * + * JRM -- 12/31/07 + * Added code supporting flash cache size increases. + * *------------------------------------------------------------------------- */ @@ -4592,6 +4664,7 @@ H5C2_insert_entry(H5C2_t * cache_ptr, void * thing, unsigned int flags) { + /* const char * fcn_name = "H5C2_insert_entry()"; */ herr_t result; herr_t ret_value = SUCCEED; /* Return value */ hbool_t insert_pinned; @@ -4628,9 +4701,9 @@ H5C2_insert_entry(H5C2_t * cache_ptr, insert_pinned = ( (flags & H5C2__PIN_ENTRY_FLAG) != 0 ); entry_ptr = (H5C2_cache_entry_t *)thing; - +#ifndef NDEBUG entry_ptr->magic = H5C2__H5C2_CACHE_ENTRY_T_MAGIC; - +#endif /* NDEBUG */ entry_ptr->addr = addr; entry_ptr->type = type; @@ -4664,6 +4737,19 @@ H5C2_insert_entry(H5C2_t * cache_ptr, H5C2__RESET_CACHE_ENTRY_STATS(entry_ptr) + if ( ( cache_ptr->flash_size_increase_possible ) && + ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) { + + result = H5C2__flash_increase_cache_size(cache_ptr, 0, + entry_ptr->size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, \ + "H5C2__flash_increase_cache_size failed.") + } + } + if ( ( cache_ptr->evictions_enabled ) && ( (cache_ptr->index_size + entry_ptr->size) > cache_ptr->max_cache_size ) ) { @@ -5151,10 +5237,13 @@ done: * * Modifications: * - * None + * Added code to do a flash cache size increase if + * appropriate. + * JRM -- 1/11/08 * *------------------------------------------------------------------------- */ + herr_t H5C2_mark_pinned_entry_dirty(H5C2_t * cache_ptr, void * thing, @@ -5162,6 +5251,8 @@ H5C2_mark_pinned_entry_dirty(H5C2_t * cache_ptr, size_t new_size) { herr_t ret_value = SUCCEED; /* Return value */ + herr_t result; + size_t size_increase; H5C2_cache_entry_t * entry_ptr; FUNC_ENTER_NOAPI(H5C2_mark_pinned_entry_dirty, FAIL) @@ -5191,10 +5282,33 @@ H5C2_mark_pinned_entry_dirty(H5C2_t * cache_ptr, /* update for change in entry size if necessary */ if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) { + /* do a flash cache size increase if appropriate */ + if ( cache_ptr->flash_size_increase_possible ) { + + if ( new_size > entry_ptr->size ) { + + size_increase = new_size - entry_ptr->size; + + if ( size_increase >= + cache_ptr->flash_size_increase_threshold ) { + + result = H5C2__flash_increase_cache_size(cache_ptr, + entry_ptr->size, + new_size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ + "H5C2__flash_increase_cache_size failed.") + } + } + } + } + /* update the protected entry list */ H5C2__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \ - (cache_ptr->pel_size), \ - (entry_ptr->size), (new_size)); + (cache_ptr->pel_size), \ + (entry_ptr->size), (new_size)); /* update the hash table */ H5C2__UPDATE_INDEX_FOR_SIZE_CHANGE((cache_ptr), (entry_ptr->size),\ @@ -5528,7 +5642,9 @@ done: * * Modifications: * - * None + * Added code to apply a flash cache size increment if + * appropriate. + * JRM -- 1/11/08 * *------------------------------------------------------------------------- */ @@ -5537,8 +5653,11 @@ H5C2_resize_pinned_entry(H5C2_t * cache_ptr, void * thing, size_t new_size) { + /* const char * fcn_name = "H5C2_resize_pinned_entry()"; */ herr_t ret_value = SUCCEED; /* Return value */ + herr_t result; H5C2_cache_entry_t * entry_ptr; + size_t size_increase; FUNC_ENTER_NOAPI(H5C2_resize_pinned_entry, FAIL) @@ -5574,6 +5693,29 @@ H5C2_resize_pinned_entry(H5C2_t * cache_ptr, /* update for change in entry size if necessary */ if ( entry_ptr->size != new_size ) { + /* do a flash cache size increase if appropriate */ + if ( cache_ptr->flash_size_increase_possible ) { + + if ( new_size > entry_ptr->size ) { + + size_increase = new_size - entry_ptr->size; + + if ( size_increase >= + cache_ptr->flash_size_increase_threshold ) { + + result = H5C2__flash_increase_cache_size(cache_ptr, + entry_ptr->size, + new_size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ + "H5C2__flash_increase_cache_size failed.") + } + } + } + } + /* update the protected entry list */ H5C2__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \ (cache_ptr->pel_size), \ @@ -5761,7 +5903,7 @@ done: * of cache entries. * * JRM -- 7/27/07 - * Added code supporting the new evictions_enabled fieled + * Added code supporting the new evictions_enabled field * in H5C2_t. * * JRM -- 7/11/07 @@ -5771,6 +5913,10 @@ done: * fields. Gained the len parameter. Also internal * changes supporting the revised API. * + * JRM -- 1/3/08 + * Added to do a flash cache size increase if appropriate + * when a large entry is loaded. + * *------------------------------------------------------------------------- */ @@ -5848,6 +5994,22 @@ H5C2_protect(H5C2_t * cache_ptr, entry_ptr = (H5C2_cache_entry_t *)thing; + /* If the entry is very large, and we are configured to allow it, + * we may wish to perform a flash cache size increase. + */ + if ( ( cache_ptr->flash_size_increase_possible ) && + ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) { + + result = H5C2__flash_increase_cache_size(cache_ptr, 0, + entry_ptr->size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, \ + "H5C2__flash_increase_cache_size failed.") + } + } + /* try to free up some space if necessary and if * evictions are permitted */ @@ -6149,6 +6311,10 @@ done: * if the new configuration forces an immediate reduction * in cache size. * + * JRM -- 12/31/07 + * Added code supporting the new flash cache size increase + * code. + * *------------------------------------------------------------------------- */ @@ -6156,6 +6322,7 @@ herr_t H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr, H5C2_auto_size_ctl_t *config_ptr) { + /* const char *fcn_name = "H5C2_set_cache_auto_resize_config()"; */ herr_t ret_value = SUCCEED; /* Return value */ herr_t result; size_t new_max_cache_size; @@ -6210,8 +6377,10 @@ H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr, "conflicting threshold fields in new config.") } - cache_ptr->size_increase_possible = TRUE; /* will set to FALSE if needed */ - cache_ptr->size_decrease_possible = TRUE; /* will set to FALSE if needed */ + /* will set the increase possible fields to FALSE later if needed */ + cache_ptr->size_increase_possible = TRUE; + cache_ptr->flash_size_increase_possible = TRUE; + cache_ptr->size_decrease_possible = TRUE; switch ( config_ptr->incr_mode ) { @@ -6233,6 +6402,11 @@ H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown incr_mode?!?!?.") } + /* logically, this is were configuration for flash cache size increases + * should go. However, this configuration depends on max_cache_size, so + * we wait until the end of the function, when this field is set. + */ + switch ( config_ptr->decr_mode ) { case H5C2_decr__off: @@ -6277,9 +6451,14 @@ H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr, if ( config_ptr->max_size == config_ptr->min_size ) { cache_ptr->size_increase_possible = FALSE; + cache_ptr->flash_size_increase_possible = FALSE; cache_ptr->size_decrease_possible = FALSE; } + /* flash_size_increase_possible is intentionally omitted from the + * following: + */ + cache_ptr->resize_enabled = cache_ptr->size_increase_possible || cache_ptr->size_decrease_possible; @@ -6367,6 +6546,37 @@ H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr, } } + /* configure flash size increase facility. We wait until the + * end of the function, as we need the max_cache_size set before + * we start to keep things simple. + * + * If we haven't already ruled out flash cache size increases above, + * go ahead and configure it. + */ + + if ( cache_ptr->flash_size_increase_possible ) { + + switch ( config_ptr->flash_incr_mode ) + { + case H5C2_flash_incr__off: + cache_ptr->flash_size_increase_possible = FALSE; + break; + + case H5C2_flash_incr__add_space: + cache_ptr->flash_size_increase_possible = TRUE; + cache_ptr->flash_size_increase_threshold = + (size_t) + (((double)(cache_ptr->max_cache_size)) * + ((cache_ptr->resize_ctl).flash_threshold)); + break; + + default: /* should be unreachable */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Unknown flash_incr_mode?!?!?.") + break; + } + } + done: FUNC_LEAVE_NOAPI(ret_value) @@ -7272,6 +7482,9 @@ done: * *cache_ptr), and one of its dxpl ids. Also internal * changes supporting the revised API. * + * JRM -- 12/31/07 + * Modified funtion to support flash cache resizes. + * *------------------------------------------------------------------------- */ herr_t @@ -7294,6 +7507,8 @@ H5C2_unprotect(H5C2_t * cache_ptr, hbool_t clear_entry = FALSE; #endif /* H5_HAVE_PARALLEL */ herr_t ret_value = SUCCEED; /* Return value */ + herr_t result; + size_t size_increase = 0; H5C2_cache_entry_t * entry_ptr; H5C2_cache_entry_t * test_entry_ptr; @@ -7439,6 +7654,29 @@ H5C2_unprotect(H5C2_t * cache_ptr, /* update for change in entry size if necessary */ if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) { + /* do a flash cache size increase if appropriate */ + if ( cache_ptr->flash_size_increase_possible ) { + + if ( new_size > entry_ptr->size ) { + + size_increase = new_size - entry_ptr->size; + + if ( size_increase >= + cache_ptr->flash_size_increase_threshold ) { + + result = H5C2__flash_increase_cache_size(cache_ptr, + entry_ptr->size, + new_size); + + if ( result < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ + "H5C2__flash_increase_cache_size failed.") + } + } + } + } + /* update the protected list */ H5C2__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pl_len), \ (cache_ptr->pl_size), \ @@ -7614,7 +7852,9 @@ done: * * Modifications: * - * None. + * Added validation for the flash increment fields. + * + * JRM -- 12/31/07 * *------------------------------------------------------------------------- */ @@ -7724,6 +7964,33 @@ H5C2_validate_resize_config(H5C2_auto_size_ctl_t * config_ptr, */ } /* H5C2_incr__threshold */ + switch ( config_ptr->flash_incr_mode ) + { + case H5C2_flash_incr__off: + /* nothing to do here */ + break; + + case H5C2_flash_incr__add_space: + if ( ( config_ptr->flash_multiple < 0.1 ) || + ( config_ptr->flash_multiple > 10.0 ) ) { + + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \ + "flash_multiple must be in the range [0.1, 10.0]"); + } + + if ( ( config_ptr->flash_threshold < 0.1 ) || + ( config_ptr->flash_threshold > 1.0 ) ) { + + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \ + "flash_threshold must be in the range [0.1, 1.0]"); + } + break; + + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \ + "Invalid flash_incr_mode"); + break; + } } /* H5C2_RESIZE_CFG__VALIDATE_INCREMENT */ @@ -7870,6 +8137,9 @@ done: * passed through to other calls, and are no longer * needed. * + * JRM -- 1/5/08 + * Added support for flash cache size increases. + * *------------------------------------------------------------------------- */ @@ -8147,6 +8417,31 @@ H5C2__auto_adjust_cache_size(H5C2_t * cache_ptr, cache_ptr->size_decreased = TRUE; } + + /* update flash cache size increase fields as appropriate */ + if ( cache_ptr->flash_size_increase_possible ) { + + switch ( (cache_ptr->resize_ctl).flash_incr_mode ) + { + case H5C2_flash_incr__off: + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "flash_size_increase_possible but H5C2_flash_incr__off?!") + break; + + case H5C2_flash_incr__add_space: + cache_ptr->flash_size_increase_threshold = + (size_t) + (((double)(cache_ptr->max_cache_size)) * + ((cache_ptr->resize_ctl).flash_threshold)); + break; + + default: /* should be unreachable */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Unknown flash_incr_mode?!?!?.") + break; + } + } } if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) { @@ -8568,7 +8863,7 @@ H5C2__autoadjust__ageout__evict_aged_out_entries(hid_t dxpl_id, } if ( prev_ptr != NULL ) { - +#ifndef NDEBUG if ( prev_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) { /* something horrible has happened to *prev_ptr -- @@ -8577,13 +8872,15 @@ H5C2__autoadjust__ageout__evict_aged_out_entries(hid_t dxpl_id, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "*prev_ptr corrupt") - } else if ( ( prev_ptr->is_dirty != prev_is_dirty ) - || - ( prev_ptr->next != next_ptr ) - || - ( prev_ptr->is_protected ) - || - ( prev_ptr->is_pinned ) ) { + } else +#endif /* NDEBUG */ + if ( ( prev_ptr->is_dirty != prev_is_dirty ) + || + ( prev_ptr->next != next_ptr ) + || + ( prev_ptr->is_protected ) + || + ( prev_ptr->is_pinned ) ) { /* something has happened to the LRU -- start over * from the tail. @@ -8951,6 +9248,177 @@ done: /*------------------------------------------------------------------------- + * + * Function: H5C2__flash_increase_cache_size + * + * Purpose: If there is not at least new_entry_size - old_entry_size + * bytes of free space in the cache and the current + * max_cache_size is less than (cache_ptr->resize_ctl).max_size, + * perform a flash increase in the cache size and then reset + * the full cache hit rate statistics, and exit. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 12/31/07 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static herr_t +H5C2__flash_increase_cache_size(H5C2_t * cache_ptr, + size_t old_entry_size, + size_t new_entry_size) +{ + /* const char * fcn_name = "H5C2__flash_increase_cache_size()";*/ + herr_t ret_value = SUCCEED; /* Return value */ + size_t new_max_cache_size = 0; + size_t old_max_cache_size = 0; + size_t new_min_clean_size = 0; + size_t old_min_clean_size = 0; + size_t space_needed; + enum H5C2_resize_status status = flash_increase2; /* may change */ + double hit_rate; + + FUNC_ENTER_NOAPI_NOINIT(H5C2__flash_increase_cache_size) + HDassert( cache_ptr ); + HDassert( cache_ptr->magic == H5C2__H5C2_T_MAGIC ); + HDassert( cache_ptr->flash_size_increase_possible ); + HDassert( new_entry_size > cache_ptr->flash_size_increase_threshold ); + HDassert( old_entry_size < new_entry_size ); + + if ( old_entry_size >= new_entry_size ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "old_entry_size >= new_entry_size") + } + + space_needed = new_entry_size - old_entry_size; + + if ( ( (cache_ptr->index_size + space_needed) > + cache_ptr->max_cache_size ) && + ( cache_ptr->max_cache_size < (cache_ptr->resize_ctl).max_size ) ) { + + /* we have work to do */ + + switch ( (cache_ptr->resize_ctl).flash_incr_mode ) + { + case H5C2_flash_incr__off: + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "flash_size_increase_possible but H5C2_flash_incr__off?!") + break; + + case H5C2_flash_incr__add_space: + if ( cache_ptr->index_size < cache_ptr->max_cache_size ) { + + HDassert( (cache_ptr->max_cache_size - + cache_ptr->index_size) + < space_needed ); + space_needed -= cache_ptr->max_cache_size - + cache_ptr->index_size; + } + space_needed = + (size_t)(((double)space_needed) * + (cache_ptr->resize_ctl).flash_multiple); + + new_max_cache_size = cache_ptr->max_cache_size + space_needed; + + break; + + default: /* should be unreachable */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Unknown flash_incr_mode?!?!?.") + break; + } + + if ( new_max_cache_size > (cache_ptr->resize_ctl).max_size ) { + + new_max_cache_size = (cache_ptr->resize_ctl).max_size; + } + + HDassert( new_max_cache_size > cache_ptr->max_cache_size ); + + new_min_clean_size = (size_t) + ((double)new_max_cache_size * + ((cache_ptr->resize_ctl).min_clean_fraction)); + + HDassert( new_min_clean_size <= new_max_cache_size ); + + old_max_cache_size = cache_ptr->max_cache_size; + old_min_clean_size = cache_ptr->min_clean_size; + + cache_ptr->max_cache_size = new_max_cache_size; + cache_ptr->min_clean_size = new_min_clean_size; + + /* update flash cache size increase fields as appropriate */ + HDassert ( cache_ptr->flash_size_increase_possible ); + + switch ( (cache_ptr->resize_ctl).flash_incr_mode ) + { + case H5C2_flash_incr__off: + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "flash_size_increase_possible but H5C2_flash_incr__off?!") + break; + + case H5C2_flash_incr__add_space: + cache_ptr->flash_size_increase_threshold = + (size_t) + (((double)(cache_ptr->max_cache_size)) * + ((cache_ptr->resize_ctl).flash_threshold)); + break; + + default: /* should be unreachable */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Unknown flash_incr_mode?!?!?.") + break; + } + + /* note that we don't cycle the epoch markers. We can + * argue either way as to whether we should, but for now + * we don't. + */ + + if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) { + + /* get the hit rate for the reporting function. Should still + * be good as we havent reset the hit rate statistics. + */ + if ( H5C2_get_cache_hit_rate(cache_ptr, &hit_rate) != SUCCEED ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't get hit rate.") + } + + (*((cache_ptr->resize_ctl).rpt_fcn)) + (cache_ptr, + H5C2__CURR_AUTO_RESIZE_RPT_FCN_VER, + hit_rate, + status, + old_max_cache_size, + new_max_cache_size, + old_min_clean_size, + new_min_clean_size); + } + + if ( H5C2_reset_cache_hit_rate_stats(cache_ptr) != SUCCEED ) { + + /* this should be impossible... */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "H5C2_reset_cache_hit_rate_stats failed.") + } + } + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5C2__flash_increase_cache_size() */ + + + +/*------------------------------------------------------------------------- * Function: H5C2_flush_invalidate_cache * * Purpose: Flush and destroy the entries contained in the target @@ -8991,12 +9459,27 @@ done: * entries being dirtied, resized, and/or renamed inside * flush callbacks. Updated function to support this. * - * -- JRM 8/27/06 - * * Reworked argument list and code to reflect the * removal of the secondary dxpl id, and the decision * to store f in H5C2_t, removing the need to pass it * in all the time. + * -- JRM 8/27/06 + * + * Added code to detect and manage the case in which a + * flush callback changes the s-list out from under + * the function. The only way I can think of in which this + * can happen is if a flush function loads an entry + * into the cache that isn't there already. Quincey tells + * me that this will never happen, but I'm not sure I + * believe him. + * + * Note that this is a pretty bad scenario if it ever + * happens. The code I have added should allow us to + * handle the situation under all but the worst conditions, + * but one can argue that I should just scream and die if I + * ever detect the condidtion. + * + * -- JRM 10/13/07 * *------------------------------------------------------------------------- */ @@ -9112,8 +9595,10 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "next_entry_ptr == NULL 1 ?!?!"); } - - HDassert( next_entry_ptr->magic == H5C2__H5C2_CACHE_ENTRY_T_MAGIC ); +#ifndef NDEBUG + HDassert( next_entry_ptr->magic == + H5C2__H5C2_CACHE_ENTRY_T_MAGIC ); +#endif /* NDEBUG */ HDassert( next_entry_ptr->is_dirty ); HDassert( next_entry_ptr->in_slist ); @@ -9180,14 +9665,16 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id, * If the entry has been evicted, we flag an error and * exit. */ - +#ifndef NDEBUG if ( entry_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) { HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "entry_ptr->magic is invalid ?!?!"); - } else if ( ( ! entry_ptr->is_dirty ) || - ( ! entry_ptr->in_slist ) ) { + } else +#endif /* NDEBUG */ + if ( ( ! entry_ptr->is_dirty ) || + ( ! entry_ptr->in_slist ) ) { /* the s-list has been modified out from under us. * break out of the loop. @@ -9208,9 +9695,10 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "next_entry_ptr == NULL 2 ?!?!"); } - +#ifndef NDEBUG HDassert( next_entry_ptr->magic == H5C2__H5C2_CACHE_ENTRY_T_MAGIC ); +#endif /* NDEBUG */ HDassert( next_entry_ptr->is_dirty ); HDassert( next_entry_ptr->in_slist ); @@ -9345,15 +9833,15 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id, while ( next_entry_ptr != NULL ) { entry_ptr = next_entry_ptr; - +#ifndef NDEBUG HDassert( entry_ptr->magic == H5C2__H5C2_CACHE_ENTRY_T_MAGIC ); - +#endif /* NDEBUG */ next_entry_ptr = entry_ptr->ht_next; - +#ifndef NDEBUG HDassert ( ( next_entry_ptr == NULL ) || ( next_entry_ptr->magic == H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) ); - +#endif /* NDEBUG */ if ( entry_ptr->is_protected ) { /* we have major problems -- but lets flush and destroy @@ -9405,6 +9893,7 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id, * we are accessing a deallocated piece of dynamically * allocated memory, so we just scream and die. */ +#ifndef NDEBUG if ( ( next_entry_ptr != NULL ) && ( next_entry_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) ) { @@ -9415,6 +9904,7 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "next_entry_ptr->magic is invalid?!?!?.") } +#endif /* NDEBUG */ } /* end while loop scanning hash table bin */ } /* end for loop scanning hash table */ @@ -10203,8 +10693,9 @@ H5C2_flush_single_entry(H5F_t * f, * set the magic field to bad magic so we can detect a * freed entry if we see one. */ +#ifndef NDEBUG entry_ptr->magic = H5C2__H5C2_CACHE_ENTRY_T_BAD_MAGIC; - +#endif /* NDEBUG */ if ( type_ptr->free_icr(entry_ptr->addr, entry_ptr->size, (void *)entry_ptr) != SUCCEED ) { @@ -10423,8 +10914,9 @@ H5C2_load_entry(H5F_t * f, HDassert( ( dirty == FALSE ) || ( type->id == 4 ) ); HDassert( entry_ptr->size < H5C2_MAX_ENTRY_SIZE ); - +#ifndef NDEBUG entry_ptr->magic = H5C2__H5C2_CACHE_ENTRY_T_MAGIC; +#endif /* NDEBUG */ entry_ptr->addr = addr; entry_ptr->size = len; entry_ptr->image_ptr = image_ptr; @@ -10547,6 +11039,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id, size_t empty_space; #endif /* H5C2_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ hbool_t prev_is_dirty = FALSE; + hbool_t entry_is_epoch_maker = FALSE; H5C2_cache_entry_t * entry_ptr; H5C2_cache_entry_t * prev_ptr; H5C2_cache_entry_t * next_ptr; @@ -10585,6 +11078,8 @@ H5C2_make_space_in_cache(hid_t dxpl_id, if ( (entry_ptr->type)->id != H5C2__EPOCH_MARKER_TYPE ) { + entry_is_epoch_maker = FALSE; + if ( entry_ptr->is_dirty ) { result = H5C2_flush_single_entry(cache_ptr->f, @@ -10610,6 +11105,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id, /* Skip epoch markers. Set result to SUCCEED to avoid * triggering the error code below. */ + entry_is_epoch_maker = TRUE; result = SUCCEED; } @@ -10620,7 +11116,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id, } if ( prev_ptr != NULL ) { - +#ifndef NDEBUG if ( prev_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) { /* something horrible has happened to *prev_ptr -- @@ -10629,6 +11125,16 @@ H5C2_make_space_in_cache(hid_t dxpl_id, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "*prev_ptr corrupt 1") + } else +#endif /* NDEBUG */ + if ( entry_is_epoch_maker ) { + + /* epoch markers don't get flushed, so the sanity checks + * on normal entries will fail -- thus just set entry_ptr + * to prev_ptr and go on. + */ + entry_ptr = prev_ptr; + } else if ( ( prev_ptr->is_dirty != prev_is_dirty ) || ( prev_ptr->next != next_ptr ) @@ -10659,6 +11165,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id, #if H5C2_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS + entries_examined = 0; initial_list_len = cache_ptr->dLRU_list_len; entry_ptr = cache_ptr->dLRU_tail_ptr; @@ -10707,7 +11214,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id, } if ( prev_ptr != NULL ) { - +#ifndef NDEBUG if (prev_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC) { /* something horrible has happened to *prev_ptr -- @@ -10717,13 +11224,15 @@ H5C2_make_space_in_cache(hid_t dxpl_id, HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ "*prev_ptr corrupt 2") - } else if ( ( ! ( prev_ptr->is_dirty ) ) - || - ( prev_ptr->aux_next != next_ptr ) - || - ( prev_ptr->is_protected ) - || - ( prev_ptr->is_pinned ) ) { + } else +#endif /* #ifndef NDEBUG */ + if ( ( ! ( prev_ptr->is_dirty ) ) + || + ( prev_ptr->aux_next != next_ptr ) + || + ( prev_ptr->is_protected ) + || + ( prev_ptr->is_pinned ) ) { /* something has happened to the dirty LRU -- start over * from the tail. diff --git a/src/H5C2journal.c b/src/H5C2journal.c new file mode 100644 index 0000000..bb64f7f --- /dev/null +++ b/src/H5C2journal.c @@ -0,0 +1,519 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5C2journal.c + * Dec 6 2007 + * John Mainzer + * + * Purpose: This file is a general catchall for functions supporting + * metadata journaling. Note that journaling must be tighly + * integrated with the metadata cache, and thus this file only + * contains only that code that can be easily separated from + * the rest of the cache code. + * + * Observe also that to minimize overhead, it is quite possible + * that many of the functions in this file will be converted + * into macros at some point in the future. + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +#define H5O_PACKAGE /*suppress error about including H5Opkg */ +#define H5C2_PACKAGE /*suppress error about including H5C2pkg */ + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Opkg.h" /* Object headers */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5C2pkg.h" /* Cache */ + +/**************************************************************************/ +/********************** super block message support ***********************/ +/**************************************************************************/ + +static void * H5O_mdj_conf_decode(H5F_t UNUSED *f, + hid_t UNUSED dxpl_id, + unsigned UNUSED mesg_flags, + const uint8_t *p); + +static herr_t H5O_mdj_conf_encode(H5F_t *f, + hbool_t UNUSED disable_shared, + uint8_t *p, + const void *_mesg); + +static void * H5O_mdj_conf_copy(const void *_mesg, + void *_dest); + +static size_t H5O_mdj_conf_size(const H5F_t UNUSED *f, + hbool_t UNUSED disable_shared, + const void *_mesg); + +static herr_t H5O_mdj_conf_reset(void *_mesg); + +static herr_t H5O_mdj_conf_debug(H5F_t UNUSED *f, + hid_t UNUSED dxpl_id, + const void *_mesg, + FILE *stream, + int indent, + int fwidth); + + +/* This message derives from H5O message class */ +const H5O_msg_class_t H5O_MSG_MDJ_CONF[1] = {{ + H5O_MDJ_CONF_ID, /* message id number */ + "metadata journaling config", /* message name for debugging */ + sizeof(H5O_mdj_conf_t), /* native message size */ + 0, /* messages are sharable? */ + H5O_mdj_conf_decode, /* decode message */ + H5O_mdj_conf_encode, /* encode message */ + H5O_mdj_conf_copy, /* copy the native value */ + H5O_mdj_conf_size, /* raw message size */ + H5O_mdj_conf_reset, /* free internal memory */ + NULL, /* free method */ + NULL, /* file delete method */ + NULL, /* link method */ + NULL, /* set share method */ + NULL, /* can share method */ + NULL, /* pre copy native value to file */ + NULL, /* copy native value to file */ + NULL, /* post copy native value to file */ + NULL, /* get creation index */ + NULL, /* set creation index */ + H5O_mdj_conf_debug /* debug the message */ +}}; + + +/* Current version of the metadata journaling configuration information */ +#define H5O_MDJ_CONF_VERSION 0 + +#define MDJ_CONF__JOURNALING_ENABLED_FLAG 0x0001 +#define MDJ_CONF__JOURNAL_IS_EXTERNAL_FLAG 0x0002 + + +/*------------------------------------------------------------------------- + * Function: H5O_mdj_conf_decode + * + * Purpose: Decode a journaling configuration message and return a + * pointer to a newly allocated H5O_mdj_conf_t struct. + * + * Return: Success: Ptr to new message in native struct. + * Failure: NULL + * + * Programmer: John Mainzer + * Dec. 14, 2007 + * + *------------------------------------------------------------------------- + */ + +static void * +H5O_mdj_conf_decode(H5F_t *f, + hid_t UNUSED dxpl_id, + unsigned UNUSED mesg_flags, + const uint8_t *p) +{ + const char * fcn_name = "H5O_mdj_conf_decode()"; + uint16_t flags = 0; /* packed boolean fields */ + H5O_mdj_conf_t *mesg; /* Native message */ + void *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5O_btreek_decode) + + HDfprintf(stdout, "%s: entering.\n", fcn_name); + + /* Sanity check */ + HDassert(f); + HDassert(p); + + /* Version of message */ + if ( *p++ != H5O_MDJ_CONF_VERSION ) { + + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, \ + "bad version number for message") + } + + /* Allocate space for message */ + + if( NULL == ( mesg = H5MM_calloc(sizeof(H5O_mdj_conf_t)))) { + + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, \ + "memory allocation failed for metadata journaling config message."); + + } + + /* retrieve packed boolean flags, upack and load them. */ + UINT16DECODE(p, flags); + + if ( (flags & MDJ_CONF__JOURNALING_ENABLED_FLAG) != 0 ) { + + mesg->journaling_enabled = TRUE; + + } else { + + mesg->journaling_enabled = FALSE; + + } + + if ( (flags & MDJ_CONF__JOURNAL_IS_EXTERNAL_FLAG) != 0 ) { + + mesg->journal_is_external = TRUE; + + } else { + + mesg->journal_is_external = FALSE; + + } + + + /* retrieve the internal journal location */ + + H5F_addr_decode(f, &p, &(mesg->internal_journal_loc)); + + + /* retrieve the size of the external journal path buffer */ + + UINT16DECODE(p, mesg->path_len); + + if ( ( ! mesg->journal_is_external ) && + ( mesg->path_len != 0 ) ) { + + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, \ + "internal journal and positive path len?!?"); + + } + + if ( mesg->path_len > 0 ) { + + /* Allocate space for buffer */ + if ( NULL == + (mesg->external_journal_file_path_ptr = + H5MM_malloc(mesg->path_len + 1)) ) { + + mesg = H5MM_xfree(mesg); + + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, \ + "memory allocation failed for journal file path buffer") + + } /* end if */ + + /* Copy encoded journal file path info into buffer */ + HDmemcpy(mesg->external_journal_file_path_ptr, p, mesg->path_len); + + } else { + + mesg->external_journal_file_path_ptr = NULL; + + } + + /* Set return value */ + ret_value = (void *)mesg; + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5O_mdj_conf_decode() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_mdj_conf_encode + * + * Purpose: Encode metadata journaling configuration message. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * Dec. 6, 2007 + * + *------------------------------------------------------------------------- + */ + +static herr_t +H5O_mdj_conf_encode(H5F_t *f, + hbool_t UNUSED disable_shared, + uint8_t *p, + const void *_mesg) +{ + const char * fcn_name = "H5O_mdj_conf_encode()"; + const H5O_mdj_conf_t *mesg = (const H5O_mdj_conf_t *)_mesg; + uint16_t flags = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_mdj_conf_encode) + + HDfprintf(stdout, "%s: entering.\n", fcn_name); + + /* Sanity check */ + HDassert(f); + HDassert(p); + HDassert(mesg); + + /* setup the flags */ + if ( mesg->journaling_enabled ) { + + flags |= MDJ_CONF__JOURNALING_ENABLED_FLAG; + + } + + if ( mesg->journal_is_external ) { + + flags |= MDJ_CONF__JOURNAL_IS_EXTERNAL_FLAG; + + } + + /* Store version, flags, internal_loc, path_len, & path buffer */ + + *p++ = H5O_MDJ_CONF_VERSION; + + UINT16ENCODE(p, flags); + + H5F_addr_encode(f, &p, mesg->internal_journal_loc); + + HDassert(mesg->path_len <= 65535); + + UINT16ENCODE(p, mesg->path_len); + + if ( mesg->path_len > 0 ) { + + HDmemcpy(p, mesg->external_journal_file_path_ptr, mesg->path_len); + + } + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* H5O_mdj_conf_encode() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_mdj_conf_copy + * + * Purpose: Copies a message from _MESG to _DEST, allocating _DEST if + * necessary. + * + * Return: Success: Ptr to _DEST + * Failure: NULL + * + * Programmer: Quincey Koziol + * Mar 1, 2007 + * + *------------------------------------------------------------------------- + */ + +static void * +H5O_mdj_conf_copy(const void *_mesg, void *_dest) +{ + const char * fcn_name = "H5O_mdj_conf_copy()"; + const H5O_mdj_conf_t *mesg = (const H5O_mdj_conf_t *)_mesg; + H5O_mdj_conf_t *dest = (H5O_mdj_conf_t *)_dest; + void *ret_value; + + FUNC_ENTER_NOAPI_NOINIT(H5O_mdj_conf_copy) + + HDfprintf(stdout, "%s: entering.\n", fcn_name); + + /* Sanity check */ + HDassert(mesg); + + if ( ( dest == NULL ) && + ( ( NULL == (dest = H5MM_malloc(sizeof(H5O_mdj_conf_t))) ) || + ( ( mesg->path_len > 0 ) && + ( ( NULL == (dest->external_journal_file_path_ptr = + H5MM_malloc(mesg->path_len + 1)) ) ) ) ) ) { + + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, \ + "memory allocation failed for metadata journaling conf message") + + } + + /* now copy the message */ + dest->journaling_enabled = mesg->journaling_enabled; + dest->journal_is_external = mesg->journal_is_external; + dest->internal_journal_loc = mesg->internal_journal_loc; + dest->path_len = mesg->path_len; + + if ( dest->path_len > 0 ) { + + HDmemcpy(dest->external_journal_file_path_ptr, + mesg->external_journal_file_path_ptr, + mesg->path_len); + + } else { + + dest->external_journal_file_path_ptr = NULL; + } + + + /* Set return value */ + ret_value = dest; + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5O_mdj_conf_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_mdj_conf_size + * + * Purpose: Returns the size of the raw message in bytes not counting the + * message type or size fields, but only the data fields. + * + * Return: Success: Message data size in bytes w/o alignment. + * Failure: 0 + * + * Programmer: John Mainzer + * Dec. 12, 2007 + * + *------------------------------------------------------------------------- + */ +static size_t +H5O_mdj_conf_size(const H5F_t *f, + hbool_t UNUSED disable_shared, + const void *_mesg) +{ + const char * fcn_name = "H5O_mdj_conf_size()"; + const H5O_mdj_conf_t * mesg = (const H5O_mdj_conf_t *)_mesg; + size_t ret_value; + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_mdj_conf_size) + + HDfprintf(stdout, "%s: entering.\n", fcn_name); + + /* Sanity check */ + HDassert(f); + HDassert(mesg); + + ret_value = 1 + /* Version number */ + 2 + /* flags */ + H5F_SIZEOF_ADDR(f) + /* addr of internal journal */ + 2 + /* length of external journal path */ + mesg->path_len; /* external journal path buffer */ + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5O_mdj_conf_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_mdj_conf_reset + * + * Purpose: Frees internal pointers and resets the message to an + * initial state. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * Dec. 13, 2007 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +static herr_t +H5O_mdj_conf_reset(void *_mesg) +{ + const char * fcn_name = "H5O_mdj_conf_reset()"; + H5O_mdj_conf_t *mesg = (H5O_mdj_conf_t *) _mesg; + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_mdj_conf_reset); + + HDfprintf(stdout, "%s: entering.\n", fcn_name); + + /* check args */ + assert(mesg); + + /* reset */ + if ( mesg->external_journal_file_path_ptr != NULL ) + { + HDassert( mesg->path_len > 0 ); + HDassert( mesg->journal_is_external ); + mesg->external_journal_file_path_ptr = + H5MM_xfree(mesg->external_journal_file_path_ptr); + mesg->path_len = 0; + } + + FUNC_LEAVE_NOAPI(SUCCEED); + +} /* H5O_mdj_conf_reset() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_mdj_conf_debug + * + * Purpose: Prints debugging info for the message. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * Dec. 7, 2007 + * + *------------------------------------------------------------------------- + */ + +static herr_t +H5O_mdj_conf_debug(H5F_t UNUSED *f, + hid_t UNUSED dxpl_id, + const void *_mesg, + FILE *stream, + int indent, + int fwidth) +{ + const char * fcn_name = "H5O_mdj_conf_debug()"; + const H5O_mdj_conf_t *mesg = (const H5O_mdj_conf_t *)_mesg; + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_btreek_debug) + + HDfprintf(stdout, "%s: entering.\n", fcn_name); + + /* Sanity check */ + HDassert(f); + HDassert(mesg); + HDassert(stream); + HDassert(indent >= 0); + HDassert(fwidth >= 0); + + HDfprintf(stream, "%*s%-*s %d\n", indent, "", fwidth, + "journaling_enabled:", + (int)(mesg->journaling_enabled)); + + HDfprintf(stream, "%*s%-*s %d\n", indent, "", fwidth, + "journal_is_external:", + (int)(mesg->journal_is_external)); + + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, + "internal_journal_loc:", + mesg->internal_journal_loc); + + HDfprintf(stream, "%*s%-*s %d\n", indent, "", fwidth, + "path_len:", + (int)(mesg->path_len)); + + if ( mesg->path_len > 0 ) { + + HDfprintf(stream, "%*s%-*s \"%s\"\n", indent, "", fwidth, + "external_journal_file_path_ptr:", + (char *)(mesg->external_journal_file_path_ptr)); + + } + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* end H5O_mdj_conf_debug() */ diff --git a/src/H5C2pkg.h b/src/H5C2pkg.h index a1753b4..71c8e15 100644 --- a/src/H5C2pkg.h +++ b/src/H5C2pkg.h @@ -471,6 +471,18 @@ * all the ways this can happen, we simply set this flag when * we receive a new configuration. * + * flash_size_increase_possible: Depending on the configuration data given + * in the resize_ctl field, it may or may not be possible + * for a flash size increase to occur. We set this flag + * whenever we receive a new configuration so as to avoid + * repeated calculations. + * + * flash_size_increase_threshold: If a flash cache size increase is possible, + * this field is used to store the minimum size of a new entry + * or size increase needed to trigger a flash cache size + * increase. Note that this field must be updated whenever + * the size of the cache is changed. + * * size_decrease_possible: Depending on the configuration data given * in the resize_ctl field, it may or may not be possible * to decrease the size of the cache. Rather than test for @@ -863,6 +875,8 @@ struct H5C2_t H5C2_cache_entry_t * dLRU_tail_ptr; hbool_t size_increase_possible; + hbool_t flash_size_increase_possible; + size_t flash_size_increase_threshold; hbool_t size_decrease_possible; hbool_t resize_enabled; hbool_t cache_full; diff --git a/src/H5C2private.h b/src/H5C2private.h index a949f6d..b70e968 100644 --- a/src/H5C2private.h +++ b/src/H5C2private.h @@ -804,12 +804,16 @@ typedef herr_t (*H5C2_log_flush_func_t)(H5C2_t * cache_ptr, * ****************************************************************************/ +#ifndef NDEBUG #define H5C2__H5C2_CACHE_ENTRY_T_MAGIC 0x005CAC0A #define H5C2__H5C2_CACHE_ENTRY_T_BAD_MAGIC 0xDeadBeef +#endif /* NDEBUG */ typedef struct H5C2_cache_entry_t { +#ifndef NDEBUG uint32_t magic; +#endif /* NDEBUG */ haddr_t addr; size_t size; void * image_ptr; @@ -954,6 +958,57 @@ typedef struct H5C2_cache_entry_t * above, this field contains the maximum number of bytes by which the * cache size can be increased in a single re-size. * + * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated + * type whose value indicates whether and by what algorithm we should + * make flash increases in the size of the cache to accomodate insertion + * of large entries and large increases in the size of a single entry. + * + * The addition of the flash increment mode was occasioned by performance + * problems that appear when a local heap is increased to a size in excess + * of the current cache size. While the existing re-size code dealt with + * this eventually, performance was very bad for the remainder of the + * epoch. + * + * At present, there are two possible values for the flash_incr_mode: + * + * H5C_flash_incr__off: Don't perform flash increases in the size of + * the cache. + * + * H5C_flash_incr__add_space: Let x be either the size of a newly + * newly inserted entry, or the number of bytes by which the + * size of an existing entry has been increased. + * + * If + * x > flash_threshold * current max cache size, + * + * increase the current maximum cache size by x * flash_multiple + * less any free space in the cache, and start a new epoch. For + * now at least, pay no attention to the maximum increment. + * + * + * With a little thought, it should be obvious that the above flash + * cache size increase algorithm is not sufficient for all + * circumstances -- for example, suppose the user round robins through + * (1/flash_threshold) +1 groups, adding one data set to each on each + * pass. Then all will increase in size at about the same time, requiring + * the max cache size to at least double to maintain acceptable + * performance, however the above flash increment algorithm will not be + * triggered. + * + * Hopefully, the add space algorithm detailed above will be sufficient + * for the performance problems encountered to date. However, we should + * expect to revisit the issue. + * + * flash_multiple: Double containing the multiple described above in the + * H5C_flash_incr__add_space section of the discussion of the + * flash_incr_mode section. This field is ignored unless flash_incr_mode + * is H5C_flash_incr__add_space. + * + * flash_threshold: Double containing the factor by which current max cache + * size is multiplied to obtain the size threshold for the add_space + * flash increment algorithm. The field is ignored unless + * flash_incr_mode is H5C2_flash_incr__add_space. + * * * Cache size decrease control fields: * @@ -1068,6 +1123,8 @@ typedef struct H5C2_cache_entry_t #define H5C2__DEF_AR_MIN_CLEAN_FRAC 0.5 #define H5C2__DEF_AR_INCREMENT 2.0 #define H5C2__DEF_AR_MAX_INCREMENT ((size_t)( 2 * 1024 * 1024)) +#define H5C2__DEF_AR_FLASH_MULTIPLE 1.0 +#define H5C2__DEV_AR_FLASH_THRESHOLD 0.25 #define H5C2__DEF_AR_DECREMENT 0.9 #define H5C2__DEF_AR_MAX_DECREMENT ((size_t)( 1 * 1024 * 1024)) #define H5C2__DEF_AR_EPCHS_B4_EVICT 3 @@ -1080,6 +1137,7 @@ enum H5C2_resize_status { in_spec2, increase2, + flash_increase2, decrease2, at_max_size2, at_min_size2, @@ -1100,45 +1158,49 @@ typedef void (*H5C2_auto_resize_rpt_fcn)(H5C2_t * cache_ptr, typedef struct H5C2_auto_size_ctl_t { /* general configuration fields: */ - int32_t version; - H5C2_auto_resize_rpt_fcn rpt_fcn; + int32_t version; + H5C2_auto_resize_rpt_fcn rpt_fcn; - hbool_t set_initial_size; - size_t initial_size; + hbool_t set_initial_size; + size_t initial_size; - double min_clean_fraction; + double min_clean_fraction; - size_t max_size; - size_t min_size; + size_t max_size; + size_t min_size; - int64_t epoch_length; + int64_t epoch_length; /* size increase control fields: */ - enum H5C2_cache_incr_mode incr_mode; + enum H5C2_cache_incr_mode incr_mode; + + double lower_hr_threshold; - double lower_hr_threshold; + double increment; - double increment; + hbool_t apply_max_increment; + size_t max_increment; - hbool_t apply_max_increment; - size_t max_increment; + enum H5C2_cache_flash_incr_mode flash_incr_mode; + double flash_multiple; + double flash_threshold; /* size decrease control fields: */ - enum H5C2_cache_decr_mode decr_mode; + enum H5C2_cache_decr_mode decr_mode; - double upper_hr_threshold; + double upper_hr_threshold; - double decrement; + double decrement; - hbool_t apply_max_decrement; - size_t max_decrement; + hbool_t apply_max_decrement; + size_t max_decrement; - int32_t epochs_before_eviction; + int32_t epochs_before_eviction; - hbool_t apply_empty_reserve; - double empty_reserve; + hbool_t apply_empty_reserve; + double empty_reserve; } H5C2_auto_size_ctl_t; diff --git a/src/H5C2public.h b/src/H5C2public.h index e1adff3..d293ccb 100644 --- a/src/H5C2public.h +++ b/src/H5C2public.h @@ -41,6 +41,13 @@ enum H5C2_cache_incr_mode H5C2_incr__threshold }; +enum H5C2_cache_flash_incr_mode +{ + H5C2_flash_incr__off, + H5C2_flash_incr__add_space +}; + + enum H5C2_cache_decr_mode { H5C2_decr__off, diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 949f923..7c04ea2 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -465,6 +465,18 @@ * all the ways this can happen, we simply set this flag when * we receive a new configuration. * + * flash_size_increase_possible: Depending on the configuration data given + * in the resize_ctl field, it may or may not be possible + * for a flash size increase to occur. We set this flag + * whenever we receive a new configuration so as to avoid + * repeated calculations. + * + * flash_size_increase_threshold: If a flash cache size increase is possible, + * this field is used to store the minimum size of a new entry + * or size increase needed to trigger a flash cache size + * increase. Note that this field must be updated whenever + * the size of the cache is changed. + * * size_decrease_possible: Depending on the configuration data given * in the resize_ctl field, it may or may not be possible * to decrease the size of the cache. Rather than test for @@ -580,10 +592,10 @@ * equal to the array index has not been in cache when * requested in the current epoch. * - * write_protects: Array of int64 of length H5C__MAX_NUM_TYPE_IDS + 1. The cells - * are used to record the number of times an entry with type id - * equal to the array index has been write protected in the - * current epoch. + * write_protects: Array of int64 of length H5C__MAX_NUM_TYPE_IDS + 1. The + * cells are used to record the number of times an entry with + * type id equal to the array index has been write protected + * in the current epoch. * * Observe that (hits + misses) = (write_protects + read_protects). * @@ -855,6 +867,8 @@ struct H5C_t H5C_cache_entry_t * dLRU_tail_ptr; hbool_t size_increase_possible; + hbool_t flash_size_increase_possible; + size_t flash_size_increase_threshold; hbool_t size_decrease_possible; hbool_t resize_enabled; hbool_t cache_full; diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index d365755..53dcf0d 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -202,6 +202,27 @@ typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr, * * JRM - 4/26/04 * + * magic: Unsigned 32 bit integer that must always be set to + * H5C__H5C_CACHE_ENTRY_T_MAGIC when the entry is valid. + * The field must be set to H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC + * just before the entry is freed. + * + * This is necessary, as the LRU list can be changed out + * from under H5C_make_space_in_cache() by the flush + * callback which may change the size of an existing entry, + * and/or load a new entry while serializing the target entry. + * + * This in turn can cause a recursive call to + * H5C_make_space_in_cache() which may either flush or evict + * the next entry that the first invocation of that function + * was about to examine. + * + * The magic field allows H5C_make_space_in_cache() to + * detect this case, and re-start its scan from the bottom + * of the LRU when this situation occurs. + * + * This field is only compiled in debug mode. + * * addr: Base address of the cache entry on disk. * * size: Length of the cache entry on disk. Note that unlike normal @@ -442,8 +463,16 @@ typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr, * ****************************************************************************/ +#ifndef NDEBUG +#define H5C__H5C_CACHE_ENTRY_T_MAGIC 0x005CAC0A +#define H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC 0xDeadBeef +#endif /* NDEBUG */ + typedef struct H5C_cache_entry_t { +#ifndef NDEBUG + uint32_t magic; +#endif /* NDEBUG */ haddr_t addr; size_t size; const H5C_class_t * type; @@ -587,7 +616,58 @@ typedef struct H5C_cache_entry_t * above, this field contains the maximum number of bytes by which the * cache size can be increased in a single re-size. * - * + * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated + * type whose value indicates whether and by what algorithm we should + * make flash increases in the size of the cache to accomodate insertion + * of large entries and large increases in the size of a single entry. + * + * The addition of the flash increment mode was occasioned by performance + * problems that appear when a local heap is increased to a size in excess + * of the current cache size. While the existing re-size code dealt with + * this eventually, performance was very bad for the remainder of the + * epoch. + * + * At present, there are two possible values for the flash_incr_mode: + * + * H5C_flash_incr__off: Don't perform flash increases in the size of + * the cache. + * + * H5C_flash_incr__add_space: Let x be either the size of a newly + * newly inserted entry, or the number of bytes by which the + * size of an existing entry has been increased. + * + * If + * x > flash_threshold * current max cache size, + * + * increase the current maximum cache size by x * flash_multiple + * less any free space in the cache, and start a new epoch. For + * now at least, pay no attention to the maximum increment. + * + * + * With a little thought, it should be obvious that the above flash + * cache size increase algorithm is not sufficient for all circumstances -- + * for example, suppose the user round robins through + * (1/flash_threshold) +1 groups, adding one data set to each on each + * pass. Then all will increase in size at about the same time, requiring + * the max cache size to at least double to maintain acceptable + * performance, however the above flash increment algorithm will not be + * triggered. + * + * Hopefully, the add space algorithm detailed above will be sufficient + * for the performance problems encountered to date. However, we should + * expect to revisit the issue. + * + * flash_multiple: Double containing the multiple described above in the + * H5C_flash_incr__add_space section of the discussion of the + * flash_incr_mode section. This field is ignored unless flash_incr_mode + * is H5C_flash_incr__add_space. + * + * flash_threshold: Double containing the factor by which current max cache size + * is multiplied to obtain the size threshold for the add_space flash + * increment algorithm. The field is ignored unless flash_incr_mode is + * H5C_flash_incr__add_space. + * + * * Cache size decrease control fields: * * decr_mode: Instance of the H5C_cache_decr_mode enumerated type whose @@ -701,6 +781,8 @@ typedef struct H5C_cache_entry_t #define H5C__DEF_AR_MIN_CLEAN_FRAC 0.5 #define H5C__DEF_AR_INCREMENT 2.0 #define H5C__DEF_AR_MAX_INCREMENT ((size_t)( 2 * 1024 * 1024)) +#define H5C__DEF_AR_FLASH_MULTIPLE 1.0 +#define H5C__DEV_AR_FLASH_THRESHOLD 0.25 #define H5C__DEF_AR_DECREMENT 0.9 #define H5C__DEF_AR_MAX_DECREMENT ((size_t)( 1 * 1024 * 1024)) #define H5C__DEF_AR_EPCHS_B4_EVICT 3 @@ -713,6 +795,7 @@ enum H5C_resize_status { in_spec, increase, + flash_increase, decrease, at_max_size, at_min_size, @@ -733,45 +816,48 @@ typedef void (*H5C_auto_resize_rpt_fcn)(H5C_t * cache_ptr, typedef struct H5C_auto_size_ctl_t { /* general configuration fields: */ - int32_t version; - H5C_auto_resize_rpt_fcn rpt_fcn; + int32_t version; + H5C_auto_resize_rpt_fcn rpt_fcn; - hbool_t set_initial_size; - size_t initial_size; + hbool_t set_initial_size; + size_t initial_size; - double min_clean_fraction; + double min_clean_fraction; - size_t max_size; - size_t min_size; + size_t max_size; + size_t min_size; - int64_t epoch_length; + int64_t epoch_length; /* size increase control fields: */ - enum H5C_cache_incr_mode incr_mode; + enum H5C_cache_incr_mode incr_mode; - double lower_hr_threshold; + double lower_hr_threshold; - double increment; + double increment; - hbool_t apply_max_increment; - size_t max_increment; + hbool_t apply_max_increment; + size_t max_increment; + enum H5C_cache_flash_incr_mode flash_incr_mode; + double flash_multiple; + double flash_threshold; /* size decrease control fields: */ - enum H5C_cache_decr_mode decr_mode; + enum H5C_cache_decr_mode decr_mode; - double upper_hr_threshold; + double upper_hr_threshold; - double decrement; + double decrement; - hbool_t apply_max_decrement; - size_t max_decrement; + hbool_t apply_max_decrement; + size_t max_decrement; - int32_t epochs_before_eviction; + int32_t epochs_before_eviction; - hbool_t apply_empty_reserve; - double empty_reserve; + hbool_t apply_empty_reserve; + double empty_reserve; } H5C_auto_size_ctl_t; diff --git a/src/H5Cpublic.h b/src/H5Cpublic.h index 082d512..7ef959a 100644 --- a/src/H5Cpublic.h +++ b/src/H5Cpublic.h @@ -41,6 +41,12 @@ enum H5C_cache_incr_mode H5C_incr__threshold }; +enum H5C_cache_flash_incr_mode +{ + H5C_flash_incr__off, + H5C_flash_incr__add_space +}; + enum H5C_cache_decr_mode { H5C_decr__off, diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 3e13b0d..5c54fdd 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -113,6 +113,11 @@ typedef struct H5F_file_t { struct H5G_t *root_grp; /* Open root group */ H5FO_t *open_objs; /* Open objects in file */ H5RC_t *grp_btree_shared; /* Ref-counted group B-tree node info */ + hbool_t journaling_enabled; /* metadata journaling configuration */ + hbool_t journal_is_external; /* fields. All fields as per those */ + haddr_t internal_journal_loc;/* of the same name in M5O_mdj_conf_t. */ + size_t path_len; + uint8_t * external_journal_file_path_ptr; } H5F_file_t; /* A record of the mount table */ diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index cbc5f72..ed3395e 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -33,6 +33,8 @@ #include "H5Iprivate.h" /* IDs */ #include "H5Pprivate.h" /* Property lists */ #include "H5SMprivate.h" /* Shared Object Header Messages */ +#include "H5MMprivate.h" /* Memory management */ + /****************/ @@ -226,6 +228,12 @@ done: * wendling@ncsa.uiuc.edu * Sept 12, 2003 * + * Changes: Johm Mainzer + * 12/14/07 + * Added code to read in the metadata journaling config + * if it is present, and to initialize + * f->shared->journaling_enabled to FALSE if it isn't. + * *------------------------------------------------------------------------- */ herr_t @@ -251,6 +259,14 @@ H5F_super_read(H5F_t *f, hid_t dxpl_id, H5G_loc_t *root_loc) shared = f->shared; lf = shared->lf; + /* initialize the metadata journaling configuration sections of the + * super block to indicate that journaling is not turned on at + * present. These initialization may be overridden shortly. + */ + shared->journaling_enabled = FALSE; + shared->path_len = 0; + shared->external_journal_file_path_ptr = NULL; + /* Get the shared file creation property list */ if(NULL == (c_plist = H5I_object(shared->fcpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") @@ -553,6 +569,9 @@ H5F_super_read(H5F_t *f, hid_t dxpl_id, H5G_loc_t *root_loc) H5O_loc_t ext_loc; /* "Object location" for superblock extension */ H5O_btreek_t btreek; /* v1 B-tree 'K' value message from superblock extension */ H5O_drvinfo_t drvinfo; /* Driver info message from superblock extension */ + H5O_mdj_conf_t mdj_conf;/* metadata journaling config message + * from superblock extension + */ /* Sanity check - superblock extension should only be defined for * superblock version >= 2. @@ -617,6 +636,48 @@ H5F_super_read(H5F_t *f, hid_t dxpl_id, H5G_loc_t *root_loc) H5O_msg_reset(H5O_DRVINFO_ID, &drvinfo); } /* end else */ + /* Read in the metadata journaling configuration message, + * if it exists. + */ + if(NULL == H5O_msg_read(&ext_loc, H5O_MDJ_CONF_ID, &mdj_conf, dxpl_id)) { + /* Reset error from "failed" message read */ + H5E_clear_stack(NULL); + } /* end if */ + else { + + shared->journaling_enabled = mdj_conf.journaling_enabled; + + if ( shared->journaling_enabled ) { + + shared->journal_is_external = mdj_conf.journal_is_external; + shared->internal_journal_loc = mdj_conf.internal_journal_loc; + shared->path_len = mdj_conf.path_len; + shared->external_journal_file_path_ptr = + mdj_conf.external_journal_file_path_ptr; + + /* for now at least, the journal file must always be + * external -- hence the following asserts. Remove them + * if we ever support an internal journal. + */ + HDassert( shared->journal_is_external ); + HDassert( shared->path_len > 0 ); + HDassert( shared->external_journal_file_path_ptr != NULL ); + + /* if there is a an external journal file, + * H5O_mdj_conf_decode() will allocate a buffer to + * store it in. Rather than allocate our own buffer, + * we will use the one created by H5O_mdj_conf_decode(), + * and modify mdj_conf so that H5O_mdj_conf_reset() will + * not discard it. + */ + mdj_conf.path_len = 0; + mdj_conf.external_journal_file_path_ptr = NULL; + } + + /* Reset metadata journaling config message */ + H5O_msg_reset(H5O_MDJ_CONF_ID, &mdj_conf); + } + /* Close the extension. Twiddle the number of open objects to avoid * closing the file (since this will be the only open object). */ @@ -645,6 +706,13 @@ done: * koziol@ncsa.uiuc.edu * Sept 15, 2003 * + * Changes: John Mainzer + * Dec. 14, 2007 + * Added initialization for the metadata journaling + * configuration fields. By default, these fields are + * initialized to indicate that we are not journaling + * metadata. + * *------------------------------------------------------------------------- */ herr_t @@ -802,6 +870,43 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update driver info header message") } /* end if */ + /* check for journaling config data to store */ + if ( f->shared->journaling_enabled ) { + + struct H5O_mdj_conf_t mdj_conf; + + mdj_conf.journaling_enabled = f->shared->journaling_enabled; + mdj_conf.journal_is_external = f->shared->journal_is_external; + mdj_conf.internal_journal_loc = f->shared->internal_journal_loc; + mdj_conf.path_len = f->shared->path_len; + + if ( f->shared->external_journal_file_path_ptr == NULL ) { + + mdj_conf.external_journal_file_path_ptr = NULL; + + } else { + + if ( ( NULL == (mdj_conf.external_journal_file_path_ptr = + H5MM_malloc(mdj_conf.path_len + 1)) ) ) { + + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, \ + "memory alloc failed for mdj path") + + } + HDmemcpy(mdj_conf.external_journal_file_path_ptr, + f->shared->external_journal_file_path_ptr, + f->shared->path_len); + } + + if ( H5O_msg_create(&ext_loc, H5O_MDJ_CONF_ID, + H5O_MSG_FLAG_CONSTANT | H5O_MSG_FLAG_DONTSHARE, + H5O_UPDATE_TIME, &mdj_conf, dxpl_id) < 0 ) { + + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, \ + "unable to update metadata journal conf header message") + } + } + /* Twiddle the number of open objects to avoid closing the file * (since this will be the only open object currently). */ @@ -77,6 +77,10 @@ static herr_t H5O_obj_type_real(H5O_t *oh, H5O_type_t *obj_type); /*********************/ /* Header message ID to class mapping */ + +/* Remember to increment H5O_MSG_TYPES in H5Opkg.h when adding a new + * message. + */ const H5O_msg_class_t *const H5O_msg_class_g[] = { H5O_MSG_NULL, /*0x0000 Null */ H5O_MSG_SDSPACE, /*0x0001 Dataspace */ @@ -105,9 +109,12 @@ const H5O_msg_class_t *const H5O_msg_class_g[] = { H5O_MSG_DRVINFO, /*0x0014 Driver info settings */ H5O_MSG_AINFO, /*0x0015 Attribute information */ H5O_MSG_REFCOUNT, /*0x0016 Object's ref. count */ - H5O_MSG_UNKNOWN, /*0x0017 Placeholder for unknown message */ + H5O_MSG_MDJ_CONF, /*0x0017 Metadata journaling config */ + H5O_MSG_UNKNOWN, /*0x0018 Placeholder for unknown message */ }; +/* HDassert(H5O_UNKNOWN_ID < H5O_MSG_TYPES); */ + /* Header object ID to class mapping */ /* * Initialize the object class info table. Begin with the most general types @@ -157,6 +164,10 @@ H5FL_EXTERN(time_t); * Programmer: Quincey Koziol * Thursday, January 18, 2007 * + * Changes: JRM -- 12/12/07 + * Added santity check verifying that H5O_msg_class_g + * is big enough. + * *------------------------------------------------------------------------- */ static herr_t @@ -168,6 +179,8 @@ H5O_init_interface(void) HDassert(H5O_MSG_TYPES == NELMTS(H5O_msg_class_g)); HDassert(sizeof(H5O_fheap_id_t) == H5O_FHEAP_ID_LEN); + HDassert(H5O_UNKNOWN_ID < H5O_MSG_TYPES); + FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5O_init_interface() */ diff --git a/src/H5Opkg.h b/src/H5Opkg.h index c90ff0a..94af802 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -30,7 +30,7 @@ #define H5O_NMESGS 8 /*initial number of messages */ #define H5O_NCHUNKS 2 /*initial number of chunks */ #define H5O_MIN_SIZE 22 /* Min. obj header data size (must be big enough for a message prefix and a continuation message) */ -#define H5O_MSG_TYPES 24 /* # of types of messages */ +#define H5O_MSG_TYPES 25 /* # of types of messages */ #define H5O_MAX_CRT_ORDER_IDX 65535 /* Max. creation order index value */ /* Versions of object header structure */ @@ -431,7 +431,10 @@ H5_DLLVAR const H5O_msg_class_t H5O_MSG_AINFO[1]; /* Reference Count Message. (0x0016) */ H5_DLLVAR const H5O_msg_class_t H5O_MSG_REFCOUNT[1]; -/* Placeholder for unknown message. (0x0017) */ +/* Metadata Journaling Config Message. (0x0017) */ +H5_DLLVAR const H5O_msg_class_t H5O_MSG_MDJ_CONF[1]; + +/* Placeholder for unknown message. (0x0018) */ H5_DLLVAR const H5O_msg_class_t H5O_MSG_UNKNOWN[1]; diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 31738a9..3188468 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -62,6 +62,8 @@ typedef struct H5O_t H5O_t; #define H5O_FHEAP_MAX_MAN_SIZE (4 * 1024) #define H5O_FHEAP_ID_LEN 8 +/* #define H5O_ENABLE_BOGUS 1 */ + /* Object header macros */ #define H5O_MESG_MAX_SIZE 65536 /*max obj header message size */ #define H5O_ALL (-1) /* Operate on all messages of type */ @@ -160,7 +162,8 @@ typedef struct H5O_copy_t { #define H5O_DRVINFO_ID 0x0014 /* Driver info message. */ #define H5O_AINFO_ID 0x0015 /* Attribute info message. */ #define H5O_REFCOUNT_ID 0x0016 /* Reference count message. */ -#define H5O_UNKNOWN_ID 0x0017 /* Placeholder message ID for unknown message. */ +#define H5O_MDJ_CONF_ID 0x0017 /* Metadata journaling config message */ +#define H5O_UNKNOWN_ID 0x0018 /* Placeholder message ID for unknown message. */ /* (this should never exist in a file) */ @@ -491,6 +494,60 @@ typedef struct H5O_ainfo_t { */ typedef uint32_t H5O_refcount_t; /* Contains # of links to object, if >1 */ + +/* + * Metadata journaling configuration message + * + * Information on whether and how metadata changes are being journaled, + * so as to reconstruct consistent metadata in the file in the event of + * a crash, stored in a superblock extension + * (Data structure in memory) + * + * The fields of the H5O_mdj_conf_t structure are discussed individually + * below: + * + * journaling_enabled: Boolean flag indicating whether metadata journaling + * is currently enabled. + * + * journal_is_external: Boolean flag indicating whether the journal is + * stored in an external file, or within the HDF5 file. + * + * If the field is TRUE, external_journal_file_path points to a + * buffer containing the path to the external journal file, and + * internal_journal_loc is undefined. + * + * If the field is FALSE, internal_journal_loc contains the base + * address of the journal within the hdf5 file, and + * external_journal_file_path_ptr must be NULL. + * + * At least for the initial implementation, journal_is_external + * will always be TRUE. + * + * internal_journal_loc: haddr_t containing the base address of the + * interal journal -- if there is one, or undefined if there + * isn't (see discussion of journal_is_external above). + * + * path_len: size_t containing the size of the buffer needed to + * contain the path to the external journal file. This field + * must contain 0 if the journal is internal. + * + * external_journal_file_path_ptr: Pointer to uint8_t. If the + * journal is stored in an external file, this field points to + * a buffer containing the path to this file. + * + * If the journal is internal, this field must be NULL. + */ +typedef struct H5O_mdj_conf_t { + + hbool_t journaling_enabled; + hbool_t journal_is_external; + haddr_t internal_journal_loc; + size_t path_len; + uint8_t * external_journal_file_path_ptr; + +} H5O_mdj_conf_t; + + /* * "Unknown" Message. * (Data structure in memory) diff --git a/src/Makefile.am b/src/Makefile.am index 7319c1c..19fde87 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -45,7 +45,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \ H5AC.c H5AC2.c H5B.c H5Bcache.c \ H5B2.c H5B2cache.c H5B2dbg.c H5B2int.c H5B2stat.c H5B2test.c \ - H5C.c H5C2.c H5CS.c \ + H5C.c H5C2.c H5C2journal.c H5CS.c \ H5D.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ H5Ddeprec.c H5Defl.c H5Dfill.c H5Dint.c \ H5Dio.c \ diff --git a/src/Makefile.in b/src/Makefile.in index 61bcbd8..a65324e 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -82,43 +82,44 @@ am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \ H5timer.lo H5trace.lo H5A.lo H5Abtree2.lo H5Adense.lo \ H5Adeprec.lo H5Aint.lo H5Atest.lo H5AC.lo H5AC2.lo H5B.lo \ H5Bcache.lo H5B2.lo H5B2cache.lo H5B2dbg.lo H5B2int.lo \ - H5B2stat.lo H5B2test.lo H5C.lo H5C2.lo H5CS.lo H5D.lo \ - H5Dcompact.lo H5Dcontig.lo H5Ddbg.lo H5Ddeprec.lo H5Defl.lo \ - H5Dfill.lo H5Dint.lo H5Dio.lo H5Distore.lo H5Dmpio.lo H5Doh.lo \ - H5Dselect.lo H5Dtest.lo H5E.lo H5Edeprec.lo H5Eint.lo H5F.lo \ - H5Fdbg.lo H5Ffake.lo H5Fmount.lo H5Fsfile.lo H5Fsuper.lo \ - H5Ftest.lo H5FD.lo H5FDcore.lo H5FDdirect.lo H5FDfamily.lo \ - H5FDlog.lo H5FDmpi.lo H5FDmpio.lo H5FDmpiposix.lo H5FDmulti.lo \ - H5FDsec2.lo H5FDstdio.lo H5FL.lo H5FO.lo H5FS.lo H5FScache.lo \ - H5FSdbg.lo H5FSsection.lo H5G.lo H5Gbtree2.lo H5Gcompact.lo \ - H5Gdense.lo H5Gdeprec.lo H5Gent.lo H5Gint.lo H5Glink.lo \ - H5Gloc.lo H5Gname.lo H5Gnode.lo H5Gobj.lo H5Goh.lo H5Gstab.lo \ - H5Gtest.lo H5Gtraverse.lo H5HF.lo H5HFbtree2.lo H5HFcache.lo \ - H5HFdbg.lo H5HFdblock.lo H5HFdtable.lo H5HFhdr.lo H5HFhuge.lo \ - H5HFiblock.lo H5HFiter.lo H5HFman.lo H5HFsection.lo \ - H5HFspace.lo H5HFstat.lo H5HFtest.lo H5HFtiny.lo H5HG.lo \ - H5HGdbg.lo H5HL.lo H5HLdbg.lo H5HP.lo H5I.lo H5L.lo \ - H5Lexternal.lo H5MF.lo H5MM.lo H5MP.lo H5MPtest.lo H5O.lo \ - H5Oainfo.lo H5Oalloc.lo H5Oattr.lo H5Oattribute.lo H5Obogus.lo \ - H5Obtreek.lo H5Ocache.lo H5Ocont.lo H5Ocopy.lo H5Odbg.lo \ - H5Odrvinfo.lo H5Odtype.lo H5Oefl.lo H5Ofill.lo H5Oginfo.lo \ - H5Olayout.lo H5Olinfo.lo H5Olink.lo H5Omessage.lo H5Omtime.lo \ - H5Oname.lo H5Onull.lo H5Opline.lo H5Orefcount.lo H5Osdspace.lo \ - H5Oshared.lo H5Ostab.lo H5Oshmesg.lo H5Otest.lo H5Ounknown.lo \ - H5P.lo H5Pacpl.lo H5Pdcpl.lo H5Pdeprec.lo H5Pdxpl.lo \ - H5Pfapl.lo H5Pfcpl.lo H5Pfmpl.lo H5Pgcpl.lo H5Pint.lo \ - H5Plapl.lo H5Plcpl.lo H5Pocpl.lo H5Pocpypl.lo H5Pstrcpl.lo \ - H5Ptest.lo H5R.lo H5Rdeprec.lo H5RC.lo H5RS.lo H5S.lo \ - H5Sall.lo H5Sdbg.lo H5Shyper.lo H5Smpio.lo H5Snone.lo \ - H5Spoint.lo H5Sselect.lo H5Stest.lo H5SL.lo H5SM.lo \ - H5SMbtree2.lo H5SMcache.lo H5SMtest.lo H5ST.lo H5T.lo \ - H5Tarray.lo H5Tbit.lo H5Tcommit.lo H5Tcompound.lo H5Tconv.lo \ - H5Tcset.lo H5Tdbg.lo H5Tdeprec.lo H5Tenum.lo H5Tfields.lo \ - H5Tfixed.lo H5Tfloat.lo H5Tinit.lo H5Tnative.lo H5Toffset.lo \ - H5Toh.lo H5Topaque.lo H5Torder.lo H5Tpad.lo H5Tprecis.lo \ - H5Tstrpad.lo H5Tvisit.lo H5Tvlen.lo H5TS.lo H5V.lo H5WB.lo \ - H5Z.lo H5Zdeflate.lo H5Zfletcher32.lo H5Znbit.lo H5Zshuffle.lo \ - H5Zszip.lo H5Zscaleoffset.lo H5Ztrans.lo + H5B2stat.lo H5B2test.lo H5C.lo H5C2.lo H5C2journal.lo H5CS.lo \ + H5D.lo H5Dcompact.lo H5Dcontig.lo H5Ddbg.lo H5Ddeprec.lo \ + H5Defl.lo H5Dfill.lo H5Dint.lo H5Dio.lo H5Distore.lo \ + H5Dmpio.lo H5Doh.lo H5Dselect.lo H5Dtest.lo H5E.lo \ + H5Edeprec.lo H5Eint.lo H5F.lo H5Fdbg.lo H5Ffake.lo H5Fmount.lo \ + H5Fsfile.lo H5Fsuper.lo H5Ftest.lo H5FD.lo H5FDcore.lo \ + H5FDdirect.lo H5FDfamily.lo H5FDlog.lo H5FDmpi.lo H5FDmpio.lo \ + H5FDmpiposix.lo H5FDmulti.lo H5FDsec2.lo H5FDstdio.lo H5FL.lo \ + H5FO.lo H5FS.lo H5FScache.lo H5FSdbg.lo H5FSsection.lo H5G.lo \ + H5Gbtree2.lo H5Gcompact.lo H5Gdense.lo H5Gdeprec.lo H5Gent.lo \ + H5Gint.lo H5Glink.lo H5Gloc.lo H5Gname.lo H5Gnode.lo H5Gobj.lo \ + H5Goh.lo H5Gstab.lo H5Gtest.lo H5Gtraverse.lo H5HF.lo \ + H5HFbtree2.lo H5HFcache.lo H5HFdbg.lo H5HFdblock.lo \ + H5HFdtable.lo H5HFhdr.lo H5HFhuge.lo H5HFiblock.lo H5HFiter.lo \ + H5HFman.lo H5HFsection.lo H5HFspace.lo H5HFstat.lo H5HFtest.lo \ + H5HFtiny.lo H5HG.lo H5HGdbg.lo H5HL.lo H5HLdbg.lo H5HP.lo \ + H5I.lo H5L.lo H5Lexternal.lo H5MF.lo H5MM.lo H5MP.lo \ + H5MPtest.lo H5O.lo H5Oainfo.lo H5Oalloc.lo H5Oattr.lo \ + H5Oattribute.lo H5Obogus.lo H5Obtreek.lo H5Ocache.lo \ + H5Ocont.lo H5Ocopy.lo H5Odbg.lo H5Odrvinfo.lo H5Odtype.lo \ + H5Oefl.lo H5Ofill.lo H5Oginfo.lo H5Olayout.lo H5Olinfo.lo \ + H5Olink.lo H5Omessage.lo H5Omtime.lo H5Oname.lo H5Onull.lo \ + H5Opline.lo H5Orefcount.lo H5Osdspace.lo H5Oshared.lo \ + H5Ostab.lo H5Oshmesg.lo H5Otest.lo H5Ounknown.lo H5P.lo \ + H5Pacpl.lo H5Pdcpl.lo H5Pdeprec.lo H5Pdxpl.lo H5Pfapl.lo \ + H5Pfcpl.lo H5Pfmpl.lo H5Pgcpl.lo H5Pint.lo H5Plapl.lo \ + H5Plcpl.lo H5Pocpl.lo H5Pocpypl.lo H5Pstrcpl.lo H5Ptest.lo \ + H5R.lo H5Rdeprec.lo H5RC.lo H5RS.lo H5S.lo H5Sall.lo H5Sdbg.lo \ + H5Shyper.lo H5Smpio.lo H5Snone.lo H5Spoint.lo H5Sselect.lo \ + H5Stest.lo H5SL.lo H5SM.lo H5SMbtree2.lo H5SMcache.lo \ + H5SMtest.lo H5ST.lo H5T.lo H5Tarray.lo H5Tbit.lo H5Tcommit.lo \ + H5Tcompound.lo H5Tconv.lo H5Tcset.lo H5Tdbg.lo H5Tdeprec.lo \ + H5Tenum.lo H5Tfields.lo H5Tfixed.lo H5Tfloat.lo H5Tinit.lo \ + H5Tnative.lo H5Toffset.lo H5Toh.lo H5Topaque.lo H5Torder.lo \ + H5Tpad.lo H5Tprecis.lo H5Tstrpad.lo H5Tvisit.lo H5Tvlen.lo \ + H5TS.lo H5V.lo H5WB.lo H5Z.lo H5Zdeflate.lo H5Zfletcher32.lo \ + H5Znbit.lo H5Zshuffle.lo H5Zszip.lo H5Zscaleoffset.lo \ + H5Ztrans.lo libhdf5_la_OBJECTS = $(am_libhdf5_la_OBJECTS) libhdf5_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ @@ -402,7 +403,7 @@ libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \ H5AC.c H5AC2.c H5B.c H5Bcache.c \ H5B2.c H5B2cache.c H5B2dbg.c H5B2int.c H5B2stat.c H5B2test.c \ - H5C.c H5C2.c H5CS.c \ + H5C.c H5C2.c H5C2journal.c H5CS.c \ H5D.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ H5Ddeprec.c H5Defl.c H5Dfill.c H5Dint.c \ H5Dio.c \ @@ -601,6 +602,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Bcache.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5C.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5C2.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5C2journal.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5CS.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5D.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dcompact.Plo@am__quote@ |