summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorJohn Mainzer <mainzer@hdfgroup.org>2008-02-18 05:28:04 (GMT)
committerJohn Mainzer <mainzer@hdfgroup.org>2008-02-18 05:28:04 (GMT)
commitcd571e4a45e5af71ae608388141e6b28a16f8171 (patch)
tree27a6b76035cd03c2b13946730225b34f0f03d98d /src
parentd3e926b897b42d27e1c5f8533a8a413c448738e2 (diff)
downloadhdf5-cd571e4a45e5af71ae608388141e6b28a16f8171.zip
hdf5-cd571e4a45e5af71ae608388141e6b28a16f8171.tar.gz
hdf5-cd571e4a45e5af71ae608388141e6b28a16f8171.tar.bz2
[svn-r14594] Ported flash cache size increase code into the journaling branch --
note that both the H5C and H5C2 code have been updated. Also checked in code to track journaling status in the super block. Note that this code has not been tested -- but as best I can tell, it does not break the existing regression tests. Tested serial (debug and production) on Phoenix. Also tested parallel on kagiso. Note that regression test fails on kagiso (but not on phoenix) if the cache2 serial tests are configured to use the core file driver. Thus this code is check in with the core file driver optimization of the cache2 tests disabled. To turn it on, set the USE_CORE_DRIVER #define to TRUE.
Diffstat (limited to 'src')
-rw-r--r--src/H5AC.c23
-rw-r--r--src/H5AC2.c27
-rw-r--r--src/H5AC2private.h4
-rw-r--r--src/H5AC2public.h60
-rw-r--r--src/H5ACprivate.h4
-rw-r--r--src/H5ACpublic.h61
-rw-r--r--src/H5C.c993
-rw-r--r--src/H5C2.c599
-rw-r--r--src/H5C2journal.c519
-rw-r--r--src/H5C2pkg.h14
-rw-r--r--src/H5C2private.h104
-rw-r--r--src/H5C2public.h7
-rw-r--r--src/H5Cpkg.h22
-rw-r--r--src/H5Cprivate.h130
-rw-r--r--src/H5Cpublic.h6
-rw-r--r--src/H5Fpkg.h5
-rw-r--r--src/H5Fsuper.c105
-rw-r--r--src/H5O.c15
-rw-r--r--src/H5Opkg.h7
-rw-r--r--src/H5Oprivate.h59
-rwxr-xr-xsrc/Makefile.am2
-rw-r--r--src/Makefile.in78
22 files changed, 2664 insertions, 180 deletions
diff --git a/src/H5AC.c b/src/H5AC.c
index 8f62095..cd79e6a 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -2546,6 +2546,10 @@ done:
* switches for functions that are only tenuously related
* to auto resize configuration.
*
+ * JRM - 1/2/08
+ * Added support for the new flash cache increment related
+ * fields.
+ *
*-------------------------------------------------------------------------
*/
@@ -2625,6 +2629,9 @@ H5AC_get_cache_auto_resize_config(H5AC_t * cache_ptr,
config_ptr->max_increment = internal_config.max_increment;
config_ptr->decr_mode = internal_config.decr_mode;
config_ptr->upper_hr_threshold = internal_config.upper_hr_threshold;
+ config_ptr->flash_incr_mode = internal_config.flash_incr_mode;
+ config_ptr->flash_multiple = internal_config.flash_multiple;
+ config_ptr->flash_threshold = internal_config.flash_threshold;
config_ptr->decrement = internal_config.decrement;
config_ptr->apply_max_decrement = internal_config.apply_max_decrement;
config_ptr->max_decrement = internal_config.max_decrement;
@@ -2816,6 +2823,10 @@ done:
* switches for functions that are only tenuously related
* to auto resize configuration.
*
+ * John Mainzer -- 1/3/07
+ * Updated trace file code to record the new flash cache
+ * size increase related fields.
+ *
*-------------------------------------------------------------------------
*/
@@ -2969,7 +2980,7 @@ done:
( trace_file_ptr != NULL ) ) {
HDfprintf(trace_file_ptr,
- "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %d %d %f %f %d %d %d %d %f %d %d\n",
+ "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %f %f %d %d %d %f %f %d %d %d %d %f %d %d\n",
"H5AC_set_cache_auto_resize_config",
trace_config.version,
(int)(trace_config.rpt_fcn_enabled),
@@ -2986,6 +2997,9 @@ done:
(int)(trace_config.incr_mode),
trace_config.lower_hr_threshold,
trace_config.increment,
+ (int)(trace_config.flash_incr_mode),
+ trace_config.flash_multiple,
+ trace_config.flash_threshold,
(int)(trace_config.apply_max_increment),
(int)(trace_config.max_increment),
(int)(trace_config.decr_mode),
@@ -3618,7 +3632,9 @@ done:
*
* Modifications:
*
- * None.
+ * Updated function for flash cache increment fields.
+ *
+ * JRM -- 1/2/08
*
*-------------------------------------------------------------------------
*/
@@ -3661,6 +3677,9 @@ H5AC_ext_config_2_int_config(H5AC_cache_config_t * ext_conf_ptr,
int_conf_ptr->increment = ext_conf_ptr->increment;
int_conf_ptr->apply_max_increment = ext_conf_ptr->apply_max_increment;
int_conf_ptr->max_increment = ext_conf_ptr->max_increment;
+ int_conf_ptr->flash_incr_mode = ext_conf_ptr->flash_incr_mode;
+ int_conf_ptr->flash_multiple = ext_conf_ptr->flash_multiple;
+ int_conf_ptr->flash_threshold = ext_conf_ptr->flash_threshold;
int_conf_ptr->decr_mode = ext_conf_ptr->decr_mode;
int_conf_ptr->upper_hr_threshold = ext_conf_ptr->upper_hr_threshold;
diff --git a/src/H5AC2.c b/src/H5AC2.c
index a053b36..49e4934 100644
--- a/src/H5AC2.c
+++ b/src/H5AC2.c
@@ -2587,6 +2587,10 @@ done:
* Modified code in support of revised cache API needed
* to permit journaling.
*
+ * JRM - 1/2/08
+ * Added support for the new flash cache increment related
+ * fields.
+ *
*-------------------------------------------------------------------------
*/
@@ -2665,6 +2669,9 @@ H5AC2_get_cache_auto_resize_config(H5AC2_t * cache_ptr,
config_ptr->increment = internal_config.increment;
config_ptr->apply_max_increment = internal_config.apply_max_increment;
config_ptr->max_increment = internal_config.max_increment;
+ config_ptr->flash_incr_mode = internal_config.flash_incr_mode;
+ config_ptr->flash_multiple = internal_config.flash_multiple;
+ config_ptr->flash_threshold = internal_config.flash_threshold;
config_ptr->decr_mode = internal_config.decr_mode;
config_ptr->upper_hr_threshold = internal_config.upper_hr_threshold;
config_ptr->decrement = internal_config.decrement;
@@ -2868,6 +2875,10 @@ done:
* Modified code in support of revised cache API needed
* to permit journaling.
*
+ * John Mainzer -- 1/3/07
+ * Updated trace file code to record the new flash cache
+ * size increase related fields.
+ *
*-------------------------------------------------------------------------
*/
@@ -3021,7 +3032,7 @@ done:
( trace_file_ptr != NULL ) ) {
HDfprintf(trace_file_ptr,
- "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %d %d %f %f %d %d %d %d %f %d %d\n",
+ "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %d %d %f %f %d %f %f %d %d %d %d %f %d %d\n",
"H5AC2_set_cache_auto_resize_config",
trace_config.version,
(int)(trace_config.rpt_fcn_enabled),
@@ -3040,6 +3051,9 @@ done:
trace_config.increment,
(int)(trace_config.apply_max_increment),
(int)(trace_config.max_increment),
+ (int)(trace_config.flash_incr_mode),
+ trace_config.flash_multiple,
+ trace_config.flash_threshold,
(int)(trace_config.decr_mode),
trace_config.upper_hr_threshold,
trace_config.decrement,
@@ -3379,7 +3393,7 @@ H5AC2_open_trace_file(H5AC2_t * cache_ptr,
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "trace file open failed.")
}
- HDfprintf(file_ptr, "### HDF5 metadata cache trace file version 1 ###\n");
+ HDfprintf(file_ptr, "### HDF5 metadata cache trace file version 2 ###\n");
if ( H5C2_set_trace_file_ptr(cache_ptr, file_ptr) < 0 ) {
@@ -3678,7 +3692,10 @@ done:
*
* Modifications:
*
- * None.
+ * Updated function for flash cache increment fields.
+ *
+ * JRM -- 1/2/08
+ *
*
*-------------------------------------------------------------------------
*/
@@ -3721,6 +3738,10 @@ H5AC2_ext_config_2_int_config(H5AC2_cache_config_t * ext_conf_ptr,
int_conf_ptr->increment = ext_conf_ptr->increment;
int_conf_ptr->apply_max_increment = ext_conf_ptr->apply_max_increment;
int_conf_ptr->max_increment = ext_conf_ptr->max_increment;
+ int_conf_ptr->flash_incr_mode = ext_conf_ptr->flash_incr_mode;
+ int_conf_ptr->flash_multiple = ext_conf_ptr->flash_multiple;
+ int_conf_ptr->flash_threshold = ext_conf_ptr->flash_threshold;
+
int_conf_ptr->decr_mode = ext_conf_ptr->decr_mode;
int_conf_ptr->upper_hr_threshold = ext_conf_ptr->upper_hr_threshold;
diff --git a/src/H5AC2private.h b/src/H5AC2private.h
index de76b40..ca2522f 100644
--- a/src/H5AC2private.h
+++ b/src/H5AC2private.h
@@ -202,6 +202,10 @@ extern hid_t H5AC2_ind_dxpl_id;
/* double increment = */ 2.0, \
/* hbool_t apply_max_increment = */ TRUE, \
/* size_t max_increment = */ (4 * 1024 * 1024), \
+ /* enum H5C2_cache_flash_incr_mode */ \
+ /* flash_incr_mode = */ H5C2_flash_incr__add_space, \
+ /* double flash_multiple = */ 1.0, \
+ /* double flash_threshold = */ 0.25, \
/* enum H5C2_cache_decr_mode decr_mode = */ \
H5C2_decr__age_out_with_threshold, \
/* double upper_hr_threshold = */ 0.999, \
diff --git a/src/H5AC2public.h b/src/H5AC2public.h
index 3c88ef1..a16ad2a 100644
--- a/src/H5AC2public.h
+++ b/src/H5AC2public.h
@@ -15,7 +15,7 @@
/*-------------------------------------------------------------------------
*
- * Created: H5ACpublic.h
+ * Created: H5AC2public.h
* Jul 10 1997
* Robb Matzke <matzke@llnl.gov>
*
@@ -208,6 +208,60 @@ extern "C" {
* above, this field contains the maximum number of bytes by which the
* cache size can be increased in a single re-size.
*
+ * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated
+ * type whose value indicates whether and by which algorithm we should
+ * make flash increases in the size of the cache to accomodate insertion
+ * of large entries and large increases in the size of a single entry.
+ *
+ * The addition of the flash increment mode was occasioned by performance
+ * problems that appear when a local heap is increased to a size in excess
+ * of the current cache size. While the existing re-size code dealt with
+ * this eventually, performance was very bad for the remainder of the
+ * epoch.
+ *
+ * At present, there are two possible values for the flash_incr_mode:
+ *
+ * H5C_flash_incr__off: Don't perform flash increases in the size of
+ * the cache.
+ *
+ * H5C_flash_incr__add_space: Let x be either the size of a newly
+ * newly inserted entry, or the number of bytes by which the
+ * size of an existing entry has been increased.
+ *
+ * If
+ * x > flash_threshold * current max cache size,
+ *
+ * increase the current maximum cache size by x * flash_multiple
+ * less any free space in the cache, and star a new epoch. For
+ * now at least, pay no attention to the maximum increment.
+ *
+ * In both of the above cases, the flash increment pays no attention to
+ * the maximum increment (at least in this first incarnation), but DOES
+ * stay within max_size.
+ *
+ * With a little thought, it should be obvious that the above flash
+ * cache size increase algorithm is not sufficient for all circumstances
+ * -- for example, suppose the user round robins through
+ * (1/flash_threshold) +1 groups, adding one data set to each on each
+ * pass. Then all will increase in size at about the same time, requiring
+ * the max cache size to at least double to maintain acceptable
+ * performance, however the above flash increment algorithm will not be
+ * triggered.
+ *
+ * Hopefully, the add space algorithms detailed above will be sufficient
+ * for the performance problems encountered to date. However, we should
+ * expect to revisit the issue.
+ *
+ * flash_multiple: Double containing the multiple described above in the
+ * H5C_flash_incr__add_space section of the discussion of the
+ * flash_incr_mode section. This field is ignored unless flash_incr_mode
+ * is H5C_flash_incr__add_space.
+ *
+ * flash_threshold: Double containing the factor by which current max cache
+ * size is multiplied to obtain the size threshold for the add_space flash
+ * increment algorithm. The field is ignored unless flash_incr_mode is
+ * H5C_flash_incr__add_space.
+ *
*
* Cache size decrease control fields:
*
@@ -364,6 +418,10 @@ typedef struct H5AC2_cache_config_t
hbool_t apply_max_increment;
size_t max_increment;
+ enum H5C2_cache_flash_incr_mode flash_incr_mode;
+ double flash_multiple;
+ double flash_threshold;
+
/* size decrease control fields: */
enum H5C2_cache_decr_mode decr_mode;
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index 628a269..bbbcf11 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -213,6 +213,10 @@ extern hid_t H5AC_ind_dxpl_id;
/* double increment = */ 2.0, \
/* hbool_t apply_max_increment = */ TRUE, \
/* size_t max_increment = */ (4 * 1024 * 1024), \
+ /* enum H5C_cache_flash_incr_mode */ \
+ /* flash_incr_mode = */ H5C_flash_incr__add_space, \
+ /* double flash_multiple = */ 1.0, \
+ /* double flash_threshold = */ 0.25, \
/* enum H5C_cache_decr_mode decr_mode = */ H5C_decr__age_out_with_threshold,\
/* double upper_hr_threshold = */ 0.999, \
/* double decrement = */ 0.9, \
diff --git a/src/H5ACpublic.h b/src/H5ACpublic.h
index 0ddef92..0e75117 100644
--- a/src/H5ACpublic.h
+++ b/src/H5ACpublic.h
@@ -15,7 +15,7 @@
/*-------------------------------------------------------------------------
*
- * Created: H5ACproto.h
+ * Created: H5ACpublic.h
* Jul 10 1997
* Robb Matzke <matzke@llnl.gov>
*
@@ -208,6 +208,61 @@ extern "C" {
* above, this field contains the maximum number of bytes by which the
* cache size can be increased in a single re-size.
*
+ * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated
+ * type whose value indicates whether and by which algorithm we should
+ * make flash increases in the size of the cache to accomodate insertion
+ * of large entries and large increases in the size of a single entry.
+ *
+ * The addition of the flash increment mode was occasioned by performance
+ * problems that appear when a local heap is increased to a size in excess
+ * of the current cache size. While the existing re-size code dealt with
+ * this eventually, performance was very bad for the remainder of the
+ * epoch.
+ *
+ * At present, there are two possible values for the flash_incr_mode:
+ *
+ * H5C_flash_incr__off: Don't perform flash increases in the size of
+ * the cache.
+ *
+ * H5C_flash_incr__add_space: Let x be either the size of a newly
+ * newly inserted entry, or the number of bytes by which the
+ * size of an existing entry has been increased.
+ *
+ * If
+ * x > flash_threshold * current max cache size,
+ *
+ * increase the current maximum cache size by x * flash_multiple
+ * less any free space in the cache, and star a new epoch. For
+ * now at least, pay no attention to the maximum increment.
+ *
+ * In both of the above cases, the flash increment pays no attention to
+ * the maximum increment (at least in this first incarnation), but DOES
+ * stay within max_size.
+ *
+ * With a little thought, it should be obvious that the above flash
+ * cache size increase algorithm is not sufficient for all circumstances --
+ * for example, suppose the user round robins through
+ * (1/flash_threshold) +1 groups, adding one data set to each on each
+ * pass. Then all will increase in size at about the same time, requiring
+ * the max cache size to at least double to maintain acceptable
+ * performance, however the above flash increment algorithm will not be
+ * triggered.
+ *
+ * Hopefully, the add space algorithms detailed above will be sufficient
+ * for the performance problems encountered to date. However, we should
+ * expect to revisit the issue.
+ *
+ * flash_multiple: Double containing the multiple described above in the
+ * H5C_flash_incr__add_space section of the discussion of the
+ * flash_incr_mode section. This field is ignored unless flash_incr_mode
+ * is H5C_flash_incr__add_space.
+ *
+ * flash_threshold: Double containing the factor by which current max cache size
+ * is multiplied to obtain the size threshold for the add_space flash
+ * increment algorithm. The field is ignored unless flash_incr_mode is
+ * H5C_flash_incr__add_space.
+ *
+ *
*
* Cache size decrease control fields:
*
@@ -364,6 +419,10 @@ typedef struct H5AC_cache_config_t
hbool_t apply_max_increment;
size_t max_increment;
+ enum H5C_cache_flash_incr_mode flash_incr_mode;
+ double flash_multiple;
+ double flash_threshold;
+
/* size decrease control fields: */
enum H5C_cache_decr_mode decr_mode;
diff --git a/src/H5C.c b/src/H5C.c
index 8ac9929..daa5c8b 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -2596,6 +2596,10 @@ static herr_t H5C__autoadjust__ageout__remove_all_markers(H5C_t * cache_ptr);
static herr_t H5C__autoadjust__ageout__remove_excess_markers(H5C_t * cache_ptr);
+static herr_t H5C__flash_increase_cache_size(H5C_t * cache_ptr,
+ size_t old_entry_size,
+ size_t new_entry_size);
+
static herr_t H5C_flush_single_entry(H5F_t * f,
hid_t primary_dxpl_id,
hid_t secondary_dxpl_id,
@@ -2836,8 +2840,12 @@ done:
* ro_ref_count fields.
*
* JRM -- 7/27/07
-* Added initialization for the new evictions_enabled
-* field of H5C_t.
+ * Added initialization for the new evictions_enabled
+ * field of H5C_t.
+ *
+ * JRM -- 12/31/07
+ * Added initialization for the new flash cache size increase
+ * related fields of H5C_t.
*
*-------------------------------------------------------------------------
*/
@@ -2954,6 +2962,8 @@ H5C_create(size_t max_cache_size,
cache_ptr->dLRU_tail_ptr = NULL;
cache_ptr->size_increase_possible = FALSE;
+ cache_ptr->flash_size_increase_possible = FALSE;
+ cache_ptr->flash_size_increase_threshold = 0;
cache_ptr->size_decrease_possible = FALSE;
cache_ptr->resize_enabled = FALSE;
cache_ptr->cache_full = FALSE;
@@ -2974,6 +2984,11 @@ H5C_create(size_t max_cache_size,
(cache_ptr->resize_ctl).apply_max_increment = TRUE;
(cache_ptr->resize_ctl).max_increment = H5C__DEF_AR_MAX_INCREMENT;
+ (cache_ptr->resize_ctl).flash_incr_mode = H5C_flash_incr__off;
+ (cache_ptr->resize_ctl).flash_multiple = 1.0;
+ (cache_ptr->resize_ctl).flash_threshold = 0.25;
+
+
(cache_ptr->resize_ctl).decr_mode = H5C_decr__off;
(cache_ptr->resize_ctl).upper_hr_threshold = H5C__DEF_AR_UPPER_THRESHHOLD;
(cache_ptr->resize_ctl).decrement = H5C__DEF_AR_DECREMENT;
@@ -2993,7 +3008,10 @@ H5C_create(size_t max_cache_size,
for ( i = 0; i < H5C__MAX_EPOCH_MARKERS; i++ )
{
(cache_ptr->epoch_marker_active)[i] = FALSE;
-
+#ifndef NDEBUG
+ ((cache_ptr->epoch_markers)[i]).magic =
+ H5C__H5C_CACHE_ENTRY_T_MAGIC;
+#endif /* NDEBUG */
((cache_ptr->epoch_markers)[i]).addr = (haddr_t)i;
((cache_ptr->epoch_markers)[i]).size = (size_t)0;
((cache_ptr->epoch_markers)[i]).type = &epoch_marker_class;
@@ -3079,6 +3097,9 @@ done:
* Updated function for display the new prefix field of
* H5C_t in output.
*
+ * JRM 12/31/07
+ * Updated function to handle flash size increases.
+ *
*-------------------------------------------------------------------------
*/
void
@@ -3125,6 +3146,24 @@ H5C_def_auto_resize_rpt_fcn(H5C_t * cache_ptr,
new_min_clean_size);
break;
+ case flash_increase:
+ HDassert( old_max_cache_size < new_max_cache_size );
+
+ HDfprintf(stdout,
+ "%sflash cache resize(%d) -- size threshold = %Zu.\n",
+ cache_ptr->prefix,
+ (int)((cache_ptr->resize_ctl).flash_incr_mode),
+ cache_ptr->flash_size_increase_threshold);
+
+ HDfprintf(stdout,
+ "%s cache size increased from (%Zu/%Zu) to (%Zu/%Zu).\n",
+ cache_ptr->prefix,
+ old_max_cache_size,
+ old_min_clean_size,
+ new_max_cache_size,
+ new_min_clean_size);
+ break;
+
case decrease:
HDassert( old_max_cache_size > new_max_cache_size );
@@ -3537,6 +3576,21 @@ done:
* callbacks. As a result, we may have to make multiple
* passes through the skip list before the cache is flushed.
*
+ * JRM -- 10/13/07
+ * Added code to detect and manage the case in which a
+ * flush callback changes the s-list out from under
+ * the function. The only way I can think of in which this
+ * can happen is if a flush function loads an entry
+ * into the cache that isn't there already. Quincey tells
+ * me that this will never happen, but I'm not sure I
+ * believe him.
+ *
+ * Note that this is a pretty bad scenario if it ever
+ * happens. The code I have added should allow us to
+ * handle the situation under all but the worst conditions,
+ * but one can argue that I should just scream and die if I
+ * ever detect the condidtion.
+ *
*-------------------------------------------------------------------------
*/
herr_t
@@ -3558,6 +3612,7 @@ H5C_flush_cache(H5F_t * f,
int32_t protected_entries = 0;
H5SL_node_t * node_ptr = NULL;
H5C_cache_entry_t * entry_ptr = NULL;
+ H5C_cache_entry_t * next_entry_ptr = NULL;
#if H5C_DO_SANITY_CHECKS
int64_t flushed_entries_count;
size_t flushed_entries_size;
@@ -3620,6 +3675,28 @@ H5C_flush_cache(H5F_t * f,
{
flushed_entries_last_pass = FALSE;
node_ptr = H5SL_first(cache_ptr->slist_ptr);
+
+ if ( node_ptr != NULL ) {
+
+ next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr);
+
+ if ( next_entry_ptr == NULL ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "next_entry_ptr == NULL 1 ?!?!");
+ }
+#ifndef NDEBUG
+ HDassert( next_entry_ptr->magic ==
+ H5C__H5C_CACHE_ENTRY_T_MAGIC );
+#endif /* NDEBUG */
+ HDassert( next_entry_ptr->is_dirty );
+ HDassert( next_entry_ptr->in_slist );
+
+ } else {
+
+ next_entry_ptr = NULL;
+
+ }
HDassert( node_ptr != NULL );
@@ -3667,13 +3744,76 @@ H5C_flush_cache(H5F_t * f,
while ( node_ptr != NULL )
{
- entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr);
+ entry_ptr = next_entry_ptr;
+
+ /* With the advent of the fractal heap, it is possible
+ * that the flush callback will dirty and/or resize
+ * other entries in the cache. In particular, while
+ * Quincey has promised me that this will never happen,
+ * it is possible that the flush callback for an
+ * entry may protect an entry that is not in the cache,
+ * perhaps causing the cache to flush and possibly
+ * evict the entry associated with node_ptr to make
+ * space for the new entry.
+ *
+ * Thus we do a bit of extra sanity checking on entry_ptr,
+ * and break out of this scan of the skip list if we
+ * detect minor problems. We have a bit of leaway on the
+ * number of passes though the skip list, so this shouldn't
+ * be an issue in the flush in and of itself, as it should
+ * be all but impossible for this to happen more than once
+ * in any flush.
+ *
+ * Observe that that breaking out of the scan early
+ * shouldn't break the sanity checks just after the end
+ * of this while loop.
+ *
+ * If an entry has merely been marked clean and removed from
+ * the s-list, we simply break out of the scan.
+ *
+ * If the entry has been evicted, we flag an error and
+ * exit.
+ */
+#ifndef NDEBUG
+ if ( entry_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "entry_ptr->magic invalid ?!?!");
+
+ } else
+#endif /* NDEBUG */
+ if ( ( ! entry_ptr->is_dirty ) ||
+ ( ! entry_ptr->in_slist ) ) {
+
+ /* the s-list has been modified out from under us.
+ * set node_ptr to NULL and break out of the loop.
+ */
+ node_ptr = NULL;
+ break;
+ }
/* increment node pointer now, before we delete its target
* from the slist.
*/
node_ptr = H5SL_next(node_ptr);
+ if ( node_ptr != NULL ) {
+ next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr);
+
+ if ( next_entry_ptr == NULL ) {
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "next_entry_ptr == NULL 2 ?!?!");
+ }
+#ifndef NDEBUG
+ HDassert( next_entry_ptr->magic ==
+ H5C__H5C_CACHE_ENTRY_T_MAGIC );
+#endif /* NDEBUG */
+ HDassert( next_entry_ptr->is_dirty );
+ HDassert( next_entry_ptr->in_slist );
+ } else {
+ next_entry_ptr = NULL;
+ }
+
HDassert( entry_ptr != NULL );
HDassert( entry_ptr->in_slist );
@@ -3689,6 +3829,40 @@ H5C_flush_cache(H5F_t * f,
tried_to_flush_protected_entry = TRUE;
protected_entries++;
+ } else if ( entry_ptr->is_pinned ) {
+ /* Test to see if we are can flush the entry now.
+ * If we can, go ahead and flush. Note that we
+ * aren't trying to do a destroy here, so that
+ * is not an issue.
+ */
+ if ( TRUE ) { /* When we get to multithreaded cache,
+ * we will need either locking code,
+ * and/or a test to see if the entry
+ * is in flushable condition here.
+ */
+#if H5C_DO_SANITY_CHECKS
+ flushed_entries_count++;
+ flushed_entries_size += entry_ptr->size;
+#endif /* H5C_DO_SANITY_CHECKS */
+ status = H5C_flush_single_entry(f,
+ primary_dxpl_id,
+ secondary_dxpl_id,
+ cache_ptr,
+ NULL,
+ entry_ptr->addr,
+ flags,
+ &first_flush,
+ FALSE);
+ if ( status < 0 ) {
+
+ /* This shouldn't happen -- if it does, we are
+ * toast so just scream and die.
+ */
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
+ "dirty pinned entry flush failed.")
+ }
+ flushed_entries_last_pass = TRUE;
+ }
} else {
#if H5C_DO_SANITY_CHECKS
flushed_entries_count++;
@@ -4434,6 +4608,9 @@ done:
* Added code to disable evictions when the new
* evictions_enabled field is FALSE.
*
+ * JRM -- 12/31/07
+ * Added code supporting flash cache size increases.
+ *
*-------------------------------------------------------------------------
*/
@@ -4447,6 +4624,7 @@ H5C_insert_entry(H5F_t * f,
void * thing,
unsigned int flags)
{
+ /* const char * fcn_name = "H5C_insert_entry()"; */
herr_t result;
herr_t ret_value = SUCCEED; /* Return value */
hbool_t first_flush = TRUE;
@@ -4486,7 +4664,9 @@ H5C_insert_entry(H5F_t * f,
insert_pinned = ( (flags & H5C__PIN_ENTRY_FLAG) != 0 );
entry_ptr = (H5C_cache_entry_t *)thing;
-
+#ifndef NDEBUG
+ entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC;
+#endif /* NDEBUG */
entry_ptr->addr = addr;
entry_ptr->type = type;
@@ -4524,6 +4704,18 @@ H5C_insert_entry(H5F_t * f,
H5C__RESET_CACHE_ENTRY_STATS(entry_ptr)
+ if ( ( cache_ptr->flash_size_increase_possible ) &&
+ ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) {
+
+ result = H5C__flash_increase_cache_size(cache_ptr, 0, entry_ptr->size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, \
+ "H5C__flash_increase_cache_size failed.")
+ }
+ }
+
if ( ( cache_ptr->evictions_enabled ) &&
( (cache_ptr->index_size + entry_ptr->size) >
cache_ptr->max_cache_size ) ) {
@@ -4999,7 +5191,9 @@ done:
*
* Modifications:
*
- * None
+ * Added code to do a flash cache size increase if
+ * appropriate.
+ * JRM -- 1/11/08
*
*-------------------------------------------------------------------------
*/
@@ -5010,6 +5204,8 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr,
size_t new_size)
{
herr_t ret_value = SUCCEED; /* Return value */
+ herr_t result;
+ size_t size_increase;
H5C_cache_entry_t * entry_ptr;
FUNC_ENTER_NOAPI(H5C_mark_pinned_entry_dirty, FAIL)
@@ -5039,6 +5235,29 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr,
/* update for change in entry size if necessary */
if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) {
+ /* do a flash cache size increase if appropriate */
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ if ( new_size > entry_ptr->size ) {
+
+ size_increase = new_size - entry_ptr->size;
+
+ if ( size_increase >=
+ cache_ptr->flash_size_increase_threshold ) {
+
+ result = H5C__flash_increase_cache_size(cache_ptr,
+ entry_ptr->size,
+ new_size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \
+ "H5C__flash_increase_cache_size failed.")
+ }
+ }
+ }
+ }
+
/* update the protected entry list */
H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \
(cache_ptr->pel_size), \
@@ -5371,7 +5590,9 @@ done:
*
* Modifications:
*
- * None
+ * Added code to apply a flash cache size increment if
+ * appropriate.
+ * JRM -- 1/11/08
*
*-------------------------------------------------------------------------
*/
@@ -5380,8 +5601,11 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr,
void * thing,
size_t new_size)
{
+ /* const char * fcn_name = "H5C_resize_pinned_entry()"; */
herr_t ret_value = SUCCEED; /* Return value */
+ herr_t result;
H5C_cache_entry_t * entry_ptr;
+ size_t size_increase;
FUNC_ENTER_NOAPI(H5C_resize_pinned_entry, FAIL)
@@ -5417,6 +5641,29 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr,
/* update for change in entry size if necessary */
if ( entry_ptr->size != new_size ) {
+ /* do a flash cache size increase if appropriate */
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ if ( new_size > entry_ptr->size ) {
+
+ size_increase = new_size - entry_ptr->size;
+
+ if ( size_increase >=
+ cache_ptr->flash_size_increase_threshold ) {
+
+ result = H5C__flash_increase_cache_size(cache_ptr,
+ entry_ptr->size,
+ new_size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \
+ "H5C__flash_increase_cache_size failed.")
+ }
+ }
+ }
+ }
+
/* update the protected entry list */
H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \
(cache_ptr->pel_size), \
@@ -5604,9 +5851,13 @@ done:
* of cache entries.
*
* JRM -- 7/27/07
- * Added code supporting the new evictions_enabled fieled
+ * Added code supporting the new evictions_enabled field
* in H5C_t.
*
+ * JRM -- 1/3/08
+ * Added to do a flash cache size increase if appropriate
+ * when a large entry is loaded.
+ *
*-------------------------------------------------------------------------
*/
@@ -5621,6 +5872,7 @@ H5C_protect(H5F_t * f,
void * udata2,
unsigned flags)
{
+ /* const char * fcn_name = "H5C_protect()"; */
hbool_t hit;
hbool_t first_flush;
hbool_t have_write_permitted = FALSE;
@@ -5678,7 +5930,25 @@ H5C_protect(H5F_t * f,
entry_ptr = (H5C_cache_entry_t *)thing;
- /* try to free up some space if necessary and if evictions are permitted */
+ /* If the entry is very large, and we are configured to allow it,
+ * we may wish to perform a flash cache size increase.
+ */
+ if ( ( cache_ptr->flash_size_increase_possible ) &&
+ ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) {
+
+ result = H5C__flash_increase_cache_size(cache_ptr, 0,
+ entry_ptr->size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, \
+ "H5C__flash_increase_cache_size failed.")
+ }
+ }
+
+ /* try to free up some space if necessary and if evictions are
+ * permitted
+ */
if ( ( cache_ptr->evictions_enabled ) &&
( (cache_ptr->index_size + entry_ptr->size) >
cache_ptr->max_cache_size ) ) {
@@ -5978,6 +6248,10 @@ done:
* if the new configuration forces an immediate reduction
* in cache size.
*
+ * JRM -- 12/31/07
+ * Added code supporting the new flash cache size increase
+ * code.
+ *
*-------------------------------------------------------------------------
*/
@@ -5985,6 +6259,7 @@ herr_t
H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
H5C_auto_size_ctl_t *config_ptr)
{
+ /* const char *fcn_name = "H5C_set_cache_auto_resize_config()"; */
herr_t ret_value = SUCCEED; /* Return value */
herr_t result;
size_t new_max_cache_size;
@@ -6039,8 +6314,10 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
"conflicting threshold fields in new config.")
}
- cache_ptr->size_increase_possible = TRUE; /* will set to FALSE if needed */
- cache_ptr->size_decrease_possible = TRUE; /* will set to FALSE if needed */
+ /* will set the increase possible fields to FALSE later if needed */
+ cache_ptr->size_increase_possible = TRUE;
+ cache_ptr->flash_size_increase_possible = TRUE;
+ cache_ptr->size_decrease_possible = TRUE;
switch ( config_ptr->incr_mode )
{
@@ -6062,6 +6339,11 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown incr_mode?!?!?.")
}
+ /* logically, this is were configuration for flash cache size increases
+ * should go. However, this configuration depends on max_cache_size, so
+ * we wait until the end of the function, when this field is set.
+ */
+
switch ( config_ptr->decr_mode )
{
case H5C_decr__off:
@@ -6106,9 +6388,13 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
if ( config_ptr->max_size == config_ptr->min_size ) {
cache_ptr->size_increase_possible = FALSE;
+ cache_ptr->flash_size_increase_possible = FALSE;
cache_ptr->size_decrease_possible = FALSE;
}
+ /* flash_size_increase_possible is intentionally omitted from the
+ * following:
+ */
cache_ptr->resize_enabled = cache_ptr->size_increase_possible ||
cache_ptr->size_decrease_possible;
@@ -6196,6 +6482,37 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
}
}
+ /* configure flash size increase facility. We wait until the
+ * end of the function, as we need the max_cache_size set before
+ * we start to keep things simple.
+ *
+ * If we haven't already ruled out flash cache size increases above,
+ * go ahead and configure it.
+ */
+
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ switch ( config_ptr->flash_incr_mode )
+ {
+ case H5C_flash_incr__off:
+ cache_ptr->flash_size_increase_possible = FALSE;
+ break;
+
+ case H5C_flash_incr__add_space:
+ cache_ptr->flash_size_increase_possible = TRUE;
+ cache_ptr->flash_size_increase_threshold =
+ (size_t)
+ (((double)(cache_ptr->max_cache_size)) *
+ ((cache_ptr->resize_ctl).flash_threshold));
+ break;
+
+ default: /* should be unreachable */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "Unknown flash_incr_mode?!?!?.")
+ break;
+ }
+ }
+
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -7095,6 +7412,9 @@ done:
* Also added sanity checks using the new is_read_only and
* ro_ref_count parameters.
*
+ * JRM -- 12/31/07
+ * Modified funtion to support flash cache resizes.
+ *
*-------------------------------------------------------------------------
*/
herr_t
@@ -7108,6 +7428,7 @@ H5C_unprotect(H5F_t * f,
unsigned int flags,
size_t new_size)
{
+ /* const char * fcn_name = "H5C_unprotect()"; */
hbool_t deleted;
hbool_t dirtied;
hbool_t set_flush_marker;
@@ -7118,6 +7439,8 @@ H5C_unprotect(H5F_t * f,
hbool_t clear_entry = FALSE;
#endif /* H5_HAVE_PARALLEL */
herr_t ret_value = SUCCEED; /* Return value */
+ herr_t result;
+ size_t size_increase = 0;
H5C_cache_entry_t * entry_ptr;
H5C_cache_entry_t * test_entry_ptr;
@@ -7265,6 +7588,29 @@ H5C_unprotect(H5F_t * f,
/* update for change in entry size if necessary */
if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) {
+ /* do a flash cache size increase if appropriate */
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ if ( new_size > entry_ptr->size ) {
+
+ size_increase = new_size - entry_ptr->size;
+
+ if ( size_increase >=
+ cache_ptr->flash_size_increase_threshold ) {
+
+ result = H5C__flash_increase_cache_size(cache_ptr,
+ entry_ptr->size,
+ new_size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \
+ "H5C__flash_increase_cache_size failed.")
+ }
+ }
+ }
+ }
+
/* update the protected list */
H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pl_len), \
(cache_ptr->pl_size), \
@@ -7458,7 +7804,9 @@ done:
*
* Modifications:
*
- * None.
+ * Added validation for the flash increment fields.
+ *
+ * JRM -- 12/31/07
*
*-------------------------------------------------------------------------
*/
@@ -7560,7 +7908,7 @@ H5C_validate_resize_config(H5C_auto_size_ctl_t * config_ptr,
( config_ptr->apply_max_increment != FALSE ) ) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
- "apply_max_increment must be either TRUE or FALSE");
+ "apply_max_increment must be either TRUE or FALSE");
}
/* no need to check max_increment, as it is a size_t,
@@ -7568,6 +7916,33 @@ H5C_validate_resize_config(H5C_auto_size_ctl_t * config_ptr,
*/
} /* H5C_incr__threshold */
+ switch ( config_ptr->flash_incr_mode )
+ {
+ case H5C_flash_incr__off:
+ /* nothing to do here */
+ break;
+
+ case H5C_flash_incr__add_space:
+ if ( ( config_ptr->flash_multiple < 0.1 ) ||
+ ( config_ptr->flash_multiple > 10.0 ) ) {
+
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
+ "flash_multiple must be in the range [0.1, 10.0]");
+ }
+
+ if ( ( config_ptr->flash_threshold < 0.1 ) ||
+ ( config_ptr->flash_threshold > 1.0 ) ) {
+
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
+ "flash_threshold must be in the range [0.1, 1.0]");
+ }
+ break;
+
+ default:
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
+ "Invalid flash_incr_mode");
+ break;
+ }
} /* H5C_RESIZE_CFG__VALIDATE_INCREMENT */
@@ -7707,6 +8082,9 @@ done:
* reduction, and to adjust to changes in the
* H5C_auto_size_ctl_t structure.
*
+ * JRM -- 1/5/08
+ * Added support for flash cache size increases.
+ *
*-------------------------------------------------------------------------
*/
@@ -7990,6 +8368,30 @@ H5C__auto_adjust_cache_size(H5C_t * cache_ptr,
cache_ptr->size_decreased = TRUE;
}
+
+ /* update flash cache size increase fields as appropriate */
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ switch ( (cache_ptr->resize_ctl).flash_incr_mode )
+ {
+ case H5C_flash_incr__off:
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "flash_size_increase_possible but H5C_flash_incr__off?!")
+ break;
+
+ case H5C_flash_incr__add_space:
+ cache_ptr->flash_size_increase_threshold =
+ (size_t)
+ (((double)(cache_ptr->max_cache_size)) *
+ ((cache_ptr->resize_ctl).flash_threshold));
+ break;
+
+ default: /* should be unreachable */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "Unknown flash_incr_mode?!?!?.")
+ break;
+ }
+ }
}
if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) {
@@ -8308,7 +8710,20 @@ done:
*
* Modifications:
*
- * None.
+ * JRM -- 10/13/07
+ * Added code to detect and manage the case in which a
+ * flush callback changes the LRU-list out from under
+ * the function. The only way I can think of in which this
+ * can happen is if a flush function loads an entry
+ * into the cache that isn't there already. Quincey tells
+ * me that this will never happen, but I'm not sure I
+ * believe him.
+ *
+ * Note that this is a pretty bad scenario if it ever
+ * happens. The code I have added should allow us to
+ * handle the situation under all but the worst conditions,
+ * but one can argue that I should just scream and die if I
+ * ever detect the condidtion.
*
*-------------------------------------------------------------------------
*/
@@ -8325,7 +8740,9 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
herr_t result;
size_t eviction_size_limit;
size_t bytes_evicted = 0;
+ hbool_t prev_is_dirty = FALSE;
H5C_cache_entry_t * entry_ptr;
+ H5C_cache_entry_t * next_ptr;
H5C_cache_entry_t * prev_ptr;
FUNC_ENTER_NOAPI_NOINIT(H5C__autoadjust__ageout__evict_aged_out_entries)
@@ -8358,8 +8775,14 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
{
HDassert( ! (entry_ptr->is_protected) );
+ next_ptr = entry_ptr->next;
prev_ptr = entry_ptr->prev;
+ if ( prev_ptr != NULL ) {
+
+ prev_is_dirty = prev_ptr->is_dirty;
+ }
+
if ( entry_ptr->is_dirty ) {
result = H5C_flush_single_entry(f,
@@ -8392,8 +8815,41 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
"unable to flush entry")
}
- entry_ptr = prev_ptr;
+ if ( prev_ptr != NULL ) {
+#ifndef NDEBUG
+ if ( prev_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) {
+
+ /* something horrible has happened to *prev_ptr --
+ * scream and die.
+ */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "*prev_ptr corrupt")
+
+ } else
+#endif /* NDEBUG */
+ if ( ( prev_ptr->is_dirty != prev_is_dirty )
+ ||
+ ( prev_ptr->next != next_ptr )
+ ||
+ ( prev_ptr->is_protected )
+ ||
+ ( prev_ptr->is_pinned ) ) {
+
+ /* something has happened to the LRU -- start over
+ * from the tail.
+ */
+ entry_ptr = cache_ptr->LRU_tail_ptr;
+
+ } else {
+
+ entry_ptr = prev_ptr;
+ }
+ } else {
+
+ entry_ptr = NULL;
+
+ }
} /* end while */
/* for now at least, don't bother to maintain the minimum clean size,
@@ -8741,6 +9197,173 @@ done:
} /* H5C__autoadjust__ageout__remove_excess_markers() */
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5C__flash_increase_cache_size
+ *
+ * Purpose: If there is not at least new_entry_size - old_entry_size
+ * bytes of free space in the cache and the current
+ * max_cache_size is less than (cache_ptr->resize_ctl).max_size,
+ * perform a flash increase in the cache size and then reset
+ * the full cache hit rate statistics, and exit.
+ *
+ * Return: Non-negative on success/Negative on failure.
+ *
+ * Programmer: John Mainzer, 12/31/07
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static herr_t
+H5C__flash_increase_cache_size(H5C_t * cache_ptr,
+ size_t old_entry_size,
+ size_t new_entry_size)
+{
+ /* const char * fcn_name = "H5C__flash_increase_cache_size()";*/
+ herr_t ret_value = SUCCEED; /* Return value */
+ size_t new_max_cache_size = 0;
+ size_t old_max_cache_size = 0;
+ size_t new_min_clean_size = 0;
+ size_t old_min_clean_size = 0;
+ size_t space_needed;
+ enum H5C_resize_status status = flash_increase; /* may change */
+ double hit_rate;
+
+ FUNC_ENTER_NOAPI_NOINIT(H5C__flash_increase_cache_size)
+ HDassert( cache_ptr );
+ HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC );
+ HDassert( cache_ptr->flash_size_increase_possible );
+ HDassert( new_entry_size > cache_ptr->flash_size_increase_threshold );
+ HDassert( old_entry_size < new_entry_size );
+
+ if ( old_entry_size >= new_entry_size ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "old_entry_size >= new_entry_size")
+ }
+
+ space_needed = new_entry_size - old_entry_size;
+
+ if ( ( (cache_ptr->index_size + space_needed) >
+ cache_ptr->max_cache_size ) &&
+ ( cache_ptr->max_cache_size < (cache_ptr->resize_ctl).max_size ) ) {
+
+ /* we have work to do */
+
+ switch ( (cache_ptr->resize_ctl).flash_incr_mode )
+ {
+ case H5C_flash_incr__off:
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "flash_size_increase_possible but H5C_flash_incr__off?!")
+ break;
+
+ case H5C_flash_incr__add_space:
+ if ( cache_ptr->index_size < cache_ptr->max_cache_size ) {
+
+ HDassert( (cache_ptr->max_cache_size - cache_ptr->index_size)
+ < space_needed );
+ space_needed -= cache_ptr->max_cache_size - cache_ptr->index_size;
+ }
+ space_needed =
+ (size_t)(((double)space_needed) *
+ (cache_ptr->resize_ctl).flash_multiple);
+
+ new_max_cache_size = cache_ptr->max_cache_size + space_needed;
+
+ break;
+
+ default: /* should be unreachable */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "Unknown flash_incr_mode?!?!?.")
+ break;
+ }
+
+ if ( new_max_cache_size > (cache_ptr->resize_ctl).max_size ) {
+
+ new_max_cache_size = (cache_ptr->resize_ctl).max_size;
+ }
+
+ HDassert( new_max_cache_size > cache_ptr->max_cache_size );
+
+ new_min_clean_size = (size_t)
+ ((double)new_max_cache_size *
+ ((cache_ptr->resize_ctl).min_clean_fraction));
+
+ HDassert( new_min_clean_size <= new_max_cache_size );
+
+ old_max_cache_size = cache_ptr->max_cache_size;
+ old_min_clean_size = cache_ptr->min_clean_size;
+
+ cache_ptr->max_cache_size = new_max_cache_size;
+ cache_ptr->min_clean_size = new_min_clean_size;
+
+ /* update flash cache size increase fields as appropriate */
+ HDassert ( cache_ptr->flash_size_increase_possible );
+
+ switch ( (cache_ptr->resize_ctl).flash_incr_mode )
+ {
+ case H5C_flash_incr__off:
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "flash_size_increase_possible but H5C_flash_incr__off?!")
+ break;
+
+ case H5C_flash_incr__add_space:
+ cache_ptr->flash_size_increase_threshold =
+ (size_t)
+ (((double)(cache_ptr->max_cache_size)) *
+ ((cache_ptr->resize_ctl).flash_threshold));
+ break;
+
+ default: /* should be unreachable */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "Unknown flash_incr_mode?!?!?.")
+ break;
+ }
+
+ /* note that we don't cycle the epoch markers. We can
+ * argue either way as to whether we should, but for now
+ * we don't.
+ */
+
+ if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) {
+
+ /* get the hit rate for the reporting function. Should still
+ * be good as we havent reset the hit rate statistics.
+ */
+ if ( H5C_get_cache_hit_rate(cache_ptr, &hit_rate) != SUCCEED ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't get hit rate.")
+ }
+
+ (*((cache_ptr->resize_ctl).rpt_fcn))
+ (cache_ptr,
+ H5C__CURR_AUTO_RESIZE_RPT_FCN_VER,
+ hit_rate,
+ status,
+ old_max_cache_size,
+ new_max_cache_size,
+ old_min_clean_size,
+ new_min_clean_size);
+ }
+
+ if ( H5C_reset_cache_hit_rate_stats(cache_ptr) != SUCCEED ) {
+
+ /* this should be impossible... */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "H5C_reset_cache_hit_rate_stats failed.")
+ }
+ }
+
+done:
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5C__flash_increase_cache_size() */
+
/*-------------------------------------------------------------------------
* Function: H5C_flush_invalidate_cache
@@ -8779,12 +9402,28 @@ done:
*
* Modifications:
*
- * To support the fractal heap, the cache must now deal with
- * entries being dirtied, resized, and/or renamed inside
- * flush callbacks. Updated function to support this.
+ * To support the fractal heap, the cache must now deal with
+ * entries being dirtied, resized, and/or renamed inside
+ * flush callbacks. Updated function to support this.
*
* -- JRM 8/27/06
*
+ * Added code to detect and manage the case in which a
+ * flush callback changes the s-list out from under
+ * the function. The only way I can think of in which this
+ * can happen is if a flush function loads an entry
+ * into the cache that isn't there already. Quincey tells
+ * me that this will never happen, but I'm not sure I
+ * believe him.
+ *
+ * Note that this is a pretty bad scenario if it ever
+ * happens. The code I have added should allow us to
+ * handle the situation under all but the worst conditions,
+ * but one can argue that I should just scream and die if I
+ * ever detect the condidtion.
+ *
+ * -- JRM 10/13/07
+ *
*-------------------------------------------------------------------------
*/
herr_t
@@ -8895,6 +9534,23 @@ H5C_flush_invalidate_cache(H5F_t * f,
node_ptr = H5SL_first(cache_ptr->slist_ptr);
+ if ( node_ptr == NULL ) {
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "slist_len != 0 && node_ptr == NULL");
+ }
+
+ next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr);
+
+ if ( next_entry_ptr == NULL ) {
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "next_entry_ptr == NULL 1 ?!?!");
+ }
+#ifndef NDEBUG
+ HDassert( next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC );
+#endif /* NDEBUG */
+ HDassert( next_entry_ptr->is_dirty );
+ HDassert( next_entry_ptr->in_slist );
+
}
#if H5C_DO_SANITY_CHECKS
/* Depending on circumstances, H5C_flush_single_entry() will
@@ -8927,23 +9583,88 @@ H5C_flush_invalidate_cache(H5F_t * f,
while ( node_ptr != NULL )
{
- /* Note that we now remove nodes from the slist as we flush
- * the associated entries, instead of leaving them there
- * until we are done, and then destroying all nodes in
- * the slist.
+ entry_ptr = next_entry_ptr;
+
+ /* With the advent of the fractal heap, it is possible
+ * that the flush callback will dirty and/or resize
+ * other entries in the cache. In particular, while
+ * Quincey has promised me that this will never happen,
+ * it is possible that the flush callback for an
+ * entry may protect an entry that is not in the cache,
+ * perhaps causing the cache to flush and possibly
+ * evict the entry associated with node_ptr to make
+ * space for the new entry.
*
- * While this optimization used to be easy, with the possibility
- * of new entries being added to the slist in the midst of the
- * flush, we must keep the slist in cannonical form at all
- * times.
+ * Thus we do a bit of extra sanity checking on entry_ptr,
+ * and break out of this scan of the skip list if we
+ * detect major problems. We have a bit of leaway on the
+ * number of passes though the skip list, so this shouldn't
+ * be an issue in the flush in and of itself, as it should
+ * be all but impossible for this to happen more than once
+ * in any flush.
+ *
+ * Observe that that breaking out of the scan early
+ * shouldn't break the sanity checks just after the end
+ * of this while loop.
+ *
+ * If an entry has merely been marked clean and removed from
+ * the s-list, we simply break out of the scan.
+ *
+ * If the entry has been evicted, we flag an error and
+ * exit.
*/
+#ifndef NDEBUG
+ if ( entry_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "entry_ptr->magic is invalid ?!?!");
+
+ } else
+#endif /* NDEBUG */
+ if ( ( ! entry_ptr->is_dirty ) ||
+ ( ! entry_ptr->in_slist ) ) {
- entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr);
+ /* the s-list has been modified out from under us.
+ * break out of the loop.
+ */
+ break;
+ }
/* increment node pointer now, before we delete its target
* from the slist.
*/
+
node_ptr = H5SL_next(node_ptr);
+ if ( node_ptr != NULL ) {
+
+ next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr);
+
+ if ( next_entry_ptr == NULL ) {
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "next_entry_ptr == NULL 2 ?!?!");
+ }
+#ifndef NDEBUG
+ HDassert( next_entry_ptr->magic ==
+ H5C__H5C_CACHE_ENTRY_T_MAGIC );
+#endif /* NDEBUG */
+ HDassert( next_entry_ptr->is_dirty );
+ HDassert( next_entry_ptr->in_slist );
+
+ } else {
+
+ next_entry_ptr = NULL;
+ }
+
+ /* Note that we now remove nodes from the slist as we flush
+ * the associated entries, instead of leaving them there
+ * until we are done, and then destroying all nodes in
+ * the slist.
+ *
+ * While this optimization used to be easy, with the possibility
+ * of new entries being added to the slist in the midst of the
+ * flush, we must keep the slist in cannonical form at all
+ * times.
+ */
HDassert( entry_ptr != NULL );
HDassert( entry_ptr->in_slist );
@@ -9030,12 +9751,19 @@ H5C_flush_invalidate_cache(H5F_t * f,
/* It is possible that entries were added to the slist during
* the scan, either before or after scan pointer. The following
* asserts take this into account.
- */
+ *
+ * Don't bother with the sanity checks if node_ptr != NULL, as
+ * in this case we broke out of the loop because it got changed
+ * out from under us.
+ */
+
+ if ( node_ptr == NULL ) {
- HDassert( (actual_slist_len + cache_ptr->slist_len) ==
- (initial_slist_len + cache_ptr->slist_len_increase) );
- HDassert( (actual_slist_size + cache_ptr->slist_size) ==
- (initial_slist_size + cache_ptr->slist_size_increase) );
+ HDassert( (actual_slist_len + cache_ptr->slist_len) ==
+ (initial_slist_len + cache_ptr->slist_len_increase) );
+ HDassert( (actual_slist_size + cache_ptr->slist_size) ==
+ (initial_slist_size + cache_ptr->slist_size_increase) );
+ }
#endif /* H5C_DO_SANITY_CHECKS */
/* Since we are doing a destroy, we must make a pass through
@@ -9056,8 +9784,13 @@ H5C_flush_invalidate_cache(H5F_t * f,
while ( next_entry_ptr != NULL )
{
entry_ptr = next_entry_ptr;
- next_entry_ptr = entry_ptr->ht_next;
+ next_entry_ptr = entry_ptr->ht_next;
+#ifndef NDEBUG
+ HDassert ( ( next_entry_ptr == NULL ) ||
+ ( next_entry_ptr->magic ==
+ H5C__H5C_CACHE_ENTRY_T_MAGIC ) );
+#endif /* NDEBUG */
if ( entry_ptr->is_protected ) {
/* we have major problems -- but lets flush and destroy
@@ -9099,6 +9832,28 @@ H5C_flush_invalidate_cache(H5F_t * f,
* of pinned entries from pass to pass. If it stops
* shrinking before it hits zero, we scream and die.
*/
+ /* if the flush function on the entry we last evicted
+ * loaded an entry into cache (as Quincey has promised me
+ * it never will), and if the cache was full, it is
+ * possible that *next_entry_ptr was flushed or evicted.
+ *
+ * Test to see if this happened here. Note that if this
+ * test is triggred, we are accessing a deallocated piece
+ * of dynamically allocated memory, so we just scream and
+ * die.
+ */
+#ifndef NDEBUG
+ if ( ( next_entry_ptr != NULL ) &&
+ ( next_entry_ptr->magic !=
+ H5C__H5C_CACHE_ENTRY_T_MAGIC ) ) {
+
+ /* Something horrible has happened to
+ * *next_entry_ptr -- scream and die.
+ */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "next_entry_ptr->magic is invalid?!?!?.")
+ }
+#endif /* NDEBUG */
} /* end while loop scanning hash table bin */
} /* end for loop scanning hash table */
@@ -9541,6 +10296,17 @@ H5C_flush_single_entry(H5F_t * f,
/* Clear the dirty flag only, if requested */
if ( clear_only ) {
+#ifndef NDEBUG
+ if ( destroy ) {
+ /* we are about to call the clear callback with the
+ * destroy flag set -- this will result in *entry_ptr
+ * being freed. Set the magic field to bad magic
+ * so we can detect a freed cache entry if we see
+ * one.
+ */
+ entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC;
+ }
+#endif /* NDEBUG */
/* Call the callback routine to clear all dirty flags for object */
if ( (entry_ptr->type->clear)(f, entry_ptr, destroy) < 0 ) {
@@ -9558,6 +10324,18 @@ H5C_flush_single_entry(H5F_t * f,
}
#endif /* H5C_DO_SANITY_CHECKS */
+#ifndef NDEBUG
+ if ( destroy ) {
+ /* we are about to call the flush callback with the
+ * destroy flag set -- this will result in *entry_ptr
+ * being freed. Set the magic field to bad magic
+ * so we can detect a freed cache entry if we see
+ * one.
+ */
+ entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC;
+ }
+#endif /* NDEBUG */
+
/* Only block for all the processes on the first piece of metadata
*/
@@ -9816,7 +10594,9 @@ H5C_load_entry(H5F_t * f,
*/
HDassert( ( entry_ptr->is_dirty == FALSE ) || ( type->id == 4 ) );
-
+#ifndef NDEBUG
+ entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC;
+#endif /* NDEBUG */
entry_ptr->addr = addr;
entry_ptr->type = type;
entry_ptr->is_protected = FALSE;
@@ -9914,6 +10694,21 @@ done:
* Added sanity checks using the new is_read_only and
* ro_ref_count fields.
*
+ * JRM -- 10/13/07
+ * Added code to detect and manage the case in which a
+ * flush callback changes the LRU-list out from under
+ * the function. The only way I can think of in which this
+ * can happen is if a flush function loads an entry
+ * into the cache that isn't there already. Quincey tells
+ * me that this will never happen, but I'm not sure I
+ * believe him.
+ *
+ * Note that this is a pretty bad scenario if it ever
+ * happens. The code I have added should allow us to
+ * handle the situation under all but the worst conditions,
+ * but one can argue that I should just scream and die if I
+ * ever detect the condidtion.
+ *
*-------------------------------------------------------------------------
*/
@@ -9933,7 +10728,10 @@ H5C_make_space_in_cache(H5F_t * f,
#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
size_t empty_space;
#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
+ hbool_t prev_is_dirty = FALSE;
+ hbool_t entry_is_epoch_maker = FALSE;
H5C_cache_entry_t * entry_ptr;
+ H5C_cache_entry_t * next_ptr;
H5C_cache_entry_t * prev_ptr;
FUNC_ENTER_NOAPI_NOINIT(H5C_make_space_in_cache)
@@ -9962,10 +10760,18 @@ H5C_make_space_in_cache(H5F_t * f,
HDassert( ! (entry_ptr->is_read_only) );
HDassert( (entry_ptr->ro_ref_count) == 0 );
- prev_ptr = entry_ptr->prev;
+ next_ptr = entry_ptr->next;
+ prev_ptr = entry_ptr->prev;
+
+ if ( prev_ptr != NULL ) {
+
+ prev_is_dirty = prev_ptr->is_dirty;
+ }
if ( (entry_ptr->type)->id != H5C__EPOCH_MARKER_TYPE ) {
+ entry_is_epoch_maker = FALSE;
+
if ( entry_ptr->is_dirty ) {
result = H5C_flush_single_entry(f,
@@ -9994,6 +10800,7 @@ H5C_make_space_in_cache(H5F_t * f,
/* Skip epoch markers. Set result to SUCCEED to avoid
* triggering the error code below.
*/
+ entry_is_epoch_maker = TRUE;
result = SUCCEED;
}
@@ -10003,11 +10810,54 @@ H5C_make_space_in_cache(H5F_t * f,
"unable to flush entry")
}
- entry_ptr = prev_ptr;
- }
+ if ( prev_ptr != NULL ) {
+#ifndef NDEBUG
+ if ( prev_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC ) {
+
+ /* something horrible has happened to *prev_ptr --
+ * scream and die.
+ */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "*prev_ptr corrupt 1")
+
+ }
+#endif /* NDEBUG */
+ if ( entry_is_epoch_maker ) {
+
+ entry_ptr = prev_ptr;
+
+ } else if ( ( prev_ptr->is_dirty != prev_is_dirty )
+ ||
+ ( prev_ptr->next != next_ptr )
+ ||
+ ( prev_ptr->is_protected )
+ ||
+ ( prev_ptr->is_pinned ) ) {
+
+ /* something has happened to the LRU -- start over
+ * from the tail.
+ */
+
+ entry_ptr = cache_ptr->LRU_tail_ptr;
+
+ } else {
+
+ entry_ptr = prev_ptr;
+
+ }
+ } else {
+
+ entry_ptr = NULL;
+
+ }
+
+ entries_examined++;
+
+ }
#if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
+ entries_examined = 0;
initial_list_len = cache_ptr->dLRU_list_len;
entry_ptr = cache_ptr->dLRU_tail_ptr;
@@ -10034,6 +10884,13 @@ H5C_make_space_in_cache(H5F_t * f,
prev_ptr = entry_ptr->aux_prev;
+ next_ptr = entry_ptr->aux_next;
+
+ if ( prev_ptr != NULL ) {
+
+ HDassert( prev_ptr->is_dirty );
+ }
+
result = H5C_flush_single_entry(f,
primary_dxpl_id,
secondary_dxpl_id,
@@ -10050,7 +10907,66 @@ H5C_make_space_in_cache(H5F_t * f,
"unable to flush entry")
}
- entry_ptr = prev_ptr;
+ if ( prev_ptr != NULL ) {
+#ifndef NDEBUG
+ if (prev_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC) {
+
+ /* something horrible has happened to *prev_ptr --
+ * scream and die.
+ */
+
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "*prev_ptr corrupt 2")
+
+ } else
+#endif /* #ifndef NDEBUG */
+ if ( ( ! ( prev_ptr->is_dirty ) )
+ ||
+ ( prev_ptr->aux_next != next_ptr )
+ ||
+ ( prev_ptr->is_protected )
+ ||
+ ( prev_ptr->is_pinned ) ) {
+
+ /* something has happened to the dirty LRU -- start over
+ * from the tail.
+ */
+
+#if 0 /* This debuging code may be useful in the future -- keep it for now. */
+ if ( ! ( prev_ptr->is_dirty ) ) {
+ HDfprintf(stdout, "%s: ! prev_ptr->is_dirty\n",
+ fcn_name);
+ }
+ if ( prev_ptr->aux_next != next_ptr ) {
+ HDfprintf(stdout, "%s: prev_ptr->next != next_ptr\n",
+ fcn_name);
+ }
+ if ( prev_ptr->is_protected ) {
+ HDfprintf(stdout, "%s: prev_ptr->is_protected\n",
+ fcn_name);
+ }
+ if ( prev_ptr->is_pinned ) {
+ HDfprintf(stdout, "%s:prev_ptr->is_pinned\n",
+ fcn_name);
+ }
+
+ HDfprintf(stdout, "%s: re-starting scan of dirty list\n",
+ fcn_name);
+#endif /* JRM */
+ entry_ptr = cache_ptr->dLRU_tail_ptr;
+
+ } else {
+
+ entry_ptr = prev_ptr;
+
+ }
+ } else {
+
+ entry_ptr = NULL;
+
+ }
+
+ entries_examined++;
}
#endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
@@ -10096,6 +11012,7 @@ H5C_make_space_in_cache(H5F_t * f,
}
entry_ptr = prev_ptr;
+ entries_examined++;
}
}
diff --git a/src/H5C2.c b/src/H5C2.c
index 373191f..b485faf 100644
--- a/src/H5C2.c
+++ b/src/H5C2.c
@@ -2588,6 +2588,10 @@ static herr_t H5C2__autoadjust__ageout__remove_all_markers(H5C2_t * cache_ptr);
static herr_t H5C2__autoadjust__ageout__remove_excess_markers(H5C2_t * cache_ptr);
+static herr_t H5C2__flash_increase_cache_size(H5C2_t * cache_ptr,
+ size_t old_entry_size,
+ size_t new_entry_size);
+
static herr_t H5C2_flush_single_entry(H5F_t * f,
hid_t dxpl_id,
H5C2_t * cache_ptr,
@@ -2850,6 +2854,11 @@ done:
* Added initialization for the new is_read_only and
* ro_ref_count fields.
*
+ * JRM -- 12/31/07
+ * Added initialization for the new flash cache size increase
+ * related fields of H5C2_t.
+ *
+ *
*-------------------------------------------------------------------------
*/
@@ -2969,6 +2978,8 @@ H5C2_create(const H5F_t * f,
cache_ptr->dLRU_tail_ptr = NULL;
cache_ptr->size_increase_possible = FALSE;
+ cache_ptr->flash_size_increase_possible = FALSE;
+ cache_ptr->flash_size_increase_threshold = 0;
cache_ptr->size_decrease_possible = FALSE;
cache_ptr->resize_enabled = FALSE;
cache_ptr->cache_full = FALSE;
@@ -2989,6 +3000,10 @@ H5C2_create(const H5F_t * f,
(cache_ptr->resize_ctl).apply_max_increment = TRUE;
(cache_ptr->resize_ctl).max_increment = H5C2__DEF_AR_MAX_INCREMENT;
+ (cache_ptr->resize_ctl).flash_incr_mode = H5C2_flash_incr__off;
+ (cache_ptr->resize_ctl).flash_multiple = 1.0;
+ (cache_ptr->resize_ctl).flash_threshold = 0.25;
+
(cache_ptr->resize_ctl).decr_mode = H5C2_decr__off;
(cache_ptr->resize_ctl).upper_hr_threshold = H5C2__DEF_AR_UPPER_THRESHHOLD;
(cache_ptr->resize_ctl).decrement = H5C2__DEF_AR_DECREMENT;
@@ -3008,9 +3023,10 @@ H5C2_create(const H5F_t * f,
for ( i = 0; i < H5C2__MAX_EPOCH_MARKERS; i++ )
{
(cache_ptr->epoch_marker_active)[i] = FALSE;
-
+#ifndef NDEBUG
((cache_ptr->epoch_markers)[i]).magic =
H5C2__H5C2_CACHE_ENTRY_T_MAGIC;
+#endif /* NDEBUG */
((cache_ptr->epoch_markers)[i]).addr = (haddr_t)i;
((cache_ptr->epoch_markers)[i]).size = (size_t)0;
((cache_ptr->epoch_markers)[i]).type = &epoch_marker_class_2;
@@ -3096,6 +3112,9 @@ done:
* Updated function for display the new prefix field of
* H5C2_t in output.
*
+ * JRM 12/31/07
+ * Updated function to handle flash size increases.
+ *
*-------------------------------------------------------------------------
*/
void
@@ -3142,6 +3161,25 @@ H5C2_def_auto_resize_rpt_fcn(H5C2_t * cache_ptr,
new_min_clean_size);
break;
+ case flash_increase2:
+ HDassert( old_max_cache_size < new_max_cache_size );
+
+ HDfprintf(stdout,
+ "%sflash cache resize(%d) -- size threshold = %Zu.\n",
+ cache_ptr->prefix,
+ (int)((cache_ptr->resize_ctl).flash_incr_mode),
+ cache_ptr->flash_size_increase_threshold);
+
+ HDfprintf(stdout,
+ "%s cache size increased from (%Zu/%Zu) to (%Zu/%Zu).\n",
+ cache_ptr->prefix,
+ old_max_cache_size,
+ old_min_clean_size,
+ new_max_cache_size,
+ new_min_clean_size);
+ break;
+
+
case decrease2:
HDassert( old_max_cache_size > new_max_cache_size );
@@ -3571,6 +3609,21 @@ done:
* handle the situation, but one can argue that I should
* just scream and die if I ever detect the condidtion.
*
+ * JRM -- 10/13/07
+ * Added code to detect and manage the case in which a
+ * flush callback changes the s-list out from under
+ * the function. The only way I can think of in which this
+ * can happen is if a flush function loads an entry
+ * into the cache that isn't there already. Quincey tells
+ * me that this will never happen, but I'm not sure I
+ * believe him.
+ *
+ * Note that this is a pretty bad scenario if it ever
+ * happens. The code I have added should allow us to
+ * handle the situation under all but the worst conditions,
+ * but one can argue that I should just scream and die if I
+ * ever detect the condidtion.
+ *
*-------------------------------------------------------------------------
*/
herr_t
@@ -3651,19 +3704,27 @@ H5C2_flush_cache(H5C2_t * cache_ptr,
{
flushed_entries_last_pass = FALSE;
node_ptr = H5SL_first(cache_ptr->slist_ptr);
+
if ( node_ptr != NULL ) {
+
next_entry_ptr = (H5C2_cache_entry_t *)H5SL_item(node_ptr);
if ( next_entry_ptr == NULL ) {
+
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"next_entry_ptr == NULL 1 ?!?!");
}
+#ifndef NDEBUG
HDassert( next_entry_ptr->magic ==
H5C2__H5C2_CACHE_ENTRY_T_MAGIC );
+#endif /* NDEBUG */
HDassert( next_entry_ptr->is_dirty );
HDassert( next_entry_ptr->in_slist );
+
} else {
+
next_entry_ptr = NULL;
+
}
HDassert( node_ptr != NULL );
@@ -3742,14 +3803,16 @@ H5C2_flush_cache(H5C2_t * cache_ptr,
* If the entry has been evicted, we flag an error and
* exit.
*/
-
+#ifndef NDEBUG
if ( entry_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) {
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"entry_ptr->magic invalid ?!?!");
- } else if ( ( ! entry_ptr->is_dirty ) ||
- ( ! entry_ptr->in_slist ) ) {
+ } else
+#endif /* NDEBUG */
+ if ( ( ! entry_ptr->is_dirty ) ||
+ ( ! entry_ptr->in_slist ) ) {
/* the s-list has been modified out from under us.
* set node_ptr to NULL and break out of the loop.
@@ -3762,15 +3825,20 @@ H5C2_flush_cache(H5C2_t * cache_ptr,
* from the slist.
*/
node_ptr = H5SL_next(node_ptr);
+
if ( node_ptr != NULL ) {
+
next_entry_ptr = (H5C2_cache_entry_t *)H5SL_item(node_ptr);
if ( next_entry_ptr == NULL ) {
+
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"next_entry_ptr == NULL 2 ?!?!");
}
+#ifndef NDEBUG
HDassert( next_entry_ptr->magic ==
H5C2__H5C2_CACHE_ENTRY_T_MAGIC );
+#endif /* NDEBUG */
HDassert( next_entry_ptr->is_dirty );
HDassert( next_entry_ptr->in_slist );
} else {
@@ -3823,6 +3891,7 @@ H5C2_flush_cache(H5C2_t * cache_ptr,
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
"dirty pinned entry flush failed.")
}
+ flushed_entries_last_pass = TRUE;
}
} else {
#if H5C2_DO_SANITY_CHECKS
@@ -4580,6 +4649,9 @@ done:
* JRM -- 10/12/07
* Added initialization for the new magic field.
*
+ * JRM -- 12/31/07
+ * Added code supporting flash cache size increases.
+ *
*-------------------------------------------------------------------------
*/
@@ -4592,6 +4664,7 @@ H5C2_insert_entry(H5C2_t * cache_ptr,
void * thing,
unsigned int flags)
{
+ /* const char * fcn_name = "H5C2_insert_entry()"; */
herr_t result;
herr_t ret_value = SUCCEED; /* Return value */
hbool_t insert_pinned;
@@ -4628,9 +4701,9 @@ H5C2_insert_entry(H5C2_t * cache_ptr,
insert_pinned = ( (flags & H5C2__PIN_ENTRY_FLAG) != 0 );
entry_ptr = (H5C2_cache_entry_t *)thing;
-
+#ifndef NDEBUG
entry_ptr->magic = H5C2__H5C2_CACHE_ENTRY_T_MAGIC;
-
+#endif /* NDEBUG */
entry_ptr->addr = addr;
entry_ptr->type = type;
@@ -4664,6 +4737,19 @@ H5C2_insert_entry(H5C2_t * cache_ptr,
H5C2__RESET_CACHE_ENTRY_STATS(entry_ptr)
+ if ( ( cache_ptr->flash_size_increase_possible ) &&
+ ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) {
+
+ result = H5C2__flash_increase_cache_size(cache_ptr, 0,
+ entry_ptr->size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, \
+ "H5C2__flash_increase_cache_size failed.")
+ }
+ }
+
if ( ( cache_ptr->evictions_enabled ) &&
( (cache_ptr->index_size + entry_ptr->size) >
cache_ptr->max_cache_size ) ) {
@@ -5151,10 +5237,13 @@ done:
*
* Modifications:
*
- * None
+ * Added code to do a flash cache size increase if
+ * appropriate.
+ * JRM -- 1/11/08
*
*-------------------------------------------------------------------------
*/
+
herr_t
H5C2_mark_pinned_entry_dirty(H5C2_t * cache_ptr,
void * thing,
@@ -5162,6 +5251,8 @@ H5C2_mark_pinned_entry_dirty(H5C2_t * cache_ptr,
size_t new_size)
{
herr_t ret_value = SUCCEED; /* Return value */
+ herr_t result;
+ size_t size_increase;
H5C2_cache_entry_t * entry_ptr;
FUNC_ENTER_NOAPI(H5C2_mark_pinned_entry_dirty, FAIL)
@@ -5191,10 +5282,33 @@ H5C2_mark_pinned_entry_dirty(H5C2_t * cache_ptr,
/* update for change in entry size if necessary */
if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) {
+ /* do a flash cache size increase if appropriate */
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ if ( new_size > entry_ptr->size ) {
+
+ size_increase = new_size - entry_ptr->size;
+
+ if ( size_increase >=
+ cache_ptr->flash_size_increase_threshold ) {
+
+ result = H5C2__flash_increase_cache_size(cache_ptr,
+ entry_ptr->size,
+ new_size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \
+ "H5C2__flash_increase_cache_size failed.")
+ }
+ }
+ }
+ }
+
/* update the protected entry list */
H5C2__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \
- (cache_ptr->pel_size), \
- (entry_ptr->size), (new_size));
+ (cache_ptr->pel_size), \
+ (entry_ptr->size), (new_size));
/* update the hash table */
H5C2__UPDATE_INDEX_FOR_SIZE_CHANGE((cache_ptr), (entry_ptr->size),\
@@ -5528,7 +5642,9 @@ done:
*
* Modifications:
*
- * None
+ * Added code to apply a flash cache size increment if
+ * appropriate.
+ * JRM -- 1/11/08
*
*-------------------------------------------------------------------------
*/
@@ -5537,8 +5653,11 @@ H5C2_resize_pinned_entry(H5C2_t * cache_ptr,
void * thing,
size_t new_size)
{
+ /* const char * fcn_name = "H5C2_resize_pinned_entry()"; */
herr_t ret_value = SUCCEED; /* Return value */
+ herr_t result;
H5C2_cache_entry_t * entry_ptr;
+ size_t size_increase;
FUNC_ENTER_NOAPI(H5C2_resize_pinned_entry, FAIL)
@@ -5574,6 +5693,29 @@ H5C2_resize_pinned_entry(H5C2_t * cache_ptr,
/* update for change in entry size if necessary */
if ( entry_ptr->size != new_size ) {
+ /* do a flash cache size increase if appropriate */
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ if ( new_size > entry_ptr->size ) {
+
+ size_increase = new_size - entry_ptr->size;
+
+ if ( size_increase >=
+ cache_ptr->flash_size_increase_threshold ) {
+
+ result = H5C2__flash_increase_cache_size(cache_ptr,
+ entry_ptr->size,
+ new_size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \
+ "H5C2__flash_increase_cache_size failed.")
+ }
+ }
+ }
+ }
+
/* update the protected entry list */
H5C2__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \
(cache_ptr->pel_size), \
@@ -5761,7 +5903,7 @@ done:
* of cache entries.
*
* JRM -- 7/27/07
- * Added code supporting the new evictions_enabled fieled
+ * Added code supporting the new evictions_enabled field
* in H5C2_t.
*
* JRM -- 7/11/07
@@ -5771,6 +5913,10 @@ done:
* fields. Gained the len parameter. Also internal
* changes supporting the revised API.
*
+ * JRM -- 1/3/08
+ * Added to do a flash cache size increase if appropriate
+ * when a large entry is loaded.
+ *
*-------------------------------------------------------------------------
*/
@@ -5848,6 +5994,22 @@ H5C2_protect(H5C2_t * cache_ptr,
entry_ptr = (H5C2_cache_entry_t *)thing;
+ /* If the entry is very large, and we are configured to allow it,
+ * we may wish to perform a flash cache size increase.
+ */
+ if ( ( cache_ptr->flash_size_increase_possible ) &&
+ ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) {
+
+ result = H5C2__flash_increase_cache_size(cache_ptr, 0,
+ entry_ptr->size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, \
+ "H5C2__flash_increase_cache_size failed.")
+ }
+ }
+
/* try to free up some space if necessary and if
* evictions are permitted
*/
@@ -6149,6 +6311,10 @@ done:
* if the new configuration forces an immediate reduction
* in cache size.
*
+ * JRM -- 12/31/07
+ * Added code supporting the new flash cache size increase
+ * code.
+ *
*-------------------------------------------------------------------------
*/
@@ -6156,6 +6322,7 @@ herr_t
H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr,
H5C2_auto_size_ctl_t *config_ptr)
{
+ /* const char *fcn_name = "H5C2_set_cache_auto_resize_config()"; */
herr_t ret_value = SUCCEED; /* Return value */
herr_t result;
size_t new_max_cache_size;
@@ -6210,8 +6377,10 @@ H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr,
"conflicting threshold fields in new config.")
}
- cache_ptr->size_increase_possible = TRUE; /* will set to FALSE if needed */
- cache_ptr->size_decrease_possible = TRUE; /* will set to FALSE if needed */
+ /* will set the increase possible fields to FALSE later if needed */
+ cache_ptr->size_increase_possible = TRUE;
+ cache_ptr->flash_size_increase_possible = TRUE;
+ cache_ptr->size_decrease_possible = TRUE;
switch ( config_ptr->incr_mode )
{
@@ -6233,6 +6402,11 @@ H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr,
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown incr_mode?!?!?.")
}
+ /* logically, this is were configuration for flash cache size increases
+ * should go. However, this configuration depends on max_cache_size, so
+ * we wait until the end of the function, when this field is set.
+ */
+
switch ( config_ptr->decr_mode )
{
case H5C2_decr__off:
@@ -6277,9 +6451,14 @@ H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr,
if ( config_ptr->max_size == config_ptr->min_size ) {
cache_ptr->size_increase_possible = FALSE;
+ cache_ptr->flash_size_increase_possible = FALSE;
cache_ptr->size_decrease_possible = FALSE;
}
+ /* flash_size_increase_possible is intentionally omitted from the
+ * following:
+ */
+
cache_ptr->resize_enabled = cache_ptr->size_increase_possible ||
cache_ptr->size_decrease_possible;
@@ -6367,6 +6546,37 @@ H5C2_set_cache_auto_resize_config(H5C2_t * cache_ptr,
}
}
+ /* configure flash size increase facility. We wait until the
+ * end of the function, as we need the max_cache_size set before
+ * we start to keep things simple.
+ *
+ * If we haven't already ruled out flash cache size increases above,
+ * go ahead and configure it.
+ */
+
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ switch ( config_ptr->flash_incr_mode )
+ {
+ case H5C2_flash_incr__off:
+ cache_ptr->flash_size_increase_possible = FALSE;
+ break;
+
+ case H5C2_flash_incr__add_space:
+ cache_ptr->flash_size_increase_possible = TRUE;
+ cache_ptr->flash_size_increase_threshold =
+ (size_t)
+ (((double)(cache_ptr->max_cache_size)) *
+ ((cache_ptr->resize_ctl).flash_threshold));
+ break;
+
+ default: /* should be unreachable */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "Unknown flash_incr_mode?!?!?.")
+ break;
+ }
+ }
+
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -7272,6 +7482,9 @@ done:
* *cache_ptr), and one of its dxpl ids. Also internal
* changes supporting the revised API.
*
+ * JRM -- 12/31/07
+ * Modified funtion to support flash cache resizes.
+ *
*-------------------------------------------------------------------------
*/
herr_t
@@ -7294,6 +7507,8 @@ H5C2_unprotect(H5C2_t * cache_ptr,
hbool_t clear_entry = FALSE;
#endif /* H5_HAVE_PARALLEL */
herr_t ret_value = SUCCEED; /* Return value */
+ herr_t result;
+ size_t size_increase = 0;
H5C2_cache_entry_t * entry_ptr;
H5C2_cache_entry_t * test_entry_ptr;
@@ -7439,6 +7654,29 @@ H5C2_unprotect(H5C2_t * cache_ptr,
/* update for change in entry size if necessary */
if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) {
+ /* do a flash cache size increase if appropriate */
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ if ( new_size > entry_ptr->size ) {
+
+ size_increase = new_size - entry_ptr->size;
+
+ if ( size_increase >=
+ cache_ptr->flash_size_increase_threshold ) {
+
+ result = H5C2__flash_increase_cache_size(cache_ptr,
+ entry_ptr->size,
+ new_size);
+
+ if ( result < 0 ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \
+ "H5C2__flash_increase_cache_size failed.")
+ }
+ }
+ }
+ }
+
/* update the protected list */
H5C2__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pl_len), \
(cache_ptr->pl_size), \
@@ -7614,7 +7852,9 @@ done:
*
* Modifications:
*
- * None.
+ * Added validation for the flash increment fields.
+ *
+ * JRM -- 12/31/07
*
*-------------------------------------------------------------------------
*/
@@ -7724,6 +7964,33 @@ H5C2_validate_resize_config(H5C2_auto_size_ctl_t * config_ptr,
*/
} /* H5C2_incr__threshold */
+ switch ( config_ptr->flash_incr_mode )
+ {
+ case H5C2_flash_incr__off:
+ /* nothing to do here */
+ break;
+
+ case H5C2_flash_incr__add_space:
+ if ( ( config_ptr->flash_multiple < 0.1 ) ||
+ ( config_ptr->flash_multiple > 10.0 ) ) {
+
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
+ "flash_multiple must be in the range [0.1, 10.0]");
+ }
+
+ if ( ( config_ptr->flash_threshold < 0.1 ) ||
+ ( config_ptr->flash_threshold > 1.0 ) ) {
+
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
+ "flash_threshold must be in the range [0.1, 1.0]");
+ }
+ break;
+
+ default:
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
+ "Invalid flash_incr_mode");
+ break;
+ }
} /* H5C2_RESIZE_CFG__VALIDATE_INCREMENT */
@@ -7870,6 +8137,9 @@ done:
* passed through to other calls, and are no longer
* needed.
*
+ * JRM -- 1/5/08
+ * Added support for flash cache size increases.
+ *
*-------------------------------------------------------------------------
*/
@@ -8147,6 +8417,31 @@ H5C2__auto_adjust_cache_size(H5C2_t * cache_ptr,
cache_ptr->size_decreased = TRUE;
}
+
+ /* update flash cache size increase fields as appropriate */
+ if ( cache_ptr->flash_size_increase_possible ) {
+
+ switch ( (cache_ptr->resize_ctl).flash_incr_mode )
+ {
+ case H5C2_flash_incr__off:
+
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "flash_size_increase_possible but H5C2_flash_incr__off?!")
+ break;
+
+ case H5C2_flash_incr__add_space:
+ cache_ptr->flash_size_increase_threshold =
+ (size_t)
+ (((double)(cache_ptr->max_cache_size)) *
+ ((cache_ptr->resize_ctl).flash_threshold));
+ break;
+
+ default: /* should be unreachable */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "Unknown flash_incr_mode?!?!?.")
+ break;
+ }
+ }
}
if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) {
@@ -8568,7 +8863,7 @@ H5C2__autoadjust__ageout__evict_aged_out_entries(hid_t dxpl_id,
}
if ( prev_ptr != NULL ) {
-
+#ifndef NDEBUG
if ( prev_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) {
/* something horrible has happened to *prev_ptr --
@@ -8577,13 +8872,15 @@ H5C2__autoadjust__ageout__evict_aged_out_entries(hid_t dxpl_id,
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"*prev_ptr corrupt")
- } else if ( ( prev_ptr->is_dirty != prev_is_dirty )
- ||
- ( prev_ptr->next != next_ptr )
- ||
- ( prev_ptr->is_protected )
- ||
- ( prev_ptr->is_pinned ) ) {
+ } else
+#endif /* NDEBUG */
+ if ( ( prev_ptr->is_dirty != prev_is_dirty )
+ ||
+ ( prev_ptr->next != next_ptr )
+ ||
+ ( prev_ptr->is_protected )
+ ||
+ ( prev_ptr->is_pinned ) ) {
/* something has happened to the LRU -- start over
* from the tail.
@@ -8951,6 +9248,177 @@ done:
/*-------------------------------------------------------------------------
+ *
+ * Function: H5C2__flash_increase_cache_size
+ *
+ * Purpose: If there is not at least new_entry_size - old_entry_size
+ * bytes of free space in the cache and the current
+ * max_cache_size is less than (cache_ptr->resize_ctl).max_size,
+ * perform a flash increase in the cache size and then reset
+ * the full cache hit rate statistics, and exit.
+ *
+ * Return: Non-negative on success/Negative on failure.
+ *
+ * Programmer: John Mainzer, 12/31/07
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static herr_t
+H5C2__flash_increase_cache_size(H5C2_t * cache_ptr,
+ size_t old_entry_size,
+ size_t new_entry_size)
+{
+ /* const char * fcn_name = "H5C2__flash_increase_cache_size()";*/
+ herr_t ret_value = SUCCEED; /* Return value */
+ size_t new_max_cache_size = 0;
+ size_t old_max_cache_size = 0;
+ size_t new_min_clean_size = 0;
+ size_t old_min_clean_size = 0;
+ size_t space_needed;
+ enum H5C2_resize_status status = flash_increase2; /* may change */
+ double hit_rate;
+
+ FUNC_ENTER_NOAPI_NOINIT(H5C2__flash_increase_cache_size)
+ HDassert( cache_ptr );
+ HDassert( cache_ptr->magic == H5C2__H5C2_T_MAGIC );
+ HDassert( cache_ptr->flash_size_increase_possible );
+ HDassert( new_entry_size > cache_ptr->flash_size_increase_threshold );
+ HDassert( old_entry_size < new_entry_size );
+
+ if ( old_entry_size >= new_entry_size ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "old_entry_size >= new_entry_size")
+ }
+
+ space_needed = new_entry_size - old_entry_size;
+
+ if ( ( (cache_ptr->index_size + space_needed) >
+ cache_ptr->max_cache_size ) &&
+ ( cache_ptr->max_cache_size < (cache_ptr->resize_ctl).max_size ) ) {
+
+ /* we have work to do */
+
+ switch ( (cache_ptr->resize_ctl).flash_incr_mode )
+ {
+ case H5C2_flash_incr__off:
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "flash_size_increase_possible but H5C2_flash_incr__off?!")
+ break;
+
+ case H5C2_flash_incr__add_space:
+ if ( cache_ptr->index_size < cache_ptr->max_cache_size ) {
+
+ HDassert( (cache_ptr->max_cache_size -
+ cache_ptr->index_size)
+ < space_needed );
+ space_needed -= cache_ptr->max_cache_size -
+ cache_ptr->index_size;
+ }
+ space_needed =
+ (size_t)(((double)space_needed) *
+ (cache_ptr->resize_ctl).flash_multiple);
+
+ new_max_cache_size = cache_ptr->max_cache_size + space_needed;
+
+ break;
+
+ default: /* should be unreachable */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "Unknown flash_incr_mode?!?!?.")
+ break;
+ }
+
+ if ( new_max_cache_size > (cache_ptr->resize_ctl).max_size ) {
+
+ new_max_cache_size = (cache_ptr->resize_ctl).max_size;
+ }
+
+ HDassert( new_max_cache_size > cache_ptr->max_cache_size );
+
+ new_min_clean_size = (size_t)
+ ((double)new_max_cache_size *
+ ((cache_ptr->resize_ctl).min_clean_fraction));
+
+ HDassert( new_min_clean_size <= new_max_cache_size );
+
+ old_max_cache_size = cache_ptr->max_cache_size;
+ old_min_clean_size = cache_ptr->min_clean_size;
+
+ cache_ptr->max_cache_size = new_max_cache_size;
+ cache_ptr->min_clean_size = new_min_clean_size;
+
+ /* update flash cache size increase fields as appropriate */
+ HDassert ( cache_ptr->flash_size_increase_possible );
+
+ switch ( (cache_ptr->resize_ctl).flash_incr_mode )
+ {
+ case H5C2_flash_incr__off:
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "flash_size_increase_possible but H5C2_flash_incr__off?!")
+ break;
+
+ case H5C2_flash_incr__add_space:
+ cache_ptr->flash_size_increase_threshold =
+ (size_t)
+ (((double)(cache_ptr->max_cache_size)) *
+ ((cache_ptr->resize_ctl).flash_threshold));
+ break;
+
+ default: /* should be unreachable */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "Unknown flash_incr_mode?!?!?.")
+ break;
+ }
+
+ /* note that we don't cycle the epoch markers. We can
+ * argue either way as to whether we should, but for now
+ * we don't.
+ */
+
+ if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) {
+
+ /* get the hit rate for the reporting function. Should still
+ * be good as we havent reset the hit rate statistics.
+ */
+ if ( H5C2_get_cache_hit_rate(cache_ptr, &hit_rate) != SUCCEED ) {
+
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't get hit rate.")
+ }
+
+ (*((cache_ptr->resize_ctl).rpt_fcn))
+ (cache_ptr,
+ H5C2__CURR_AUTO_RESIZE_RPT_FCN_VER,
+ hit_rate,
+ status,
+ old_max_cache_size,
+ new_max_cache_size,
+ old_min_clean_size,
+ new_min_clean_size);
+ }
+
+ if ( H5C2_reset_cache_hit_rate_stats(cache_ptr) != SUCCEED ) {
+
+ /* this should be impossible... */
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+ "H5C2_reset_cache_hit_rate_stats failed.")
+ }
+ }
+
+done:
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5C2__flash_increase_cache_size() */
+
+
+
+/*-------------------------------------------------------------------------
* Function: H5C2_flush_invalidate_cache
*
* Purpose: Flush and destroy the entries contained in the target
@@ -8991,12 +9459,27 @@ done:
* entries being dirtied, resized, and/or renamed inside
* flush callbacks. Updated function to support this.
*
- * -- JRM 8/27/06
- *
* Reworked argument list and code to reflect the
* removal of the secondary dxpl id, and the decision
* to store f in H5C2_t, removing the need to pass it
* in all the time.
+ * -- JRM 8/27/06
+ *
+ * Added code to detect and manage the case in which a
+ * flush callback changes the s-list out from under
+ * the function. The only way I can think of in which this
+ * can happen is if a flush function loads an entry
+ * into the cache that isn't there already. Quincey tells
+ * me that this will never happen, but I'm not sure I
+ * believe him.
+ *
+ * Note that this is a pretty bad scenario if it ever
+ * happens. The code I have added should allow us to
+ * handle the situation under all but the worst conditions,
+ * but one can argue that I should just scream and die if I
+ * ever detect the condidtion.
+ *
+ * -- JRM 10/13/07
*
*-------------------------------------------------------------------------
*/
@@ -9112,8 +9595,10 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id,
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"next_entry_ptr == NULL 1 ?!?!");
}
-
- HDassert( next_entry_ptr->magic == H5C2__H5C2_CACHE_ENTRY_T_MAGIC );
+#ifndef NDEBUG
+ HDassert( next_entry_ptr->magic ==
+ H5C2__H5C2_CACHE_ENTRY_T_MAGIC );
+#endif /* NDEBUG */
HDassert( next_entry_ptr->is_dirty );
HDassert( next_entry_ptr->in_slist );
@@ -9180,14 +9665,16 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id,
* If the entry has been evicted, we flag an error and
* exit.
*/
-
+#ifndef NDEBUG
if ( entry_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) {
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"entry_ptr->magic is invalid ?!?!");
- } else if ( ( ! entry_ptr->is_dirty ) ||
- ( ! entry_ptr->in_slist ) ) {
+ } else
+#endif /* NDEBUG */
+ if ( ( ! entry_ptr->is_dirty ) ||
+ ( ! entry_ptr->in_slist ) ) {
/* the s-list has been modified out from under us.
* break out of the loop.
@@ -9208,9 +9695,10 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id,
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"next_entry_ptr == NULL 2 ?!?!");
}
-
+#ifndef NDEBUG
HDassert( next_entry_ptr->magic ==
H5C2__H5C2_CACHE_ENTRY_T_MAGIC );
+#endif /* NDEBUG */
HDassert( next_entry_ptr->is_dirty );
HDassert( next_entry_ptr->in_slist );
@@ -9345,15 +9833,15 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id,
while ( next_entry_ptr != NULL )
{
entry_ptr = next_entry_ptr;
-
+#ifndef NDEBUG
HDassert( entry_ptr->magic == H5C2__H5C2_CACHE_ENTRY_T_MAGIC );
-
+#endif /* NDEBUG */
next_entry_ptr = entry_ptr->ht_next;
-
+#ifndef NDEBUG
HDassert ( ( next_entry_ptr == NULL ) ||
( next_entry_ptr->magic ==
H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) );
-
+#endif /* NDEBUG */
if ( entry_ptr->is_protected ) {
/* we have major problems -- but lets flush and destroy
@@ -9405,6 +9893,7 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id,
* we are accessing a deallocated piece of dynamically
* allocated memory, so we just scream and die.
*/
+#ifndef NDEBUG
if ( ( next_entry_ptr != NULL ) &&
( next_entry_ptr->magic !=
H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) ) {
@@ -9415,6 +9904,7 @@ H5C2_flush_invalidate_cache(hid_t dxpl_id,
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"next_entry_ptr->magic is invalid?!?!?.")
}
+#endif /* NDEBUG */
} /* end while loop scanning hash table bin */
} /* end for loop scanning hash table */
@@ -10203,8 +10693,9 @@ H5C2_flush_single_entry(H5F_t * f,
* set the magic field to bad magic so we can detect a
* freed entry if we see one.
*/
+#ifndef NDEBUG
entry_ptr->magic = H5C2__H5C2_CACHE_ENTRY_T_BAD_MAGIC;
-
+#endif /* NDEBUG */
if ( type_ptr->free_icr(entry_ptr->addr, entry_ptr->size,
(void *)entry_ptr) != SUCCEED )
{
@@ -10423,8 +10914,9 @@ H5C2_load_entry(H5F_t * f,
HDassert( ( dirty == FALSE ) || ( type->id == 4 ) );
HDassert( entry_ptr->size < H5C2_MAX_ENTRY_SIZE );
-
+#ifndef NDEBUG
entry_ptr->magic = H5C2__H5C2_CACHE_ENTRY_T_MAGIC;
+#endif /* NDEBUG */
entry_ptr->addr = addr;
entry_ptr->size = len;
entry_ptr->image_ptr = image_ptr;
@@ -10547,6 +11039,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id,
size_t empty_space;
#endif /* H5C2_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */
hbool_t prev_is_dirty = FALSE;
+ hbool_t entry_is_epoch_maker = FALSE;
H5C2_cache_entry_t * entry_ptr;
H5C2_cache_entry_t * prev_ptr;
H5C2_cache_entry_t * next_ptr;
@@ -10585,6 +11078,8 @@ H5C2_make_space_in_cache(hid_t dxpl_id,
if ( (entry_ptr->type)->id != H5C2__EPOCH_MARKER_TYPE ) {
+ entry_is_epoch_maker = FALSE;
+
if ( entry_ptr->is_dirty ) {
result = H5C2_flush_single_entry(cache_ptr->f,
@@ -10610,6 +11105,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id,
/* Skip epoch markers. Set result to SUCCEED to avoid
* triggering the error code below.
*/
+ entry_is_epoch_maker = TRUE;
result = SUCCEED;
}
@@ -10620,7 +11116,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id,
}
if ( prev_ptr != NULL ) {
-
+#ifndef NDEBUG
if ( prev_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC ) {
/* something horrible has happened to *prev_ptr --
@@ -10629,6 +11125,16 @@ H5C2_make_space_in_cache(hid_t dxpl_id,
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"*prev_ptr corrupt 1")
+ } else
+#endif /* NDEBUG */
+ if ( entry_is_epoch_maker ) {
+
+ /* epoch markers don't get flushed, so the sanity checks
+ * on normal entries will fail -- thus just set entry_ptr
+ * to prev_ptr and go on.
+ */
+ entry_ptr = prev_ptr;
+
} else if ( ( prev_ptr->is_dirty != prev_is_dirty )
||
( prev_ptr->next != next_ptr )
@@ -10659,6 +11165,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id,
#if H5C2_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS
+ entries_examined = 0;
initial_list_len = cache_ptr->dLRU_list_len;
entry_ptr = cache_ptr->dLRU_tail_ptr;
@@ -10707,7 +11214,7 @@ H5C2_make_space_in_cache(hid_t dxpl_id,
}
if ( prev_ptr != NULL ) {
-
+#ifndef NDEBUG
if (prev_ptr->magic != H5C2__H5C2_CACHE_ENTRY_T_MAGIC) {
/* something horrible has happened to *prev_ptr --
@@ -10717,13 +11224,15 @@ H5C2_make_space_in_cache(hid_t dxpl_id,
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
"*prev_ptr corrupt 2")
- } else if ( ( ! ( prev_ptr->is_dirty ) )
- ||
- ( prev_ptr->aux_next != next_ptr )
- ||
- ( prev_ptr->is_protected )
- ||
- ( prev_ptr->is_pinned ) ) {
+ } else
+#endif /* #ifndef NDEBUG */
+ if ( ( ! ( prev_ptr->is_dirty ) )
+ ||
+ ( prev_ptr->aux_next != next_ptr )
+ ||
+ ( prev_ptr->is_protected )
+ ||
+ ( prev_ptr->is_pinned ) ) {
/* something has happened to the dirty LRU -- start over
* from the tail.
diff --git a/src/H5C2journal.c b/src/H5C2journal.c
new file mode 100644
index 0000000..bb64f7f
--- /dev/null
+++ b/src/H5C2journal.c
@@ -0,0 +1,519 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the files COPYING and Copyright.html. COPYING can be found at the root *
+ * of the source code distribution tree; Copyright.html can be found at the *
+ * root level of an installed copy of the electronic HDF5 document set and *
+ * is linked from the top-level documents page. It can also be found at *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+ * access to either file, you may request a copy from help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: H5C2journal.c
+ * Dec 6 2007
+ * John Mainzer
+ *
+ * Purpose: This file is a general catchall for functions supporting
+ * metadata journaling. Note that journaling must be tighly
+ * integrated with the metadata cache, and thus this file only
+ * contains only that code that can be easily separated from
+ * the rest of the cache code.
+ *
+ * Observe also that to minimize overhead, it is quite possible
+ * that many of the functions in this file will be converted
+ * into macros at some point in the future.
+ *
+ * Modifications:
+ *
+ * None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#define H5O_PACKAGE /*suppress error about including H5Opkg */
+#define H5C2_PACKAGE /*suppress error about including H5C2pkg */
+
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Opkg.h" /* Object headers */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5C2pkg.h" /* Cache */
+
+/**************************************************************************/
+/********************** super block message support ***********************/
+/**************************************************************************/
+
+static void * H5O_mdj_conf_decode(H5F_t UNUSED *f,
+ hid_t UNUSED dxpl_id,
+ unsigned UNUSED mesg_flags,
+ const uint8_t *p);
+
+static herr_t H5O_mdj_conf_encode(H5F_t *f,
+ hbool_t UNUSED disable_shared,
+ uint8_t *p,
+ const void *_mesg);
+
+static void * H5O_mdj_conf_copy(const void *_mesg,
+ void *_dest);
+
+static size_t H5O_mdj_conf_size(const H5F_t UNUSED *f,
+ hbool_t UNUSED disable_shared,
+ const void *_mesg);
+
+static herr_t H5O_mdj_conf_reset(void *_mesg);
+
+static herr_t H5O_mdj_conf_debug(H5F_t UNUSED *f,
+ hid_t UNUSED dxpl_id,
+ const void *_mesg,
+ FILE *stream,
+ int indent,
+ int fwidth);
+
+
+/* This message derives from H5O message class */
+const H5O_msg_class_t H5O_MSG_MDJ_CONF[1] = {{
+ H5O_MDJ_CONF_ID, /* message id number */
+ "metadata journaling config", /* message name for debugging */
+ sizeof(H5O_mdj_conf_t), /* native message size */
+ 0, /* messages are sharable? */
+ H5O_mdj_conf_decode, /* decode message */
+ H5O_mdj_conf_encode, /* encode message */
+ H5O_mdj_conf_copy, /* copy the native value */
+ H5O_mdj_conf_size, /* raw message size */
+ H5O_mdj_conf_reset, /* free internal memory */
+ NULL, /* free method */
+ NULL, /* file delete method */
+ NULL, /* link method */
+ NULL, /* set share method */
+ NULL, /* can share method */
+ NULL, /* pre copy native value to file */
+ NULL, /* copy native value to file */
+ NULL, /* post copy native value to file */
+ NULL, /* get creation index */
+ NULL, /* set creation index */
+ H5O_mdj_conf_debug /* debug the message */
+}};
+
+
+/* Current version of the metadata journaling configuration information */
+#define H5O_MDJ_CONF_VERSION 0
+
+#define MDJ_CONF__JOURNALING_ENABLED_FLAG 0x0001
+#define MDJ_CONF__JOURNAL_IS_EXTERNAL_FLAG 0x0002
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_mdj_conf_decode
+ *
+ * Purpose: Decode a journaling configuration message and return a
+ * pointer to a newly allocated H5O_mdj_conf_t struct.
+ *
+ * Return: Success: Ptr to new message in native struct.
+ * Failure: NULL
+ *
+ * Programmer: John Mainzer
+ * Dec. 14, 2007
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void *
+H5O_mdj_conf_decode(H5F_t *f,
+ hid_t UNUSED dxpl_id,
+ unsigned UNUSED mesg_flags,
+ const uint8_t *p)
+{
+ const char * fcn_name = "H5O_mdj_conf_decode()";
+ uint16_t flags = 0; /* packed boolean fields */
+ H5O_mdj_conf_t *mesg; /* Native message */
+ void *ret_value; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT(H5O_btreek_decode)
+
+ HDfprintf(stdout, "%s: entering.\n", fcn_name);
+
+ /* Sanity check */
+ HDassert(f);
+ HDassert(p);
+
+ /* Version of message */
+ if ( *p++ != H5O_MDJ_CONF_VERSION ) {
+
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, \
+ "bad version number for message")
+ }
+
+ /* Allocate space for message */
+
+ if( NULL == ( mesg = H5MM_calloc(sizeof(H5O_mdj_conf_t)))) {
+
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, \
+ "memory allocation failed for metadata journaling config message.");
+
+ }
+
+ /* retrieve packed boolean flags, upack and load them. */
+ UINT16DECODE(p, flags);
+
+ if ( (flags & MDJ_CONF__JOURNALING_ENABLED_FLAG) != 0 ) {
+
+ mesg->journaling_enabled = TRUE;
+
+ } else {
+
+ mesg->journaling_enabled = FALSE;
+
+ }
+
+ if ( (flags & MDJ_CONF__JOURNAL_IS_EXTERNAL_FLAG) != 0 ) {
+
+ mesg->journal_is_external = TRUE;
+
+ } else {
+
+ mesg->journal_is_external = FALSE;
+
+ }
+
+
+ /* retrieve the internal journal location */
+
+ H5F_addr_decode(f, &p, &(mesg->internal_journal_loc));
+
+
+ /* retrieve the size of the external journal path buffer */
+
+ UINT16DECODE(p, mesg->path_len);
+
+ if ( ( ! mesg->journal_is_external ) &&
+ ( mesg->path_len != 0 ) ) {
+
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, \
+ "internal journal and positive path len?!?");
+
+ }
+
+ if ( mesg->path_len > 0 ) {
+
+ /* Allocate space for buffer */
+ if ( NULL ==
+ (mesg->external_journal_file_path_ptr =
+ H5MM_malloc(mesg->path_len + 1)) ) {
+
+ mesg = H5MM_xfree(mesg);
+
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, \
+ "memory allocation failed for journal file path buffer")
+
+ } /* end if */
+
+ /* Copy encoded journal file path info into buffer */
+ HDmemcpy(mesg->external_journal_file_path_ptr, p, mesg->path_len);
+
+ } else {
+
+ mesg->external_journal_file_path_ptr = NULL;
+
+ }
+
+ /* Set return value */
+ ret_value = (void *)mesg;
+
+done:
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5O_mdj_conf_decode() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_mdj_conf_encode
+ *
+ * Purpose: Encode metadata journaling configuration message.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: John Mainzer
+ * Dec. 6, 2007
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static herr_t
+H5O_mdj_conf_encode(H5F_t *f,
+ hbool_t UNUSED disable_shared,
+ uint8_t *p,
+ const void *_mesg)
+{
+ const char * fcn_name = "H5O_mdj_conf_encode()";
+ const H5O_mdj_conf_t *mesg = (const H5O_mdj_conf_t *)_mesg;
+ uint16_t flags = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_mdj_conf_encode)
+
+ HDfprintf(stdout, "%s: entering.\n", fcn_name);
+
+ /* Sanity check */
+ HDassert(f);
+ HDassert(p);
+ HDassert(mesg);
+
+ /* setup the flags */
+ if ( mesg->journaling_enabled ) {
+
+ flags |= MDJ_CONF__JOURNALING_ENABLED_FLAG;
+
+ }
+
+ if ( mesg->journal_is_external ) {
+
+ flags |= MDJ_CONF__JOURNAL_IS_EXTERNAL_FLAG;
+
+ }
+
+ /* Store version, flags, internal_loc, path_len, & path buffer */
+
+ *p++ = H5O_MDJ_CONF_VERSION;
+
+ UINT16ENCODE(p, flags);
+
+ H5F_addr_encode(f, &p, mesg->internal_journal_loc);
+
+ HDassert(mesg->path_len <= 65535);
+
+ UINT16ENCODE(p, mesg->path_len);
+
+ if ( mesg->path_len > 0 ) {
+
+ HDmemcpy(p, mesg->external_journal_file_path_ptr, mesg->path_len);
+
+ }
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* H5O_mdj_conf_encode() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_mdj_conf_copy
+ *
+ * Purpose: Copies a message from _MESG to _DEST, allocating _DEST if
+ * necessary.
+ *
+ * Return: Success: Ptr to _DEST
+ * Failure: NULL
+ *
+ * Programmer: Quincey Koziol
+ * Mar 1, 2007
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void *
+H5O_mdj_conf_copy(const void *_mesg, void *_dest)
+{
+ const char * fcn_name = "H5O_mdj_conf_copy()";
+ const H5O_mdj_conf_t *mesg = (const H5O_mdj_conf_t *)_mesg;
+ H5O_mdj_conf_t *dest = (H5O_mdj_conf_t *)_dest;
+ void *ret_value;
+
+ FUNC_ENTER_NOAPI_NOINIT(H5O_mdj_conf_copy)
+
+ HDfprintf(stdout, "%s: entering.\n", fcn_name);
+
+ /* Sanity check */
+ HDassert(mesg);
+
+ if ( ( dest == NULL ) &&
+ ( ( NULL == (dest = H5MM_malloc(sizeof(H5O_mdj_conf_t))) ) ||
+ ( ( mesg->path_len > 0 ) &&
+ ( ( NULL == (dest->external_journal_file_path_ptr =
+ H5MM_malloc(mesg->path_len + 1)) ) ) ) ) ) {
+
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, \
+ "memory allocation failed for metadata journaling conf message")
+
+ }
+
+ /* now copy the message */
+ dest->journaling_enabled = mesg->journaling_enabled;
+ dest->journal_is_external = mesg->journal_is_external;
+ dest->internal_journal_loc = mesg->internal_journal_loc;
+ dest->path_len = mesg->path_len;
+
+ if ( dest->path_len > 0 ) {
+
+ HDmemcpy(dest->external_journal_file_path_ptr,
+ mesg->external_journal_file_path_ptr,
+ mesg->path_len);
+
+ } else {
+
+ dest->external_journal_file_path_ptr = NULL;
+ }
+
+
+ /* Set return value */
+ ret_value = dest;
+
+done:
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5O_mdj_conf_copy() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_mdj_conf_size
+ *
+ * Purpose: Returns the size of the raw message in bytes not counting the
+ * message type or size fields, but only the data fields.
+ *
+ * Return: Success: Message data size in bytes w/o alignment.
+ * Failure: 0
+ *
+ * Programmer: John Mainzer
+ * Dec. 12, 2007
+ *
+ *-------------------------------------------------------------------------
+ */
+static size_t
+H5O_mdj_conf_size(const H5F_t *f,
+ hbool_t UNUSED disable_shared,
+ const void *_mesg)
+{
+ const char * fcn_name = "H5O_mdj_conf_size()";
+ const H5O_mdj_conf_t * mesg = (const H5O_mdj_conf_t *)_mesg;
+ size_t ret_value;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_mdj_conf_size)
+
+ HDfprintf(stdout, "%s: entering.\n", fcn_name);
+
+ /* Sanity check */
+ HDassert(f);
+ HDassert(mesg);
+
+ ret_value = 1 + /* Version number */
+ 2 + /* flags */
+ H5F_SIZEOF_ADDR(f) + /* addr of internal journal */
+ 2 + /* length of external journal path */
+ mesg->path_len; /* external journal path buffer */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5O_mdj_conf_size() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_mdj_conf_reset
+ *
+ * Purpose: Frees internal pointers and resets the message to an
+ * initial state.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: John Mainzer
+ * Dec. 13, 2007
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static herr_t
+H5O_mdj_conf_reset(void *_mesg)
+{
+ const char * fcn_name = "H5O_mdj_conf_reset()";
+ H5O_mdj_conf_t *mesg = (H5O_mdj_conf_t *) _mesg;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_mdj_conf_reset);
+
+ HDfprintf(stdout, "%s: entering.\n", fcn_name);
+
+ /* check args */
+ assert(mesg);
+
+ /* reset */
+ if ( mesg->external_journal_file_path_ptr != NULL )
+ {
+ HDassert( mesg->path_len > 0 );
+ HDassert( mesg->journal_is_external );
+ mesg->external_journal_file_path_ptr =
+ H5MM_xfree(mesg->external_journal_file_path_ptr);
+ mesg->path_len = 0;
+ }
+
+ FUNC_LEAVE_NOAPI(SUCCEED);
+
+} /* H5O_mdj_conf_reset() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_mdj_conf_debug
+ *
+ * Purpose: Prints debugging info for the message.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: John Mainzer
+ * Dec. 7, 2007
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static herr_t
+H5O_mdj_conf_debug(H5F_t UNUSED *f,
+ hid_t UNUSED dxpl_id,
+ const void *_mesg,
+ FILE *stream,
+ int indent,
+ int fwidth)
+{
+ const char * fcn_name = "H5O_mdj_conf_debug()";
+ const H5O_mdj_conf_t *mesg = (const H5O_mdj_conf_t *)_mesg;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_btreek_debug)
+
+ HDfprintf(stdout, "%s: entering.\n", fcn_name);
+
+ /* Sanity check */
+ HDassert(f);
+ HDassert(mesg);
+ HDassert(stream);
+ HDassert(indent >= 0);
+ HDassert(fwidth >= 0);
+
+ HDfprintf(stream, "%*s%-*s %d\n", indent, "", fwidth,
+ "journaling_enabled:",
+ (int)(mesg->journaling_enabled));
+
+ HDfprintf(stream, "%*s%-*s %d\n", indent, "", fwidth,
+ "journal_is_external:",
+ (int)(mesg->journal_is_external));
+
+ HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth,
+ "internal_journal_loc:",
+ mesg->internal_journal_loc);
+
+ HDfprintf(stream, "%*s%-*s %d\n", indent, "", fwidth,
+ "path_len:",
+ (int)(mesg->path_len));
+
+ if ( mesg->path_len > 0 ) {
+
+ HDfprintf(stream, "%*s%-*s \"%s\"\n", indent, "", fwidth,
+ "external_journal_file_path_ptr:",
+ (char *)(mesg->external_journal_file_path_ptr));
+
+ }
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* end H5O_mdj_conf_debug() */
diff --git a/src/H5C2pkg.h b/src/H5C2pkg.h
index a1753b4..71c8e15 100644
--- a/src/H5C2pkg.h
+++ b/src/H5C2pkg.h
@@ -471,6 +471,18 @@
* all the ways this can happen, we simply set this flag when
* we receive a new configuration.
*
+ * flash_size_increase_possible: Depending on the configuration data given
+ * in the resize_ctl field, it may or may not be possible
+ * for a flash size increase to occur. We set this flag
+ * whenever we receive a new configuration so as to avoid
+ * repeated calculations.
+ *
+ * flash_size_increase_threshold: If a flash cache size increase is possible,
+ * this field is used to store the minimum size of a new entry
+ * or size increase needed to trigger a flash cache size
+ * increase. Note that this field must be updated whenever
+ * the size of the cache is changed.
+ *
* size_decrease_possible: Depending on the configuration data given
* in the resize_ctl field, it may or may not be possible
* to decrease the size of the cache. Rather than test for
@@ -863,6 +875,8 @@ struct H5C2_t
H5C2_cache_entry_t * dLRU_tail_ptr;
hbool_t size_increase_possible;
+ hbool_t flash_size_increase_possible;
+ size_t flash_size_increase_threshold;
hbool_t size_decrease_possible;
hbool_t resize_enabled;
hbool_t cache_full;
diff --git a/src/H5C2private.h b/src/H5C2private.h
index a949f6d..b70e968 100644
--- a/src/H5C2private.h
+++ b/src/H5C2private.h
@@ -804,12 +804,16 @@ typedef herr_t (*H5C2_log_flush_func_t)(H5C2_t * cache_ptr,
*
****************************************************************************/
+#ifndef NDEBUG
#define H5C2__H5C2_CACHE_ENTRY_T_MAGIC 0x005CAC0A
#define H5C2__H5C2_CACHE_ENTRY_T_BAD_MAGIC 0xDeadBeef
+#endif /* NDEBUG */
typedef struct H5C2_cache_entry_t
{
+#ifndef NDEBUG
uint32_t magic;
+#endif /* NDEBUG */
haddr_t addr;
size_t size;
void * image_ptr;
@@ -954,6 +958,57 @@ typedef struct H5C2_cache_entry_t
* above, this field contains the maximum number of bytes by which the
* cache size can be increased in a single re-size.
*
+ * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated
+ * type whose value indicates whether and by what algorithm we should
+ * make flash increases in the size of the cache to accomodate insertion
+ * of large entries and large increases in the size of a single entry.
+ *
+ * The addition of the flash increment mode was occasioned by performance
+ * problems that appear when a local heap is increased to a size in excess
+ * of the current cache size. While the existing re-size code dealt with
+ * this eventually, performance was very bad for the remainder of the
+ * epoch.
+ *
+ * At present, there are two possible values for the flash_incr_mode:
+ *
+ * H5C_flash_incr__off: Don't perform flash increases in the size of
+ * the cache.
+ *
+ * H5C_flash_incr__add_space: Let x be either the size of a newly
+ * newly inserted entry, or the number of bytes by which the
+ * size of an existing entry has been increased.
+ *
+ * If
+ * x > flash_threshold * current max cache size,
+ *
+ * increase the current maximum cache size by x * flash_multiple
+ * less any free space in the cache, and start a new epoch. For
+ * now at least, pay no attention to the maximum increment.
+ *
+ *
+ * With a little thought, it should be obvious that the above flash
+ * cache size increase algorithm is not sufficient for all
+ * circumstances -- for example, suppose the user round robins through
+ * (1/flash_threshold) +1 groups, adding one data set to each on each
+ * pass. Then all will increase in size at about the same time, requiring
+ * the max cache size to at least double to maintain acceptable
+ * performance, however the above flash increment algorithm will not be
+ * triggered.
+ *
+ * Hopefully, the add space algorithm detailed above will be sufficient
+ * for the performance problems encountered to date. However, we should
+ * expect to revisit the issue.
+ *
+ * flash_multiple: Double containing the multiple described above in the
+ * H5C_flash_incr__add_space section of the discussion of the
+ * flash_incr_mode section. This field is ignored unless flash_incr_mode
+ * is H5C_flash_incr__add_space.
+ *
+ * flash_threshold: Double containing the factor by which current max cache
+ * size is multiplied to obtain the size threshold for the add_space
+ * flash increment algorithm. The field is ignored unless
+ * flash_incr_mode is H5C2_flash_incr__add_space.
+ *
*
* Cache size decrease control fields:
*
@@ -1068,6 +1123,8 @@ typedef struct H5C2_cache_entry_t
#define H5C2__DEF_AR_MIN_CLEAN_FRAC 0.5
#define H5C2__DEF_AR_INCREMENT 2.0
#define H5C2__DEF_AR_MAX_INCREMENT ((size_t)( 2 * 1024 * 1024))
+#define H5C2__DEF_AR_FLASH_MULTIPLE 1.0
+#define H5C2__DEV_AR_FLASH_THRESHOLD 0.25
#define H5C2__DEF_AR_DECREMENT 0.9
#define H5C2__DEF_AR_MAX_DECREMENT ((size_t)( 1 * 1024 * 1024))
#define H5C2__DEF_AR_EPCHS_B4_EVICT 3
@@ -1080,6 +1137,7 @@ enum H5C2_resize_status
{
in_spec2,
increase2,
+ flash_increase2,
decrease2,
at_max_size2,
at_min_size2,
@@ -1100,45 +1158,49 @@ typedef void (*H5C2_auto_resize_rpt_fcn)(H5C2_t * cache_ptr,
typedef struct H5C2_auto_size_ctl_t
{
/* general configuration fields: */
- int32_t version;
- H5C2_auto_resize_rpt_fcn rpt_fcn;
+ int32_t version;
+ H5C2_auto_resize_rpt_fcn rpt_fcn;
- hbool_t set_initial_size;
- size_t initial_size;
+ hbool_t set_initial_size;
+ size_t initial_size;
- double min_clean_fraction;
+ double min_clean_fraction;
- size_t max_size;
- size_t min_size;
+ size_t max_size;
+ size_t min_size;
- int64_t epoch_length;
+ int64_t epoch_length;
/* size increase control fields: */
- enum H5C2_cache_incr_mode incr_mode;
+ enum H5C2_cache_incr_mode incr_mode;
+
+ double lower_hr_threshold;
- double lower_hr_threshold;
+ double increment;
- double increment;
+ hbool_t apply_max_increment;
+ size_t max_increment;
- hbool_t apply_max_increment;
- size_t max_increment;
+ enum H5C2_cache_flash_incr_mode flash_incr_mode;
+ double flash_multiple;
+ double flash_threshold;
/* size decrease control fields: */
- enum H5C2_cache_decr_mode decr_mode;
+ enum H5C2_cache_decr_mode decr_mode;
- double upper_hr_threshold;
+ double upper_hr_threshold;
- double decrement;
+ double decrement;
- hbool_t apply_max_decrement;
- size_t max_decrement;
+ hbool_t apply_max_decrement;
+ size_t max_decrement;
- int32_t epochs_before_eviction;
+ int32_t epochs_before_eviction;
- hbool_t apply_empty_reserve;
- double empty_reserve;
+ hbool_t apply_empty_reserve;
+ double empty_reserve;
} H5C2_auto_size_ctl_t;
diff --git a/src/H5C2public.h b/src/H5C2public.h
index e1adff3..d293ccb 100644
--- a/src/H5C2public.h
+++ b/src/H5C2public.h
@@ -41,6 +41,13 @@ enum H5C2_cache_incr_mode
H5C2_incr__threshold
};
+enum H5C2_cache_flash_incr_mode
+{
+ H5C2_flash_incr__off,
+ H5C2_flash_incr__add_space
+};
+
+
enum H5C2_cache_decr_mode
{
H5C2_decr__off,
diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h
index 949f923..7c04ea2 100644
--- a/src/H5Cpkg.h
+++ b/src/H5Cpkg.h
@@ -465,6 +465,18 @@
* all the ways this can happen, we simply set this flag when
* we receive a new configuration.
*
+ * flash_size_increase_possible: Depending on the configuration data given
+ * in the resize_ctl field, it may or may not be possible
+ * for a flash size increase to occur. We set this flag
+ * whenever we receive a new configuration so as to avoid
+ * repeated calculations.
+ *
+ * flash_size_increase_threshold: If a flash cache size increase is possible,
+ * this field is used to store the minimum size of a new entry
+ * or size increase needed to trigger a flash cache size
+ * increase. Note that this field must be updated whenever
+ * the size of the cache is changed.
+ *
* size_decrease_possible: Depending on the configuration data given
* in the resize_ctl field, it may or may not be possible
* to decrease the size of the cache. Rather than test for
@@ -580,10 +592,10 @@
* equal to the array index has not been in cache when
* requested in the current epoch.
*
- * write_protects: Array of int64 of length H5C__MAX_NUM_TYPE_IDS + 1. The cells
- * are used to record the number of times an entry with type id
- * equal to the array index has been write protected in the
- * current epoch.
+ * write_protects: Array of int64 of length H5C__MAX_NUM_TYPE_IDS + 1. The
+ * cells are used to record the number of times an entry with
+ * type id equal to the array index has been write protected
+ * in the current epoch.
*
* Observe that (hits + misses) = (write_protects + read_protects).
*
@@ -855,6 +867,8 @@ struct H5C_t
H5C_cache_entry_t * dLRU_tail_ptr;
hbool_t size_increase_possible;
+ hbool_t flash_size_increase_possible;
+ size_t flash_size_increase_threshold;
hbool_t size_decrease_possible;
hbool_t resize_enabled;
hbool_t cache_full;
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index d365755..53dcf0d 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -202,6 +202,27 @@ typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr,
*
* JRM - 4/26/04
*
+ * magic: Unsigned 32 bit integer that must always be set to
+ * H5C__H5C_CACHE_ENTRY_T_MAGIC when the entry is valid.
+ * The field must be set to H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC
+ * just before the entry is freed.
+ *
+ * This is necessary, as the LRU list can be changed out
+ * from under H5C_make_space_in_cache() by the flush
+ * callback which may change the size of an existing entry,
+ * and/or load a new entry while serializing the target entry.
+ *
+ * This in turn can cause a recursive call to
+ * H5C_make_space_in_cache() which may either flush or evict
+ * the next entry that the first invocation of that function
+ * was about to examine.
+ *
+ * The magic field allows H5C_make_space_in_cache() to
+ * detect this case, and re-start its scan from the bottom
+ * of the LRU when this situation occurs.
+ *
+ * This field is only compiled in debug mode.
+ *
* addr: Base address of the cache entry on disk.
*
* size: Length of the cache entry on disk. Note that unlike normal
@@ -442,8 +463,16 @@ typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr,
*
****************************************************************************/
+#ifndef NDEBUG
+#define H5C__H5C_CACHE_ENTRY_T_MAGIC 0x005CAC0A
+#define H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC 0xDeadBeef
+#endif /* NDEBUG */
+
typedef struct H5C_cache_entry_t
{
+#ifndef NDEBUG
+ uint32_t magic;
+#endif /* NDEBUG */
haddr_t addr;
size_t size;
const H5C_class_t * type;
@@ -587,7 +616,58 @@ typedef struct H5C_cache_entry_t
* above, this field contains the maximum number of bytes by which the
* cache size can be increased in a single re-size.
*
- *
+ * flash_incr_mode: Instance of the H5C_cache_flash_incr_mode enumerated
+ * type whose value indicates whether and by what algorithm we should
+ * make flash increases in the size of the cache to accomodate insertion
+ * of large entries and large increases in the size of a single entry.
+ *
+ * The addition of the flash increment mode was occasioned by performance
+ * problems that appear when a local heap is increased to a size in excess
+ * of the current cache size. While the existing re-size code dealt with
+ * this eventually, performance was very bad for the remainder of the
+ * epoch.
+ *
+ * At present, there are two possible values for the flash_incr_mode:
+ *
+ * H5C_flash_incr__off: Don't perform flash increases in the size of
+ * the cache.
+ *
+ * H5C_flash_incr__add_space: Let x be either the size of a newly
+ * newly inserted entry, or the number of bytes by which the
+ * size of an existing entry has been increased.
+ *
+ * If
+ * x > flash_threshold * current max cache size,
+ *
+ * increase the current maximum cache size by x * flash_multiple
+ * less any free space in the cache, and start a new epoch. For
+ * now at least, pay no attention to the maximum increment.
+ *
+ *
+ * With a little thought, it should be obvious that the above flash
+ * cache size increase algorithm is not sufficient for all circumstances --
+ * for example, suppose the user round robins through
+ * (1/flash_threshold) +1 groups, adding one data set to each on each
+ * pass. Then all will increase in size at about the same time, requiring
+ * the max cache size to at least double to maintain acceptable
+ * performance, however the above flash increment algorithm will not be
+ * triggered.
+ *
+ * Hopefully, the add space algorithm detailed above will be sufficient
+ * for the performance problems encountered to date. However, we should
+ * expect to revisit the issue.
+ *
+ * flash_multiple: Double containing the multiple described above in the
+ * H5C_flash_incr__add_space section of the discussion of the
+ * flash_incr_mode section. This field is ignored unless flash_incr_mode
+ * is H5C_flash_incr__add_space.
+ *
+ * flash_threshold: Double containing the factor by which current max cache size
+ * is multiplied to obtain the size threshold for the add_space flash
+ * increment algorithm. The field is ignored unless flash_incr_mode is
+ * H5C_flash_incr__add_space.
+ *
+ *
* Cache size decrease control fields:
*
* decr_mode: Instance of the H5C_cache_decr_mode enumerated type whose
@@ -701,6 +781,8 @@ typedef struct H5C_cache_entry_t
#define H5C__DEF_AR_MIN_CLEAN_FRAC 0.5
#define H5C__DEF_AR_INCREMENT 2.0
#define H5C__DEF_AR_MAX_INCREMENT ((size_t)( 2 * 1024 * 1024))
+#define H5C__DEF_AR_FLASH_MULTIPLE 1.0
+#define H5C__DEV_AR_FLASH_THRESHOLD 0.25
#define H5C__DEF_AR_DECREMENT 0.9
#define H5C__DEF_AR_MAX_DECREMENT ((size_t)( 1 * 1024 * 1024))
#define H5C__DEF_AR_EPCHS_B4_EVICT 3
@@ -713,6 +795,7 @@ enum H5C_resize_status
{
in_spec,
increase,
+ flash_increase,
decrease,
at_max_size,
at_min_size,
@@ -733,45 +816,48 @@ typedef void (*H5C_auto_resize_rpt_fcn)(H5C_t * cache_ptr,
typedef struct H5C_auto_size_ctl_t
{
/* general configuration fields: */
- int32_t version;
- H5C_auto_resize_rpt_fcn rpt_fcn;
+ int32_t version;
+ H5C_auto_resize_rpt_fcn rpt_fcn;
- hbool_t set_initial_size;
- size_t initial_size;
+ hbool_t set_initial_size;
+ size_t initial_size;
- double min_clean_fraction;
+ double min_clean_fraction;
- size_t max_size;
- size_t min_size;
+ size_t max_size;
+ size_t min_size;
- int64_t epoch_length;
+ int64_t epoch_length;
/* size increase control fields: */
- enum H5C_cache_incr_mode incr_mode;
+ enum H5C_cache_incr_mode incr_mode;
- double lower_hr_threshold;
+ double lower_hr_threshold;
- double increment;
+ double increment;
- hbool_t apply_max_increment;
- size_t max_increment;
+ hbool_t apply_max_increment;
+ size_t max_increment;
+ enum H5C_cache_flash_incr_mode flash_incr_mode;
+ double flash_multiple;
+ double flash_threshold;
/* size decrease control fields: */
- enum H5C_cache_decr_mode decr_mode;
+ enum H5C_cache_decr_mode decr_mode;
- double upper_hr_threshold;
+ double upper_hr_threshold;
- double decrement;
+ double decrement;
- hbool_t apply_max_decrement;
- size_t max_decrement;
+ hbool_t apply_max_decrement;
+ size_t max_decrement;
- int32_t epochs_before_eviction;
+ int32_t epochs_before_eviction;
- hbool_t apply_empty_reserve;
- double empty_reserve;
+ hbool_t apply_empty_reserve;
+ double empty_reserve;
} H5C_auto_size_ctl_t;
diff --git a/src/H5Cpublic.h b/src/H5Cpublic.h
index 082d512..7ef959a 100644
--- a/src/H5Cpublic.h
+++ b/src/H5Cpublic.h
@@ -41,6 +41,12 @@ enum H5C_cache_incr_mode
H5C_incr__threshold
};
+enum H5C_cache_flash_incr_mode
+{
+ H5C_flash_incr__off,
+ H5C_flash_incr__add_space
+};
+
enum H5C_cache_decr_mode
{
H5C_decr__off,
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index 3e13b0d..5c54fdd 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -113,6 +113,11 @@ typedef struct H5F_file_t {
struct H5G_t *root_grp; /* Open root group */
H5FO_t *open_objs; /* Open objects in file */
H5RC_t *grp_btree_shared; /* Ref-counted group B-tree node info */
+ hbool_t journaling_enabled; /* metadata journaling configuration */
+ hbool_t journal_is_external; /* fields. All fields as per those */
+ haddr_t internal_journal_loc;/* of the same name in M5O_mdj_conf_t. */
+ size_t path_len;
+ uint8_t * external_journal_file_path_ptr;
} H5F_file_t;
/* A record of the mount table */
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index cbc5f72..ed3395e 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -33,6 +33,8 @@
#include "H5Iprivate.h" /* IDs */
#include "H5Pprivate.h" /* Property lists */
#include "H5SMprivate.h" /* Shared Object Header Messages */
+#include "H5MMprivate.h" /* Memory management */
+
/****************/
@@ -226,6 +228,12 @@ done:
* wendling@ncsa.uiuc.edu
* Sept 12, 2003
*
+ * Changes: Johm Mainzer
+ * 12/14/07
+ * Added code to read in the metadata journaling config
+ * if it is present, and to initialize
+ * f->shared->journaling_enabled to FALSE if it isn't.
+ *
*-------------------------------------------------------------------------
*/
herr_t
@@ -251,6 +259,14 @@ H5F_super_read(H5F_t *f, hid_t dxpl_id, H5G_loc_t *root_loc)
shared = f->shared;
lf = shared->lf;
+ /* initialize the metadata journaling configuration sections of the
+ * super block to indicate that journaling is not turned on at
+ * present. These initialization may be overridden shortly.
+ */
+ shared->journaling_enabled = FALSE;
+ shared->path_len = 0;
+ shared->external_journal_file_path_ptr = NULL;
+
/* Get the shared file creation property list */
if(NULL == (c_plist = H5I_object(shared->fcpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
@@ -553,6 +569,9 @@ H5F_super_read(H5F_t *f, hid_t dxpl_id, H5G_loc_t *root_loc)
H5O_loc_t ext_loc; /* "Object location" for superblock extension */
H5O_btreek_t btreek; /* v1 B-tree 'K' value message from superblock extension */
H5O_drvinfo_t drvinfo; /* Driver info message from superblock extension */
+ H5O_mdj_conf_t mdj_conf;/* metadata journaling config message
+ * from superblock extension
+ */
/* Sanity check - superblock extension should only be defined for
* superblock version >= 2.
@@ -617,6 +636,48 @@ H5F_super_read(H5F_t *f, hid_t dxpl_id, H5G_loc_t *root_loc)
H5O_msg_reset(H5O_DRVINFO_ID, &drvinfo);
} /* end else */
+ /* Read in the metadata journaling configuration message,
+ * if it exists.
+ */
+ if(NULL == H5O_msg_read(&ext_loc, H5O_MDJ_CONF_ID, &mdj_conf, dxpl_id)) {
+ /* Reset error from "failed" message read */
+ H5E_clear_stack(NULL);
+ } /* end if */
+ else {
+
+ shared->journaling_enabled = mdj_conf.journaling_enabled;
+
+ if ( shared->journaling_enabled ) {
+
+ shared->journal_is_external = mdj_conf.journal_is_external;
+ shared->internal_journal_loc = mdj_conf.internal_journal_loc;
+ shared->path_len = mdj_conf.path_len;
+ shared->external_journal_file_path_ptr =
+ mdj_conf.external_journal_file_path_ptr;
+
+ /* for now at least, the journal file must always be
+ * external -- hence the following asserts. Remove them
+ * if we ever support an internal journal.
+ */
+ HDassert( shared->journal_is_external );
+ HDassert( shared->path_len > 0 );
+ HDassert( shared->external_journal_file_path_ptr != NULL );
+
+ /* if there is a an external journal file,
+ * H5O_mdj_conf_decode() will allocate a buffer to
+ * store it in. Rather than allocate our own buffer,
+ * we will use the one created by H5O_mdj_conf_decode(),
+ * and modify mdj_conf so that H5O_mdj_conf_reset() will
+ * not discard it.
+ */
+ mdj_conf.path_len = 0;
+ mdj_conf.external_journal_file_path_ptr = NULL;
+ }
+
+ /* Reset metadata journaling config message */
+ H5O_msg_reset(H5O_MDJ_CONF_ID, &mdj_conf);
+ }
+
/* Close the extension. Twiddle the number of open objects to avoid
* closing the file (since this will be the only open object).
*/
@@ -645,6 +706,13 @@ done:
* koziol@ncsa.uiuc.edu
* Sept 15, 2003
*
+ * Changes: John Mainzer
+ * Dec. 14, 2007
+ * Added initialization for the metadata journaling
+ * configuration fields. By default, these fields are
+ * initialized to indicate that we are not journaling
+ * metadata.
+ *
*-------------------------------------------------------------------------
*/
herr_t
@@ -802,6 +870,43 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update driver info header message")
} /* end if */
+ /* check for journaling config data to store */
+ if ( f->shared->journaling_enabled ) {
+
+ struct H5O_mdj_conf_t mdj_conf;
+
+ mdj_conf.journaling_enabled = f->shared->journaling_enabled;
+ mdj_conf.journal_is_external = f->shared->journal_is_external;
+ mdj_conf.internal_journal_loc = f->shared->internal_journal_loc;
+ mdj_conf.path_len = f->shared->path_len;
+
+ if ( f->shared->external_journal_file_path_ptr == NULL ) {
+
+ mdj_conf.external_journal_file_path_ptr = NULL;
+
+ } else {
+
+ if ( ( NULL == (mdj_conf.external_journal_file_path_ptr =
+ H5MM_malloc(mdj_conf.path_len + 1)) ) ) {
+
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, \
+ "memory alloc failed for mdj path")
+
+ }
+ HDmemcpy(mdj_conf.external_journal_file_path_ptr,
+ f->shared->external_journal_file_path_ptr,
+ f->shared->path_len);
+ }
+
+ if ( H5O_msg_create(&ext_loc, H5O_MDJ_CONF_ID,
+ H5O_MSG_FLAG_CONSTANT | H5O_MSG_FLAG_DONTSHARE,
+ H5O_UPDATE_TIME, &mdj_conf, dxpl_id) < 0 ) {
+
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, \
+ "unable to update metadata journal conf header message")
+ }
+ }
+
/* Twiddle the number of open objects to avoid closing the file
* (since this will be the only open object currently).
*/
diff --git a/src/H5O.c b/src/H5O.c
index 94435f5..021b69e 100644
--- a/src/H5O.c
+++ b/src/H5O.c
@@ -77,6 +77,10 @@ static herr_t H5O_obj_type_real(H5O_t *oh, H5O_type_t *obj_type);
/*********************/
/* Header message ID to class mapping */
+
+/* Remember to increment H5O_MSG_TYPES in H5Opkg.h when adding a new
+ * message.
+ */
const H5O_msg_class_t *const H5O_msg_class_g[] = {
H5O_MSG_NULL, /*0x0000 Null */
H5O_MSG_SDSPACE, /*0x0001 Dataspace */
@@ -105,9 +109,12 @@ const H5O_msg_class_t *const H5O_msg_class_g[] = {
H5O_MSG_DRVINFO, /*0x0014 Driver info settings */
H5O_MSG_AINFO, /*0x0015 Attribute information */
H5O_MSG_REFCOUNT, /*0x0016 Object's ref. count */
- H5O_MSG_UNKNOWN, /*0x0017 Placeholder for unknown message */
+ H5O_MSG_MDJ_CONF, /*0x0017 Metadata journaling config */
+ H5O_MSG_UNKNOWN, /*0x0018 Placeholder for unknown message */
};
+/* HDassert(H5O_UNKNOWN_ID < H5O_MSG_TYPES); */
+
/* Header object ID to class mapping */
/*
* Initialize the object class info table. Begin with the most general types
@@ -157,6 +164,10 @@ H5FL_EXTERN(time_t);
* Programmer: Quincey Koziol
* Thursday, January 18, 2007
*
+ * Changes: JRM -- 12/12/07
+ * Added santity check verifying that H5O_msg_class_g
+ * is big enough.
+ *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -168,6 +179,8 @@ H5O_init_interface(void)
HDassert(H5O_MSG_TYPES == NELMTS(H5O_msg_class_g));
HDassert(sizeof(H5O_fheap_id_t) == H5O_FHEAP_ID_LEN);
+ HDassert(H5O_UNKNOWN_ID < H5O_MSG_TYPES);
+
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5O_init_interface() */
diff --git a/src/H5Opkg.h b/src/H5Opkg.h
index c90ff0a..94af802 100644
--- a/src/H5Opkg.h
+++ b/src/H5Opkg.h
@@ -30,7 +30,7 @@
#define H5O_NMESGS 8 /*initial number of messages */
#define H5O_NCHUNKS 2 /*initial number of chunks */
#define H5O_MIN_SIZE 22 /* Min. obj header data size (must be big enough for a message prefix and a continuation message) */
-#define H5O_MSG_TYPES 24 /* # of types of messages */
+#define H5O_MSG_TYPES 25 /* # of types of messages */
#define H5O_MAX_CRT_ORDER_IDX 65535 /* Max. creation order index value */
/* Versions of object header structure */
@@ -431,7 +431,10 @@ H5_DLLVAR const H5O_msg_class_t H5O_MSG_AINFO[1];
/* Reference Count Message. (0x0016) */
H5_DLLVAR const H5O_msg_class_t H5O_MSG_REFCOUNT[1];
-/* Placeholder for unknown message. (0x0017) */
+/* Metadata Journaling Config Message. (0x0017) */
+H5_DLLVAR const H5O_msg_class_t H5O_MSG_MDJ_CONF[1];
+
+/* Placeholder for unknown message. (0x0018) */
H5_DLLVAR const H5O_msg_class_t H5O_MSG_UNKNOWN[1];
diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h
index 31738a9..3188468 100644
--- a/src/H5Oprivate.h
+++ b/src/H5Oprivate.h
@@ -62,6 +62,8 @@ typedef struct H5O_t H5O_t;
#define H5O_FHEAP_MAX_MAN_SIZE (4 * 1024)
#define H5O_FHEAP_ID_LEN 8
+/* #define H5O_ENABLE_BOGUS 1 */
+
/* Object header macros */
#define H5O_MESG_MAX_SIZE 65536 /*max obj header message size */
#define H5O_ALL (-1) /* Operate on all messages of type */
@@ -160,7 +162,8 @@ typedef struct H5O_copy_t {
#define H5O_DRVINFO_ID 0x0014 /* Driver info message. */
#define H5O_AINFO_ID 0x0015 /* Attribute info message. */
#define H5O_REFCOUNT_ID 0x0016 /* Reference count message. */
-#define H5O_UNKNOWN_ID 0x0017 /* Placeholder message ID for unknown message. */
+#define H5O_MDJ_CONF_ID 0x0017 /* Metadata journaling config message */
+#define H5O_UNKNOWN_ID 0x0018 /* Placeholder message ID for unknown message. */
/* (this should never exist in a file) */
@@ -491,6 +494,60 @@ typedef struct H5O_ainfo_t {
*/
typedef uint32_t H5O_refcount_t; /* Contains # of links to object, if >1 */
+
+/*
+ * Metadata journaling configuration message
+ *
+ * Information on whether and how metadata changes are being journaled,
+ * so as to reconstruct consistent metadata in the file in the event of
+ * a crash, stored in a superblock extension
+ * (Data structure in memory)
+ *
+ * The fields of the H5O_mdj_conf_t structure are discussed individually
+ * below:
+ *
+ * journaling_enabled: Boolean flag indicating whether metadata journaling
+ * is currently enabled.
+ *
+ * journal_is_external: Boolean flag indicating whether the journal is
+ * stored in an external file, or within the HDF5 file.
+ *
+ * If the field is TRUE, external_journal_file_path points to a
+ * buffer containing the path to the external journal file, and
+ * internal_journal_loc is undefined.
+ *
+ * If the field is FALSE, internal_journal_loc contains the base
+ * address of the journal within the hdf5 file, and
+ * external_journal_file_path_ptr must be NULL.
+ *
+ * At least for the initial implementation, journal_is_external
+ * will always be TRUE.
+ *
+ * internal_journal_loc: haddr_t containing the base address of the
+ * interal journal -- if there is one, or undefined if there
+ * isn't (see discussion of journal_is_external above).
+ *
+ * path_len: size_t containing the size of the buffer needed to
+ * contain the path to the external journal file. This field
+ * must contain 0 if the journal is internal.
+ *
+ * external_journal_file_path_ptr: Pointer to uint8_t. If the
+ * journal is stored in an external file, this field points to
+ * a buffer containing the path to this file.
+ *
+ * If the journal is internal, this field must be NULL.
+ */
+typedef struct H5O_mdj_conf_t {
+
+ hbool_t journaling_enabled;
+ hbool_t journal_is_external;
+ haddr_t internal_journal_loc;
+ size_t path_len;
+ uint8_t * external_journal_file_path_ptr;
+
+} H5O_mdj_conf_t;
+
+
/*
* "Unknown" Message.
* (Data structure in memory)
diff --git a/src/Makefile.am b/src/Makefile.am
index 7319c1c..19fde87 100755
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -45,7 +45,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \
H5AC.c H5AC2.c H5B.c H5Bcache.c \
H5B2.c H5B2cache.c H5B2dbg.c H5B2int.c H5B2stat.c H5B2test.c \
- H5C.c H5C2.c H5CS.c \
+ H5C.c H5C2.c H5C2journal.c H5CS.c \
H5D.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \
H5Ddeprec.c H5Defl.c H5Dfill.c H5Dint.c \
H5Dio.c \
diff --git a/src/Makefile.in b/src/Makefile.in
index 61bcbd8..a65324e 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -82,43 +82,44 @@ am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \
H5timer.lo H5trace.lo H5A.lo H5Abtree2.lo H5Adense.lo \
H5Adeprec.lo H5Aint.lo H5Atest.lo H5AC.lo H5AC2.lo H5B.lo \
H5Bcache.lo H5B2.lo H5B2cache.lo H5B2dbg.lo H5B2int.lo \
- H5B2stat.lo H5B2test.lo H5C.lo H5C2.lo H5CS.lo H5D.lo \
- H5Dcompact.lo H5Dcontig.lo H5Ddbg.lo H5Ddeprec.lo H5Defl.lo \
- H5Dfill.lo H5Dint.lo H5Dio.lo H5Distore.lo H5Dmpio.lo H5Doh.lo \
- H5Dselect.lo H5Dtest.lo H5E.lo H5Edeprec.lo H5Eint.lo H5F.lo \
- H5Fdbg.lo H5Ffake.lo H5Fmount.lo H5Fsfile.lo H5Fsuper.lo \
- H5Ftest.lo H5FD.lo H5FDcore.lo H5FDdirect.lo H5FDfamily.lo \
- H5FDlog.lo H5FDmpi.lo H5FDmpio.lo H5FDmpiposix.lo H5FDmulti.lo \
- H5FDsec2.lo H5FDstdio.lo H5FL.lo H5FO.lo H5FS.lo H5FScache.lo \
- H5FSdbg.lo H5FSsection.lo H5G.lo H5Gbtree2.lo H5Gcompact.lo \
- H5Gdense.lo H5Gdeprec.lo H5Gent.lo H5Gint.lo H5Glink.lo \
- H5Gloc.lo H5Gname.lo H5Gnode.lo H5Gobj.lo H5Goh.lo H5Gstab.lo \
- H5Gtest.lo H5Gtraverse.lo H5HF.lo H5HFbtree2.lo H5HFcache.lo \
- H5HFdbg.lo H5HFdblock.lo H5HFdtable.lo H5HFhdr.lo H5HFhuge.lo \
- H5HFiblock.lo H5HFiter.lo H5HFman.lo H5HFsection.lo \
- H5HFspace.lo H5HFstat.lo H5HFtest.lo H5HFtiny.lo H5HG.lo \
- H5HGdbg.lo H5HL.lo H5HLdbg.lo H5HP.lo H5I.lo H5L.lo \
- H5Lexternal.lo H5MF.lo H5MM.lo H5MP.lo H5MPtest.lo H5O.lo \
- H5Oainfo.lo H5Oalloc.lo H5Oattr.lo H5Oattribute.lo H5Obogus.lo \
- H5Obtreek.lo H5Ocache.lo H5Ocont.lo H5Ocopy.lo H5Odbg.lo \
- H5Odrvinfo.lo H5Odtype.lo H5Oefl.lo H5Ofill.lo H5Oginfo.lo \
- H5Olayout.lo H5Olinfo.lo H5Olink.lo H5Omessage.lo H5Omtime.lo \
- H5Oname.lo H5Onull.lo H5Opline.lo H5Orefcount.lo H5Osdspace.lo \
- H5Oshared.lo H5Ostab.lo H5Oshmesg.lo H5Otest.lo H5Ounknown.lo \
- H5P.lo H5Pacpl.lo H5Pdcpl.lo H5Pdeprec.lo H5Pdxpl.lo \
- H5Pfapl.lo H5Pfcpl.lo H5Pfmpl.lo H5Pgcpl.lo H5Pint.lo \
- H5Plapl.lo H5Plcpl.lo H5Pocpl.lo H5Pocpypl.lo H5Pstrcpl.lo \
- H5Ptest.lo H5R.lo H5Rdeprec.lo H5RC.lo H5RS.lo H5S.lo \
- H5Sall.lo H5Sdbg.lo H5Shyper.lo H5Smpio.lo H5Snone.lo \
- H5Spoint.lo H5Sselect.lo H5Stest.lo H5SL.lo H5SM.lo \
- H5SMbtree2.lo H5SMcache.lo H5SMtest.lo H5ST.lo H5T.lo \
- H5Tarray.lo H5Tbit.lo H5Tcommit.lo H5Tcompound.lo H5Tconv.lo \
- H5Tcset.lo H5Tdbg.lo H5Tdeprec.lo H5Tenum.lo H5Tfields.lo \
- H5Tfixed.lo H5Tfloat.lo H5Tinit.lo H5Tnative.lo H5Toffset.lo \
- H5Toh.lo H5Topaque.lo H5Torder.lo H5Tpad.lo H5Tprecis.lo \
- H5Tstrpad.lo H5Tvisit.lo H5Tvlen.lo H5TS.lo H5V.lo H5WB.lo \
- H5Z.lo H5Zdeflate.lo H5Zfletcher32.lo H5Znbit.lo H5Zshuffle.lo \
- H5Zszip.lo H5Zscaleoffset.lo H5Ztrans.lo
+ H5B2stat.lo H5B2test.lo H5C.lo H5C2.lo H5C2journal.lo H5CS.lo \
+ H5D.lo H5Dcompact.lo H5Dcontig.lo H5Ddbg.lo H5Ddeprec.lo \
+ H5Defl.lo H5Dfill.lo H5Dint.lo H5Dio.lo H5Distore.lo \
+ H5Dmpio.lo H5Doh.lo H5Dselect.lo H5Dtest.lo H5E.lo \
+ H5Edeprec.lo H5Eint.lo H5F.lo H5Fdbg.lo H5Ffake.lo H5Fmount.lo \
+ H5Fsfile.lo H5Fsuper.lo H5Ftest.lo H5FD.lo H5FDcore.lo \
+ H5FDdirect.lo H5FDfamily.lo H5FDlog.lo H5FDmpi.lo H5FDmpio.lo \
+ H5FDmpiposix.lo H5FDmulti.lo H5FDsec2.lo H5FDstdio.lo H5FL.lo \
+ H5FO.lo H5FS.lo H5FScache.lo H5FSdbg.lo H5FSsection.lo H5G.lo \
+ H5Gbtree2.lo H5Gcompact.lo H5Gdense.lo H5Gdeprec.lo H5Gent.lo \
+ H5Gint.lo H5Glink.lo H5Gloc.lo H5Gname.lo H5Gnode.lo H5Gobj.lo \
+ H5Goh.lo H5Gstab.lo H5Gtest.lo H5Gtraverse.lo H5HF.lo \
+ H5HFbtree2.lo H5HFcache.lo H5HFdbg.lo H5HFdblock.lo \
+ H5HFdtable.lo H5HFhdr.lo H5HFhuge.lo H5HFiblock.lo H5HFiter.lo \
+ H5HFman.lo H5HFsection.lo H5HFspace.lo H5HFstat.lo H5HFtest.lo \
+ H5HFtiny.lo H5HG.lo H5HGdbg.lo H5HL.lo H5HLdbg.lo H5HP.lo \
+ H5I.lo H5L.lo H5Lexternal.lo H5MF.lo H5MM.lo H5MP.lo \
+ H5MPtest.lo H5O.lo H5Oainfo.lo H5Oalloc.lo H5Oattr.lo \
+ H5Oattribute.lo H5Obogus.lo H5Obtreek.lo H5Ocache.lo \
+ H5Ocont.lo H5Ocopy.lo H5Odbg.lo H5Odrvinfo.lo H5Odtype.lo \
+ H5Oefl.lo H5Ofill.lo H5Oginfo.lo H5Olayout.lo H5Olinfo.lo \
+ H5Olink.lo H5Omessage.lo H5Omtime.lo H5Oname.lo H5Onull.lo \
+ H5Opline.lo H5Orefcount.lo H5Osdspace.lo H5Oshared.lo \
+ H5Ostab.lo H5Oshmesg.lo H5Otest.lo H5Ounknown.lo H5P.lo \
+ H5Pacpl.lo H5Pdcpl.lo H5Pdeprec.lo H5Pdxpl.lo H5Pfapl.lo \
+ H5Pfcpl.lo H5Pfmpl.lo H5Pgcpl.lo H5Pint.lo H5Plapl.lo \
+ H5Plcpl.lo H5Pocpl.lo H5Pocpypl.lo H5Pstrcpl.lo H5Ptest.lo \
+ H5R.lo H5Rdeprec.lo H5RC.lo H5RS.lo H5S.lo H5Sall.lo H5Sdbg.lo \
+ H5Shyper.lo H5Smpio.lo H5Snone.lo H5Spoint.lo H5Sselect.lo \
+ H5Stest.lo H5SL.lo H5SM.lo H5SMbtree2.lo H5SMcache.lo \
+ H5SMtest.lo H5ST.lo H5T.lo H5Tarray.lo H5Tbit.lo H5Tcommit.lo \
+ H5Tcompound.lo H5Tconv.lo H5Tcset.lo H5Tdbg.lo H5Tdeprec.lo \
+ H5Tenum.lo H5Tfields.lo H5Tfixed.lo H5Tfloat.lo H5Tinit.lo \
+ H5Tnative.lo H5Toffset.lo H5Toh.lo H5Topaque.lo H5Torder.lo \
+ H5Tpad.lo H5Tprecis.lo H5Tstrpad.lo H5Tvisit.lo H5Tvlen.lo \
+ H5TS.lo H5V.lo H5WB.lo H5Z.lo H5Zdeflate.lo H5Zfletcher32.lo \
+ H5Znbit.lo H5Zshuffle.lo H5Zszip.lo H5Zscaleoffset.lo \
+ H5Ztrans.lo
libhdf5_la_OBJECTS = $(am_libhdf5_la_OBJECTS)
libhdf5_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
@@ -402,7 +403,7 @@ libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \
H5AC.c H5AC2.c H5B.c H5Bcache.c \
H5B2.c H5B2cache.c H5B2dbg.c H5B2int.c H5B2stat.c H5B2test.c \
- H5C.c H5C2.c H5CS.c \
+ H5C.c H5C2.c H5C2journal.c H5CS.c \
H5D.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \
H5Ddeprec.c H5Defl.c H5Dfill.c H5Dint.c \
H5Dio.c \
@@ -601,6 +602,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Bcache.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5C.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5C2.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5C2journal.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5CS.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5D.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dcompact.Plo@am__quote@