[svn-r14442] Modified metadata cache to adapt more quickly to the insertion of a

large entry, or to a large increases in the size of an existing entry.
This required some additions to the cache configuration structure, and
thus will require changes in the metadata cache documentation.

The basic idea is to monitor the size of entries as they are loaded,
inserted, or increased in size.  If the size of the entry (or increase)
exceeds some user selected fraction of the size of the cache, increase
the size of the cache.

Note that this fix was designed quickly -- while it deals with the 
use case that exposed the problem, we may have to revisit the issue
later.

Tested serial and parallel on Phoenix, and h5committest.

9 files changed, 648 insertions, 41 deletions

diff --git a/src/H5AC.c b/src/H5AC.c
index 8f62095..cd79e6a 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -2546,6 +2546,10 @@ done:
  *		switches for functions that are only tenuously related 
  *		to auto resize configuration.
  *
+ *		JRM - 1/2/08
+ *		Added support for the new flash cache increment related
+ *		fields.
+ *
  *-------------------------------------------------------------------------
  */
 
@@ -2625,6 +2629,9 @@ H5AC_get_cache_auto_resize_config(H5AC_t * cache_ptr,
     config_ptr->max_increment          = internal_config.max_increment;
     config_ptr->decr_mode              = internal_config.decr_mode;
     config_ptr->upper_hr_threshold     = internal_config.upper_hr_threshold;
+    config_ptr->flash_incr_mode	       = internal_config.flash_incr_mode;
+    config_ptr->flash_multiple	       = internal_config.flash_multiple;
+    config_ptr->flash_threshold	       = internal_config.flash_threshold;
     config_ptr->decrement              = internal_config.decrement;
     config_ptr->apply_max_decrement    = internal_config.apply_max_decrement;
     config_ptr->max_decrement          = internal_config.max_decrement;
@@ -2816,6 +2823,10 @@ done:
  *		switches for functions that are only tenuously related 
  *		to auto resize configuration.
  *
+ *		John Mainzer -- 1/3/07
+ *		Updated trace file code to record the new flash cache 
+ *		size increase related fields.
+ *
  *-------------------------------------------------------------------------
  */
 
@@ -2969,7 +2980,7 @@ done:
          ( trace_file_ptr != NULL ) ) {
 
 	HDfprintf(trace_file_ptr, 
-                  "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %d %d %f %f %d %d %d %d %f %d %d\n", 
+                  "%s %d %d %d %d \"%s\" %d %d %d %f %d %d %ld %d %f %f %d %f %f %d %d %d %f %f %d %d %d %d %f %d %d\n", 
 		  "H5AC_set_cache_auto_resize_config",
 		  trace_config.version,
 		  (int)(trace_config.rpt_fcn_enabled),
@@ -2986,6 +2997,9 @@ done:
 		  (int)(trace_config.incr_mode),
 		  trace_config.lower_hr_threshold,
 		  trace_config.increment,
+		  (int)(trace_config.flash_incr_mode),
+		  trace_config.flash_multiple,
+		  trace_config.flash_threshold,
 		  (int)(trace_config.apply_max_increment),
 		  (int)(trace_config.max_increment),
 		  (int)(trace_config.decr_mode),
@@ -3618,7 +3632,9 @@ done:
  *
  * Modifications:
  *
- *              None.
+ *              Updated function for flash cache increment fields.
+ *
+ *              				JRM -- 1/2/08
  *
  *-------------------------------------------------------------------------
  */
@@ -3661,6 +3677,9 @@ H5AC_ext_config_2_int_config(H5AC_cache_config_t * ext_conf_ptr,
     int_conf_ptr->increment              = ext_conf_ptr->increment;
     int_conf_ptr->apply_max_increment    = ext_conf_ptr->apply_max_increment;
     int_conf_ptr->max_increment          = ext_conf_ptr->max_increment;
+    int_conf_ptr->flash_incr_mode	 = ext_conf_ptr->flash_incr_mode;
+    int_conf_ptr->flash_multiple	 = ext_conf_ptr->flash_multiple;
+    int_conf_ptr->flash_threshold	 = ext_conf_ptr->flash_threshold;
 
     int_conf_ptr->decr_mode              = ext_conf_ptr->decr_mode;
     int_conf_ptr->upper_hr_threshold     = ext_conf_ptr->upper_hr_threshold;
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index 628a269..bbbcf11 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -213,6 +213,10 @@ extern hid_t H5AC_ind_dxpl_id;
   /* double      increment              = */ 2.0,                             \
   /* hbool_t     apply_max_increment    = */ TRUE,                            \
   /* size_t      max_increment          = */ (4 * 1024 * 1024),               \
+  /* enum H5C_cache_flash_incr_mode       */                                  \
+  /*                    flash_incr_mode = */ H5C_flash_incr__add_space,       \
+  /* double      flash_multiple         = */ 1.0,                             \
+  /* double      flash_threshold        = */ 0.25,                            \
   /* enum H5C_cache_decr_mode decr_mode = */ H5C_decr__age_out_with_threshold,\
   /* double      upper_hr_threshold     = */ 0.999,                           \
   /* double      decrement              = */ 0.9,                             \
diff --git a/src/H5ACpublic.h b/src/H5ACpublic.h
index 0ddef92..0e75117 100644
--- a/src/H5ACpublic.h
+++ b/src/H5ACpublic.h
@@ -15,7 +15,7 @@
 
 /*-------------------------------------------------------------------------
  *
- * Created:             H5ACproto.h
+ * Created:             H5ACpublic.h
  *                      Jul 10 1997
  *                      Robb Matzke <matzke@llnl.gov>
  *
@@ -208,6 +208,61 @@ extern "C" {
  *      above, this field contains the maximum number of bytes by which the
  *      cache size can be increased in a single re-size.
  *
+ * flash_incr_mode:  Instance of the H5C_cache_flash_incr_mode enumerated
+ *      type whose value indicates whether and by which algorithm we should
+ *      make flash increases in the size of the cache to accomodate insertion
+ *      of large entries and large increases in the size of a single entry.
+ *
+ *      The addition of the flash increment mode was occasioned by performance
+ *      problems that appear when a local heap is increased to a size in excess
+ *      of the current cache size.  While the existing re-size code dealt with
+ *      this eventually, performance was very bad for the remainder of the 
+ *      epoch.
+ *
+ *      At present, there are two possible values for the flash_incr_mode:
+ *
+ *      H5C_flash_incr__off:  Don't perform flash increases in the size of
+ *              the cache.
+ *
+ *      H5C_flash_incr__add_space:  Let x be either the size of a newly
+ *              newly inserted entry, or the number of bytes by which the
+ *              size of an existing entry has been increased.
+ *
+ *              If
+ *                      x > flash_threshold * current max cache size,
+ *
+ *              increase the current maximum cache size by x * flash_multiple
+ *              less any free space in the cache, and star a new epoch.  For
+ *              now at least, pay no attention to the maximum increment.
+ *
+ *      In both of the above cases, the flash increment pays no attention to
+ *      the maximum increment (at least in this first incarnation), but DOES
+ *      stay within max_size.
+ *
+ *      With a little thought, it should be obvious that the above flash
+ *      cache size increase algorithm is not sufficient for all circumstances --
+ *      for example, suppose the user round robins through 
+ *      (1/flash_threshold) +1 groups, adding one data set to each on each
+ *      pass.  Then all will increase in size at about the same time, requiring
+ *      the max cache size to at least double to maintain acceptable
+ *      performance, however the above flash increment algorithm will not be
+ *      triggered.
+ *
+ *      Hopefully, the add space algorithms detailed above will be sufficient 
+ *      for the performance problems encountered to date.  However, we should 
+ *      expect to revisit the issue.
+ *
+ * flash_multiple: Double containing the multiple described above in the
+ *      H5C_flash_incr__add_space section of the discussion of the 
+ *      flash_incr_mode section.  This field is ignored unless flash_incr_mode 
+ *      is H5C_flash_incr__add_space.
+ *
+ * flash_threshold: Double containing the factor by which current max cache size
+ *      is multiplied to obtain the size threshold for the add_space flash
+ *      increment algorithm.  The field is ignored unless flash_incr_mode is
+ *      H5C_flash_incr__add_space.
+ *
+ *
  *
  * Cache size decrease control fields:
  *
@@ -364,6 +419,10 @@ typedef struct H5AC_cache_config_t
     hbool_t                  apply_max_increment;
     size_t                   max_increment;
 
+    enum H5C_cache_flash_incr_mode      flash_incr_mode;
+    double                              flash_multiple;
+    double                              flash_threshold;
+
 
     /* size decrease control fields: */
     enum H5C_cache_decr_mode decr_mode;
diff --git a/src/H5C.c b/src/H5C.c
index 8ac9929..a865270 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -2596,6 +2596,10 @@ static herr_t H5C__autoadjust__ageout__remove_all_markers(H5C_t * cache_ptr);
 
 static herr_t H5C__autoadjust__ageout__remove_excess_markers(H5C_t * cache_ptr);
 
+static herr_t H5C__flash_increase_cache_size(H5C_t * cache_ptr,
+                                             size_t old_entry_size,
+                                             size_t new_entry_size);
+
 static herr_t H5C_flush_single_entry(H5F_t *             f,
                                      hid_t               primary_dxpl_id,
                                      hid_t               secondary_dxpl_id,
@@ -2836,8 +2840,12 @@ done:
  *		ro_ref_count fields.
  *
  *		JRM -- 7/27/07
-*		Added initialization for the new evictions_enabled 
-*		field of H5C_t.
+ *		Added initialization for the new evictions_enabled 
+ *		field of H5C_t.
+ *
+ *		JRM -- 12/31/07
+ *		Added initialization for the new flash cache size increase
+ *		related fields of H5C_t.
  *
  *-------------------------------------------------------------------------
  */
@@ -2954,6 +2962,8 @@ H5C_create(size_t		      max_cache_size,
     cache_ptr->dLRU_tail_ptr			= NULL;
 
     cache_ptr->size_increase_possible		= FALSE;
+    cache_ptr->flash_size_increase_possible     = FALSE;
+    cache_ptr->flash_size_increase_threshold    = 0;
     cache_ptr->size_decrease_possible		= FALSE;
     cache_ptr->resize_enabled			= FALSE;
     cache_ptr->cache_full			= FALSE;
@@ -2974,6 +2984,11 @@ H5C_create(size_t		      max_cache_size,
     (cache_ptr->resize_ctl).apply_max_increment	= TRUE;
     (cache_ptr->resize_ctl).max_increment	= H5C__DEF_AR_MAX_INCREMENT;
 
+    (cache_ptr->resize_ctl).flash_incr_mode	= H5C_flash_incr__off;
+    (cache_ptr->resize_ctl).flash_multiple	= 1.0;
+    (cache_ptr->resize_ctl).flash_threshold	= 0.25;
+
+
     (cache_ptr->resize_ctl).decr_mode		= H5C_decr__off;
     (cache_ptr->resize_ctl).upper_hr_threshold	= H5C__DEF_AR_UPPER_THRESHHOLD;
     (cache_ptr->resize_ctl).decrement	        = H5C__DEF_AR_DECREMENT;
@@ -3079,6 +3094,9 @@ done:
  *		Updated function for display the new prefix field of
  *		H5C_t in output.
  *
+ *		JRM 12/31/07
+ *		Updated function to handle flash size increases.
+ *
  *-------------------------------------------------------------------------
  */
 void
@@ -3125,6 +3143,24 @@ H5C_def_auto_resize_rpt_fcn(H5C_t * cache_ptr,
                     new_min_clean_size);
             break;
 
+        case flash_increase:
+            HDassert( old_max_cache_size < new_max_cache_size );
+
+            HDfprintf(stdout,
+                      "%sflash cache resize(%d) -- size threshold = %Zu.\n",
+                      cache_ptr->prefix, 
+                      (int)((cache_ptr->resize_ctl).flash_incr_mode),
+                      cache_ptr->flash_size_increase_threshold); 
+
+            HDfprintf(stdout,
+                    "%s	cache size increased from (%Zu/%Zu) to (%Zu/%Zu).\n",
+                    cache_ptr->prefix,
+                    old_max_cache_size,
+                    old_min_clean_size,
+                    new_max_cache_size,
+                    new_min_clean_size);
+            break;
+
         case decrease:
             HDassert( old_max_cache_size > new_max_cache_size );
 
@@ -4434,6 +4470,9 @@ done:
  *		Added code to disable evictions when the new 
  *		evictions_enabled field is FALSE.
  *
+ *		JRM -- 12/31/07
+ *		Added code supporting flash cache size increases.
+ *
  *-------------------------------------------------------------------------
  */
 
@@ -4447,6 +4486,7 @@ H5C_insert_entry(H5F_t * 	     f,
                  void *		     thing,
                  unsigned int        flags)
 {
+    const char *	fcn_name = "H5C_insert_entry()";
     herr_t		result;
     herr_t		ret_value = SUCCEED;    /* Return value */
     hbool_t		first_flush = TRUE;
@@ -4524,6 +4564,18 @@ H5C_insert_entry(H5F_t * 	     f,
 
     H5C__RESET_CACHE_ENTRY_STATS(entry_ptr)
 
+    if ( ( cache_ptr->flash_size_increase_possible ) &&
+         ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) {
+
+        result = H5C__flash_increase_cache_size(cache_ptr, 0, entry_ptr->size);
+
+        if ( result < 0 ) {
+
+            HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, \
+                        "H5C__flash_increase_cache_size failed.")
+        }
+    }
+
     if ( ( cache_ptr->evictions_enabled ) &&
          ( (cache_ptr->index_size + entry_ptr->size) > 
 	    cache_ptr->max_cache_size ) ) {
@@ -4999,7 +5051,9 @@ done:
  *
  * Modifications:
  *
- * 		None
+ * 		Added code to do a flash cache size increase if 
+ * 		appropriate.
+ * 						JRM -- 1/11/08
  *
  *-------------------------------------------------------------------------
  */
@@ -5010,6 +5064,8 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr,
                             size_t  new_size)
 {
     herr_t              ret_value = SUCCEED;    /* Return value */
+    herr_t		result;
+    size_t		size_increase;
     H5C_cache_entry_t *	entry_ptr;
 
     FUNC_ENTER_NOAPI(H5C_mark_pinned_entry_dirty, FAIL)
@@ -5039,6 +5095,29 @@ H5C_mark_pinned_entry_dirty(H5C_t * cache_ptr,
     /* update for change in entry size if necessary */
     if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) {
 
+        /* do a flash cache size increase if appropriate */
+        if ( cache_ptr->flash_size_increase_possible ) {
+
+            if ( new_size > entry_ptr->size ) {
+
+                size_increase = new_size - entry_ptr->size;
+
+                if ( size_increase >= 
+		     cache_ptr->flash_size_increase_threshold ) {
+
+                    result = H5C__flash_increase_cache_size(cache_ptr,
+                                                            entry_ptr->size,
+                                                            new_size);
+
+                    if ( result < 0 ) {
+
+                        HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \
+                                    "H5C__flash_increase_cache_size failed.")
+                    }
+                }
+            }
+        }
+
         /* update the protected entry list */
         H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \
                                         (cache_ptr->pel_size), \
@@ -5371,7 +5450,9 @@ done:
  *
  * Modifications:
  *
- * 		None
+ * 		Added code to apply a flash cache size increment if 
+ * 		appropriate.
+ * 						JRM -- 1/11/08
  *
  *-------------------------------------------------------------------------
  */
@@ -5380,8 +5461,11 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr,
                         void *  thing,
                         size_t  new_size)
 {
+    const char *	fcn_name = "H5C_resize_pinned_entry()";
     herr_t              ret_value = SUCCEED;    /* Return value */
+    herr_t		result;
     H5C_cache_entry_t *	entry_ptr;
+    size_t 		size_increase;
 
     FUNC_ENTER_NOAPI(H5C_resize_pinned_entry, FAIL)
 
@@ -5417,6 +5501,29 @@ H5C_resize_pinned_entry(H5C_t * cache_ptr,
     /* update for change in entry size if necessary */
     if ( entry_ptr->size != new_size ) {
 
+        /* do a flash cache size increase if appropriate */
+        if ( cache_ptr->flash_size_increase_possible ) {
+
+            if ( new_size > entry_ptr->size ) {
+
+                size_increase = new_size - entry_ptr->size;
+
+                if ( size_increase >= 
+		     cache_ptr->flash_size_increase_threshold ) {
+
+                    result = H5C__flash_increase_cache_size(cache_ptr,
+                                                            entry_ptr->size,
+                                                            new_size);
+
+                    if ( result < 0 ) {
+
+                        HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \
+                                    "H5C__flash_increase_cache_size failed.")
+                    }
+                }
+            }
+        }
+
         /* update the protected entry list */
         H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pel_len), \
                                         (cache_ptr->pel_size), \
@@ -5604,9 +5711,13 @@ done:
  * 		of cache entries.
  *
  * 		JRM -- 7/27/07
- * 		Added code supporting the new evictions_enabled fieled
+ * 		Added code supporting the new evictions_enabled field
  * 		in H5C_t.
  *
+ * 		JRM -- 1/3/08
+ * 		Added to do a flash cache size increase if appropriate 
+ * 		when a large entry is loaded.
+ *
  *-------------------------------------------------------------------------
  */
 
@@ -5621,6 +5732,7 @@ H5C_protect(H5F_t *	        f,
             void *	        udata2,
 	    unsigned		flags)
 {
+    const char *	fcn_name = "H5C_protect()";
     hbool_t		hit;
     hbool_t		first_flush;
     hbool_t		have_write_permitted = FALSE;
@@ -5678,7 +5790,25 @@ H5C_protect(H5F_t *	        f,
 
         entry_ptr = (H5C_cache_entry_t *)thing;
 
-        /* try to free up some space if necessary and if evictions are permitted */
+	/* If the entry is very large, and we are configured to allow it,
+	 * we may wish to perform a flash cache size increase.
+	 */
+        if ( ( cache_ptr->flash_size_increase_possible ) &&
+             ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) {
+
+            result = H5C__flash_increase_cache_size(cache_ptr, 0, 
+			                            entry_ptr->size);
+
+            if ( result < 0 ) {
+
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, NULL, \
+                            "H5C__flash_increase_cache_size failed.")
+            }
+        }
+
+        /* try to free up some space if necessary and if evictions are 
+	 * permitted 
+	 */
         if ( ( cache_ptr->evictions_enabled ) &&
 	     ( (cache_ptr->index_size + entry_ptr->size) > 
 	       cache_ptr->max_cache_size ) ) {
@@ -5978,6 +6108,10 @@ done:
  *		if the new configuration forces an immediate reduction
  *		in cache size.
  *
+ *		JRM -- 12/31/07
+ *		Added code supporting the new flash cache size increase
+ *		code.
+ *
  *-------------------------------------------------------------------------
  */
 
@@ -5985,6 +6119,7 @@ herr_t
 H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
                                  H5C_auto_size_ctl_t *config_ptr)
 {
+    const char *fcn_name = "H5C_set_cache_auto_resize_config()";
     herr_t	ret_value = SUCCEED;      /* Return value */
     herr_t	result;
     size_t      new_max_cache_size;
@@ -6039,8 +6174,10 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
                     "conflicting threshold fields in new config.")
     }
 
-    cache_ptr->size_increase_possible = TRUE; /* will set to FALSE if needed */
-    cache_ptr->size_decrease_possible = TRUE; /* will set to FALSE if needed */
+    /* will set the increase possible fields to FALSE later if needed */
+    cache_ptr->size_increase_possible       = TRUE;
+    cache_ptr->flash_size_increase_possible = TRUE; 
+    cache_ptr->size_decrease_possible       = TRUE; 
 
     switch ( config_ptr->incr_mode )
     {
@@ -6062,6 +6199,11 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
             HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown incr_mode?!?!?.")
     }
 
+    /* logically, this is were configuration for flash cache size increases
+     * should go.  However, this configuration depends on max_cache_size, so 
+     * we wait until the end of the function, when this field is set.
+     */
+
     switch ( config_ptr->decr_mode )
     {
         case H5C_decr__off:
@@ -6106,9 +6248,13 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
     if ( config_ptr->max_size == config_ptr->min_size ) {
 
         cache_ptr->size_increase_possible = FALSE;
+        cache_ptr->flash_size_increase_possible = FALSE; 
         cache_ptr->size_decrease_possible = FALSE;
     }
 
+    /* flash_size_increase_possible is intentionally omitted from the
+     * following:
+     */
     cache_ptr->resize_enabled = cache_ptr->size_increase_possible ||
                                 cache_ptr->size_decrease_possible;
 
@@ -6196,6 +6342,37 @@ H5C_set_cache_auto_resize_config(H5C_t * cache_ptr,
         }
     }
 
+    /* configure flash size increase facility.  We wait until the
+     * end of the function, as we need the max_cache_size set before 
+     * we start to keep things simple.
+     *
+     * If we haven't already ruled out flash cache size increases above,
+     * go ahead and configure it.
+     */
+
+    if ( cache_ptr->flash_size_increase_possible ) {
+
+        switch ( config_ptr->flash_incr_mode )
+        {
+            case H5C_flash_incr__off:
+                cache_ptr->flash_size_increase_possible = FALSE; 
+	        break;
+
+            case H5C_flash_incr__add_space:
+                cache_ptr->flash_size_increase_possible = TRUE; 
+                cache_ptr->flash_size_increase_threshold = 
+                    (size_t)
+                    (((double)(cache_ptr->max_cache_size)) *
+                     ((cache_ptr->resize_ctl).flash_threshold));
+                break;
+
+            default: /* should be unreachable */
+                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+                            "Unknown flash_incr_mode?!?!?.")
+                break;
+        }
+    }	
+
 done:
 
     FUNC_LEAVE_NOAPI(ret_value)
@@ -7095,6 +7272,9 @@ done:
  *		Also added sanity checks using the new is_read_only and
  *		ro_ref_count parameters.
  *
+ *		JRM -- 12/31/07
+ *		Modified funtion to support flash cache resizes.
+ *
  *-------------------------------------------------------------------------
  */
 herr_t
@@ -7108,6 +7288,7 @@ H5C_unprotect(H5F_t *		  f,
               unsigned int        flags,
               size_t              new_size)
 {
+    const char *	fcn_name = "H5C_unprotect()";
     hbool_t		deleted;
     hbool_t		dirtied;
     hbool_t             set_flush_marker;
@@ -7118,6 +7299,8 @@ H5C_unprotect(H5F_t *		  f,
     hbool_t		clear_entry = FALSE;
 #endif /* H5_HAVE_PARALLEL */
     herr_t              ret_value = SUCCEED;    /* Return value */
+    herr_t              result;
+    size_t		size_increase = 0;
     H5C_cache_entry_t *	entry_ptr;
     H5C_cache_entry_t *	test_entry_ptr;
 
@@ -7265,6 +7448,29 @@ H5C_unprotect(H5F_t *		  f,
         /* update for change in entry size if necessary */
         if ( ( size_changed ) && ( entry_ptr->size != new_size ) ) {
 
+            /* do a flash cache size increase if appropriate */
+            if ( cache_ptr->flash_size_increase_possible ) {
+
+                if ( new_size > entry_ptr->size ) {
+
+                    size_increase = new_size - entry_ptr->size;
+
+                    if ( size_increase >= 
+                         cache_ptr->flash_size_increase_threshold ) {
+
+                        result = H5C__flash_increase_cache_size(cache_ptr, 
+                                                              entry_ptr->size,
+                                                              new_size);
+
+                        if ( result < 0 ) {
+
+                            HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \
+                                        "H5C__flash_increase_cache_size failed.")
+                        }
+                    }
+                }
+            }
+
             /* update the protected list */
             H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr->pl_len), \
                                             (cache_ptr->pl_size), \
@@ -7458,7 +7664,9 @@ done:
  *
  * Modifications:
  *
- *		None.
+ *		Added validation for the flash increment fields.
+ *
+ *						JRM -- 12/31/07
  *
  *-------------------------------------------------------------------------
  */
@@ -7560,7 +7768,7 @@ H5C_validate_resize_config(H5C_auto_size_ctl_t * config_ptr,
                  ( config_ptr->apply_max_increment != FALSE ) ) {
 
                 HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
-                            "apply_max_increment must be either TRUE or FALSE");
+                          "apply_max_increment must be either TRUE or FALSE");
             }
 
             /* no need to check max_increment, as it is a size_t,
@@ -7568,6 +7776,33 @@ H5C_validate_resize_config(H5C_auto_size_ctl_t * config_ptr,
              */
         } /* H5C_incr__threshold */
 
+        switch ( config_ptr->flash_incr_mode )
+        {
+	    case H5C_flash_incr__off:
+                /* nothing to do here */
+		break;
+
+	    case H5C_flash_incr__add_space:
+                if ( ( config_ptr->flash_multiple < 0.1 ) ||
+                     ( config_ptr->flash_multiple > 10.0 ) ) {
+
+                    HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
+                        "flash_multiple must be in the range [0.1, 10.0]");
+                }
+
+                if ( ( config_ptr->flash_threshold < 0.1 ) ||
+                     ( config_ptr->flash_threshold > 1.0 ) ) {
+
+                    HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
+                        "flash_threshold must be in the range [0.1, 1.0]");
+                }
+		break;
+
+	    default:
+                HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, \
+			    "Invalid flash_incr_mode");
+		break;
+	}
     } /* H5C_RESIZE_CFG__VALIDATE_INCREMENT */
 
 
@@ -7707,6 +7942,9 @@ done:
  *		reduction, and to adjust to changes in the
  *		H5C_auto_size_ctl_t structure.
  *
+ *		JRM -- 1/5/08
+ *		Added support for flash cache size increases.
+ *
  *-------------------------------------------------------------------------
  */
 
@@ -7990,6 +8228,30 @@ H5C__auto_adjust_cache_size(H5C_t * cache_ptr,
 
             cache_ptr->size_decreased = TRUE;
         }
+
+        /* update flash cache size increase fields as appropriate */
+        if ( cache_ptr->flash_size_increase_possible ) {
+
+            switch ( (cache_ptr->resize_ctl).flash_incr_mode )
+            {
+                case H5C_flash_incr__off:
+                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+                       "flash_size_increase_possible but H5C_flash_incr__off?!")
+                    break;
+
+                case H5C_flash_incr__add_space:
+                    cache_ptr->flash_size_increase_threshold =
+                        (size_t)
+                        (((double)(cache_ptr->max_cache_size)) *
+                         ((cache_ptr->resize_ctl).flash_threshold));
+                    break;
+
+                default: /* should be unreachable */
+                    HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+                                "Unknown flash_incr_mode?!?!?.")
+                    break;
+            }
+        }
     }
 
     if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) {
@@ -8741,6 +9003,177 @@ done:
 
 } /* H5C__autoadjust__ageout__remove_excess_markers() */
 
+/*-------------------------------------------------------------------------
+ *
+ * Function:	H5C__flash_increase_cache_size
+ *
+ * Purpose:    	If there is not at least new_entry_size - old_entry_size
+ *		bytes of free space in the cache and the current 
+ *              max_cache_size is less than (cache_ptr->resize_ctl).max_size, 
+ *              perform a flash increase in the cache size and then reset 
+ *              the full cache hit rate statistics, and exit.
+ *
+ * Return:      Non-negative on success/Negative on failure.
+ *
+ * Programmer:  John Mainzer, 12/31/07
+ *
+ * Modifications:
+ *
+ *		None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static herr_t
+H5C__flash_increase_cache_size(H5C_t * cache_ptr,
+                               size_t old_entry_size,
+                               size_t new_entry_size)
+{
+    const char *		fcn_name = "H5C__flash_increase_cache_size()";
+    herr_t			ret_value = SUCCEED;      /* Return value */
+    size_t			new_max_cache_size = 0;
+    size_t			old_max_cache_size = 0;
+    size_t			new_min_clean_size = 0;
+    size_t			old_min_clean_size = 0;
+    size_t                      space_needed;
+    enum H5C_resize_status	status = flash_increase; /* may change */
+    double			hit_rate;
+
+    FUNC_ENTER_NOAPI_NOINIT(H5C__flash_increase_cache_size)
+#if 0
+    HDfprintf(stdout, "%s: old = %ld, new = %ld.\n", fcn_name,
+	      (long)old_entry_size, (long)new_entry_size);
+#endif
+    HDassert( cache_ptr );
+    HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC );
+    HDassert( cache_ptr->flash_size_increase_possible );
+    HDassert( new_entry_size > cache_ptr->flash_size_increase_threshold );
+    HDassert( old_entry_size < new_entry_size );
+
+    if ( old_entry_size >= new_entry_size ) {
+
+        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+                    "old_entry_size >= new_entry_size")
+    }
+
+    space_needed = new_entry_size - old_entry_size;
+
+    if ( ( (cache_ptr->index_size + space_needed) > 
+                           cache_ptr->max_cache_size ) &&
+         ( cache_ptr->max_cache_size < (cache_ptr->resize_ctl).max_size ) ) {
+
+        /* we have work to do */
+
+        switch ( (cache_ptr->resize_ctl).flash_incr_mode ) 
+        {
+            case H5C_flash_incr__off:
+                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+                   "flash_size_increase_possible but H5C_flash_incr__off?!")
+                break;
+
+            case H5C_flash_incr__add_space:
+		if ( cache_ptr->index_size < cache_ptr->max_cache_size ) {
+                
+		    HDassert( (cache_ptr->max_cache_size - cache_ptr->index_size)
+				    < space_needed );
+                    space_needed -= cache_ptr->max_cache_size - cache_ptr->index_size;
+		}
+		space_needed = 
+		    (size_t)(((double)space_needed) * 
+			     (cache_ptr->resize_ctl).flash_multiple);
+
+                new_max_cache_size = cache_ptr->max_cache_size + space_needed;
+
+                break;
+
+            default: /* should be unreachable */
+                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+                            "Unknown flash_incr_mode?!?!?.")
+                break;
+        }
+
+        if ( new_max_cache_size > (cache_ptr->resize_ctl).max_size ) {
+
+	    new_max_cache_size = (cache_ptr->resize_ctl).max_size;
+        }
+
+        HDassert( new_max_cache_size > cache_ptr->max_cache_size );
+
+        new_min_clean_size = (size_t)
+                             ((double)new_max_cache_size *
+                              ((cache_ptr->resize_ctl).min_clean_fraction));
+
+        HDassert( new_min_clean_size <= new_max_cache_size );
+
+        old_max_cache_size = cache_ptr->max_cache_size;
+        old_min_clean_size = cache_ptr->min_clean_size;
+
+        cache_ptr->max_cache_size = new_max_cache_size;
+        cache_ptr->min_clean_size = new_min_clean_size;
+
+        /* update flash cache size increase fields as appropriate */
+        HDassert ( cache_ptr->flash_size_increase_possible );
+
+        switch ( (cache_ptr->resize_ctl).flash_incr_mode )
+        {
+            case H5C_flash_incr__off:
+                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+                   "flash_size_increase_possible but H5C_flash_incr__off?!")
+                break;
+
+            case H5C_flash_incr__add_space:
+                cache_ptr->flash_size_increase_threshold =
+                    (size_t)
+                    (((double)(cache_ptr->max_cache_size)) *
+                     ((cache_ptr->resize_ctl).flash_threshold));
+                break;
+
+            default: /* should be unreachable */
+                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+                            "Unknown flash_incr_mode?!?!?.")
+                break;
+        }
+
+        /* note that we don't cycle the epoch markers.  We can 
+         * argue either way as to whether we should, but for now
+         * we don't.
+         */
+
+        if ( (cache_ptr->resize_ctl).rpt_fcn != NULL ) {
+
+	    /* get the hit rate for the reporting function.  Should still
+	     * be good as we havent reset the hit rate statistics.
+	     */
+            if ( H5C_get_cache_hit_rate(cache_ptr, &hit_rate) != SUCCEED ) {
+
+                HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't get hit rate.")
+            }
+
+            (*((cache_ptr->resize_ctl).rpt_fcn))
+                (cache_ptr,
+                 H5C__CURR_AUTO_RESIZE_RPT_FCN_VER,
+                 hit_rate,
+                 status,
+                 old_max_cache_size,
+                 new_max_cache_size,
+                 old_min_clean_size,
+                 new_min_clean_size);
+        }
+
+        if ( H5C_reset_cache_hit_rate_stats(cache_ptr) != SUCCEED ) {
+
+            /* this should be impossible... */
+            HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \
+                        "H5C_reset_cache_hit_rate_stats failed.")
+        }
+    }
+
+done:
+
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5C__flash_increase_cache_size() */
+
 
 /*-------------------------------------------------------------------------
  * Function:    H5C_flush_invalidate_cache
diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h
index 949f923..7c04ea2 100644
--- a/src/H5Cpkg.h
+++ b/src/H5Cpkg.h
@@ -465,6 +465,18 @@
  *		all the ways this can happen, we simply set this flag when
  *		we receive a new configuration.
  *
+ * flash_size_increase_possible: Depending on the configuration data given
+ * 		in the resize_ctl field, it may or may not be possible
+ * 		for a flash size increase to occur.  We set this flag
+ * 		whenever we receive a new configuration so as to avoid 
+ * 		repeated calculations.
+ *
+ * flash_size_increase_threshold: If a flash cache size increase is possible,
+ * 		this field is used to store the minimum size of a new entry
+ * 		or size increase needed to trigger a flash cache size 
+ * 		increase.  Note that this field must be updated whenever 
+ * 		the size of the cache is changed.
+ *
  * size_decrease_possible:  Depending on the configuration data given
  *              in the resize_ctl field, it may or may not be possible
  *              to decrease the size of the cache.  Rather than test for
@@ -580,10 +592,10 @@
  *		equal to the array index has not been in cache when
  *		requested in the current epoch.
  *
- * write_protects:  Array of int64 of length H5C__MAX_NUM_TYPE_IDS + 1.  The cells
- * 		are used to record the number of times an entry with type id
- * 		equal to the array index has been write protected in the 
- * 		current epoch.
+ * write_protects:  Array of int64 of length H5C__MAX_NUM_TYPE_IDS + 1.  The 
+ *		cells are used to record the number of times an entry with 
+ *		type id equal to the array index has been write protected 
+ *		in the current epoch.
  *
  * 		Observe that (hits + misses) = (write_protects + read_protects).
  *
@@ -855,6 +867,8 @@ struct H5C_t
     H5C_cache_entry_t *	        dLRU_tail_ptr;
 
     hbool_t			size_increase_possible;
+    hbool_t			flash_size_increase_possible;
+    size_t			flash_size_increase_threshold; 
     hbool_t			size_decrease_possible;
     hbool_t			resize_enabled;
     hbool_t			cache_full;
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index d365755..10585a3 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -37,7 +37,7 @@
 #include "H5Fprivate.h"		/* File access				*/
 
 
-#define H5C_DO_SANITY_CHECKS		0
+#define H5C_DO_SANITY_CHECKS		1
 #define H5C_DO_EXTREME_SANITY_CHECKS	0
 
 /* This sanity checking constant was picked out of the air.  Increase
@@ -51,7 +51,7 @@
 /* H5C_COLLECT_CACHE_STATS controls overall collection of statistics
  * on cache activity.  In general, this #define should be set to 0.
  */
-#define H5C_COLLECT_CACHE_STATS	0
+#define H5C_COLLECT_CACHE_STATS	1
 
 /* H5C_COLLECT_CACHE_ENTRY_STATS controls collection of statistics
  * in individual cache entries.
@@ -587,7 +587,58 @@ typedef struct H5C_cache_entry_t
  *	above, this field contains the maximum number of bytes by which the
  *	cache size can be increased in a single re-size.
  *
- *
+ * flash_incr_mode:  Instance of the H5C_cache_flash_incr_mode enumerated
+ * 	type whose value indicates whether and by what algorithm we should
+ * 	make flash increases in the size of the cache to accomodate insertion
+ * 	of large entries and large increases in the size of a single entry.
+ *
+ * 	The addition of the flash increment mode was occasioned by performance
+ * 	problems that appear when a local heap is increased to a size in excess
+ * 	of the current cache size.  While the existing re-size code dealt with 
+ * 	this eventually, performance was very bad for the remainder of the 
+ * 	epoch.
+ * 	
+ * 	At present, there are two possible values for the flash_incr_mode:
+ *
+ * 	H5C_flash_incr__off:  Don't perform flash increases in the size of 
+ * 		the cache. 
+ *
+ *	H5C_flash_incr__add_space:  Let x be either the size of a newly 
+ *	        newly inserted entry, or the number of bytes by which the 
+ *	        size of an existing entry has been increased.
+ *
+ *	        If 
+ *	        	x > flash_threshold * current max cache size, 
+ *
+ *	       	increase the current maximum cache size by x * flash_multiple
+ *	       	less any free space in the cache, and start a new epoch.  For
+ *	       	now at least, pay no attention to the maximum increment.
+ *
+ *
+ *	With a little thought, it should be obvious that the above flash
+ *	cache size increase algorithm is not sufficient for all circumstances --
+ *	for example, suppose the user round robins through 
+ *	(1/flash_threshold) +1 groups, adding one data set to each on each 
+ *	pass.  Then all will increase in size at about the same time, requiring 
+ *	the max cache size to at least double to maintain acceptable 
+ *      performance, however the above flash increment algorithm will not be 
+ *	triggered.
+ *
+ *	Hopefully, the add space algorithm detailed above will be sufficient 
+ *	for the performance problems encountered to date.  However, we should 
+ *	expect to revisit the issue.
+ *
+ * flash_multiple: Double containing the multiple described above in the 
+ * 	H5C_flash_incr__add_space section of the discussion of the 
+ * 	flash_incr_mode	section.  This field is ignored unless flash_incr_mode 
+ * 	is H5C_flash_incr__add_space.
+ *
+ * flash_threshold: Double containing the factor by which current max cache size
+ *      is multiplied to obtain the size threshold for the add_space flash 
+ *      increment algorithm.  The field is ignored unless flash_incr_mode is 
+ *	H5C_flash_incr__add_space.
+ *
+ *	
  * Cache size decrease control fields:
  *
  * decr_mode: Instance of the H5C_cache_decr_mode enumerated type whose
@@ -701,6 +752,8 @@ typedef struct H5C_cache_entry_t
 #define H5C__DEF_AR_MIN_CLEAN_FRAC		0.5
 #define H5C__DEF_AR_INCREMENT			2.0
 #define H5C__DEF_AR_MAX_INCREMENT		((size_t)( 2 * 1024 * 1024))
+#define	H5C__DEF_AR_FLASH_MULTIPLE		1.0
+#define H5C__DEV_AR_FLASH_THRESHOLD		0.25
 #define H5C__DEF_AR_DECREMENT			0.9
 #define H5C__DEF_AR_MAX_DECREMENT		((size_t)( 1 * 1024 * 1024))
 #define H5C__DEF_AR_EPCHS_B4_EVICT		3
@@ -713,6 +766,7 @@ enum H5C_resize_status
 {
     in_spec,
     increase,
+    flash_increase,
     decrease,
     at_max_size,
     at_min_size,
@@ -733,45 +787,48 @@ typedef void (*H5C_auto_resize_rpt_fcn)(H5C_t * cache_ptr,
 typedef struct H5C_auto_size_ctl_t
 {
     /* general configuration fields: */
-    int32_t			version;
-    H5C_auto_resize_rpt_fcn	rpt_fcn;
+    int32_t				version;
+    H5C_auto_resize_rpt_fcn		rpt_fcn;
 
-    hbool_t			set_initial_size;
-    size_t			initial_size;
+    hbool_t				set_initial_size;
+    size_t				initial_size;
 
-    double			min_clean_fraction;
+    double				min_clean_fraction;
 
-    size_t			max_size;
-    size_t			min_size;
+    size_t				max_size;
+    size_t				min_size;
 
-    int64_t			epoch_length;
+    int64_t				epoch_length;
 
 
     /* size increase control fields: */
-    enum H5C_cache_incr_mode	incr_mode;
+    enum H5C_cache_incr_mode		incr_mode;
 
-    double			lower_hr_threshold;
+    double				lower_hr_threshold;
 
-    double			increment;
+    double				increment;
 
-    hbool_t			apply_max_increment;
-    size_t			max_increment;
+    hbool_t				apply_max_increment;
+    size_t				max_increment;
 
+    enum H5C_cache_flash_incr_mode	flash_incr_mode;
+    double				flash_multiple;
+    double				flash_threshold;
 
     /* size decrease control fields: */
-    enum H5C_cache_decr_mode	decr_mode;
+    enum H5C_cache_decr_mode		decr_mode;
 
-    double			upper_hr_threshold;
+    double				upper_hr_threshold;
 
-    double			decrement;
+    double				decrement;
 
-    hbool_t			apply_max_decrement;
-    size_t			max_decrement;
+    hbool_t				apply_max_decrement;
+    size_t				max_decrement;
 
-    int32_t			epochs_before_eviction;
+    int32_t				epochs_before_eviction;
 
-    hbool_t			apply_empty_reserve;
-    double			empty_reserve;
+    hbool_t				apply_empty_reserve;
+    double				empty_reserve;
 
 } H5C_auto_size_ctl_t;
 
diff --git a/src/H5Cpublic.h b/src/H5Cpublic.h
index 082d512..7ef959a 100644
--- a/src/H5Cpublic.h
+++ b/src/H5Cpublic.h
@@ -41,6 +41,12 @@ enum H5C_cache_incr_mode
     H5C_incr__threshold
 };
 
+enum H5C_cache_flash_incr_mode
+{
+    H5C_flash_incr__off,
+    H5C_flash_incr__add_space
+};
+
 enum H5C_cache_decr_mode
 {
     H5C_decr__off,
diff --git a/src/H5HL.c b/src/H5HL.c
index 20cac4e..358816b 100644
--- a/src/H5HL.c
+++ b/src/H5HL.c
@@ -967,6 +967,9 @@ H5HL_insert(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t buf_size, const void *
             need_more = need_size;
 
 	new_heap_alloc = heap->heap_alloc + need_more;
+#if 0 /* JRM */ /* delete this once we are convinced that the general 
+		 * fix will do the job.
+		 */
 /*
  * XXX: This is a _total_ hack, a real kludge. :-/  The metadata cache currently
  *      responds very poorly when an object is inserted into the cache (or
@@ -1002,6 +1005,7 @@ H5HL_insert(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t buf_size, const void *
 	        HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, (size_t)-1, "H5AC_set_cache_auto_resize_config() failed.")
         } /* end if */
 }
+#endif /* JRM */
 	HDassert(heap->heap_alloc < new_heap_alloc);
 	H5_CHECK_OVERFLOW(heap->heap_alloc, size_t, hsize_t);
 	H5_CHECK_OVERFLOW(new_heap_alloc, size_t, hsize_t);
diff --git a/src/H5O.c b/src/H5O.c
index c615437..b5c74ff 100644
--- a/src/H5O.c
+++ b/src/H5O.c
@@ -91,6 +91,11 @@ static herr_t H5O_visit(hid_t loc_id, const char *obj_name, H5_index_t idx_type,
 /*********************/
 
 /* Header message ID to class mapping */
+
+/* Remember to increment H5O_MSG_TYPES in H5Opkg.h when adding a new
+ * message.
+ */
+
 const H5O_msg_class_t *const H5O_msg_class_g[] = {
     H5O_MSG_NULL,		/*0x0000 Null				*/
     H5O_MSG_SDSPACE,		/*0x0001 Dataspace			*/
@@ -174,6 +179,10 @@ H5FL_EXTERN(H5_obj_t);
  * Programmer:	Quincey Koziol
  *              Thursday, January 18, 2007
  *
+ * Changes:     JRM -- 12/12/07
+ *              Added santity check verifying that H5O_msg_class_g
+ *              is big enough.
+ *
  *-------------------------------------------------------------------------
  */
 static herr_t
@@ -185,6 +194,8 @@ H5O_init_interface(void)
     HDassert(H5O_MSG_TYPES == NELMTS(H5O_msg_class_g));
     HDassert(sizeof(H5O_fheap_id_t) == H5O_FHEAP_ID_LEN);
 
+    HDassert(H5O_UNKNOWN_ID < H5O_MSG_TYPES);
+
     FUNC_LEAVE_NOAPI(SUCCEED)
 } /* end H5O_init_interface() */