Checkin of additions to cache image parallel test code and

associated bug fixes.

Also, modifications to H5PB_dest() to flush the page buffer before
destroying the page buffer.  This is necessary, as when persistant
free space managers are enabled, the page buffer will typically contain
dirty FSM data at page buffer destroy time.

Tested serial/debug, serial/production, serial/check-vfd/debug,
       parallel/debug, parallel/production on Jelly.

7 files changed, 2046 insertions, 46 deletions

diff --git a/src/H5AC.c b/src/H5AC.c
index a561852..be41b6a 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -611,6 +611,7 @@ H5AC_dest(H5F_t *f, hid_t dxpl_id)
 
     /* Sanity check */
     HDassert(f);
+    HDassert(f->shared);
     HDassert(f->shared->cache);
 
 #if H5AC_DUMP_STATS_ON_CLOSE
@@ -641,9 +642,17 @@ H5AC_dest(H5F_t *f, hid_t dxpl_id)
         /* Sanity check */
         HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC);
 
-    /* Attempt to flush all entries from rank 0 & Bcast clean list to other ranks */
-    if(H5AC__flush_entries(f, dxpl_id) < 0)
+    /* If the file was opened R/W, attempt to flush all entries 
+     * from rank 0 & Bcast clean list to other ranks.
+     *
+     * Must not flush in the R/O case, as this will trigger the 
+     * free space manager settle routines.
+     */
+    if ( ( H5F_ACC_RDWR & H5F_INTENT(f) ) && 
+         ( H5AC__flush_entries(f, dxpl_id) < 0 ) )
+
         HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush")
+
 #endif /* H5_HAVE_PARALLEL */
 
     /* Destroy the cache */
diff --git a/src/H5C.c b/src/H5C.c
index 120abb8..2ba9f2d 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -761,7 +761,10 @@ H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id)
         HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "can't create cache image")
 
 #ifdef H5_HAVE_PARALLEL
-    if(!image_generated && cache_ptr->aux_ptr != NULL && f->shared->fs_persist) {
+    if ( ( H5F_INTENT(f) & H5F_ACC_RDWR ) &&
+         ( ! image_generated ) && 
+         ( cache_ptr->aux_ptr != NULL ) && 
+         ( f->shared->fs_persist ) ) {
         /* If persistent free space managers are enabled, flushing the
          * metadata cache may result in the deletion, insertion, and/or
          * dirtying of entries.
@@ -7295,14 +7298,20 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
 
             prev_ptr = entry_ptr->aux_prev;
 
+            if ( ( !(entry_ptr->prefetched_dirty) ) 
 #ifdef H5_HAVE_PARALLEL
-            if(!(entry_ptr->coll_access)) {
+                 && ( ! (entry_ptr->coll_access) )
 #endif /* H5_HAVE_PARALLEL */
-                if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
-                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry")
-#ifdef H5_HAVE_PARALLEL
+               ) {
+
+                if ( H5C__flush_single_entry(f, dxpl_id, entry_ptr, 
+                                  H5C__FLUSH_INVALIDATE_FLAG | 
+                                  H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0 )
+
+                    HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \
+                                "unable to flush entry")
+
             } /* end if */
-#endif /* H5_HAVE_PARALLEL */
 
 	    /* we are scanning the clean LRU, so the serialize function
 	     * will not be called on any entry -- thus there is no
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 794be50..3a2b422 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -1006,8 +1006,15 @@ H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush)
             /* Push error, but keep going*/
             HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file")
 
+        /* Set up I/O info for operation */
+        fio_info.f = f;
+        if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id)))
+            HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+        if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
+            HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+
         /* Shutdown the page buffer cache */
-        if(H5PB_dest(f) < 0)
+        if(H5PB_dest(f, &fio_info) < 0)
             /* Push error, but keep going*/
             HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing page buffer cache")
 
@@ -1027,13 +1034,6 @@ H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush)
             f->shared->root_grp = NULL;
         } /* end if */
 
-        /* Set up I/O info for operation */
-        fio_info.f = f;
-        if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id)))
-            HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
-        if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
-            HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
-
         /* Destroy other components of the file */
         if(H5F__accum_reset(&fio_info, TRUE) < 0)
             /* Push error, but keep going*/
diff --git a/src/H5MF.c b/src/H5MF.c
index 358e326..87c910c 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -1347,6 +1347,12 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r
     HDassert(f);
     HDassert(H5F_INTENT(f) & H5F_ACC_RDWR);
 
+    if(f->shared->first_alloc_dealloc) {
+        HDassert(! H5AC_cache_image_pending(f));
+        if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
+    } /* end if */
+
     /* Set mapped type, treating global heap as raw data */
     map_type = (alloc_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : alloc_type;
 
diff --git a/src/H5PB.c b/src/H5PB.c
index c67ae59..ad61657 100644
--- a/src/H5PB.c
+++ b/src/H5PB.c
@@ -468,7 +468,7 @@ H5PB__dest_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data)
 /*-------------------------------------------------------------------------
  * Function:	H5PB_dest
  *
- * Purpose:	destroy the PB on the file.
+ * Purpose:	Flush and destroy the PB on the file if it exists.
  *
  * Return:	Non-negative on success/Negative on failure
  *
@@ -477,7 +477,7 @@ H5PB__dest_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data)
  *-------------------------------------------------------------------------
  */
 herr_t
-H5PB_dest(H5F_t *f)
+H5PB_dest(H5F_t *f, const H5F_io_info2_t *fio_info)
 {
     herr_t  ret_value = SUCCEED;        /* Return value */
 
@@ -486,11 +486,14 @@ H5PB_dest(H5F_t *f)
     /* Sanity checks */
     HDassert(f);
 
-    /* Destroy page buffer info, if there is any */
+    /* flush and destroy the page buffer, if it exists */
     if(f->shared->page_buf) {
         H5PB_t *page_buf = f->shared->page_buf;
         H5PB_ud1_t op_data;                 /* Iteration context */
 
+        if(H5PB_flush(fio_info)<0)
+            HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTFLUSH, FAIL, "can't flush page buffer")
+
         /* Set up context info */
         op_data.page_buf = page_buf;
 
diff --git a/src/H5PBprivate.h b/src/H5PBprivate.h
index 3b5dcae..3933029 100644
--- a/src/H5PBprivate.h
+++ b/src/H5PBprivate.h
@@ -89,7 +89,7 @@ typedef struct H5PB_t {
 /* General routines */
 H5_DLL herr_t H5PB_create(H5F_t *file, size_t page_buffer_size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc);
 H5_DLL herr_t H5PB_flush(const H5F_io_info2_t *fio_info);
-H5_DLL herr_t H5PB_dest(H5F_t *file);
+H5_DLL herr_t H5PB_dest(H5F_t *file, const H5F_io_info2_t *fio_info);
 H5_DLL herr_t H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr);
 H5_DLL herr_t H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf);
 H5_DLL herr_t H5PB_remove_entry(const H5F_t *f, haddr_t addr);
diff --git a/testpar/t_cache_image.c b/testpar/t_cache_image.c
index 7283fa7..9e9efea 100644
--- a/testpar/t_cache_image.c
+++ b/testpar/t_cache_image.c
@@ -25,10 +25,17 @@
 #include "cache_common.h"
 #include "genall5.h"
 
+#if 1
+#define TEST_FILES_TO_CONSTRUCT 2
+#else
+#define TEST_FILES_TO_CONSTRUCT 1
+#endif
 #define CHUNK_SIZE              10
 #define DSET_SIZE               (40 * CHUNK_SIZE)
 #define MAX_NUM_DSETS           256
-
+#define PAR_NUM_DSETS           32
+#define PAGE_SIZE		(4 * 1024)
+#define PB_SIZE                 (64 * PAGE_SIZE)
 
 /* global variable declarations: */
 
@@ -36,6 +43,7 @@
 const char *FILENAMES[] = {
         "t_cache_image_00",
         "t_cache_image_01",
+        "t_cache_image_02",
         NULL
 };
 
@@ -45,28 +53,48 @@ static void create_data_sets(hid_t file_id, int min_dset, int max_dset);
 static void delete_data_sets(hid_t file_id, int min_dset, int max_dset);
 
 static void open_hdf5_file(const hbool_t create_file,
-    const hbool_t mdci_sbem_expected, const hbool_t read_only, 
-    const hbool_t set_mdci_fapl, const hbool_t config_fsm,
-    const char * hdf_file_name, const unsigned cache_image_flags, 
-    hid_t * file_id_ptr, H5F_t ** file_ptr_ptr, H5C_t ** cache_ptr_ptr,
-    MPI_Comm comm, MPI_Info info, int l_facc_type,
-    const hbool_t all_coll_metadata_ops, const hbool_t coll_metadata_write,
-    const int md_write_strat);
+    const hbool_t mdci_sbem_expected,
+    const hbool_t read_only,
+    const hbool_t set_mdci_fapl,
+    const hbool_t config_fsm,
+    const hbool_t enable_page_buffer,
+    const char * hdf_file_name,
+    const unsigned cache_image_flags,
+    hid_t * file_id_ptr,
+    H5F_t ** file_ptr_ptr,
+    H5C_t ** cache_ptr_ptr,
+    MPI_Comm comm, 
+    MPI_Info info, 
+   int l_facc_type,
+   const hbool_t all_coll_metadata_ops,
+   const hbool_t coll_metadata_write,
+   const int md_write_strat);
 
 static void verify_data_sets(hid_t file_id, int min_dset, int max_dset);
 
 /* local test function declarations */
 
 static hbool_t parse_flags(int argc, char * argv[], hbool_t * setup_ptr, 
-    hbool_t display);
+    hbool_t * ici_ptr, int * file_idx_ptr, int * mpi_size_ptr, hbool_t display);
 static void usage(void);
 static unsigned construct_test_file(int test_file_index);
+static void par_create_dataset(int dset_num, hid_t file_id, int mpi_rank, 
+    int mpi_size);
+static void par_delete_dataset(int dset_num, hid_t file_id, int mpi_rank);
+static void par_verify_dataset(int dset_num, hid_t file_id, int mpi_rank);
 
+static hbool_t serial_insert_cache_image(int file_name_idx, int mpi_size);
+static void serial_verify_dataset(int dset_num, hid_t file_id, int mpi_size);
 
 /* top level test function declarations */
+static unsigned verify_cache_image_RO(int file_name_id, 
+    int md_write_strat, int mpi_rank);
 static unsigned verify_cache_image_RW(int file_name_id, 
     int md_write_strat, int mpi_rank);
 
+static hbool_t smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, 
+    int mpi_rank, int mpi_size);
+
 
 /****************************************************************************/
 /***************************** Utility Functions ****************************/
@@ -187,6 +215,7 @@ construct_test_file(int test_file_index)
                        /* read_only             */ FALSE,
                        /* set_mdci_fapl         */ TRUE,
 		       /* config_fsm            */ TRUE,
+                       /* enable_page_buffer    */ FALSE,
                        /* hdf_file_name         */ filename,
                        /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
                        /* file_id_ptr           */ &file_id,
@@ -258,6 +287,7 @@ construct_test_file(int test_file_index)
                            /* read_only             */ FALSE,
                            /* set_mdci_fapl         */ TRUE,
 		           /* config_fsm            */ FALSE,
+                           /* enable_page_buffer    */ FALSE,
                            /* hdf_file_name         */ filename,
                            /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
                            /* file_id_ptr           */ &file_id,
@@ -331,6 +361,7 @@ construct_test_file(int test_file_index)
                        /* read_only             */ TRUE,
                        /* set_mdci_fapl         */ FALSE,
 	               /* config_fsm            */ FALSE,
+                       /* enable_page_buffer    */ FALSE,
                        /* hdf_file_name         */ filename,
                        /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
                        /* file_id_ptr           */ &file_id,
@@ -863,6 +894,8 @@ delete_data_sets(hid_t file_id, int min_dset, int max_dset)
  *              
  *                                                 JRM -- 2/1/17
  *
+ *              Modified function to handle 
+ *
  *-------------------------------------------------------------------------
  */
 
@@ -872,6 +905,7 @@ open_hdf5_file(const hbool_t create_file,
 	       const hbool_t read_only,
 	       const hbool_t set_mdci_fapl,
 	       const hbool_t config_fsm,
+               const hbool_t enable_page_buffer,
 	       const char * hdf_file_name,
                const unsigned cache_image_flags,
                hid_t * file_id_ptr,
@@ -901,6 +935,8 @@ open_hdf5_file(const hbool_t create_file,
         FALSE,
         H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE};
 
+    HDassert(!create_file || config_fsm);
+
     if ( pass )
     {
 	/* opening the file both read only and with a cache image 
@@ -910,6 +946,7 @@ open_hdf5_file(const hbool_t create_file,
         if ( ( create_file && mdci_sbem_expected ) ||
              ( create_file && read_only ) ||
              ( config_fsm && !create_file ) ||
+             ( create_file && enable_page_buffer && ! config_fsm ) ||
              ( hdf_file_name == NULL ) ||
              ( ( set_mdci_fapl ) && ( cache_image_flags == 0 ) ) ||
              ( ( set_mdci_fapl ) &&
@@ -1029,9 +1066,32 @@ open_hdf5_file(const hbool_t create_file,
         }
     }
 
+    if ( ( pass ) && ( config_fsm ) ) {
+
+        if ( H5Pset_file_space_page_size(fcpl_id, PAGE_SIZE) == FAIL ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Pset_file_space_page_size() failed.\n";
+        }
+    }
+
     if ( show_progress ) 
         HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
 
+    /* setup the page buffer if indicated */
+    if ( ( pass ) && ( enable_page_buffer ) ) {
+
+        if ( H5Pset_page_buffer_size(fapl_id, PB_SIZE, 0, 0) < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Pset_page_buffer_size() failed.\n";
+        } 
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
     if ( ( pass ) && ( l_facc_type == FACC_MPIO ) ) {
 
         /* set Parallel access with communicator */
@@ -1162,6 +1222,31 @@ open_hdf5_file(const hbool_t create_file,
     if ( show_progress ) 
         HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
 
+
+    /* verify expected page buffer status.  At present, page buffering 
+     * must be disabled in parallel -- hopefully this will change in the 
+     * future.
+     */
+    if ( pass ) {
+
+        if ( ( file_ptr->shared->page_buf ) &&
+             ( ( ! enable_page_buffer ) || ( l_facc_type == FACC_MPIO ) ) ) {
+
+            pass = FALSE;
+            failure_mssg = "page buffer unexepectedly enabled.";
+
+        } else if ( ( file_ptr->shared->page_buf != NULL ) &&
+             ( ( enable_page_buffer ) || ( l_facc_type != FACC_MPIO ) ) ) {
+
+            pass = FALSE;
+            failure_mssg = "page buffer unexepectedly disabled.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
     /* verify expected metadata cache status */
 
     /* get the cache image control structure from the cache, and verify
@@ -1181,6 +1266,7 @@ open_hdf5_file(const hbool_t create_file,
     if ( show_progress ) 
         HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
 
+
     if ( pass ) {
 
         if ( set_mdci_fapl ) {
@@ -1308,6 +1394,1244 @@ open_hdf5_file(const hbool_t create_file,
 
 
 /*-------------------------------------------------------------------------
+ * Function:    par_create_dataset()
+ *
+ * Purpose:     Collectively create a chunked dataset, and fill it with 
+ *              known values.
+ *
+ *              On failure, set pass to FALSE, and set failure_mssg
+ *              to point to an appropriate failure message.
+ *
+ *              Do nothing if pass is FALSE on entry.
+ *
+ * Return:      void
+ *
+ * Programmer:  John Mainzer
+ *              3/4/17
+ *
+ * Modifications:
+ *
+ *              None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void
+par_create_dataset(int dset_num,
+                   hid_t file_id,
+                   int mpi_rank,
+                   int mpi_size)
+{
+    const char * fcn_name = "par_create_dataset()";
+    char dset_name[256];
+    hbool_t show_progress = FALSE;
+    hbool_t valid_chunk;
+    hbool_t verbose = FALSE;
+    int cp = 0;
+    int i, j, k, l;
+    int data_chunk[1][CHUNK_SIZE][CHUNK_SIZE];
+    hsize_t dims[3];
+    hsize_t a_size[3];
+    hsize_t offset[3];
+    hsize_t chunk_size[3];
+    hid_t status;
+    hid_t dataspace_id = -1;
+    hid_t memspace_id = -1;
+    hid_t dset_id = -1;
+    hid_t filespace_id = -1;
+    hid_t dcpl_id = -1;
+    hid_t dxpl_id = -1;
+
+    show_progress = (show_progress && (mpi_rank == 0));
+    verbose       = (verbose && (mpi_rank == 0));
+
+    sprintf(dset_name, "/dset%03d", dset_num);
+
+    if ( show_progress ) {
+        HDfprintf(stdout, "%s: dset name = \"%s\".\n", fcn_name, dset_name);
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+    }
+
+    if ( pass ) {
+
+        /* create a dataspace for the chunked dataset */
+        dims[0] = (hsize_t)mpi_size;
+        dims[1] = DSET_SIZE;
+        dims[2] = DSET_SIZE;
+        dataspace_id = H5Screate_simple(3, dims, NULL);
+
+        if ( dataspace_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Screate_simple() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* set the dataset creation plist to specify that the raw data is
+     * to be partioned into 1X10X10 element chunks.
+     */
+
+    if ( pass ) {
+
+        dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
+
+        if ( dcpl_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Pcreate(H5P_DATASET_CREATE) failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    if ( pass ) {
+
+        chunk_size[0] = 1;
+        chunk_size[1] = CHUNK_SIZE;
+        chunk_size[2] = CHUNK_SIZE;
+
+        if ( H5Pset_chunk(dcpl_id, 3, chunk_size) < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Pset_chunk() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* create the dataset */
+    if ( pass ) {
+
+        dset_id = H5Dcreate2(file_id, dset_name, H5T_STD_I32BE,
+                             dataspace_id, H5P_DEFAULT,
+                             dcpl_id, H5P_DEFAULT);
+
+        if ( dset_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Dcreate() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* get the file space ID */
+    if ( pass ) {
+
+        filespace_id = H5Dget_space(dset_id);
+
+        if ( filespace_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Dget_space() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* create the mem space to be used to read and write chunks */
+    if ( pass ) {
+
+        dims[0] = 1;
+        dims[1] = CHUNK_SIZE;
+        dims[2] = CHUNK_SIZE;
+        memspace_id = H5Screate_simple(3, dims, NULL);
+
+        if ( memspace_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Screate_simple() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* select in memory hyperslab */
+    if ( pass ) {
+
+        offset[0] = 0;                  /* offset of hyperslab in memory */
+        offset[1] = 0;
+        offset[2] = 0;
+        a_size[0] = 1;                  /* size of hyperslab */
+        a_size[1] = CHUNK_SIZE;
+        a_size[2] = CHUNK_SIZE;
+        status = H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, offset, NULL,
+                                     a_size, NULL);
+
+        if ( status < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Sselect_hyperslab() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* setup the DXPL for collective I/O */
+    if ( pass ) {
+
+        dxpl_id = H5Pcreate(H5P_DATASET_XFER);
+
+        if ( dxpl_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Pcreate(H5P_DATASET_XFER) failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    if ( pass ) {
+
+        if ( H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Pset_dxpl_mpio() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* initialize the dataset with collective writes */
+    i = 0;
+    while ( ( pass ) && ( i < DSET_SIZE ) )
+    {
+        j = 0;
+        while ( ( pass ) && ( j < DSET_SIZE ) )
+        {
+
+            if ( show_progress ) 
+                HDfprintf(stdout, "%s: cp = %d.0, pass = %d.\n", 
+                          fcn_name, cp, pass);
+
+            /* initialize the slab */
+            for ( k = 0; k < CHUNK_SIZE; k++ )
+            {
+                for ( l = 0; l < CHUNK_SIZE; l++ )
+                {
+                    data_chunk[0][k][l] = (DSET_SIZE * DSET_SIZE * mpi_rank) +
+                                          (DSET_SIZE * (i + k)) + j + l +
+                                          dset_num;
+                }
+            }
+
+            if ( show_progress ) 
+                HDfprintf(stdout, "%s: cp = %d.1, pass = %d.\n", 
+                          fcn_name, cp, pass);
+
+            /* select on disk hyperslab */
+            offset[0] = (hsize_t)mpi_rank; /* offset of hyperslab in file */
+            offset[1] = (hsize_t)i;
+            offset[2] = (hsize_t)j;
+            a_size[0] = (hsize_t)1;        /* size of hyperslab */
+            a_size[1] = CHUNK_SIZE;  
+            a_size[2] = CHUNK_SIZE;
+            status = H5Sselect_hyperslab(filespace_id, H5S_SELECT_SET,
+                                         offset, NULL, a_size, NULL);
+
+            if ( status < 0 ) {
+
+                pass = FALSE;
+                failure_mssg = "disk H5Sselect_hyperslab() failed.";
+            }
+
+            if ( show_progress ) 
+                HDfprintf(stdout, "%s: cp = %d.2, pass = %d.\n", 
+                          fcn_name, cp, pass);
+
+            /* write the chunk to file */
+            status = H5Dwrite(dset_id, H5T_NATIVE_INT, memspace_id,
+                              filespace_id, dxpl_id, data_chunk);
+
+            if ( status < 0 ) {
+
+                pass = FALSE;
+                failure_mssg = "H5Dwrite() failed.";
+            }
+
+            if ( show_progress ) 
+                HDfprintf(stdout, "%s: cp = %d.3, pass = %d.\n", 
+                          fcn_name, cp, pass);
+
+            j += CHUNK_SIZE;
+        }
+
+        i += CHUNK_SIZE;
+    }
+
+    cp++;
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* read data from data sets and validate it */
+    i = 0;
+    while ( ( pass ) && ( i < DSET_SIZE ) )
+    {
+        j = 0;
+        while ( ( pass ) && ( j < DSET_SIZE ) )
+        {
+            /* select on disk hyperslab */
+            offset[0] = (hsize_t)mpi_rank;
+            offset[1] = (hsize_t)i; /* offset of hyperslab in file */
+            offset[2] = (hsize_t)j;
+            a_size[0] = (hsize_t)1;
+            a_size[1] = CHUNK_SIZE; /* size of hyperslab */
+            a_size[2] = CHUNK_SIZE;
+
+            status = H5Sselect_hyperslab(filespace_id, H5S_SELECT_SET,
+                                         offset, NULL, a_size, NULL);
+
+            if ( status < 0 ) {
+
+               pass = FALSE;
+               failure_mssg = "disk hyperslab create failed.";
+            }
+
+            /* read the chunk from file */
+            if ( pass ) {
+
+                status = H5Dread(dset_id, H5T_NATIVE_INT,
+                                 memspace_id, filespace_id,
+                                 dxpl_id, data_chunk);
+
+                if ( status < 0 ) {
+
+                   pass = FALSE;
+                   failure_mssg = "chunk read failed.";
+                }
+            }
+
+            /* validate the slab */
+            if ( pass ) {
+
+                valid_chunk = TRUE;
+                for ( k = 0; k < CHUNK_SIZE; k++ )
+                {
+                    for ( l = 0; l < CHUNK_SIZE; l++ )
+                    {
+                        if ( data_chunk[0][k][l]
+                             !=
+                             ((DSET_SIZE * DSET_SIZE * mpi_rank) +
+                              (DSET_SIZE * (i + k)) + j + l + dset_num) ) {
+
+                            valid_chunk = FALSE;
+
+                            if ( verbose ) {
+
+                                HDfprintf(stdout,
+                                    "data_chunk[%0d][%0d] = %0d, expect %0d.\n",
+                                    k, l, data_chunk[0][k][l],
+                                    ((DSET_SIZE * DSET_SIZE * mpi_rank) +
+                                     (DSET_SIZE * (i + k)) + j + l + dset_num));
+                                    HDfprintf(stdout,
+                                    "dset_num = %d, i = %d, j = %d, k = %d, l = %d\n",
+                                    dset_num, i, j, k, l);
+                            }
+                        }
+                    }
+                }
+
+                if ( ! valid_chunk ) {
+
+                    pass = FALSE;
+                    failure_mssg = "slab validation failed.";
+
+                    if ( verbose ) {
+
+                        fprintf(stdout,
+                                "Chunk (%0d, %0d) in /dset%03d is invalid.\n",
+                                i, j, dset_num);
+                    }
+                }
+            }
+            j += CHUNK_SIZE;
+        }
+        i += CHUNK_SIZE;
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* close the data space */
+    if ( ( pass ) && ( H5Sclose(dataspace_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Sclose(dataspace_id) failed.";
+    }
+
+    /* close the file space */
+    if ( ( pass ) && ( H5Sclose(filespace_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Sclose(filespace_id) failed.";
+    }
+
+    /* close the dataset */
+    if ( ( pass ) && ( H5Dclose(dset_id) < 0 ) )  {
+
+        pass = FALSE;
+        failure_mssg = "H5Dclose(dset_id) failed.";
+    }
+
+    /* close the mem space */
+    if ( ( pass ) && ( H5Sclose(memspace_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Sclose(memspace_id) failed.";
+    }
+
+    /* close the dataset creation property list */
+    if ( ( pass ) && ( H5Pclose(dcpl_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Pclose(dcpl) failed.";
+    }
+
+    /* close the data access property list */
+    if ( ( pass ) && ( H5Pclose(dxpl_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Pclose(dxpl) failed.";
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    return;
+
+} /* par_create_dataset() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    par_delete_dataset()
+ *
+ * Purpose:     Collectively delete the specified dataset.
+ *
+ *              On failure, set pass to FALSE, and set failure_mssg
+ *              to point to an appropriate failure message.
+ *
+ *              Do nothing if pass is FALSE on entry.
+ *
+ * Return:      void
+ *
+ * Programmer:  John Mainzer
+ *              3/6/17
+ *
+ * Modifications:
+ *
+ *              None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void
+par_delete_dataset(int dset_num,
+                   hid_t file_id,
+                   int mpi_rank)
+{
+    const char * fcn_name = "par_delete_dataset()";
+    char dset_name[256];
+    hbool_t show_progress = FALSE;
+    int cp = 0;
+
+    show_progress = (show_progress && (mpi_rank == 0));
+
+    sprintf(dset_name, "/dset%03d", dset_num);
+
+    if ( show_progress ) {
+        HDfprintf(stdout, "%s: dset name = \"%s\".\n", fcn_name, dset_name);
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+    }
+
+    /* verify the target dataset */
+    if ( pass ) {
+
+        par_verify_dataset(dset_num, file_id, mpi_rank);
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* delete the target dataset */
+    if ( pass ) {
+
+        if ( H5Ldelete(file_id, dset_name, H5P_DEFAULT) < 0) {
+
+            pass = FALSE;
+            failure_mssg = "H5Ldelete() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    return;
+
+} /* par_delete_dataset() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    par_insert_cache_image()
+ *
+ * Purpose:     Insert a cache image in the supplied file.
+ *
+ *              At present, cache image is not enabled in the parallel 
+ *              so we have to insert the cache image with a serial 
+ *              process.  Do this via a fork and an execv from process 0.
+ *              All processes wait until the child process completes, and
+ *              then return.
+ *
+ *              On failure, set pass to FALSE, and set failure_mssg
+ *              to point to an appropriate failure message.
+ *
+ *              Do nothing if pass is FALSE on entry.
+ *
+ * Return:      void
+ *
+ * Programmer:  John Mainzer
+ *              3/8/17
+ *
+ * Modifications:
+ *
+ *              None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void
+par_insert_cache_image(int file_name_idx, int mpi_rank, int mpi_size )
+{
+    hbool_t show_progress = FALSE;
+
+    if ( pass ) {
+
+        if ( mpi_rank == 0 ) { /* insert cache image in supplied test file */
+
+            char file_name_idx_str[32];
+            char mpi_size_str[32];
+            int child_status;
+            pid_t child_pid;
+
+            sprintf(file_name_idx_str, "%d", file_name_idx);
+            sprintf(mpi_size_str, "%d", mpi_size);
+
+            child_pid = fork();
+
+            if ( child_pid == 0 ) { /* this is the child process */
+
+                /* fun and games to shutup the compiler */
+                char param0[32] = "t_cache_image";
+                char param1[32] = "ici";
+                char * child_argv[] = {param0, 
+                                       param1, 
+                                       file_name_idx_str, 
+                                       mpi_size_str, 
+                                       NULL};
+
+                /* we may need to play with the path here */
+                if ( execv("t_cache_image", child_argv) == -1 ) {
+
+                    HDfprintf(stdout, 
+                             "execl() of ici process failed. errno = %d(%s)\n",
+                             errno, strerror(errno));
+                    exit(1);
+                }
+
+            } else if ( child_pid != -1 ) {  
+                /* this is the parent process -- wait until child is done */
+                if ( -1 == waitpid(child_pid, &child_status, WUNTRACED)) {
+
+                    HDfprintf(stdout, "can't wait on ici process.\n");
+                    pass = FALSE;
+
+                } else if ( ! WIFEXITED(child_status) ) {
+
+                    HDfprintf(stdout, "ici process hasn't exitied.\n");
+                    pass = FALSE;
+
+                } else if ( WEXITSTATUS(child_status) != 0 ) {
+
+                    HDfprintf(stdout, "ici process reports failure.\n");
+                    pass = FALSE;
+
+                } else if ( show_progress ) {
+
+                    HDfprintf(stdout, "cache image insertion complete.\n");
+                }
+            } else { /* fork failed */
+
+                HDfprintf(stdout, 
+                          "can't create process to insert cache image.\n");
+                pass = FALSE;
+            }
+        }
+    }
+
+    if ( pass ) {
+
+        /* make sure insertion of the cache image is complete 
+         * before proceeding 
+         */
+        MPI_Barrier(MPI_COMM_WORLD);
+    }
+
+    return;
+
+} /* par_insert_cache_image() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    par_verify_dataset()
+ *
+ * Purpose:     Collectively verify the contents of a chunked dataset.
+ *
+ *              On failure, set pass to FALSE, and set failure_mssg
+ *              to point to an appropriate failure message.
+ *
+ *              Do nothing if pass is FALSE on entry.
+ *
+ * Return:      void
+ *
+ * Programmer:  John Mainzer
+ *              3/6/17
+ *
+ * Modifications:
+ *
+ *              None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void
+par_verify_dataset(int dset_num,
+                   hid_t file_id,
+                   int mpi_rank)
+{
+    const char * fcn_name = "par_verify_dataset()";
+    char dset_name[256];
+    hbool_t show_progress = FALSE;
+    hbool_t valid_chunk;
+    hbool_t verbose = FALSE;
+    int cp = 0;
+    int i, j, k, l;
+    int data_chunk[1][CHUNK_SIZE][CHUNK_SIZE];
+    hsize_t dims[3];
+    hsize_t a_size[3];
+    hsize_t offset[3];
+    hid_t status;
+    hid_t memspace_id = -1;
+    hid_t dset_id = -1;
+    hid_t filespace_id = -1;
+    hid_t dxpl_id = -1;
+
+    show_progress = (show_progress && (mpi_rank == 0));
+    verbose       = (verbose && (mpi_rank == 0));
+
+    sprintf(dset_name, "/dset%03d", dset_num);
+
+    if ( show_progress ) {
+        HDfprintf(stdout, "%s: dset name = \"%s\".\n", fcn_name, dset_name);
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+    }
+
+    if ( pass ) {
+
+        /* open the dataset */
+
+        dset_id = H5Dopen2(file_id, dset_name, H5P_DEFAULT);
+
+        if ( dset_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Dopen2() failed.";
+        }
+    }
+
+    /* get the file space ID */
+    if ( pass ) {
+
+        filespace_id = H5Dget_space(dset_id);
+
+        if ( filespace_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Dget_space() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* create the mem space to be used to read */
+    if ( pass ) {
+
+        dims[0] = 1;
+        dims[1] = CHUNK_SIZE;
+        dims[2] = CHUNK_SIZE;
+        memspace_id = H5Screate_simple(3, dims, NULL);
+
+        if ( memspace_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Screate_simple() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* select in memory hyperslab */
+    if ( pass ) {
+
+        offset[0] = 0;                  /* offset of hyperslab in memory */
+        offset[1] = 0;
+        offset[2] = 0;
+        a_size[0] = 1;                  /* size of hyperslab */
+        a_size[1] = CHUNK_SIZE;
+        a_size[2] = CHUNK_SIZE;
+        status = H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, offset, NULL,
+                                     a_size, NULL);
+
+        if ( status < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Sselect_hyperslab() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* setup the DXPL for collective I/O */
+    if ( pass ) {
+
+        dxpl_id = H5Pcreate(H5P_DATASET_XFER);
+
+        if ( dxpl_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Pcreate(H5P_DATASET_XFER) failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    if ( pass ) {
+
+        if ( H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Pset_dxpl_mpio() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* read data from data sets and validate it */
+    i = 0;
+    while ( ( pass ) && ( i < DSET_SIZE ) )
+    {
+        j = 0;
+        while ( ( pass ) && ( j < DSET_SIZE ) )
+        {
+            /* select on disk hyperslab */
+            offset[0] = (hsize_t)mpi_rank;
+            offset[1] = (hsize_t)i; /* offset of hyperslab in file */
+            offset[2] = (hsize_t)j;
+            a_size[0] = (hsize_t)1;
+            a_size[1] = CHUNK_SIZE; /* size of hyperslab */
+            a_size[2] = CHUNK_SIZE;
+
+            status = H5Sselect_hyperslab(filespace_id, H5S_SELECT_SET,
+                                         offset, NULL, a_size, NULL);
+
+            if ( status < 0 ) {
+
+               pass = FALSE;
+               failure_mssg = "disk hyperslab create failed.";
+            }
+
+            /* read the chunk from file */
+            if ( pass ) {
+
+                status = H5Dread(dset_id, H5T_NATIVE_INT,
+                                 memspace_id, filespace_id,
+                                 dxpl_id, data_chunk);
+
+                if ( status < 0 ) {
+
+                   pass = FALSE;
+                   failure_mssg = "chunk read failed.";
+                }
+            }
+
+            /* validate the slab */
+            if ( pass ) {
+
+                valid_chunk = TRUE;
+                for ( k = 0; k < CHUNK_SIZE; k++ )
+                {
+                    for ( l = 0; l < CHUNK_SIZE; l++ )
+                    {
+                        if ( data_chunk[0][k][l]
+                             !=
+                             ((DSET_SIZE * DSET_SIZE * mpi_rank) +
+                              (DSET_SIZE * (i + k)) + j + l + dset_num) ) {
+
+                            valid_chunk = FALSE;
+
+                            if ( verbose ) {
+
+                                HDfprintf(stdout,
+                                    "data_chunk[%0d][%0d] = %0d, expect %0d.\n",
+                                    k, l, data_chunk[0][k][l],
+                                    ((DSET_SIZE * DSET_SIZE * mpi_rank) +
+                                     (DSET_SIZE * (i + k)) + j + l + dset_num));
+                                    HDfprintf(stdout,
+                                    "dset_num = %d, i = %d, j = %d, k = %d, l = %d\n",
+                                    dset_num, i, j, k, l);
+                            }
+                        }
+                    }
+                }
+
+                if ( ! valid_chunk ) {
+
+                    pass = FALSE;
+                    failure_mssg = "slab validation failed.";
+
+                    if ( verbose ) {
+
+                        fprintf(stdout,
+                                "Chunk (%0d, %0d) in /dset%03d is invalid.\n",
+                                i, j, dset_num);
+                    }
+                }
+            }
+            j += CHUNK_SIZE;
+        }
+        i += CHUNK_SIZE;
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* close the file space */
+    if ( ( pass ) && ( H5Sclose(filespace_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Sclose(filespace_id) failed.";
+    }
+
+    /* close the dataset */
+    if ( ( pass ) && ( H5Dclose(dset_id) < 0 ) )  {
+
+        pass = FALSE;
+        failure_mssg = "H5Dclose(dset_id) failed.";
+    }
+
+    /* close the mem space */
+    if ( ( pass ) && ( H5Sclose(memspace_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Sclose(memspace_id) failed.";
+    }
+
+    /* close the data access property list */
+    if ( ( pass ) && ( H5Pclose(dxpl_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Pclose(dxpl) failed.";
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    return;
+
+} /* par_verify_dataset() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    serial_insert_cache_image()
+ *
+ * Purpose:     Insert a cache image in the supplied file.
+ *
+ * 		To populate the cache image, validate the contents 
+ * 		of the file before closing.
+ *
+ *              On failure, print an appropriate error message and 
+ *              return FALSE.
+ *
+ * Return:      TRUE if succussful, FALSE otherwise.
+ *
+ * Programmer:  John Mainzer
+ *              3/8/17
+ *
+ * Modifications:
+ *
+ *              None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static hbool_t
+serial_insert_cache_image(int file_name_idx, int mpi_size )
+{
+    const char * fcn_name = "serial_insert_cache_image()";
+    char filename[512];
+    hbool_t show_progress = FALSE;
+    int cp = 0;
+    int i;
+    int num_dsets = PAR_NUM_DSETS;
+    hid_t file_id = -1;
+    H5F_t *file_ptr = NULL;
+    H5C_t *cache_ptr = NULL;
+    MPI_Comm dummy_comm = MPI_COMM_WORLD;
+    MPI_Info dummy_info = MPI_INFO_NULL;
+
+    pass = TRUE;
+
+    if ( show_progress )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 1) setup the file name */
+    if ( pass ) {
+
+        HDassert(FILENAMES[file_name_idx]);
+
+        if ( h5_fixname(FILENAMES[file_name_idx], H5P_DEFAULT,
+                                  filename, sizeof(filename))
+            == NULL ) {
+
+            pass = FALSE;
+            HDfprintf(stdout, "h5_fixname() failed.\n");
+        }
+    }
+
+    if ( show_progress )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 2) Open the PHDF5 file with the cache image FAPL entry. 
+     */
+
+    if ( pass ) {
+
+        open_hdf5_file(/* create_file           */ FALSE,
+                       /* mdci_sbem_expected    */ FALSE,
+                       /* read_only             */ FALSE,
+                       /* set_mdci_fapl         */ TRUE,
+                       /* config_fsm            */ FALSE,
+                       /* enable_page_buffer    */ FALSE,
+                       /* hdf_file_name         */ filename,
+                       /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
+                       /* file_id_ptr           */ &file_id,
+                       /* file_ptr_ptr          */ &file_ptr,
+                       /* cache_ptr_ptr         */ &cache_ptr,
+                       /* comm                  */ dummy_comm,
+                       /* info                  */ dummy_info,
+                       /* l_facc_type           */ 0,
+                       /* all_coll_metadata_ops */ FALSE,
+                       /* coll_metadata_write   */ FALSE,
+                       /* md_write_strat        */ 1);
+    }
+
+    if ( show_progress )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 3) Validate contents of the file */
+
+    i = 0;
+    while ( ( pass ) && ( i < num_dsets ) ) {
+
+        serial_verify_dataset(i, file_id, mpi_size);
+        i++;
+    }
+
+    if ( show_progress )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 4) Close the file */
+
+    if ( pass ) {
+
+        if ( H5Fclose(file_id) < 0  ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Fclose() failed.\n";
+        }
+    }
+
+    if ( show_progress )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    return pass;
+
+} /* serial_insert_cache_image() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    serial_verify_dataset()
+ *
+ * Purpose:     Verify the contents of a chunked dataset.
+ *
+ *              On failure, set pass to FALSE, and set failure_mssg
+ *              to point to an appropriate failure message.
+ *
+ *              Do nothing if pass is FALSE on entry.
+ *
+ * Return:      void
+ *
+ * Programmer:  John Mainzer
+ *              3/6/17
+ *
+ * Modifications:
+ *
+ *              None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static void
+serial_verify_dataset(int dset_num,
+                      hid_t file_id,
+                      int mpi_size)
+{
+    const char * fcn_name = "serial_verify_dataset()";
+    char dset_name[256];
+    hbool_t show_progress = FALSE;
+    hbool_t valid_chunk;
+    hbool_t verbose = FALSE;
+    int cp = 0;
+    int i, j, k, l, m;
+    int data_chunk[1][CHUNK_SIZE][CHUNK_SIZE];
+    hsize_t dims[3];
+    hsize_t a_size[3];
+    hsize_t offset[3];
+    hid_t status;
+    hid_t memspace_id = -1;
+    hid_t dset_id = -1;
+    hid_t filespace_id = -1;
+
+    sprintf(dset_name, "/dset%03d", dset_num);
+
+    if ( show_progress ) {
+        HDfprintf(stdout, "%s: dset name = \"%s\".\n", fcn_name, dset_name);
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+    }
+
+    if ( pass ) {
+
+        /* open the dataset */
+
+        dset_id = H5Dopen2(file_id, dset_name, H5P_DEFAULT);
+
+        if ( dset_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Dopen2() failed.";
+        }
+    }
+
+    /* get the file space ID */
+    if ( pass ) {
+
+        filespace_id = H5Dget_space(dset_id);
+
+        if ( filespace_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Dget_space() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* create the mem space to be used to read */
+    if ( pass ) {
+
+        dims[0] = 1;
+        dims[1] = CHUNK_SIZE;
+        dims[2] = CHUNK_SIZE;
+        memspace_id = H5Screate_simple(3, dims, NULL);
+
+        if ( memspace_id < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Screate_simple() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* select in memory hyperslab */
+    if ( pass ) {
+
+        offset[0] = 0;                  /* offset of hyperslab in memory */
+        offset[1] = 0;
+        offset[2] = 0;
+        a_size[0] = 1;                  /* size of hyperslab */
+        a_size[1] = CHUNK_SIZE;
+        a_size[2] = CHUNK_SIZE;
+        status = H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, offset, NULL,
+                                     a_size, NULL);
+
+        if ( status < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Sselect_hyperslab() failed.";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* read data from data sets and validate it */
+    i = 0;
+    while ( ( pass ) && ( i < mpi_size ) )
+    {
+        j = 0;
+        while ( ( pass ) && ( j < DSET_SIZE ) )
+        {
+            k = 0;
+            while ( ( pass ) && ( k < DSET_SIZE ) )
+            {
+                /* select on disk hyperslab */
+                offset[0] = (hsize_t)i; /* offset of hyperslab in file */
+                offset[1] = (hsize_t)j; /* offset of hyperslab in file */
+                offset[2] = (hsize_t)k;
+                a_size[0] = (hsize_t)1;
+                a_size[1] = CHUNK_SIZE; /* size of hyperslab */
+                a_size[2] = CHUNK_SIZE;
+
+                status = H5Sselect_hyperslab(filespace_id, H5S_SELECT_SET,
+                                             offset, NULL, a_size, NULL);
+
+                if ( status < 0 ) {
+
+                   pass = FALSE;
+                   failure_mssg = "disk hyperslab create failed.";
+                }
+
+                /* read the chunk from file */
+                if ( pass ) {
+
+                    status = H5Dread(dset_id, H5T_NATIVE_INT,
+                                     memspace_id, filespace_id,
+                                     H5P_DEFAULT, data_chunk);
+
+                    if ( status < 0 ) {
+
+                       pass = FALSE;
+                       failure_mssg = "chunk read failed.";
+                    }
+                }
+
+                /* validate the slab */
+                if ( pass ) {
+
+                    valid_chunk = TRUE;
+
+                    for ( l = 0; l < CHUNK_SIZE; l++ )
+                    {
+                        for ( m = 0; m < CHUNK_SIZE; m++ )
+                        {
+                            if ( data_chunk[0][l][m]
+                                 !=
+                                 ((DSET_SIZE * DSET_SIZE * i) +
+                                  (DSET_SIZE * (j + l)) + k + m + dset_num) ) {
+
+                                valid_chunk = FALSE;
+
+                                if ( verbose ) {
+
+                                    HDfprintf(stdout,
+                                    "data_chunk[%0d][%0d] = %0d, expect %0d.\n",
+                                    j, k, data_chunk[0][j][k],
+                                    ((DSET_SIZE * DSET_SIZE * i) +
+                                     (DSET_SIZE * (j + l)) + k + m + dset_num));
+                                    HDfprintf(stdout,
+                                    "dset_num = %d, i = %d, j = %d, k = %d, l = %d, m = %d\n",
+                                    dset_num, i, j, k, l, m);
+                                }
+                            }
+                        }
+                    }
+
+                    if ( ! valid_chunk ) {
+
+                        pass = FALSE;
+                        failure_mssg = "slab validation failed.";
+
+                        if ( verbose ) {
+
+                            fprintf(stdout,
+                                  "Chunk (%0d, %0d) in /dset%03d is invalid.\n",
+                                   j, k, dset_num);
+                        }
+                    }
+                }
+                k += CHUNK_SIZE;
+            }
+            j += CHUNK_SIZE;
+        }
+        i++;
+    }
+
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* close the file space */
+    if ( ( pass ) && ( H5Sclose(filespace_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Sclose(filespace_id) failed.";
+    }
+
+    /* close the dataset */
+    if ( ( pass ) && ( H5Dclose(dset_id) < 0 ) )  {
+
+        pass = FALSE;
+        failure_mssg = "H5Dclose(dset_id) failed.";
+    }
+
+    /* close the mem space */
+    if ( ( pass ) && ( H5Sclose(memspace_id) < 0 ) ) {
+
+        pass = FALSE;
+        failure_mssg = "H5Sclose(memspace_id) failed.";
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    return;
+
+} /* serial_verify_dataset() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    parse_flags
  *
  * Purpose:     Parse the flags passed to this program, and load the 
@@ -1321,50 +2645,80 @@ open_hdf5_file(const hbool_t create_file,
  *
  *-------------------------------------------------------------------------
  */
-hbool_t
-parse_flags(int argc, char * argv[], hbool_t * setup_ptr, hbool_t display)
+static hbool_t
+parse_flags(int argc, char * argv[], hbool_t * setup_ptr, 
+    hbool_t * ici_ptr, int * file_idx_ptr, int * mpi_size_ptr, hbool_t display)
 {
     const char * fcn_name = "parse_flags()";
-    const char * (ops[]) = {"setup"};
+    const char * (ops[]) = {"setup", "ici"};
     int success = TRUE;
 
+    HDassert(setup_ptr);
+    HDassert(*setup_ptr == FALSE);
+    HDassert(ici_ptr);
+    HDassert(*ici_ptr == FALSE);
+    HDassert(file_idx_ptr);
+    HDassert(mpi_size_ptr);
+
     if ( setup_ptr == NULL ) {
 
         success = FALSE;
         HDfprintf(stdout, "%s: bad arg(s) on entry.\n", fcn_name);
     }
 
+
     if ( ( success ) && 
-         ( ( argc < 1 ) || ( argc > 2 ) ) ) {
+         ( ( argc != 1 ) && ( argc != 2 ) && ( argc != 4 ) ) ) {
 
         success = FALSE;
         usage();
     }
 
-    if ( success ) {
 
-        if ( argc == 2 ) {
+    if ( ( success ) && ( argc >= 2 ) ) {
 
-            if ( strcmp(argv[1], ops[0]) == 0 ) {
+        if ( strcmp(argv[1], ops[0]) == 0 ) {
 
-                *setup_ptr = TRUE;
+            if ( argc != 2 ) {
+
+                success = FALSE;
+                usage();
 
             } else {
 
+                *setup_ptr = TRUE;
+
+            }
+        } else if ( strcmp(argv[1], ops[1]) == 0 ) {
+
+            if ( argc != 4 ) {
+
                 success = FALSE;
                 usage();
-            }
-        } else {
 
-            *setup_ptr = FALSE;
+            } else {
+
+                *ici_ptr = TRUE;
+                *file_idx_ptr = atoi(argv[2]);
+                *mpi_size_ptr = atoi(argv[3]);
+
+            }
         }
     }
 
     if ( ( success ) && ( display ) ) {
 
         if ( *setup_ptr )
+
             HDfprintf(stdout, "t_cache_image setup\n");
+
+        else if ( *ici_ptr )
+
+            HDfprintf(stdout, "t_cache_image ici %d %d\n", 
+                      *file_idx_ptr, *mpi_size_ptr);
+
         else
+
             HDfprintf(stdout, "t_cache_image\n");
     }
 
@@ -1694,6 +3048,266 @@ verify_data_sets(hid_t file_id, int min_dset, int max_dset)
 /****************************************************************************/
 
 /*-------------------------------------------------------------------------
+ * Function:    verify_cache_image_RO()
+ *
+ * Purpose:     Verify that a HDF5 file containing a cache image is 
+ *              opened R/O and read correctly by PHDF5 with the specified 
+ *              metadata write strategy.
+ *              
+ *              Basic cycle of operation is as follows:
+ *
+ *		1) Open the test file created at the beginning of this 
+ *		   test read only.
+ *
+ *		   Verify that the file contains a cache image.
+ *
+ *		   Verify that only process 0 reads the cache image.
+ *
+ *		   Verify that all other processes receive the cache 
+ *		   image block from process 0.
+ *
+ *              2) Verify that the file contains the expected data.
+ *
+ *              3) Close the file.
+ *
+ *              4) Open the file R/O, and verify that it still contains
+ *                 a cache image.
+ *
+ *              5) Verify that the file contains the expected data.
+ *
+ *              6) Close the file.
+ *
+ * Return:      void
+ *
+ * Programmer:  John Mainzer
+ *              3/11/17
+ *
+ * Modifications:
+ *
+ *		None.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static unsigned
+verify_cache_image_RO(int file_name_id, int md_write_strat, int mpi_rank)
+{
+    const char * fcn_name = "verify_cache_image_RO()";
+    char filename[512];
+    hbool_t show_progress = FALSE;
+    hid_t file_id = -1;
+    H5F_t *file_ptr = NULL;
+    H5C_t *cache_ptr = NULL;
+    int cp = 0;
+
+    pass = TRUE;
+
+    if ( mpi_rank == 0 ) {
+
+        switch(md_write_strat) {
+
+            case H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY:
+                TESTING("parallel CI load test -- proc0 md write -- R/O");
+                break;
+
+            case H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED:
+                TESTING("parallel CI load test -- dist md write -- R/O");
+                break;
+
+            default:
+                TESTING("parallel CI load test -- unknown md write -- R/o");
+                pass = FALSE;
+                break;
+        }
+    }
+
+    show_progress = ( ( show_progress ) && ( mpi_rank == 0 ) );
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* setup the file name */
+    if ( pass ) {
+
+        if ( h5_fixname(FILENAMES[file_name_id], H5P_DEFAULT, 
+                        filename, sizeof(filename)) == NULL ) {
+
+            pass = FALSE;
+            failure_mssg = "h5_fixname() failed.\n";
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 1) Open the test file created at the beginning of this test.
+     *
+     *    Verify that the file contains a cache image.
+     *
+     *    Verify that only process 0 reads the cache image.
+     *
+     *    Verify that all other processes receive the cache 
+     *    image block from process 0.
+     */
+
+    if ( pass ) {
+
+        open_hdf5_file(/* create_file           */ FALSE,
+                       /* mdci_sbem_expected    */ TRUE,
+                       /* read_only             */ TRUE,
+                       /* set_mdci_fapl         */ FALSE,
+		       /* config_fsm            */ FALSE,
+                       /* enable_page_buffer    */ FALSE,
+                       /* hdf_file_name         */ filename,
+                       /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
+                       /* file_id_ptr           */ &file_id,
+                       /* file_ptr_ptr          */ &file_ptr,
+                       /* cache_ptr_ptr         */ &cache_ptr,
+                       /* comm                  */ MPI_COMM_WORLD,
+                       /* info                  */ MPI_INFO_NULL,
+                       /* l_facc_type           */ FACC_MPIO,
+                       /* all_coll_metadata_ops */ FALSE,
+                       /* coll_metadata_write   */ FALSE,
+                       /* md_write_strat        */ md_write_strat);
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* Verify that only process 0 reads the cache image. */
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* Verify that all other processes receive the cache image block 
+     * from process 0.
+     */
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 2) Verify that the file contains the expected data. */
+    if ( pass ) {
+
+       verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1);
+    }
+
+#if H5C_COLLECT_CACHE_STATS
+    if ( pass ) {
+
+        if ( cache_ptr->images_loaded == 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "metadata cache image block not loaded(1).";
+        }
+    }
+#endif /* H5C_COLLECT_CACHE_STATS */
+
+
+    /* 3) Close the file. */
+ 
+    if ( pass ) {
+
+        if ( H5Fclose(file_id) < 0  ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Fclose() failed.\n";
+
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 4) Open the file, and verify that it doesn't contain a cache image. */
+
+    if ( pass ) {
+
+        open_hdf5_file(/* create_file           */ FALSE,
+                       /* mdci_sbem_expected    */ TRUE,
+                       /* read_only             */ TRUE,
+                       /* set_mdci_fapl         */ FALSE,
+		       /* config_fsm            */ FALSE,
+                       /* enable_page_buffer    */ FALSE,
+                       /* hdf_file_name         */ filename,
+                       /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
+                       /* file_id_ptr           */ &file_id,
+                       /* file_ptr_ptr          */ &file_ptr,
+                       /* cache_ptr_ptr         */ &cache_ptr,
+                       /* comm                  */ MPI_COMM_WORLD,
+                       /* info                  */ MPI_INFO_NULL,
+                       /* l_facc_type           */ FACC_MPIO,
+                       /* all_coll_metadata_ops */ FALSE,
+                       /* coll_metadata_write   */ FALSE,
+                       /* md_write_strat        */ md_write_strat);
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 5) Verify that the file contains the expected data. */
+ 
+    if ( pass ) {
+
+       verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1);
+    }
+
+#if H5C_COLLECT_CACHE_STATS
+    if ( pass ) {
+
+        if ( cache_ptr->images_loaded != 1 ) {
+
+            pass = FALSE;
+            failure_mssg = "metadata cache image block not loaded(2).";
+        }
+    }
+#endif /* H5C_COLLECT_CACHE_STATS */
+
+
+    /* 6) Close the file. */
+ 
+    if ( pass ) {
+
+        if ( H5Fclose(file_id) < 0  ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Fclose() failed.\n";
+
+        }
+    }
+
+    if ( show_progress ) 
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* report results */
+    if ( mpi_rank == 0 ) {
+
+        if ( pass ) {
+
+            PASSED();
+
+        } else {
+
+            H5_FAILED();
+
+            if ( show_progress ) 
+                HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", failure_mssg);
+        }
+    }
+
+
+    return !pass;
+
+} /* verify_cache_image_RO() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    verify_cache_image_RW()
  *
  * Purpose:     Verify that a HDF5 file containing a cache image is 
@@ -1747,7 +3361,6 @@ verify_cache_image_RW(int file_name_id, int md_write_strat, int mpi_rank)
     H5F_t *file_ptr = NULL;
     H5C_t *cache_ptr = NULL;
     int cp = 0;
-    int i;
 
     pass = TRUE;
 
@@ -1808,6 +3421,7 @@ verify_cache_image_RW(int file_name_id, int md_write_strat, int mpi_rank)
                        /* read_only             */ FALSE,
                        /* set_mdci_fapl         */ FALSE,
 		       /* config_fsm            */ FALSE,
+                       /* enable_page_buffer    */ FALSE,
                        /* hdf_file_name         */ filename,
                        /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
                        /* file_id_ptr           */ &file_id,
@@ -1880,6 +3494,7 @@ verify_cache_image_RW(int file_name_id, int md_write_strat, int mpi_rank)
                        /* read_only             */ FALSE,
                        /* set_mdci_fapl         */ FALSE,
 		       /* config_fsm            */ FALSE,
+                       /* enable_page_buffer    */ FALSE,
                        /* hdf_file_name         */ filename,
                        /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
                        /* file_id_ptr           */ &file_id,
@@ -1969,6 +3584,338 @@ verify_cache_image_RW(int file_name_id, int md_write_strat, int mpi_rank)
 } /* verify_cache_imageRW() */
 
 
+/*****************************************************************************
+ *
+ * Function:    smoke_check_1()
+ *
+ * Purpose:     Initial smoke check to verify correct behaviour of cache 
+ *              image in combination with parallel.
+ *              
+ *              As cache image is currently disabled in the parallel case,
+ *              we construct a test file in parallel, verify it in serial
+ *              and generate a cache image in passing, and then verify
+ *              it again in parallel.
+ *
+ *              In passing, also verify that page buffering is silently 
+ *              disabled in the parallel case.  Needless to say, this part
+ *              of the test will have to be re-worked when and if page
+ *              buffering is supported in parallel.
+ *
+ * Return:      Success:        TRUE
+ *
+ *              Failure:        FALSE
+ *
+ * Programmer:  JRM -- 3/6/17
+ *
+ *****************************************************************************/
+static hbool_t
+smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, int mpi_rank, int mpi_size)
+{
+    const char * fcn_name = "smoke_check_1()";
+    char filename[512];
+    hbool_t show_progress = FALSE;
+    hid_t file_id = -1;
+    H5F_t *file_ptr = NULL;
+    H5C_t *cache_ptr = NULL;
+    int cp = 0;
+    int i;
+    int num_dsets = PAR_NUM_DSETS;
+    int test_file_index = 2;
+    h5_stat_size_t file_size;
+
+    pass = TRUE;
+
+    if ( mpi_rank == 0 ) {
+
+        TESTING("parallel cache image smoke check 1");
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+    /* setup the file name */
+    if ( pass ) {
+
+        HDassert(FILENAMES[test_file_index]);
+
+        if ( h5_fixname(FILENAMES[test_file_index], H5P_DEFAULT,
+                                  filename, sizeof(filename))
+            == NULL ) {
+
+            pass = FALSE;
+            failure_mssg = "h5_fixname() failed.\n";
+        }
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 1) Create a PHDF5 file without the cache image FAPL entry. 
+     *
+     *    Verify that a cache image is not requested
+     */
+
+    if ( pass ) {
+
+        open_hdf5_file(/* create_file           */ TRUE,
+                       /* mdci_sbem_expected    */ FALSE,
+                       /* read_only             */ FALSE,
+                       /* set_mdci_fapl         */ FALSE,
+                       /* config_fsm            */ TRUE,
+                       /* enable_page_buffer    */ FALSE,
+                       /* hdf_file_name         */ filename,
+                       /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
+                       /* file_id_ptr           */ &file_id,
+                       /* file_ptr_ptr          */ &file_ptr,
+                       /* cache_ptr_ptr         */ &cache_ptr,
+                       /* comm                  */ mpi_comm,
+                       /* info                  */ mpi_info,
+                       /* l_facc_type           */ FACC_MPIO,
+                       /* all_coll_metadata_ops */ FALSE,
+                       /* coll_metadata_write   */ TRUE,
+                       /* md_write_strat        */ 1);
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 2) Create datasets in the file */
+
+    i = 0;
+    while ( ( pass ) && ( i < num_dsets ) ) {
+
+        par_create_dataset(i, file_id, mpi_rank, mpi_size);
+        i++;
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 3) Verify the datasets in the file */
+
+    i = 0;
+    while ( ( pass ) && ( i < num_dsets ) ) {
+
+        par_verify_dataset(i, file_id, mpi_rank);
+        i++;
+    }
+
+
+    /* 4) Close the file */
+
+    if ( pass ) {
+
+        if ( H5Fclose(file_id) < 0  ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Fclose() failed.\n";
+
+        }
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 5 Insert a cache image into the file */
+
+    if ( pass ) {
+
+        par_insert_cache_image(test_file_index, mpi_rank,  mpi_size);
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 6) Open the file R/O */
+
+    if ( pass ) {
+
+        open_hdf5_file(/* create_file           */ FALSE,
+                       /* mdci_sbem_expected    */ TRUE,
+                       /* read_only             */ TRUE,
+                       /* set_mdci_fapl         */ FALSE,
+                       /* config_fsm            */ FALSE,
+                       /* enable_page_buffer    */ FALSE,
+                       /* hdf_file_name         */ filename,
+                       /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
+                       /* file_id_ptr           */ &file_id,
+                       /* file_ptr_ptr          */ &file_ptr,
+                       /* cache_ptr_ptr         */ &cache_ptr,
+                       /* comm                  */ mpi_comm,
+                       /* info                  */ mpi_info,
+                       /* l_facc_type           */ FACC_MPIO,
+                       /* all_coll_metadata_ops */ FALSE,
+                       /* coll_metadata_write   */ TRUE,
+                       /* md_write_strat        */ 1);
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 7) Verify the datasets in the file backwards */
+
+    i = num_dsets - 1;
+    while ( ( pass ) && ( i >= 0 ) ) {
+
+        par_verify_dataset(i, file_id, mpi_rank);
+        i--;
+    }
+
+
+    /* 8) Close the file */
+
+    if ( pass ) {
+
+        if ( H5Fclose(file_id) < 0  ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Fclose() failed.";
+
+        }
+    }
+
+
+    /* 9) Open the file */
+
+    if ( pass ) {
+
+        open_hdf5_file(/* create_file           */ FALSE,
+                       /* mdci_sbem_expected    */ TRUE,
+                       /* read_only             */ FALSE,
+                       /* set_mdci_fapl         */ FALSE,
+                       /* config_fsm            */ FALSE,
+                       /* enable_page_buffer    */ FALSE,
+                       /* hdf_file_name         */ filename,
+                       /* cache_image_flags     */ H5C_CI__ALL_FLAGS,
+                       /* file_id_ptr           */ &file_id,
+                       /* file_ptr_ptr          */ &file_ptr,
+                       /* cache_ptr_ptr         */ &cache_ptr,
+                       /* comm                  */ mpi_comm,
+                       /* info                  */ mpi_info,
+                       /* l_facc_type           */ FACC_MPIO,
+                       /* all_coll_metadata_ops */ FALSE,
+                       /* coll_metadata_write   */ TRUE,
+                       /* md_write_strat        */ 1);
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 10) Verify the datasets in the file */
+
+    i = 0;
+    while ( ( pass ) && ( i < num_dsets ) ) {
+
+        par_verify_dataset(i, file_id, mpi_rank);
+        i++;
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 11) Delete the datasets in the file */
+
+    i = 0;
+    while ( ( pass ) && ( i < num_dsets ) ) {
+
+        par_delete_dataset(i, file_id, mpi_rank);
+        i++;
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 12) Close the file */
+
+    if ( pass ) {
+
+        if ( H5Fclose(file_id) < 0  ) {
+
+            pass = FALSE;
+            failure_mssg = "H5Fclose() failed.";
+
+        }
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 13) Get the size of the file.  Verify that it is less
+     *     than 20 KB.  Without deletions and persistant free
+     *     space managers, size size is about 30 MB, so this
+     *     is sufficient to verify that the persistant free
+     *     space managers are more or less doing their job.
+     *
+     *     Note that this test will have to change if we use
+     *     a larger page size.
+     */
+    if ( pass ) {
+
+        if ( ( file_size = h5_get_file_size(filename, H5P_DEFAULT) ) < 0 ) {
+
+            pass = FALSE;
+            failure_mssg = "h5_get_file_size() failed.";
+
+        } else if ( file_size > 20 * 1024 ) {
+
+            pass = FALSE;
+            failure_mssg = "unexpectedly large file size.";
+        }
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* 14) Delete the file */
+
+    if ( pass ) {
+
+        /* wait for everyone to close the file */
+        MPI_Barrier(MPI_COMM_WORLD);
+
+        if ( ( mpi_rank == 0 ) && ( HDremove(filename) < 0 ) ) {
+
+            pass = FALSE;
+            failure_mssg = "HDremove() failed.\n";
+        }
+    }
+
+    if ( ( mpi_rank == 0 ) && ( show_progress ) )
+        HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
+
+
+    /* report results */
+    if ( mpi_rank == 0 ) {
+
+        if ( pass ) {
+
+            PASSED();
+
+        } else {
+
+            H5_FAILED();
+
+            HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", 
+                      fcn_name, failure_mssg);
+        }
+    }
+
+    return !pass;
+    
+} /* smoke_check_1() */
+
+
 /*-------------------------------------------------------------------------
  * Function:    main
  *
@@ -1999,22 +3946,26 @@ int
 main(int argc, char **argv)
 {
     hbool_t setup = FALSE;
+    hbool_t ici = FALSE;
     unsigned nerrs = 0;
+    MPI_Comm comm = MPI_COMM_WORLD;
+    MPI_Info info = MPI_INFO_NULL;
+    int file_idx;
     int i;
     int mpi_size;
     int mpi_rank;
 
-    if ( ! parse_flags(argc, argv, &setup, FALSE) ) 
+    if ( ! parse_flags(argc, argv, &setup, &ici, &file_idx, &mpi_size, FALSE) )
         exit(1);  /* exit now if unable to parse flags */
 
-    if ( setup ) { /* construct test file and exit */
+    if ( setup ) { /* construct test files and exit */
 
         H5open();
         HDfprintf(stdout, "Constructing test files: \n");
         HDfflush(stdout);
 
         i = 0;
-        while ( FILENAMES[i] != NULL ) { 
+        while ( ( FILENAMES[i] != NULL ) && ( i < TEST_FILES_TO_CONSTRUCT ) ) { 
 
             HDfprintf(stdout, "   writing %s ... ", FILENAMES[i]);
             HDfflush(stdout);
@@ -2032,11 +3983,26 @@ main(int argc, char **argv)
             }
             i++;
         }
+
         HDfprintf(stdout, "Test file construction complete.\n");
         exit(0);
+
+    } else if ( ici ) {
+
+        if ( serial_insert_cache_image(file_idx, mpi_size) ) {
+
+            exit(0);
+
+        } else {
+
+            HDfprintf(stderr, "\n\nCache image insertion failed.\n");
+            HDfprintf(stderr, "  failure mssg = \"%s\"\n", failure_mssg);
+            exit(1);
+        }
     }
 
     HDassert(!setup);
+    HDassert(!ici);
 
     MPI_Init(&argc, &argv);
     MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
@@ -2118,14 +4084,21 @@ main(int argc, char **argv)
         }
     }
 
-    /* can't start test until test file exists */
+    /* can't start test until test files exist */
     MPI_Barrier(MPI_COMM_WORLD);
 
+
+    nerrs += verify_cache_image_RO(0, 
+                     H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY, mpi_rank);
+#if 1
+    nerrs += verify_cache_image_RO(1, 
+                     H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, mpi_rank);
     nerrs += verify_cache_image_RW(0, 
                      H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY, mpi_rank);
     nerrs += verify_cache_image_RW(1, 
                      H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, mpi_rank);
-
+    nerrs += smoke_check_1(comm, info, mpi_rank, mpi_size);
+#endif
 finish:
 
     /* make sure all processes are finished before final report, cleanup