Merge pull request #50 from HDFGroup/feature/vfd_swmr

Feature/vfd swmr

6 files changed, 360 insertions, 12 deletions

diff --git a/src/H5AC.c b/src/H5AC.c
index c913805..e89c22d 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -2777,3 +2777,34 @@ H5AC_get_mdc_image_info(const H5AC_t *cache_ptr, haddr_t *image_addr, hsize_t *i
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* H5AC_get_mdc_image_info() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5AC_set_vfd_swmr_reader
+ *
+ * Purpose:     Wrapper function for H5C_set_vfd_swmr_reader().
+ *
+ * Return:      SUCCEED on success, and FAIL on failure.
+ *
+ * Programmer:  Vailin Choi; Dec 2021
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5AC_set_vfd_swmr_reader(H5AC_t *cache_ptr, hbool_t vfd_swmr_reader, hsize_t page_size)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Sanity checks */
+    HDassert(cache_ptr);
+
+    if (cache_ptr->page_size != page_size) {
+
+        if (H5C_set_vfd_swmr_reader((H5C_t *)cache_ptr, vfd_swmr_reader, page_size) < 0)
+            HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set page_size for VFD SWMR reader")
+    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_set_vfd_swmr_reader() */
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index f7bed0c..ff632d0 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -433,6 +433,7 @@ H5_DLL herr_t H5AC_unsettle_ring(H5F_t *f, H5AC_ring_t ring);
 H5_DLL herr_t H5AC_expunge_tag_type_metadata(H5F_t *f, haddr_t tag, int type_id, unsigned flags,
                                              hbool_t type_match);
 H5_DLL herr_t H5AC_get_tag(const void *thing, /*OUT*/ haddr_t *tag);
+H5_DLL herr_t H5AC_set_vfd_swmr_reader(H5AC_t *cache_ptr, hbool_t vfd_swmr_reader, hsize_t page_size);
 
 /* Virtual entry routines */
 H5_DLL H5AC_proxy_entry_t *H5AC_proxy_entry_create(void);
diff --git a/src/H5C.c b/src/H5C.c
index 8e72767..802b4e3 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -2801,8 +2801,21 @@ H5C_protect(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *udata, unsign
 #ifdef H5_HAVE_PARALLEL
                                              coll_access,
 #endif /* H5_HAVE_PARALLEL */
-                                             type, addr, udata)))
+                                             type, addr, udata))) {
+            /* Print out meaningful message for VFD SWMR reader */
+            if(f->shared->vfd_swmr && !f->shared->vfd_swmr_writer) {
+                uint64_t                   tmp_tick_num = 0;
+
+                if (H5FD_vfd_swmr_get_tick_and_idx(f->shared->lf, TRUE, &tmp_tick_num, NULL, NULL) < 0)
+                    HGOTO_ERROR(H5E_ARGS, H5E_CANTGET, NULL, "error in retrieving tick_num from driver");
+
+                if (tmp_tick_num >= f->shared->tick_num + f->shared->vfd_swmr_config.max_lag)
+                    HDONE_ERROR(H5E_FILE, H5E_SYSTEM, NULL,
+                      "Reader's API time exceeds max_lag ticks, suggest to increase the value of max_lag.");
+            }
+
             HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "can't load entry")
+        }
 
         entry_ptr = (H5C_cache_entry_t *)thing;
         cache_ptr->entries_loaded_counter++;
diff --git a/src/H5Fvfd_swmr.c b/src/H5Fvfd_swmr.c
index 43567ac..372ae19 100644
--- a/src/H5Fvfd_swmr.c
+++ b/src/H5Fvfd_swmr.c
@@ -169,7 +169,7 @@ H5FL_DEFINE(eot_queue_entry_t);
  * Return:      Success:        SUCCEED
  *              Failure:        FAIL
  *
- * Programmer:  Vailin Choi -- 11/??/18
+ * Programmer:  Vailin Choi -- 10/??/18
  *
  * Changes:     None.
  *
@@ -252,6 +252,21 @@ H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create)
 
         HDassert(!shared->vfd_swmr_config.writer);
 
+        HDassert(shared->fs_page_size > 0);
+        /* This is a bug uncovered by issue #3 of the group test failures.
+         *  See Kent's documentation "Designed to Fail Tests and Issues".
+         *  The file opening process in H5F__new() initializes the cache copy of
+         *  page_size via H5AC_create().  However, later on H5F__super_read()
+         *  may change page size due to non-default setting of
+         *  'free-space manager info' in superblock extension.
+         *  Fix: set the cache copy of page_size again if different from
+         *  f->shared->fs_page_size.
+         */
+        if (shared->cache) {
+            if (H5AC_set_vfd_swmr_reader(shared->cache, TRUE, shared->fs_page_size) < 0)
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set page size in cache for VFD SWMR reader");
+        }
+
         shared->vfd_swmr_writer = FALSE;
         shared->max_jump_ticks  = 0;
 
@@ -1121,6 +1136,7 @@ H5F_vfd_swmr_reader_end_of_tick(H5F_t *f, hbool_t entering_api)
     uint32_t                   tmp_mdf_idx_len;
     uint32_t                   tmp_mdf_idx_entries_used;
     uint32_t                   mdf_idx_entries_used;
+    uint32_t                   vfd_entries;
     H5F_shared_t *             shared = f->shared;
     struct {
         uint64_t pgno;
@@ -1142,8 +1158,12 @@ H5F_vfd_swmr_reader_end_of_tick(H5F_t *f, hbool_t entering_api)
      *
      *    If the tick reported has not increased since the last
      *    call, do nothing and exit.
+     *
+     *    Also retrieve the current number of entries in the index
+     *    so as to detect the need to allocate more space for the
+     *    index.
      */
-    if (H5FD_vfd_swmr_get_tick_and_idx(file, TRUE, &tmp_tick_num, NULL, NULL) < 0)
+    if (H5FD_vfd_swmr_get_tick_and_idx(file, TRUE, &tmp_tick_num, &vfd_entries, NULL) < 0)
 
         HGOTO_ERROR(H5E_ARGS, H5E_CANTGET, FAIL, "error in retrieving tick_num from driver")
 
@@ -1157,7 +1177,7 @@ H5F_vfd_swmr_reader_end_of_tick(H5F_t *f, hbool_t entering_api)
      *               */
     if (!entering_api && tmp_tick_num >= shared->tick_num + shared->vfd_swmr_config.max_lag) {
         HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL,
-                    "Reader's API time exceeds max_lag of ticks, may increase the value of max_lag.");
+                    "Reader's API time exceeds max_lag ticks, suggest to increase the value of max_lag.");
     }
 #if 0 /* Kent */
     /* The original code */
@@ -1192,6 +1212,33 @@ H5F_vfd_swmr_reader_end_of_tick(H5F_t *f, hbool_t entering_api)
         if (shared->mdf_idx == NULL && H5F__vfd_swmr_create_index(shared) < 0)
             HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, "unable to allocate metadata file index");
 
+        /* Check if more space is needed for the index */
+        if (shared->mdf_idx_len < vfd_entries) {
+            uint32_t                   inc_mdf_idx_len;
+            H5FD_vfd_swmr_idx_entry_t *new_idx;
+            H5FD_vfd_swmr_idx_entry_t *old_idx;
+
+            HDassert(shared->old_mdf_idx_len <= shared->mdf_idx_len);
+
+            /* Determine the size needed for the index */
+            inc_mdf_idx_len = shared->mdf_idx_len * 2;
+            while (vfd_entries > inc_mdf_idx_len)
+                inc_mdf_idx_len *= 2;
+
+            /* Allocate more space for shared->mdf_idx and shared->old_mdf_idx */
+            new_idx = H5MM_realloc(shared->mdf_idx, inc_mdf_idx_len * sizeof(shared->mdf_idx[0]));
+            if (new_idx == NULL)
+                HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, "new index null after H5MM_realloc()")
+            shared->mdf_idx     = new_idx;
+            shared->mdf_idx_len = inc_mdf_idx_len;
+
+            old_idx = H5MM_realloc(shared->old_mdf_idx, inc_mdf_idx_len * sizeof(shared->old_mdf_idx[0]));
+            if (old_idx == NULL)
+                HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, "old index null after H5MM_realloc()")
+            shared->old_mdf_idx     = old_idx;
+            shared->old_mdf_idx_len = inc_mdf_idx_len;
+        }
+
         mdf_idx_entries_used = shared->mdf_idx_len;
 
 #if 0  /* JRM */
@@ -1258,6 +1305,14 @@ H5F_vfd_swmr_reader_end_of_tick(H5F_t *f, hbool_t entering_api)
 #if 0 /*Kent*/
                     HDassert(oent->length == nent->length);
 #endif
+                    /* This is a bug uncovered by issue #1 of the
+                     * group test failures.  See Kent's documentation
+                     * "Designed to Fail Tests and Issues".
+                     * nent->length can be <, =, > to oent->length.
+                     * Fix: HDassert the 1st two cases: < and =.
+                     * John will address the > case.
+                     */
+                    HDassert(nent->length <= oent->length);
 
                     /* the page has been altered -- evict it and
                      * any contained metadata cache entries.
@@ -1864,6 +1919,7 @@ vfd_swmr_enlarge_shadow_index(H5F_t *f)
     H5F_shared_t *             shared    = f->shared;
     H5FD_vfd_swmr_idx_entry_t *ret_value = NULL;
     haddr_t                    idx_addr;
+    haddr_t                    old_writer_index_offset;
     hsize_t                    idx_size;
     H5FD_vfd_swmr_idx_entry_t *new_mdf_idx = NULL, *old_mdf_idx;
     uint32_t                   new_mdf_idx_len, old_mdf_idx_len;
@@ -1901,6 +1957,7 @@ vfd_swmr_enlarge_shadow_index(H5F_t *f)
      */
     H5MM_memcpy(new_mdf_idx, old_mdf_idx, sizeof(new_mdf_idx[0]) * old_mdf_idx_len);
 
+    old_writer_index_offset     = shared->writer_index_offset;
     shared->writer_index_offset = idx_addr;
     ret_value = shared->mdf_idx = new_mdf_idx;
     shared->mdf_idx_len         = new_mdf_idx_len;
@@ -1916,8 +1973,8 @@ vfd_swmr_enlarge_shadow_index(H5F_t *f)
      * past what is strictly necessary, but it seems like a reasonable
      * trade-off for simplicity.
      */
-    if (shadow_range_defer_free(shared, shared->writer_index_offset, H5FD_MD_INDEX_SIZE(old_mdf_idx_len)) ==
-        -1) {
+    /* Fix: use the saved old_writer_index_offset not the current one */
+    if (shadow_range_defer_free(shared, old_writer_index_offset, H5FD_MD_INDEX_SIZE(old_mdf_idx_len)) == -1) {
         HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "could not schedule index reclamation");
     }
 done:
diff --git a/src/H5private.h b/src/H5private.h
index 761ad17..58a4100 100644
--- a/src/H5private.h
+++ b/src/H5private.h
@@ -2145,7 +2145,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props);
         else if (TAILQ_EMPTY(&eot_queue_g))                                                                  \
             ; /* Nothing to do. */                                                                           \
         else if (H5F_vfd_swmr_process_eot_queue(true) < 0) {                                                 \
-            HGOTO_ERROR(H5E_FUNC, H5E_CANTSET, err, "error processing EOT queue")                            \
+            /* Report error instead of "err" */                                                              \
+            HGOTO_ERROR(H5E_FUNC, H5E_CANTSET, FALSE, "error processing EOT queue")                          \
         }                                                                                                    \
     } while (0)
 
@@ -2160,7 +2161,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props);
         else if (TAILQ_EMPTY(&eot_queue_g))                                                                  \
             ; /* Nothing to do. */                                                                           \
         else if (H5F_vfd_swmr_process_eot_queue(false) < 0) {                                                \
-            HDONE_ERROR(H5E_FUNC, H5E_CANTSET, err, "error processing EOT queue")                            \
+            /* Report error instead of "err" */                                                              \
+            HDONE_ERROR(H5E_FUNC, H5E_CANTSET, FALSE, "error processing EOT queue")                          \
         }                                                                                                    \
     } while (0)
 
diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in
index d52527b..176e1e5 100644
--- a/test/testvfdswmr.sh.in
+++ b/test/testvfdswmr.sh.in
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/us/bin/env bash
 #
 # Copyright by The HDF Group.
 # Copyright by the Board of Trustees of the University of Illinois.
@@ -75,6 +75,13 @@ DSETCHKS_FIFO_WRITER_TO_READER=fifo_dsetchks_writer_to_reader
 DSETCHKS_FIFO_READER_TO_WRITER=fifo_dsetchks_reader_to_writer
 
 ###############################################################################
+## For gfail{1,2,3,4} test: definitions for fifo files to coordinate test runs
+###############################################################################
+#
+GFAIL_FIFO_WRITER_TO_READER=fifo_group_writer_to_reader
+GFAIL_FIFO_READER_TO_WRITER=fifo_group_reader_to_writer
+
+###############################################################################
 ## short hands and function definitions
 ###############################################################################
 #
@@ -142,11 +149,16 @@ if [ $rc -ne 0 ] ; then
 fi
 
 all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo"
-all_tests="${all_tests} groups groups_attrs groups_ops few_big many_small attrdset"
+all_tests="${all_tests} groups groups_attrs groups_ops few_big many_small"
 
-# For exhaustive run, add: os_groups_attrs, os_groups_ops, os_groups_seg, dsetops, dsetchks,independ_wr
+# For exhaustive run, add: 
+#   attrdset, dsetops, dsetchks,
+#   os_groups_attrs, os_groups_ops, os_groups_seg, independ_wr,
+#   gfail_entry_length, gfail_checksum, gfail_page_size, gfail_index_space
 if [[ "$HDF5TestExpress" -eq 0 ]] ; then         # exhaustive run
-    all_tests="${all_tests} os_groups_attrs os_groups_ops os_groups_seg dsetops dsetchks independ_wr"
+    all_tests="${all_tests} attrdset dsetops dsetchks"
+    all_tests="${all_tests} os_groups_attrs os_groups_ops os_groups_seg independ_wr"
+    all_tests="${all_tests} gfail_entry_length gfail_checksum gfail_page_size gfail_index_space"
 fi
 
 tests=${all_tests}
@@ -1388,6 +1400,238 @@ fi
 
 
 ###############################################################################
+#
+# "gfail_entry_length" test
+#
+# Only for exhaustive run
+#
+# Verify that the assertion failure for old and new entry length is fixed.
+# (See issue #1 in Kent's documentation "Designed to Fail Tests and Issues".
+#
+###############################################################################
+#
+#
+if [ ${do_gfail_entry_length:-no} = yes ]; then
+    #
+    # Clean up any existing fifo files from previous runs
+    if [ -e ./$GFAIL_FIFO_WRITER_TO_READER ]; then     # If writer fifo file is found
+        rm -f ./$GFAIL_FIFO_WRITER_TO_READER
+    fi
+    if [ -e ./$GFAIL_FIFO_READER_TO_WRITER ]; then      # If reader fifo file is found
+        rm -f ./$GFAIL_FIFO_READER_TO_WRITER
+    fi
+    #
+	echo launch vfd_swmr_gfail_writer -q -m 10 -n 340000
+	catch_out_err_and_rc vfd_swmr_gfail_entry_length_writer \
+	    ../vfd_swmr_gfail_writer -q -m 10 -n 340000 &
+	pid_writer=$!
+
+	echo launch vfd_swmr_gfail_reader -m 10 -n 340000
+	catch_out_err_and_rc vfd_swmr_gfail_entry_length_reader \
+	    ../vfd_swmr_gfail_reader -m 10 -n 340000 &
+	pid_reader=$!
+
+	# Wait for the reader to finish before signaling the
+	# writer to quit: the writer holds the file open so that the
+	# reader will find the shadow file when it opens
+	# the .h5 file.
+	wait $pid_reader
+	wait $pid_writer
+
+	# Collect exit code of the reader
+    if [ $(cat vfd_swmr_gfail_entry_length_reader.rc) -ne 0 ]; then
+	    echo reader had error
+	    nerrors=$((nerrors + 1))
+    fi
+
+	# Collect exit code of the writer
+	if [ $(cat vfd_swmr_gfail_entry_length_writer.rc) -ne 0 ]; then
+	    echo writer had error
+	    nerrors=$((nerrors + 1))
+	fi
+
+	# Clean up output files
+	rm -f vfd_swmr_gfail_entry_length_writer.{out,rc}
+	rm -f vfd_swmr_gfail_entry_length_reader.*.{out,rc}
+fi
+
+###############################################################################
+#
+# "gfail_checksum" test
+#
+# Only for exhaustive run
+#
+# Verify that meaningful message about increasing max_lag is printed when
+# incorrect metadata checksum error is enountered when loading the cache entry.
+#
+# Note that there will be messages from the error stack printed out for this test.
+# This is expected.
+#
+# (See issue #2 in Kent's documentation "Designed to Fail Tests and Issues".
+#
+###############################################################################
+#
+#
+if [ ${do_gfail_checksum:-no} = yes ]; then
+    #
+    # Clean up any existing fifo files from previous runs
+    if [ -e ./$GFAIL_FIFO_WRITER_TO_READER ]; then     # If writer fifo file is found
+        rm -f ./$GFAIL_FIFO_WRITER_TO_READER
+    fi
+    if [ -e ./$GFAIL_FIFO_READER_TO_WRITER ]; then      # If reader fifo file is found
+        rm -f ./$GFAIL_FIFO_READER_TO_WRITER
+    fi
+    #
+	echo launch vfd_swmr_gfail_writer -q -n 420000 ......may take some time......
+	catch_out_err_and_rc vfd_swmr_gfail_checksum_writer \
+	    ../vfd_swmr_gfail_writer -q -m 50 -t 10 -n 4000000 &
+	pid_writer=$!
+
+	echo launch vfd_swmr_gfail_reader -n 420000 ......may take some time......
+	catch_out_err_and_rc vfd_swmr_gfail_checksum_reader \
+	    ../vfd_swmr_gfail_reader -q -m 50 -t 10 -n 4000000 &
+	pid_reader=$!
+
+	# Wait for the reader to finish before signaling the
+	# writer to quit: the writer holds the file open so that the
+	# reader will find the shadow file when it opens
+	# the .h5 file.
+	wait $pid_reader
+	wait $pid_writer
+
+	# Collect exit code of the reader
+    if [ $(cat vfd_swmr_gfail_checksum_reader.rc) -ne 0 ]; then
+        grep "suggest to increase the value of max_lag" vfd_swmr_gfail_checksum_reader.out >/dev/null 2>&1
+        if test $? -ne 0; then
+            echo reader had error
+	        nerrors=$((nerrors + 1))
+        fi
+    fi
+
+	# Collect exit code of the writer
+	if [ $(cat vfd_swmr_gfail_checksum_writer.rc) -ne 0 ]; then
+	    echo writer had error
+	    nerrors=$((nerrors + 1))
+	fi
+
+	# Clean up output files
+	rm -f vfd_swmr_gfail_checksum_writer.{out,rc}
+	rm -f vfd_swmr_gfail_checksum_reader.*.{out,rc}
+fi
+
+###############################################################################
+#
+# "gfail_page_size" test
+#
+# Only for exhaustive run
+#
+# Verify that the assertion failure is fixed when non-default page size is set.
+# (See issue #3 in Kent's documentation "Designed to Fail Tests and Issues".
+#
+###############################################################################
+#
+#
+if [ ${do_gfail_page_size:-no} = yes ]; then
+    #
+    # Clean up any existing fifo files from previous runs
+    if [ -e ./$GFAIL_FIFO_WRITER_TO_READER ]; then     # If writer fifo file is found
+        rm -f ./$GFAIL_FIFO_WRITER_TO_READER
+    fi
+    if [ -e ./$GFAIL_FIFO_READER_TO_WRITER ]; then      # If reader fifo file is found
+        rm -f ./$GFAIL_FIFO_READER_TO_WRITER
+    fi
+    #
+	echo launch vfd_swmr_gfail_writer -q -m 10 -B 8192 -s 8192 -n 320000
+	catch_out_err_and_rc vfd_swmr_gfail_page_size_writer \
+	    ../vfd_swmr_gfail_writer -q -m 10 -B 8192 -s 8192 -n 320000 &
+	pid_writer=$!
+
+	echo launch vfd_swmr_gfail_reader -m 10 -B 8192 -s 8192 -n 320000
+	catch_out_err_and_rc vfd_swmr_gfail_page_size_reader  \
+	    ../vfd_swmr_gfail_reader -m 10 -B 8192 -s 8192 -n 320000 &
+	pid_reader=$!
+
+	# Wait for the reader to finish before signaling the
+	# writer to quit: the writer holds the file open so that the
+	# reader will find the shadow file when it opens
+	# the .h5 file.
+	wait $pid_reader
+	wait $pid_writer
+
+	# Collect exit code of the reader
+	if [ $(cat vfd_swmr_gfail_page_size_reader.rc) -ne 0 ]; then
+	    echo reader had error
+	    nerrors=$((nerrors + 1))
+	fi
+
+	# Collect exit code of the writer
+	if [ $(cat vfd_swmr_gfail_page_size_writer.rc) -ne 0 ]; then
+	    echo writer had error
+	    nerrors=$((nerrors + 1))
+	fi
+
+	# Clean up output files
+	rm -f vfd_swmr_gfail_page_size_writer.{out,rc}
+	rm -f vfd_swmr_gfail_page_size_reader.*.{out,rc}
+fi
+
+###############################################################################
+#
+# "gfail_index_space" test
+#
+# Only for exhaustive run
+#
+# Verify that the failure is fixed when there is not enough space to copy the index.
+# (See issue #4 in Kent's documentation "Designed to Fail Tests and Issues".
+#
+###############################################################################
+#
+#
+if [ ${do_gfail_index_space:-no} = yes ]; then
+    #
+    # Clean up any existing fifo files from previous runs
+    if [ -e ./$GFAIL_FIFO_WRITER_TO_READER ]; then     # If writer fifo file is found
+        rm -f ./$GFAIL_FIFO_WRITER_TO_READER
+    fi
+    if [ -e ./$GFAIL_FIFO_READER_TO_WRITER ]; then      # If reader fifo file is found
+        rm -f ./$GFAIL_FIFO_READER_TO_WRITER
+    fi
+    #
+	echo launch vfd_swmr_gfail_writer -q -m 50 -t 10 -n 1000000 ......may take some time......
+	catch_out_err_and_rc vfd_swmr_gfail_index_space_writer \
+	    ../vfd_swmr_gfail_writer -q -m 50 -t 10 -n 1000000 &
+	pid_writer=$!
+
+	echo launch vfd_swmr_gfail_reader -m 50 -t 10 -n 1000000 ......may take some time......
+	catch_out_err_and_rc vfd_swmr_gfail_index_space_reader  \
+	    ../vfd_swmr_gfail_reader -m 50 -t 10 -n 1000000 &
+	pid_reader=$!
+
+	# Wait for the reader to finish before signaling the
+	# writer to quit: the writer holds the file open so that the
+	# reader will find the shadow file when it opens
+	# the .h5 file.
+	wait $pid_reader
+	wait $pid_writer
+
+	# Collect exit code of the reader
+	if [ $(cat vfd_swmr_gfail_index_space_reader.rc) -ne 0 ]; then
+	    echo reader had error
+	    nerrors=$((nerrors + 1))
+	fi
+
+	# Collect exit code of the writer
+	if [ $(cat vfd_swmr_gfail_index_space_writer.rc) -ne 0 ]; then
+	    echo writer had error
+	    nerrors=$((nerrors + 1))
+	fi
+
+	# Clean up output files
+	rm -f vfd_swmr_gfail_index_space_writer.{out,rc}
+	rm -f vfd_swmr_gfail_index_space_reader.*.{out,rc}
+fi
+
+###############################################################################
 ## Report and exit
 ###############################################################################
 cd ..