Merge pull request #1333 from raylu-hdf/raylu_aux_process

Changes Related to the Auxiliary Process

8 files changed, 215 insertions, 11 deletions

diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml
index 33608b5..62fd0d8 100644
--- a/.github/workflows/pr-check.yml
+++ b/.github/workflows/pr-check.yml
@@ -174,6 +174,8 @@ jobs:
     # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
     - name: Get Sources
       uses: actions/checkout@v2
+      with:
+        submodules: 'true'
 
     - name: Autotools Configure 
       if: matrix.generator == 'autogen'
diff --git a/MANIFEST b/MANIFEST
index bad97be..29109c1 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -3048,6 +3048,11 @@
 ./utils/mirror_vfd/mirror_server_stop.c
 ./utils/mirror_vfd/mirror_writer.c
 
+# VFD SWMR utilities
+./utils/vfd_swmr/README.md
+./utils/vfd_swmr/Makefile.am
+./utils/vfd_swmr/aux_process.c
+
 # high level libraries
 ./hl/Makefile.am
 ./hl/examples/Makefile.am
diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in
index d80b0f8..a7c5f37 100644
--- a/config/cmake/libhdf5.settings.cmake.in
+++ b/config/cmake/libhdf5.settings.cmake.in
@@ -81,6 +81,7 @@ Dimension scales w/ new references: @DIMENSION_SCALES_WITH_NEW_REF@
                         Mirror VFD: @H5_HAVE_MIRROR_VFD@
                 (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@
               (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@
+                 Auxiliary Process: @H5_HAVE_AUX_PROCESS@
                            dmalloc: @H5_HAVE_LIBDMALLOC@
     Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@
                        API Tracing: @HDF5_ENABLE_TRACE@
diff --git a/configure.ac b/configure.ac
index ab55f3e..f8a1e4d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3171,6 +3171,30 @@ fi
 AM_CONDITIONAL([MIRROR_VFD_CONDITIONAL], [test "X$MIRROR_VFD" = "Xyes"])
 
 ## ----------------------------------------------------------------------
+## Check whether the auxiliary process for the VFD SWMR should be built.
+##
+AC_SUBST([AUX_PROCESS])
+
+## Default is no auxiliary process
+AUX_PROCESS=no
+
+AC_MSG_CHECKING([if the auxiliary process is enabled])
+
+AC_ARG_ENABLE([aux-process],
+              [AS_HELP_STRING([--enable-aux-process],
+                              [Build the auxiliary process for the VFD SWMR.
+                               [default=no]])],
+              [AUX_PROCESS=$enableval], [AUX_PROCESS=no])
+
+if test "X$AUX_PROCESS" = "Xyes"; then
+    AC_MSG_RESULT([yes])
+    AC_DEFINE([HAVE_AUX_PROCESS], [1],
+            [Define whether the auxiliary process is compiled])
+else
+    AC_MSG_RESULT([no])
+fi
+
+## ----------------------------------------------------------------------
 ## Check if Read-Only S3 virtual file driver is enabled by --enable-ros3-vfd
 ##
 AC_SUBST([ROS3_VFD])
@@ -4063,6 +4087,7 @@ AC_CONFIG_FILES([src/libhdf5.settings
                  testpar/testpflush.sh
                  utils/Makefile
                  utils/mirror_vfd/Makefile
+                 utils/vfd_swmr/Makefile
                  tools/Makefile
                  tools/lib/Makefile
                  tools/libtest/Makefile
diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in
index 9453284..7ab324e 100644
--- a/test/testvfdswmr.sh.in
+++ b/test/testvfdswmr.sh.in
@@ -1095,6 +1095,16 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t
         # first dimension (up to 25 1x16x16)
         #
 
+        # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration.
+        # If it isn't enabled, the auxiliary process simply exit without executing any code.  For the testing case of
+        # VDS across multiple files (-M option), the program also skips for future support.
+        if [[ $options == *"-M"* ]]; then
+	    catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater &
+        else
+	    catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater &
+        fi
+        pid_aux_proc=$!
+
 	echo launch vfd_swmr_bigset_writer many small, options $options
 	catch_out_err_and_rc vfd_swmr_bigset_writer \
 	    ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 1 -r 16 -c 16 -q &
@@ -1107,10 +1117,17 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t
 	# Wait for the reader to finish before signalling the
 	# writer to quit: the writer holds the file open so that the
 	# reader will find the shadow file when it opens
-	# the .h5 file.
+	# the .h5 file. Also wait for the auxiliary process to finish.
+        wait $pid_aux_proc
 	wait $pid_reader
 	wait $pid_writer
 
+	# Collect exit code of the auxiliary process
+	if [ $(cat aux_process.rc) -ne 0 ]; then
+		echo the auxiliary process had error
+		nerrors=$((nerrors + 1))
+	fi
+
 	# Collect exit code of the reader
 	if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then
 		echo reader had error
@@ -1126,6 +1143,8 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t
 	# Clean up output files
 	rm -f vfd_swmr_bigset_writer.{out,rc}
 	rm -f vfd_swmr_bigset_reader.*.{out,rc}
+        rm -f aux_process.{out,rc}
+        rm -f mdfile bigset_updater.* bigset-shadow-*
 done
 
 # bigset test for bigger chunks
@@ -1149,6 +1168,17 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d
 		continue
 	fi
 	echo launch vfd_swmr_bigset_writer few big, options $options ......may take some time......
+
+        # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration.
+        # If it isn't enabled, the auxiliary process simply exit without executing any code.  For the testing case of
+        # VDS across multiple files (-M option), the program also skips for future support.
+        if [[ $options == *"-M"* ]]; then
+	    catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater &
+        else
+	    catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater &
+        fi
+        pid_aux_proc=$!
+
 	catch_out_err_and_rc vfd_swmr_bigset_writer \
 	    ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -e 8 -r 256 -c 256 -q &
 	pid_writer=$!
@@ -1160,10 +1190,17 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d
 	# Wait for the reader to finish before signalling the
 	# writer to quit: the writer holds the file open so that the
 	# reader will find the shadow file when it opens
-	# the .h5 file.
+	# the .h5 file. Also wait for the auxiliary process to finish.
+        wait $pid_aux_proc
 	wait $pid_reader
 	wait $pid_writer
 
+	# Collect exit code of the auxiliary process
+	if [ $(cat aux_process.rc) -ne 0 ]; then
+		echo the auxiliary process had error
+		nerrors=$((nerrors + 1))
+	fi
+
 	# Collect exit code of the reader
 	if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then
 		echo reader had error
@@ -1179,6 +1216,8 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d
 	# Clean up output files
 	rm -f vfd_swmr_bigset_writer.{out,rc}
 	rm -f vfd_swmr_bigset_reader.*.{out,rc}
+        rm -f aux_process.{out,rc}
+        rm -f mdfile bigset_updater.* bigset-shadow-*
 done
 
 ###############################################################################
diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c
index 14cbedf..9b33ee8 100644
--- a/test/vfd_swmr_bigset_writer.c
+++ b/test/vfd_swmr_bigset_writer.c
@@ -151,6 +151,7 @@ typedef struct {
     bool            use_vfd_swmr;
     bool            use_legacy_swmr;
     bool            use_named_pipe;
+    bool            use_aux_proc;
     bool            do_perf;
     bool            cross_chunk_read;
     bool            writer;
@@ -217,6 +218,7 @@ state_initializer(void)
                      .use_vfd_swmr     = true,
                      .use_legacy_swmr  = false,
                      .use_named_pipe   = true,
+                     .use_aux_proc     = false,
                      .do_perf          = false,
                      .cross_chunk_read = false,
                      .writer           = true,
@@ -245,11 +247,12 @@ usage(const char *progname)
 {
     HDfprintf(
         stderr,
-        "usage: %s [-C] [-F] [-M] [-P] [-R] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n"
+        "usage: %s [-A] [-C] [-F] [-M] [-P] [-R] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n"
         "    [-d dims] [-e depth] [-f tick_len] [-g max_lag] [-j skip_chunk] [-k part_chunk]\n"
         "    [-l tick_num] [-n iterations] [-o page_buf_size] [-p fsp_size] [-r rows]\n"
         "    [-s datasets] [-t] [-u over_extend] [-v chunk_cache_size] [-w deflate_level]\n"
         "\n"
+        "-A:                   use the auxiliary process to update the metadata file\n"
         "-C:                   cross-over chunk read during chunk verification\n"
         "-F:                   fixed maximal dimension for the chunked datasets\n"
         "-M:	               use virtual datasets and many source\n"
@@ -342,8 +345,11 @@ state_init(state_t *s, int argc, char **argv)
     if (tfile)
         HDfree(tfile);
 
-    while ((ch = getopt(argc, argv, "CFMNPRSTVa:bc:d:e:f:g:j:k:l:m:n:o:p:qr:s:tu:v:w:")) != -1) {
+    while ((ch = getopt(argc, argv, "ACFMNPRSTVa:bc:d:e:f:g:j:k:l:m:n:o:p:qr:s:tu:v:w:")) != -1) {
         switch (ch) {
+            case 'A':
+                s->use_aux_proc = true;
+                break;
             case 'C':
                 /* This flag indicates cross-over chunk read during data validation */
                 s->cross_chunk_read = true;
@@ -488,6 +494,11 @@ state_init(state_t *s, int argc, char **argv)
         TEST_ERROR;
     }
 
+#ifdef H5_HAVE_AUX_PROCESS
+    if (s->vds == vds_multi)
+        exit(EXIT_SUCCESS);
+#endif
+
     if (s->vds != vds_off && s->expand_2d) {
         HDfprintf(stderr, "virtual datasets and 2D datasets are mutually exclusive\n");
         TEST_ERROR;
@@ -1148,6 +1159,101 @@ error:
     return -1;
 }
 
+/*-------------------------------------------------------------------------
+ * Function:    md_ck_cb()
+ *
+ * Purpose:     This is the callback function for debugging only.  It's used
+ *              when the H5F_ACS_GENERATE_MD_CK_CB_NAME property is set in fapl.
+ *                  --Opens and read the metadata file into a buffer.
+ *                  --Generate checksum for the metadata file
+ *                  --Write the tick number and the checksum to the checksum file
+ *
+ * Return:      0 if test is sucessful
+ *              1 if test fails
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+md_ck_cb(char *md_file_path, uint64_t updater_seq_num)
+{
+    FILE *   md_fp  = NULL;      /* Metadata file pointer */
+    FILE *   chk_fp = NULL;      /* Checksum file pointer */
+    long     size   = 0;         /* File size returned from HDftell() */
+    void *   buf    = NULL;      /* Buffer for holding the metadata file content */
+    uint32_t chksum = 0;         /* The checksum generated for the metadata file */
+    char     chk_name[1024 + 4]; /* Buffer for the checksum file name */
+    size_t   ret;                /* Return value */
+
+    /* Open the metadata file */
+    if ((md_fp = HDfopen(md_file_path, "r")) == NULL)
+        FAIL_STACK_ERROR;
+
+    /* Set file pointer at end of file.*/
+    if (HDfseek(md_fp, 0, SEEK_END) < 0)
+        FAIL_STACK_ERROR;
+
+    /* Get the current position of the file pointer.*/
+    if ((size = HDftell(md_fp)) < 0)
+        FAIL_STACK_ERROR;
+
+    if (size != 0) {
+
+        HDrewind(md_fp);
+
+        if ((buf = HDmalloc((size_t)size)) == NULL)
+            FAIL_STACK_ERROR;
+
+        /* Read the metadata file to buf */
+        if ((ret = HDfread(buf, 1, (size_t)size, md_fp)) != (size_t)size)
+            FAIL_STACK_ERROR;
+
+        /* Calculate checksum of the metadata file */
+        chksum = H5_checksum_metadata(buf, (size_t)size, 0);
+    }
+
+    /* Close the metadata file */
+    if (md_fp && HDfclose(md_fp) < 0)
+        FAIL_STACK_ERROR;
+
+    /*
+     *  Checksum file
+     */
+
+    /* Generate checksum file name: <md_file_path>.chk */
+    HDsprintf(chk_name, "%s.chk", md_file_path);
+
+    /* Open checksum file for append */
+    if ((chk_fp = HDfopen(chk_name, "a")) == NULL)
+        FAIL_STACK_ERROR;
+
+    /* Write the updater sequence number to the checksum file */
+    if ((ret = HDfwrite(&updater_seq_num, sizeof(uint64_t), 1, chk_fp)) != 1)
+        FAIL_STACK_ERROR;
+
+    /* Write the checksum to the checksum file */
+    if ((ret = HDfwrite(&chksum, sizeof(uint32_t), 1, chk_fp)) != 1)
+        FAIL_STACK_ERROR;
+
+    /* Close the checksum file */
+    if (chk_fp && HDfclose(chk_fp) != 0)
+        FAIL_STACK_ERROR;
+
+    if (buf)
+        HDfree(buf);
+
+    return 0;
+
+error:
+    if (buf)
+        HDfree(buf);
+    if (md_fp)
+        HDfclose(md_fp);
+    if (chk_fp)
+        HDfclose(chk_fp);
+
+    return -1;
+} /* md_ck_cb() */
+
 static bool
 create_extensible_dset(state_t *s, unsigned int which)
 {
@@ -2437,12 +2543,13 @@ error:
 int
 main(int argc, char **argv)
 {
-    mat_t *    mat;
-    hid_t      fcpl = H5I_INVALID_HID;
-    unsigned   which;
-    state_t    s;
-    np_state_t np;
-    size_t     i;
+    mat_t *                 mat;
+    hid_t                   fcpl = H5I_INVALID_HID;
+    unsigned                which;
+    state_t                 s;
+    np_state_t              np;
+    size_t                  i;
+    H5F_generate_md_ck_cb_t cb_info; /* Callback */
 
     if (!state_init(&s, argc, argv)) {
         HDfprintf(stderr, "state_init failed\n");
@@ -2472,8 +2579,21 @@ main(int argc, char **argv)
 
         /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files,
          * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */
+#ifdef H5_HAVE_AUX_PROCESS
+        /* If using the auxiliary process, the writer creates the updater files.
+         * The reader uses the metadata file generated by the auxiliary process. */
+        if (s.writer) {
+            init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, FALSE, TRUE, s.flush_raw_data, 128,
+                                 "./bigset-shadow-%zu", "bigset_updater", i);
+        }
+        else {
+            init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, TRUE, FALSE, s.flush_raw_data, 128,
+                                 "./mdfile", NULL);
+        }
+#else
         init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, TRUE, FALSE, s.flush_raw_data, 128,
                              "./bigset-shadow-%zu", NULL, i);
+#endif
 
         /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */
         if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, s.page_buf_size, &config)) < 0) {
@@ -2502,6 +2622,17 @@ main(int argc, char **argv)
             }
         }
 
+        /* This part is for debugging only */
+#ifdef TMP
+        {
+            /* Set up callback to generate checksums for updater's metadata files */
+            cb_info.func = md_ck_cb;
+
+            /* Activate private property to generate checksums for updater's metadata file */
+            H5Pset(fapl, H5F_ACS_GENERATE_MD_CK_CB_NAME, &cb_info);
+        }
+#endif
+
         s.file[i] = s.writer ? H5Fcreate(s.filename[i], H5F_ACC_TRUNC, fcpl, fapl)
                              : H5Fopen(s.filename[i], H5F_ACC_RDONLY, fapl);
 
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
index 2d5626e..d6a5afd 100644
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@@ -2,3 +2,4 @@ cmake_minimum_required (VERSION 3.10)
 project (HDF5_UTILS C)
 
 add_subdirectory (mirror_vfd)
+add_subdirectory (vfd_swmr)
diff --git a/utils/Makefile.am b/utils/Makefile.am
index 288da37..fa877ef 100644
--- a/utils/Makefile.am
+++ b/utils/Makefile.am
@@ -21,6 +21,6 @@ include $(top_srcdir)/config/commence.am
 CONFIG=ordered
 
 # All subdirectories
-SUBDIRS=mirror_vfd
+SUBDIRS=mirror_vfd vfd_swmr
 
 include $(top_srcdir)/config/conclude.am