From 7fa3e66be58958b556d3d929211ffd0faf8b5f40 Mon Sep 17 00:00:00 2001 From: Ray Lu Date: Mon, 3 Jan 2022 20:55:33 -0600 Subject: Rearragned how the auxiliary process is tested: let the test script launch it if the --enable-aux-process option is enabled during configure. --- MANIFEST | 5 ++++ config/cmake/libhdf5.settings.cmake.in | 1 + configure.ac | 25 +++++++++++++++++++ test/testvfdswmr.sh.in | 43 ++++++++++++++++++++++++++++++-- test/vfd_swmr_bigset_writer.c | 45 ++++++++++++---------------------- utils/CMakeLists.txt | 1 + utils/Makefile.am | 2 +- 7 files changed, 90 insertions(+), 32 deletions(-) diff --git a/MANIFEST b/MANIFEST index bad97be..29109c1 100644 --- a/MANIFEST +++ b/MANIFEST @@ -3048,6 +3048,11 @@ ./utils/mirror_vfd/mirror_server_stop.c ./utils/mirror_vfd/mirror_writer.c +# VFD SWMR utilities +./utils/vfd_swmr/README.md +./utils/vfd_swmr/Makefile.am +./utils/vfd_swmr/aux_process.c + # high level libraries ./hl/Makefile.am ./hl/examples/Makefile.am diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index d80b0f8..a7c5f37 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -81,6 +81,7 @@ Dimension scales w/ new references: @DIMENSION_SCALES_WITH_NEW_REF@ Mirror VFD: @H5_HAVE_MIRROR_VFD@ (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@ (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@ + Auxiliary Process: @H5_HAVE_AUX_PROCESS@ dmalloc: @H5_HAVE_LIBDMALLOC@ Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@ API Tracing: @HDF5_ENABLE_TRACE@ diff --git a/configure.ac b/configure.ac index ab55f3e..f8a1e4d 100644 --- a/configure.ac +++ b/configure.ac @@ -3171,6 +3171,30 @@ fi AM_CONDITIONAL([MIRROR_VFD_CONDITIONAL], [test "X$MIRROR_VFD" = "Xyes"]) ## ---------------------------------------------------------------------- +## Check whether the auxiliary process for the VFD SWMR should be built. +## +AC_SUBST([AUX_PROCESS]) + +## Default is no auxiliary process +AUX_PROCESS=no + +AC_MSG_CHECKING([if the auxiliary process is enabled]) + +AC_ARG_ENABLE([aux-process], + [AS_HELP_STRING([--enable-aux-process], + [Build the auxiliary process for the VFD SWMR. + [default=no]])], + [AUX_PROCESS=$enableval], [AUX_PROCESS=no]) + +if test "X$AUX_PROCESS" = "Xyes"; then + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_AUX_PROCESS], [1], + [Define whether the auxiliary process is compiled]) +else + AC_MSG_RESULT([no]) +fi + +## ---------------------------------------------------------------------- ## Check if Read-Only S3 virtual file driver is enabled by --enable-ros3-vfd ## AC_SUBST([ROS3_VFD]) @@ -4063,6 +4087,7 @@ AC_CONFIG_FILES([src/libhdf5.settings testpar/testpflush.sh utils/Makefile utils/mirror_vfd/Makefile + utils/vfd_swmr/Makefile tools/Makefile tools/lib/Makefile tools/libtest/Makefile diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index d52527b..c0b98fd 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -1083,6 +1083,16 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t # first dimension (up to 25 1x16x16) # + # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration. + # If it isn't enabled, the auxiliary process simply exit without executing any code. For the testing case of + # VDS across multiple files (-M option), the program also skips for future support. + if [[ $options == *"-M"* ]]; then + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater & + else + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater & + fi + pid_aux_proc=$! + echo launch vfd_swmr_bigset_writer many small, options $options catch_out_err_and_rc vfd_swmr_bigset_writer \ ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 1 -r 16 -c 16 -q & @@ -1095,10 +1105,17 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t # Wait for the reader to finish before signalling the # writer to quit: the writer holds the file open so that the # reader will find the shadow file when it opens - # the .h5 file. + # the .h5 file. Also wait for the auxiliary process to finish. + wait $pid_aux_proc wait $pid_reader wait $pid_writer + # Collect exit code of the auxiliary process + if [ $(cat aux_process.rc) -ne 0 ]; then + echo the auxiliary process had error + nerrors=$((nerrors + 1)) + fi + # Collect exit code of the reader if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then echo reader had error @@ -1114,6 +1131,8 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t # Clean up output files rm -f vfd_swmr_bigset_writer.{out,rc} rm -f vfd_swmr_bigset_reader.*.{out,rc} + rm -f aux_process.{out,rc} + rm -f mdfile bigset_updater.* bigset-shadow-* done # bigset test for bigger chunks @@ -1137,6 +1156,17 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d continue fi echo launch vfd_swmr_bigset_writer few big, options $options ......may take some time...... + + # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration. + # If it isn't enabled, the auxiliary process simply exit without executing any code. For the testing case of + # VDS across multiple files (-M option), the program also skips for future support. + if [[ $options == *"-M"* ]]; then + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater & + else + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater & + fi + pid_aux_proc=$! + catch_out_err_and_rc vfd_swmr_bigset_writer \ ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -e 8 -r 256 -c 256 -q & pid_writer=$! @@ -1148,10 +1178,17 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d # Wait for the reader to finish before signalling the # writer to quit: the writer holds the file open so that the # reader will find the shadow file when it opens - # the .h5 file. + # the .h5 file. Also wait for the auxiliary process to finish. + wait $pid_aux_proc wait $pid_reader wait $pid_writer + # Collect exit code of the auxiliary process + if [ $(cat aux_process.rc) -ne 0 ]; then + echo the auxiliary process had error + nerrors=$((nerrors + 1)) + fi + # Collect exit code of the reader if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then echo reader had error @@ -1167,6 +1204,8 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d # Clean up output files rm -f vfd_swmr_bigset_writer.{out,rc} rm -f vfd_swmr_bigset_reader.*.{out,rc} + rm -f aux_process.{out,rc} + rm -f mdfile bigset_updater.* bigset-shadow-* done ############################################################################### diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 1284b75..9cbc3c2 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -494,6 +494,11 @@ state_init(state_t *s, int argc, char **argv) TEST_ERROR; } +#ifdef H5_HAVE_AUX_PROCESS + if (s->vds == vds_multi) + exit(EXIT_SUCCESS); +#endif + if (s->vds != vds_off && s->expand_2d) { HDfprintf(stderr, "virtual datasets and 2D datasets are mutually exclusive\n"); TEST_ERROR; @@ -2544,7 +2549,6 @@ main(int argc, char **argv) state_t s; np_state_t np; size_t i; - pid_t pid; H5F_generate_md_ck_cb_t cb_info; /* Callback */ if (!state_init(&s, argc, argv)) { @@ -2552,23 +2556,6 @@ main(int argc, char **argv) TEST_ERROR; } - /* If using the auxiliary process, the reader creates a child process to run it */ - if (s.use_aux_proc && !s.writer) { - pid = fork(); - - if (pid == 0) { - /* Run the auxiliary process to apply the updater files to the metadata file. - * 'aux' is the executable of the auxiliary process which should be in the shell - * path. Otherwise, use its absolute path. 'mdfile' is the metadata file created - * by the auxiliary process. 'bigset_updater.*' are the updater files created - * by the writer. */ - system("aux mdfile bigset_updater"); - - /* Exit this process after the auxiliary process finishes */ - exit(0); - } - } - if ((mat = newmat(s)) == NULL) { HDfprintf(stderr, "could not allocate matrix\n"); TEST_ERROR; @@ -2592,20 +2579,20 @@ main(int argc, char **argv) /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ - if (s.use_aux_proc) { - /* If using the auxiliary process, the writer creates the updater files. - * The reader uses the metadata file generated by the auxiliary process. */ - if (s.writer) { - init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, FALSE, TRUE, s.flush_raw_data, 128, - "./bigset-shadow-%zu", "bigset_updater", i); - } else { - init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, TRUE, FALSE, s.flush_raw_data, 128, - "./mdfile", NULL); - } +#ifdef H5_HAVE_AUX_PROCESS + /* If using the auxiliary process, the writer creates the updater files. + * The reader uses the metadata file generated by the auxiliary process. */ + if (s.writer) { + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, FALSE, TRUE, s.flush_raw_data, 128, + "./bigset-shadow-%zu", "bigset_updater", i); } else { init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, TRUE, FALSE, s.flush_raw_data, 128, - "./bigset-shadow-%zu", NULL, i); + "./mdfile", NULL); } +#else + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, TRUE, FALSE, s.flush_raw_data, 128, + "./bigset-shadow-%zu", NULL, i); +#endif /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, s.page_buf_size, &config)) < 0) { diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 2d5626e..d6a5afd 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -2,3 +2,4 @@ cmake_minimum_required (VERSION 3.10) project (HDF5_UTILS C) add_subdirectory (mirror_vfd) +add_subdirectory (vfd_swmr) diff --git a/utils/Makefile.am b/utils/Makefile.am index 288da37..fa877ef 100644 --- a/utils/Makefile.am +++ b/utils/Makefile.am @@ -21,6 +21,6 @@ include $(top_srcdir)/config/commence.am CONFIG=ordered # All subdirectories -SUBDIRS=mirror_vfd +SUBDIRS=mirror_vfd vfd_swmr include $(top_srcdir)/config/conclude.am -- cgit v0.12