Plugin test cleanup (#1479)

* Autotools plugin test cleanup

* Combines filter, VFD, and VOL plugin tests
* Adds VFD plugin tests to the Autotools
* Implements a uniform shell script naming scheme in test/

* codespell fix

* Changes after code review

1 files changed, 563 insertions, 0 deletions

diff --git a/test/test_swmr.sh.in b/test/test_swmr.sh.in
new file mode 100644
index 0000000..771fe4a
--- /dev/null
+++ b/test/test_swmr.sh.in
@@ -0,0 +1,563 @@
+#! /bin/bash
+#
+# Copyright by The HDF Group.
+# Copyright by the Board of Trustees of the University of Illinois.
+# All rights reserved.
+#
+# This file is part of HDF5.  The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+# Tests for the swmr feature.
+#
+# Created:
+#   Albert Cheng, 2009/07/22
+
+srcdir=@srcdir@
+utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@
+testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@
+
+###############################################################################
+## test parameters
+###############################################################################
+
+Nreaders=5              # number of readers to launch
+Nrdrs_spa=3             # number of sparse readers to launch
+Nrecords=200000         # number of records to write
+Nrecs_rem=40000         # number of times to shrink
+Nrecs_spa=20000         # number of records to write in the sparse test
+Nsecs_add=5             # number of seconds per read interval
+Nsecs_rem=3             # number of seconds per read interval
+Nsecs_addrem=8          # number of seconds per read interval
+nerrors=0
+
+###############################################################################
+## definitions for message file to coordinate test runs
+###############################################################################
+WRITER_MESSAGE=SWMR_WRITER_MESSAGE      # The message file created by writer that the open is complete
+                                        # This should be the same as the define in "test/swmr_common.h"
+MESSAGE_TIMEOUT=300                     # Message timeout length in secs
+                                        # This should be the same as the define in "test/h5test.h"
+
+###############################################################################
+## short hands and function definitions
+###############################################################################
+DPRINT=:                # Set to "echo Debug:" for debugging printing,
+                        # else ":" for noop.
+IFDEBUG=:               # Set to null to turn on debugging, else ":" for noop.
+
+# Print a line-line message left justified in a field of 70 characters
+# beginning with the word "Testing".
+#
+TESTING() {
+   SPACES="                                                               "
+   echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
+}
+
+# To wait for the writer message file or till the maximum # of seconds is reached
+# $1 is the message file to wait for
+# This performs similar function as the routine h5_wait_message() in test/h5test.c
+WAIT_MESSAGE() {
+    message=$1                                  # Get the name of the message file to wait for
+    t0=`date +%s`                               # Get current time in seconds
+    difft=0                                     # Initialize the time difference
+    mexist=0                                    # Indicate whether the message file is found
+    while [ $difft -lt $MESSAGE_TIMEOUT ] ;     # Loop till message times out
+    do
+        t1=`date +%s`                           # Get current time in seconds
+        difft=`expr $t1 - $t0`                  # Calculate the time difference
+        if [ -e $message ]; then                # If message file is found:
+            mexist=1                            #       indicate the message file is found
+            rm $message                         #       remove the message file
+            break                               #       get out of the while loop
+        fi
+    done;
+    if test $mexist -eq 0; then
+        # Issue warning that the writer message file is not found, continue with launching the reader(s)
+        echo warning: $WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds
+    else
+        echo $WRITER_MESSAGE is found
+    fi
+}
+
+###############################################################################
+## Main
+##
+## Modifications:
+##   Vailin Choi; July 2013
+##     Add waiting of message file before launching the reader(s).
+##     Due to the implementation of file locking, coordination
+##     is needed in file opening for the writer/reader tests
+##     to proceed as expected.
+##
+###############################################################################
+
+# Check to see if the VFD specified by the HDF5_DRIVER environment variable
+# supports SWMR.
+$utils_testdir/swmr_check_compat_vfd
+rc=$?
+if [ $rc -ne 0 ] ; then
+    echo
+    echo "The VFD specified by the HDF5_DRIVER environment variable"
+    echo "does not support SWMR."
+    echo
+    echo "SWMR acceptance tests skipped"
+    echo
+    exit 0
+fi
+
+# Parse options (none accepted at this time)
+while [ $# -gt 0 ]; do
+    case "$1" in
+    *)  # unknown option
+        echo "$0: Unknown option ($1)"
+        exit 1
+        ;;
+    esac
+done
+
+# HDF5 has several tests that create and delete signal files to communicate
+# between processes, and it seems that even though the names of the files are
+# different, occasionally the wrong file is deleted, interrupting the flow of
+# the test.  Running each of these tests in its own directory should eliminate
+# the problem.
+mkdir swmr_test
+for FILE in swmr*; do
+    case "$FILE" in
+        *.o) continue ;;    ## don't copy the .o files
+    esac
+    if test -f "$FILE" ; then
+        cp $FILE swmr_test
+    fi
+done
+
+# With the --disable-shared option, swmr program files are built in the test
+# directory, otherwise they are in test/.libs with a corresponding wrapper
+# script in the test directory.  The programs or wrapper scripts in test should
+# always be copied, swmr files in .libs should be copied only if they exists.
+if [ -f .libs/swmr ]; then
+    mkdir swmr_test/.libs
+    for FILE in .libs/swmr*; do
+        case "$FILE" in
+            *.o) continue ;;    ## don't copy the .o files
+        esac
+        if test -f "$FILE" ; then
+            cp $FILE swmr_test/.libs
+        fi
+    done
+fi
+
+cd swmr_test
+
+
+# Loop over index types
+for index_type in "-i ea" "-i b2"
+do
+    # Try with and without compression
+    for compress in "" "-c 5"
+    do
+        echo
+        echo "*******************************************************************************"
+        echo "** Loop testing parameters: $index_type $compress"
+        echo "*******************************************************************************"
+        echo
+        echo
+        echo "###############################################################################"
+        echo "## Generator test"
+        echo "###############################################################################"
+        # Launch the Generator without SWMR_WRITE
+        echo launch the swmr_generator
+        $testdir/swmr_generator $compress $index_type
+        if test $? -ne 0; then
+            echo generator had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Launch the Generator with SWMR_WRITE
+        echo launch the swmr_generator with SWMR_WRITE
+        $testdir/swmr_generator -s $compress $index_type
+        if test $? -ne 0; then
+            echo generator had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Check for error and exit if one occurred
+        $DPRINT nerrors=$nerrors
+        if test $nerrors -ne 0 ; then
+            echo "SWMR tests failed with $nerrors errors."
+            exit 1
+        fi
+
+        echo
+        echo "###############################################################################"
+        echo "## Use H5Fstart_swmr_write() to enable SWMR writing mode"
+        echo "###############################################################################"
+
+        # Remove any possible writer message file before launching writer
+        rm -f $WRITER_MESSAGE
+        #
+        # Launch the Writer
+        echo launch the swmr_start_writer
+        seed="" # Put -r <random seed> command here
+        $testdir/swmr_start_write $compress $index_type $Nrecords $seed 2>&1 |tee swmr_writer.out &
+        pid_writer=$!
+        $DPRINT pid_writer=$pid_writer
+
+        # Wait for message from writer process before starting reader(s)
+        WAIT_MESSAGE $WRITER_MESSAGE
+
+        #
+        # Launch the Readers
+        #declare -a seeds=(<seed1> <seed2> <seed3> ... )
+        echo launch $Nreaders swmr_readers
+        pid_readers=""
+        n=0
+        while [ $n -lt $Nreaders ]; do
+            #seed="-r ${seeds[$n]}"
+            seed=""
+            $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n &
+            pid_readers="$pid_readers $!"
+            n=`expr $n + 1`
+        done
+        $DPRINT pid_readers=$pid_readers
+        $IFDEBUG ps
+
+        # Collect exit code of the readers first because they usually finish
+        # before the writer.
+        for xpid in $pid_readers; do
+            $DPRINT checked reader $xpid
+            wait $xpid
+            if test $? -ne 0; then
+                echo reader had error
+                nerrors=`expr $nerrors + 1`
+            fi
+        done
+
+        # Collect exit code of the writer
+        $DPRINT checked writer $pid_writer
+        wait $pid_writer
+        if test $? -ne 0; then
+            echo writer had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Check for error and exit if one occurred
+        $DPRINT nerrors=$nerrors
+        if test $nerrors -ne 0 ; then
+            echo "SWMR tests failed with $nerrors errors."
+            echo "(Writer and reader output preserved)"
+            exit 1
+        fi
+
+        # Clean up output files
+        rm -f swmr_writer.out
+        rm -f swmr_reader.out.*
+
+        echo
+        echo "###############################################################################"
+        echo "## Writer test - test expanding the dataset"
+        echo "###############################################################################"
+
+        # Launch the Generator
+        echo launch the swmr_generator
+        $testdir/swmr_generator -s $compress $index_type
+        if test $? -ne 0; then
+            echo generator had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Remove any possible writer message file before launching writer
+        rm -f $WRITER_MESSAGE
+        #
+        # Launch the Writer
+        echo launch the swmr_writer
+        seed="" # Put -r <random seed> command here
+        $testdir/swmr_writer -o $Nrecords $seed 2>&1 |tee swmr_writer.out &
+        pid_writer=$!
+        $DPRINT pid_writer=$pid_writer
+
+        # Wait for message from writer process before starting reader(s)
+        WAIT_MESSAGE $WRITER_MESSAGE
+        #
+        # Launch the Readers
+        #declare -a seeds=(<seed1> <seed2> <seed3> ... )
+        echo launch $Nreaders swmr_readers
+        pid_readers=""
+        n=0
+        while [ $n -lt $Nreaders ]; do
+            #seed="-r ${seeds[$n]}"
+            seed=""
+            $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n &
+            pid_readers="$pid_readers $!"
+            n=`expr $n + 1`
+        done
+        $DPRINT pid_readers=$pid_readers
+        $IFDEBUG ps
+
+        # Collect exit code of the readers first because they usually finish
+        # before the writer.
+        for xpid in $pid_readers; do
+            $DPRINT checked reader $xpid
+            wait $xpid
+            if test $? -ne 0; then
+                echo reader had error
+                nerrors=`expr $nerrors + 1`
+            fi
+        done
+
+        # Collect exit code of the writer
+        $DPRINT checked writer $pid_writer
+        wait $pid_writer
+        if test $? -ne 0; then
+            echo writer had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Check for error and exit if one occurred
+        $DPRINT nerrors=$nerrors
+        if test $nerrors -ne 0 ; then
+            echo "SWMR tests failed with $nerrors errors."
+            echo "(Writer and reader output preserved)"
+            exit 1
+        fi
+
+        # Clean up output files
+        rm -f swmr_writer.out
+        rm -f swmr_reader.out.*
+
+        echo
+        echo "###############################################################################"
+        echo "## Remove test - test shrinking the dataset"
+        echo "###############################################################################"
+
+        # Remove any possible writer message file before launching writer
+        rm -f $WRITER_MESSAGE
+        # Launch the Remove Writer
+        echo launch the swmr_remove_writer
+        seed="" # Put -r <random seed> command here
+        $testdir/swmr_remove_writer -o $Nrecs_rem $seed 2>&1 |tee swmr_writer.out &
+        pid_writer=$!
+        $DPRINT pid_writer=$pid_writer
+
+        # Wait for message from writer process before starting reader(s)
+        WAIT_MESSAGE $WRITER_MESSAGE
+        #
+        # Launch the Remove Readers
+        #declare -a seeds=(<seed1> <seed2> <seed3> ... )
+        n=0
+        pid_readers=""
+        echo launch $Nreaders swmr_remove_readers
+        while [ $n -lt $Nreaders ]; do
+            #seed="-r ${seeds[$n]}"
+            seed=""
+            $testdir/swmr_remove_reader $Nsecs_rem $seed 2>&1 |tee swmr_reader.out.$n &
+            pid_readers="$pid_readers $!"
+            n=`expr $n + 1`
+        done
+        $DPRINT pid_readers=$pid_readers
+        $IFDEBUG ps
+
+        # Collect exit code of the readers first because they usually finish
+        # before the writer.
+        for xpid in $pid_readers; do
+            $DPRINT checked reader $xpid
+            wait $xpid
+            if test $? -ne 0; then
+                echo reader had error
+                nerrors=`expr $nerrors + 1`
+            fi
+        done
+
+        # Collect exit code of the writer
+        $DPRINT checked writer $pid_writer
+        wait $pid_writer
+        if test $? -ne 0; then
+            echo writer had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Check for error and exit if one occurred
+        $DPRINT nerrors=$nerrors
+        if test $nerrors -ne 0 ; then
+            echo "SWMR tests failed with $nerrors errors."
+            echo "(Writer and reader output preserved)"
+            exit 1
+        fi
+
+        # Clean up output files
+        rm -f swmr_writer.out
+        rm -f swmr_reader.out.*
+
+        echo
+        echo "###############################################################################"
+        echo "## Add/remove test - randomly grow or shrink the dataset"
+        echo "###############################################################################"
+
+        # Launch the Generator
+        echo launch the swmr_generator
+        $testdir/swmr_generator $compress $index_type
+        if test $? -ne 0; then
+            echo generator had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Launch the Writer (not in parallel - just to rebuild the datasets)
+        echo launch the swmr_writer
+        seed="" # Put -r <random seed> command here
+        $testdir/swmr_writer $Nrecords $seed
+        if test $? -ne 0; then
+            echo writer had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Remove any possible writer message file before launching writer
+        rm -f $WRITER_MESSAGE
+        #
+        # Launch the Add/Remove Writer
+        echo launch the swmr_addrem_writer
+        seed="" # Put -r <random seed> command here
+        $testdir/swmr_addrem_writer $Nrecords $seed 2>&1 |tee swmr_writer.out &
+        pid_writer=$!
+        $DPRINT pid_writer=$pid_writer
+
+        # Wait for message from writer process before starting reader(s)
+        WAIT_MESSAGE $WRITER_MESSAGE
+        #
+        # Launch the Add/Remove Readers
+        #declare -a seeds=(<seed1> <seed2> <seed3> ... )
+        n=0
+        pid_readers=""
+        echo launch $Nreaders swmr_remove_readers
+        while [ $n -lt $Nreaders ]; do
+            #seed="-r ${seeds[$n]}"
+            seed=""
+            $testdir/swmr_remove_reader $Nsecs_addrem $seed 2>&1 |tee swmr_reader.out.$n &
+            pid_readers="$pid_readers $!"
+            n=`expr $n + 1`
+        done
+        $DPRINT pid_readers=$pid_readers
+        $IFDEBUG ps
+
+        # Collect exit code of the readers first because they usually finish
+        # before the writer.
+        for xpid in $pid_readers; do
+            $DPRINT checked reader $xpid
+            wait $xpid
+            if test $? -ne 0; then
+                echo reader had error
+                nerrors=`expr $nerrors + 1`
+            fi
+        done
+
+        # Collect exit code of the writer
+        $DPRINT checked writer $pid_writer
+        wait $pid_writer
+        if test $? -ne 0; then
+            echo writer had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Check for error and exit if one occurred
+        $DPRINT nerrors=$nerrors
+        if test $nerrors -ne 0 ; then
+            echo "SWMR tests failed with $nerrors errors."
+            echo "(Writer and reader output preserved)"
+            exit 1
+        fi
+
+        # Clean up output files
+        rm -f swmr_writer.out
+        rm -f swmr_reader.out.*
+
+        echo
+        echo "###############################################################################"
+        echo "## Sparse writer test - test writing to random locations in the dataset"
+        echo "###############################################################################"
+
+        # Launch the Generator
+        # NOTE: Random seed is shared between readers and writers and is
+        #       created by the generator.
+        echo launch the swmr_generator
+        seed="" # Put -r <random seed> command here
+        $testdir/swmr_generator $compress $index_type $seed
+        if test $? -ne 0; then
+            echo generator had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Remove any possible writer message file before launching writer
+        rm -f $WRITER_MESSAGE
+        # Launch the Sparse writer
+        echo launch the swmr_sparse_writer
+        nice -n 20 $testdir/swmr_sparse_writer $Nrecs_spa 2>&1 |tee swmr_writer.out &
+        pid_writer=$!
+        $DPRINT pid_writer=$pid_writer
+
+        # Wait for message from writer process before starting reader(s)
+        WAIT_MESSAGE $WRITER_MESSAGE
+        #
+        # Launch the Sparse readers
+        n=0
+        pid_readers=""
+        echo launch $Nrdrs_spa swmr_sparse_readers
+        while [ $n -lt $Nrdrs_spa ]; do
+            # The sparse reader spits out a LOT of data so it's set to 'quiet'
+            $testdir/swmr_sparse_reader -q $Nrecs_spa 2>&1 |tee swmr_reader.out.$n &
+            pid_readers="$pid_readers $!"
+            n=`expr $n + 1`
+        done
+        $DPRINT pid_readers=$pid_readers
+        $IFDEBUG ps
+
+        # Collect exit code of the writer
+        $DPRINT checked writer $pid_writer
+        wait $pid_writer
+        if test $? -ne 0; then
+            echo writer had error
+            nerrors=`expr $nerrors + 1`
+        fi
+
+        # Collect exit code of the readers
+        for xpid in $pid_readers; do
+            $DPRINT checked reader $xpid
+            wait $xpid
+            if test $? -ne 0; then
+                echo reader had error
+                nerrors=`expr $nerrors + 1`
+            fi
+        done
+
+        # Check for error and exit if one occurred
+        $DPRINT nerrors=$nerrors
+        if test $nerrors -ne 0 ; then
+            echo "SWMR tests failed with $nerrors errors."
+            echo "(Writer and reader output preserved)"
+            exit 1
+        fi
+
+        # Clean up output files
+        rm -f swmr_writer.out
+        rm -f swmr_reader.out.*
+    done
+done
+
+###############################################################################
+## Report and exit
+###############################################################################
+cd ..
+$DPRINT nerrors=$nerrors
+if test $nerrors -eq 0 ; then
+    echo "SWMR tests passed."
+    if test -z "$HDF5_NOCLEANUP"; then
+        # delete the test directory
+        rm -rf swmr_test
+    fi
+    exit 0
+else
+    echo "SWMR tests failed with $nerrors errors."
+    exit 1
+fi
+