Plugin test cleanup (#1479)

* Autotools plugin test cleanup

* Combines filter, VFD, and VOL plugin tests
* Adds VFD plugin tests to the Autotools
* Implements a uniform shell script naming scheme in test/

* codespell fix

* Changes after code review

1 files changed, 519 insertions, 0 deletions

diff --git a/test/test_swmr.pwsh.in b/test/test_swmr.pwsh.in
new file mode 100644
index 0000000..de7a57a
--- /dev/null
+++ b/test/test_swmr.pwsh.in
@@ -0,0 +1,519 @@
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5.  The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+# Tests for the swmr feature.
+
+$srcdir  = '@srcdir@'
+$utils_testdir=@H5_UTILS_TEST_BUILDDIR@
+$testdir=@H5_TEST_BUILDDIR@
+
+###############################################################################
+## test parameters
+###############################################################################
+
+$Nreaders     = 5          # number of readers to launch
+$Nrdrs_spa    = 3          # number of sparse readers to launch
+$Nrecords     = 200000     # number of records to write
+$Nrecs_rem    = 40000      # number of times to shrink
+$Nrecs_spa    = 20000      # number of records to write in the sparse test
+$Nsecs_add    = 5          # number of seconds per read interval
+$Nsecs_rem    = 3          # number of seconds per read interval
+$Nsecs_addrem = 8          # number of seconds per read interval
+$nerrors      = 0
+
+###############################################################################
+## definitions for message file to coordinate test runs
+###############################################################################
+$WRITER_MESSAGE = 'SWMR_WRITER_MESSAGE'    # The message file created by writer that the open is complete
+                                           # This should be the same as the define in "test/swmr_common.h"
+$MESSAGE_TIMEOUT = 300                     # Message timeout length in secs
+                                           # This should be the same as the define in "test/h5test.h"
+
+# Print a line-line message left justified in a field of 70 characters
+# beginning with the word "Testing".
+#
+function Test-WithSpaces {
+   $SPACES='                                                               '
+   #Write-Output  "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
+}
+
+# To wait for the writer message file or till the maximum # of seconds is reached
+# $Message is the message file to wait for
+# This performs similar function as the routine h5_wait_message() in test/h5test.c
+function Wait-Message {
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory, ValueFromPipeline)]
+        [string]$TestPath,                             # Get the path of the message file to wait for
+        [Parameter(Mandatory, ValueFromPipeline)]
+        [string]$Message                              # Get the name of the message file to wait for
+    )
+
+    PROCESS {
+        $message = Join-Path -Path $TestPath -ChildPath $Message
+        $t0 = Get-Date                                # Get current time
+        $t1 = $t0
+        $difft = New-Timespan -Start $t0 -End $t1     # Initialize the time difference
+        $mexist = 0                                   # Indicate whether the message file is found
+        while ($difft.TotalSeconds -lt $MESSAGE_TIMEOUT) {    # Loop till message times out
+            $t1 = Get-Date                                    # Get current time in seconds
+            $difft = New-Timespan -Start $t0 -End $t1         # Calculate the time difference
+            #Write-Output "Check for $message : time=$difft"
+            if ([System.IO.File]::Exists($message)) {         # If message file is found:
+                $mexist = 1                                   #       indicate the message file is found
+                Remove-Item $message                          #       remove the message file
+                break                                         #       get out of the while loop
+            }
+        }
+        if ($mexist -eq 0) {
+            # Issue warning that the writer message file is not found, continue with launching the reader(s)
+            Write-Warning -Message "$WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds"
+        }
+        else {
+            Write-Output "$WRITER_MESSAGE is found"
+        }
+    }
+}
+
+###############################################################################
+## Main
+##
+## Modifications:
+##   Vailin Choi; July 2013
+##     Add waiting of message file before launching the reader(s).
+##     Due to the implementation of file locking, coordination
+##     is needed in file opening for the writer/reader tests
+##     to proceed as expected.
+##
+###############################################################################
+
+# Check to see if the VFD specified by the HDF5_DRIVER environment variable
+# supports SWMR.
+$testprog = Join-Path -Path $utils_testdir -ChildPath swmr_check_compat_vfd.exe
+$rp = Start-Process -FilePath $testprog -PassThru -Wait -NoNewWindow
+if ($rp.ExitCode -ne 0) {
+    Write-Output ""
+    Write-Output "The VFD specified by the HDF5_DRIVER environment variable"
+    Write-Output "does not support SWMR."
+    Write-Output ""
+    Write-Output "SWMR acceptance tests skipped"
+    Write-Output ""
+    exit 0
+}
+
+Set-Location -Path swmr_test
+$testdir = Join-Path -Path $testdir -ChildPath swmr_test
+
+# Loop over index types
+$IndexTypeLoop = '-i ea', '-i b2'
+foreach ($index_type in $IndexTypeLoop) {
+    # Try with and without compression
+    $CompressLoop = '', '-c 5'
+    foreach ($compress in $CompressLoop) {
+        Write-Output ""
+        Write-Output "*******************************************************************************"
+        Write-Output "** Loop testing parameters: $index_type $compress"
+        Write-Output "*******************************************************************************"
+        Write-Output ""
+        Write-Output ""
+        Write-Output "###############################################################################"
+        Write-Output "## Generator test"
+        Write-Output "###############################################################################"
+        # Launch the Generator without SWMR_WRITE
+        Write-Output "launch the swmr_generator"
+        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "$compress $index_type" -PassThru -Wait
+        if ($rp.ExitCode -ne 0) {
+            Write-Warning "generator had error"
+            $nerrors += 1
+        }
+
+        # Launch the Generator with SWMR_WRITE
+        Write-Output "launch the swmr_generator with SWMR_WRITE"
+        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "-s $compress $index_type" -PassThru -Wait
+        if ($rp.ExitCode -ne 0) {
+            Write-Warning "generator had error"
+            $nerrors += 1
+        }
+
+        # Check for error and exit if one occurred
+        #Write-Output "nerrors=$nerrors"
+        if ($nerrors -ne 0) {
+            Write-Warning "SWMR tests failed with $nerrors errors."
+            exit 1
+        }
+
+        Write-Output ""
+        Write-Output "###############################################################################"
+        Write-Output "## Use H5Fstart_swmr_write() to enable SWMR writing mode"
+        Write-Output "###############################################################################"
+
+        # Remove any possible writer message file before launching writer
+        if ([System.IO.File]::Exists($WRITER_MESSAGE)) {                # If message file is found
+            Remove-Item $WRITER_MESSAGE
+        }
+        #
+        # Launch the Writer
+        Write-Output "launch the swmr_start_writer"
+        $seed = "" # Put -r <random seed> command here
+        $rp = Start-Process -FilePath $testdir/swmr_start_write.exe -NoNewWindow -PassThru -ArgumentList "$compress $index_type $Nrecords $seed" 2>&1 |tee swmr_writer.out
+        $pid_writer = $rp.id
+        #Write-Output "pid_writer=$pid_writer"
+
+        # Wait for message from writer process before starting reader(s)
+        Wait-Message $testdir $WRITER_MESSAGE
+
+        #
+        # Launch the Readers
+        #declare -a seeds = (<seed1> <seed2> <seed3> ... )
+        Write-Output "launch $Nreaders swmr_readers"
+        $pid_readers = @()
+        $n = 0
+        while ($n -lt $Nreaders) {
+            #seed = "-r ${seeds[$n]}"
+            $seed = ""
+            $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 | tee swmr_reader.out.$n
+            $pid_readers +=  $rp.id
+            $n += 1
+        }
+        #Write-Output "pid_readers=$pid_readers"
+
+        # Collect exit code of the readers first because they usually finish
+        # before the writer.
+        foreach ($xpid in $pid_readers) {
+            #Write-Output "checked reader $xpid"
+            $result = Wait-Process -Id  $xpid
+            if ($result.ExitCode -ne 0) {
+                Write-Warning "reader had error"
+                $nerrors += 1
+            }
+        }
+
+        # Collect exit code of the writer
+        #Write-Output "checked writer $pid_writer"
+        $result = Wait-Process -Id  $pid_writer
+        if ($result.ExitCode -ne 0) {
+            Write-Warning "writer had error"
+            $nerrors += 1
+        }
+
+        # Check for error and exit if one occurred
+        #Write-Output "nerrors=$nerrors"
+        if ($nerrors -ne 0) {
+            Write-Warning "SWMR tests failed with $nerrors errors."
+            Write-Warning "(Writer and reader output preserved)"
+            exit 1
+        }
+
+        # Clean up output files
+        Remove-Item swmr_writer.out
+        Remove-Item swmr_reader.out.*
+
+        Write-Output ""
+        Write-Output "###############################################################################"
+        Write-Output "## Writer test - test expanding the dataset"
+        Write-Output "###############################################################################"
+
+        # Launch the Generator
+        Write-Output "launch the swmr_generator"
+        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "-s $compress $index_type"
+        if ($rp.ExitCode -ne 0) {
+            Write-Warning "generator had error"
+            $nerrors += 1
+        }
+
+        # Remove any possible writer message file before launching writer
+        Remove-Item $WRITER_MESSAGE
+        #
+        # Launch the Writer
+        Write-Output "launch the swmr_writer"
+        $seed = "" # Put -r <random seed> command here
+        $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecords $seed" 2>&1 |tee swmr_writer.out
+        $pid_writer = $rp.id
+        #Write-Output "pid_writer=$pid_writer"
+
+        # Wait for message from writer process before starting reader(s)
+        Wait-Message $testdir $WRITER_MESSAGE
+        #
+        # Launch the Readers
+        #declare -a seeds = (<seed1> <seed2> <seed3> ... )
+        Write-Output "launch $Nreaders swmr_readers"
+        $pid_readers = @()
+        $n = 0
+        while ($n -lt $Nreaders) {
+            #seed = "-r ${seeds[$n]}"
+            $seed = ""
+            $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 |tee swmr_reader.out.$n
+            $pid_readers += $rp.id
+            $n += 1
+        }
+        #Write-Output "pid_readers=$pid_readers"
+
+        # Collect exit code of the readers first because they usually finish
+        # before the writer.
+        foreach ($xpid in $pid_readers) {
+            #Write-Output "checked reader $xpid"
+            $result = Wait-Process -Id  $xpid
+            if ($result.ExitCode -ne 0) {
+                Write-Warning "reader had error"
+                $nerrors += 1
+            }
+        }
+
+        # Collect exit code of the writer
+        #Write-Output "checked writer $pid_writer"
+        $result = Wait-Process -Id  $pid_writer
+        if ($result.ExitCode -ne 0) {
+            Write-Warning "writer had error"
+            $nerrors += 1
+        }
+
+        # Check for error and exit if one occurred
+        #Write-Output "nerrors=$nerrors"
+        if ($nerrors -ne 0) {
+            Write-Warning "SWMR tests failed with $nerrors errors."
+            Write-Warning "(Writer and reader output preserved)"
+            exit 1
+        }
+
+        # Clean up output files
+        Remove-Item swmr_writer.out
+        Remove-Item swmr_reader.out.*
+
+        Write-Output ""
+        Write-Output "###############################################################################"
+        Write-Output "## Remove test - test shrinking the dataset"
+        Write-Output "###############################################################################"
+
+        # Remove any possible writer message file before launching writer
+        Remove-Item $WRITER_MESSAGE
+        # Launch the Remove Writer
+        Write-Output "launch the swmr_remove_writer"
+        $seed = "" # Put -r <random seed> command here
+        $rp = Start-Process -FilePath $testdir/swmr_remove_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecs_rem $seed" 2>&1 |tee swmr_writer.out
+        $pid_writer = $rp.id
+        #Write-Output "pid_writer=$pid_writer"
+
+        # Wait for message from writer process before starting reader(s)
+        Wait-Message $testdir $WRITER_MESSAGE
+        #
+        # Launch the Remove Readers
+        #declare -a seeds = (<seed1> <seed2> <seed3> ... )
+        $n = 0
+        $pid_readers = @()
+        Write-Output "launch $Nreaders swmr_remove_readers"
+        while ($n -lt $Nreaders) {
+            #seed = "-r ${seeds[$n]}"
+            $seed = ""
+            $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_rem $seed" 2>&1 |tee swmr_reader.out.$n
+            $pid_readers += $rp.id
+            $n += 1
+        }
+        #Write-Output "pid_readers=$pid_readers"
+
+        # Collect exit code of the readers first because they usually finish
+        # before the writer.
+        foreach ($xpid in $pid_readers) {
+            #Write-Output "checked reader $xpid"
+            $result = Wait-Process -Id  $xpid
+            if ($result.ExitCode -ne 0) {
+                Write-Warning "reader had error"
+                $nerrors += 1
+            }
+        }
+
+        # Collect exit code of the writer
+        #Write-Output "checked writer $pid_writer"
+        $result = Wait-Process -Id  $pid_writer
+        if ($result.ExitCode -ne 0) {
+            Write-Warning "writer had error"
+            $nerrors += 1
+        }
+
+        # Check for error and exit if one occurred
+        #Write-Output "nerrors=$nerrors"
+        if ($nerrors -ne 0) {
+            Write-Warning "SWMR tests failed with $nerrors errors."
+            Write-Warning "(Writer and reader output preserved)"
+            exit 1
+        }
+
+        # Clean up output files
+        Remove-Item swmr_writer.out
+        Remove-Item swmr_reader.out.*
+
+        Write-Output ""
+        Write-Output "###############################################################################"
+        Write-Output "## Add/remove test - randomly grow or shrink the dataset"
+        Write-Output "###############################################################################"
+
+        # Launch the Generator
+        Write-Output "launch the swmr_generator"
+        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow-PassThru -Wait -ArgumentList "$compress $index_type"
+        if ($rp.ExitCode -ne 0) {
+            Write-Warning "generator had error"
+            $nerrors += 1
+        }
+
+        # Launch the Writer (not in parallel - just to rebuild the datasets)
+        Write-Output "launch the swmr_writer"
+        $seed = "" # Put -r <random seed> command here
+        $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -Wait -ArgumentList "$Nrecords $seed"
+        if ($rp.ExitCode -ne 0) {
+            Write-Warning "writer had error"
+            $nerrors += 1
+        }
+
+        # Remove any possible writer message file before launching writer
+        Remove-Item $WRITER_MESSAGE
+        #
+        # Launch the Add/Remove Writer
+        Write-Output "launch the swmr_addrem_writer"
+        $seed = "" # Put -r <random seed> command here
+        $rp = Start-Process -FilePath $testdir/swmr_addrem_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecords $seed" 2>&1 |tee swmr_writer.out
+        $pid_writer = $rp.id
+        #Write-Output "pid_writer=$pid_writer"
+
+        # Wait for message from writer process before starting reader(s)
+        Wait-Message $testdir $WRITER_MESSAGE
+        #
+        # Launch the Add/Remove Readers
+        #declare -a seeds = (<seed1> <seed2> <seed3> ... )
+        $n = 0
+        $pid_readers = @()
+        Write-Output "launch $Nreaders swmr_remove_readers"
+        while ($n -lt $Nreaders) {
+            #seed = "-r ${seeds[$n]}"
+            $seed = ""
+            $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_addrem $seed" 2>&1 |tee swmr_reader.out.$n
+            $pid_readers += $rp.id
+            $n += 1
+        }
+        #Write-Output "pid_readers=$pid_readers"
+
+        # Collect exit code of the readers first because they usually finish
+        # before the writer.
+        foreach ($xpid in $pid_readers) {
+            #Write-Output "checked reader $xpid"
+            $result = Wait-Process -Id  $xpid
+            if ($result.ExitCode -ne 0) {
+                Write-Warning "reader had error"
+                $nerrors += 1
+            }
+        }
+
+        # Collect exit code of the writer
+        #Write-Output "checked writer $pid_writer"
+        $result = Wait-Process -Id  $pid_writer
+        if ($result.ExitCode -ne 0) {
+            Write-Warning "writer had error"
+            $nerrors += 1
+        }
+
+        # Check for error and exit if one occurred
+        #Write-Output "nerrors=$nerrors"
+        if ($nerrors -ne 0) {
+            Write-Warning "SWMR tests failed with $nerrors errors."
+            Write-Warning "(Writer and reader output preserved)"
+            exit 1
+        }
+
+        # Clean up output files
+        Remove-Item swmr_writer.out
+        Remove-Item swmr_reader.out.*
+
+        Write-Output ""
+        Write-Output "###############################################################################"
+        Write-Output "## Sparse writer test - test writing to random locations in the dataset"
+        Write-Output "###############################################################################"
+
+        # Launch the Generator
+        # NOTE: Random seed is shared between readers and writers and is
+        #       created by the generator.
+        Write-Output "launch the swmr_generator"
+        $seed = "" # Put -r <random seed> command here
+        $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "$compress $index_type $seed"
+        if ($rp.ExitCode -ne 0) {
+            Write-Warning "generator had error"
+            $nerrors += 1
+        }
+
+        # Remove any possible writer message file before launching writer
+        Remove-Item $WRITER_MESSAGE
+        # Launch the Sparse writer
+        Write-Output "launch the swmr_sparse_writer"
+        $rp = Start-Process -FilePath $testdir/swmr_sparse_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecs_spa" 2>&1 |tee swmr_writer.out
+        $pid_writer = $rp.Id
+        #Write-Output "pid_writer=$pid_writer"
+
+        # Wait for message from writer process before starting reader(s)
+        Wait-Message $testdir $WRITER_MESSAGE
+        #
+        # Launch the Sparse readers
+        $n = 0
+        $pid_readers = @()
+        Write-Output "launch $Nrdrs_spa swmr_sparse_readers"
+        while ($n -lt $Nrdrs_spa) {
+            # The sparse reader spits out a LOT of data so it's set to 'quiet'
+            $rp = Start-Process -FilePath $testdir/swmr_sparse_reader.exe -NoNewWindow -PassThru -ArgumentList "-q $Nrecs_spa" 2>&1 |tee swmr_reader.out.$n
+            $pid_readers += $rp.id
+            $n += 1
+        }
+        #Write-Output "pid_readers=$pid_readers"
+
+        # Collect exit code of the writer
+        #Write-Output "checked writer $pid_writer"
+        $result = Wait-Process -Id  $pid_writer
+        if ($result.ExitCode -ne 0) {
+            Write-Warning "writer had error"
+            $nerrors += 1
+        }
+
+        # Collect exit code of the readers
+        foreach ($xpid in $pid_readers) {
+            #Write-Output "checked reader $xpid"
+            $result = Wait-Process -Id  $xpid
+            if ($result.ExitCode -ne 0) {
+                Write-Warning "reader had error"
+                $nerrors += 1
+            }
+        }
+
+        # Check for error and exit if one occurred
+        #Write-Output "nerrors=$nerrors"
+        if ($nerrors -ne 0) {
+            Write-Warning "SWMR tests failed with $nerrors errors."
+            Write-Warning "(Writer and reader output preserved)"
+            exit 1
+        }
+
+        # Clean up output files
+        Remove-Item swmr_writer.out
+        Remove-Item swmr_reader.out.*
+    }
+}
+
+###############################################################################
+## Report and exit
+###############################################################################
+cd ..
+#Write-Output "nerrors=$nerrors"
+if ($nerrors -eq 0) {
+    Write-Output "SWMR tests passed."
+#    if test -z "$HDF5_NOCLEANUP"; then
+#        # delete the test directory
+#        Remove-Item swmr_test -Recurse
+#    fi
+    exit 0
+}
+else {
+    Write-Warning "SWMR tests failed with $nerrors errors."
+    exit 1
+}
+