diff options
author | Dana Robinson <43805+derobins@users.noreply.github.com> | 2022-03-23 13:48:05 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-03-23 13:48:05 (GMT) |
commit | fbb532cd633e216f47ce846493b38af02cfbb43a (patch) | |
tree | 1ef7bc9f25ec0832f8b25bb8ea4be34d53bd144d /test/test_swmr.sh.in | |
parent | 78375882485a99a81caa933928ed08d7a38ef88b (diff) | |
download | hdf5-fbb532cd633e216f47ce846493b38af02cfbb43a.zip hdf5-fbb532cd633e216f47ce846493b38af02cfbb43a.tar.gz hdf5-fbb532cd633e216f47ce846493b38af02cfbb43a.tar.bz2 |
VFD SWMR: Normalization with develop (#1506)
* Brought over plugin and test script changes
* Removes remaining register keywords (#1481)
* Fixed warnings in the aux process code
* Minor fixes from develop
* Minor changes from develop, fixed format warnings
* Formatted source
* Added HD prefix to timespec_get
* Cleanup in new files
* Removes the MANIFEST file and unused release files (#1497)
* Removes the MANIFEST file and unused release files
* Updated tar command
* checkposix corrections
* More checkposix fixes
* Ripped out unused instrumentation functionality
* Brought over cache tagging changes from develop
* Changes to tagged metadata expulsion iteration
* Fixed typo
* Brought over H5O__free() changes from develop
* Brings (unused) parallel page buffer test in line with develop
* Moved the functionality in supervise.subr to test_vfd_swmr.sh
* Tools VFD parameter updates
* Committing clang-format changes
* H5F VFD SWMR refactoring
* Committing clang-format changes
* Misc changes
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Diffstat (limited to 'test/test_swmr.sh.in')
-rw-r--r-- | test/test_swmr.sh.in | 563 |
1 files changed, 563 insertions, 0 deletions
diff --git a/test/test_swmr.sh.in b/test/test_swmr.sh.in new file mode 100644 index 0000000..771fe4a --- /dev/null +++ b/test/test_swmr.sh.in @@ -0,0 +1,563 @@ +#! /bin/bash +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the swmr feature. +# +# Created: +# Albert Cheng, 2009/07/22 + +srcdir=@srcdir@ +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ +testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ + +############################################################################### +## test parameters +############################################################################### + +Nreaders=5 # number of readers to launch +Nrdrs_spa=3 # number of sparse readers to launch +Nrecords=200000 # number of records to write +Nrecs_rem=40000 # number of times to shrink +Nrecs_spa=20000 # number of records to write in the sparse test +Nsecs_add=5 # number of seconds per read interval +Nsecs_rem=3 # number of seconds per read interval +Nsecs_addrem=8 # number of seconds per read interval +nerrors=0 + +############################################################################### +## definitions for message file to coordinate test runs +############################################################################### +WRITER_MESSAGE=SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete + # This should be the same as the define in "test/swmr_common.h" +MESSAGE_TIMEOUT=300 # Message timeout length in secs + # This should be the same as the define in "test/h5test.h" + +############################################################################### +## short hands and function definitions +############################################################################### +DPRINT=: # Set to "echo Debug:" for debugging printing, + # else ":" for noop. +IFDEBUG=: # Set to null to turn on debugging, else ":" for noop. + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# To wait for the writer message file or till the maximum # of seconds is reached +# $1 is the message file to wait for +# This performs similar function as the routine h5_wait_message() in test/h5test.c +WAIT_MESSAGE() { + message=$1 # Get the name of the message file to wait for + t0=`date +%s` # Get current time in seconds + difft=0 # Initialize the time difference + mexist=0 # Indicate whether the message file is found + while [ $difft -lt $MESSAGE_TIMEOUT ] ; # Loop till message times out + do + t1=`date +%s` # Get current time in seconds + difft=`expr $t1 - $t0` # Calculate the time difference + if [ -e $message ]; then # If message file is found: + mexist=1 # indicate the message file is found + rm $message # remove the message file + break # get out of the while loop + fi + done; + if test $mexist -eq 0; then + # Issue warning that the writer message file is not found, continue with launching the reader(s) + echo warning: $WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds + else + echo $WRITER_MESSAGE is found + fi +} + +############################################################################### +## Main +## +## Modifications: +## Vailin Choi; July 2013 +## Add waiting of message file before launching the reader(s). +## Due to the implementation of file locking, coordination +## is needed in file opening for the writer/reader tests +## to proceed as expected. +## +############################################################################### + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. +$utils_testdir/swmr_check_compat_vfd +rc=$? +if [ $rc -ne 0 ] ; then + echo + echo "The VFD specified by the HDF5_DRIVER environment variable" + echo "does not support SWMR." + echo + echo "SWMR acceptance tests skipped" + echo + exit 0 +fi + +# Parse options (none accepted at this time) +while [ $# -gt 0 ]; do + case "$1" in + *) # unknown option + echo "$0: Unknown option ($1)" + exit 1 + ;; + esac +done + +# HDF5 has several tests that create and delete signal files to communicate +# between processes, and it seems that even though the names of the files are +# different, occasionally the wrong file is deleted, interrupting the flow of +# the test. Running each of these tests in its own directory should eliminate +# the problem. +mkdir swmr_test +for FILE in swmr*; do + case "$FILE" in + *.o) continue ;; ## don't copy the .o files + esac + if test -f "$FILE" ; then + cp $FILE swmr_test + fi +done + +# With the --disable-shared option, swmr program files are built in the test +# directory, otherwise they are in test/.libs with a corresponding wrapper +# script in the test directory. The programs or wrapper scripts in test should +# always be copied, swmr files in .libs should be copied only if they exists. +if [ -f .libs/swmr ]; then + mkdir swmr_test/.libs + for FILE in .libs/swmr*; do + case "$FILE" in + *.o) continue ;; ## don't copy the .o files + esac + if test -f "$FILE" ; then + cp $FILE swmr_test/.libs + fi + done +fi + +cd swmr_test + + +# Loop over index types +for index_type in "-i ea" "-i b2" +do + # Try with and without compression + for compress in "" "-c 5" + do + echo + echo "*******************************************************************************" + echo "** Loop testing parameters: $index_type $compress" + echo "*******************************************************************************" + echo + echo + echo "###############################################################################" + echo "## Generator test" + echo "###############################################################################" + # Launch the Generator without SWMR_WRITE + echo launch the swmr_generator + $testdir/swmr_generator $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Launch the Generator with SWMR_WRITE + echo launch the swmr_generator with SWMR_WRITE + $testdir/swmr_generator -s $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + exit 1 + fi + + echo + echo "###############################################################################" + echo "## Use H5Fstart_swmr_write() to enable SWMR writing mode" + echo "###############################################################################" + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # + # Launch the Writer + echo launch the swmr_start_writer + seed="" # Put -r <random seed> command here + $testdir/swmr_start_write $compress $index_type $Nrecords $seed 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + + # + # Launch the Readers + #declare -a seeds=(<seed1> <seed2> <seed3> ... ) + echo launch $Nreaders swmr_readers + pid_readers="" + n=0 + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + + echo + echo "###############################################################################" + echo "## Writer test - test expanding the dataset" + echo "###############################################################################" + + # Launch the Generator + echo launch the swmr_generator + $testdir/swmr_generator -s $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # + # Launch the Writer + echo launch the swmr_writer + seed="" # Put -r <random seed> command here + $testdir/swmr_writer -o $Nrecords $seed 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Readers + #declare -a seeds=(<seed1> <seed2> <seed3> ... ) + echo launch $Nreaders swmr_readers + pid_readers="" + n=0 + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + + echo + echo "###############################################################################" + echo "## Remove test - test shrinking the dataset" + echo "###############################################################################" + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # Launch the Remove Writer + echo launch the swmr_remove_writer + seed="" # Put -r <random seed> command here + $testdir/swmr_remove_writer -o $Nrecs_rem $seed 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Remove Readers + #declare -a seeds=(<seed1> <seed2> <seed3> ... ) + n=0 + pid_readers="" + echo launch $Nreaders swmr_remove_readers + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + $testdir/swmr_remove_reader $Nsecs_rem $seed 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + + echo + echo "###############################################################################" + echo "## Add/remove test - randomly grow or shrink the dataset" + echo "###############################################################################" + + # Launch the Generator + echo launch the swmr_generator + $testdir/swmr_generator $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Launch the Writer (not in parallel - just to rebuild the datasets) + echo launch the swmr_writer + seed="" # Put -r <random seed> command here + $testdir/swmr_writer $Nrecords $seed + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # + # Launch the Add/Remove Writer + echo launch the swmr_addrem_writer + seed="" # Put -r <random seed> command here + $testdir/swmr_addrem_writer $Nrecords $seed 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Add/Remove Readers + #declare -a seeds=(<seed1> <seed2> <seed3> ... ) + n=0 + pid_readers="" + echo launch $Nreaders swmr_remove_readers + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + $testdir/swmr_remove_reader $Nsecs_addrem $seed 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + + echo + echo "###############################################################################" + echo "## Sparse writer test - test writing to random locations in the dataset" + echo "###############################################################################" + + # Launch the Generator + # NOTE: Random seed is shared between readers and writers and is + # created by the generator. + echo launch the swmr_generator + seed="" # Put -r <random seed> command here + $testdir/swmr_generator $compress $index_type $seed + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # Launch the Sparse writer + echo launch the swmr_sparse_writer + nice -n 20 $testdir/swmr_sparse_writer $Nrecs_spa 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Sparse readers + n=0 + pid_readers="" + echo launch $Nrdrs_spa swmr_sparse_readers + while [ $n -lt $Nrdrs_spa ]; do + # The sparse reader spits out a LOT of data so it's set to 'quiet' + $testdir/swmr_sparse_reader -q $Nrecs_spa 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Collect exit code of the readers + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + done +done + +############################################################################### +## Report and exit +############################################################################### +cd .. +$DPRINT nerrors=$nerrors +if test $nerrors -eq 0 ; then + echo "SWMR tests passed." + if test -z "$HDF5_NOCLEANUP"; then + # delete the test directory + rm -rf swmr_test + fi + exit 0 +else + echo "SWMR tests failed with $nerrors errors." + exit 1 +fi + |