path: root/bin/runtest

#! /bin/sh
#
# Copyright by the Board of Trustees of the University of Illinois.
# All rights reserved.
#
# This file is part of HDF5.  The full HDF5 copyright notice, including
# terms governing use, modification, and redistribution, is contained in
# the files COPYING and Copyright.html.  COPYING can be found at the root
# of the source code distribution tree; Copyright.html can be found at the
# root level of an installed copy of the electronic HDF5 document set and
# is linked from the top-level documents page.  It can also be found at
# http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html.  If you do not have
# access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu.
#

# run the hdf5/bin/snapshot
# Usage:
#	runtest		run the test for the local host
#	runtest <hostname>	run the test for <hostname>
#	runtest -all		run the test for all predefined hosts
#
# Assumptions in knowing where to find the right scripts to execute.
# 1. assume we are at the top level of the hdf5 source.  So, bin/* are
#    where the script files are.
# 2. after the cvs update is completed, we can go to the snapshot area
#    hdf5 source tree and use the bin/* there.
# 3. Cannot use the snapshot area scripts from the beginning because
#    for one, the current directory is renamed as previous right after
#    a snapshot release; and for another, some scripts may be changed
#    by the cvs update while it is being used.

# local setup
DEBUGMODE=""
test -n "$DEBUGMODE" && echo "******** DEBUGMODE is $DEBUGMODE ************"
WHEREAMI='pwd'
CMD=

# the name of this program
PROGNAME="bin/runtest $DEBUGMODE"

# Setup
HOSTNAME=`hostname | cut -f1 -d.`	# no domain part
TODAY=`date +%y%m%d`
WEEKDAY=`date +%a`
H5VER=			# default to current CVS version
H5VERSION=		# default to current CVS version

# Default to do checkout (only once) and test, no release.
# Will run test only if there is significant differences from previous version.
# If srcdir is not used, don't launched multiple tests
SNAPSHOT="${DEBUGMODE:+echo }bin/snapshot"
SRCDIR="srcdir"
# Default standard Snaptest commands
SNAPCMD="$SRCDIR test clean"
# Default Standard snaptest command options
STANDARD_OPT=""
ENABLE_PARALLEL="--enable-parallel"
CONFIGNAME=$HOSTNAME	# Name used in the SNAPTESTCFG file

# test host default as local host.
TESTHOST=""

#################################
# Function definitions
#################################

# Print messages to stdout
# Use this to show output heading to stdout
PRINT()
{
    echo "$*"
}

# Show seconds since midnight.
# This is used to calculate seconds elapsed
SecOfDay()
{
    set `date '+%H %M %S'`
    t_sec=`expr $1 \* 3600 + $2 \* 60 + $3`
    echo $t_sec
}

# Calculated the elapsed time (in seconds) between the first
# and second time.  If second time is smaller than the first,
# we assume the clock has passed midnight and calculate appropriately.
ElapsedTime()
{
    if [ $2 -lt $1 ]; then
	t_sec=`expr 3600 \* 24 - $1 + $2`
    else
	t_sec=`expr $2 - $1`
    fi
    echo `expr $t_sec / 60`m `expr $t_sec % 60`s
}

# Report errors
# $1--an error message to be printed
REPORT_ERR()
{
    ERRMSG=$1
    # print it with a banner shifted right a bit
    PRINT "	*************************************"
    PRINT "	`date`"
    PRINT "	$ERRMSG"
    PRINT "	*************************************"
    # report it in the FAILED-LOG file too
    PRINT "$ERRMSG" >> $FAILEDLOG
}

#
# Report results of the last test done
REPORT_RESULT()
{
    if [ $retcode -eq 0 ]; then
	if [ $skiptest = yes ]; then
	    PRINT "SKIPPED ${HOSTNAME}: $TEST_TYPE" | tee -a $SKIPPEDLOG
	else
	    PRINT "PASSED ${HOSTNAME}: $TEST_TYPE" | tee -a $PASSEDLOG
	fi
    else
	# test failed.
	REPORT_ERR "****FAILED ${HOSTNAME}: $TEST_TYPE****"
    fi
}

# Print a blank line
PRINT_BLANK()
{
    PRINT
}

# Print test trailer
PRINT_TEST_TRAILER()
{
    PRINT "*** finished $TEST_TYPE tests for $HOSTNAME ***"
    date; EndTime=`SecOfDay`
    PRINT Total time = `ElapsedTime $StartTime $EndTime`
    PRINT_BLANK
}

# Print trailer summary
PRINT_TRAILER()
{
    PRINT "*** finished tests in $HOSTNAME ***"
    date; TotalEndTime=`SecOfDay`
    PRINT "${HOSTNAME}: Ran $n_test $runtest_type, Grand total test time = " \
          "`ElapsedTime $TotalStartTime $TotalEndTime`" | tee -a $TIMELOG
    PRINT_BLANK
}

# Figure out which remote command to use to reach a host.
# Try rsh first, then ssh.
# $1--hostname to reach.
CHECK_RSH()
{
    # Figure out how to use ping command in this host.
    # Some hosts use "ping host count", some use "ping -c count host".
    # Test "ping -c 3 -w 5" since it has timeout feature.
    # Test "ping -c ..." style before "ping host 3" because some machines
    # that recognize -c treat 'ping localhost 3' as to ping host '3'.
    if [ -z "$PING" ]; then
	if ping -c 3 -w 5 localhost >/dev/null 2>&1; then
	    PING='ping -c 3 -w 5'
	    PINGCOUNT=
	elif ping -c 3 localhost >/dev/null 2>&1; then
	    PING='ping -c 3'
	    PINGCOUNT=
	elif ping localhost 3 >/dev/null 2>&1; then
	    PING=ping
	    PINGCOUNT=3
	else	# don't know how to use ping.
	    PING=no_ping
	    PINGCOUNT=
	fi
    fi
    #
    host=$1
    # Try remote command with host if it responds to ping.
    # Still try it if we don't know how to do ping.
    if [ no_ping = "$PING" ] || $PING $host $PINGCOUNT >/dev/null 2>&1; then
	if rsh $host -n hostname >/dev/null 2>&1; then
	    RSH=rsh
	elif ssh $host -n hostname >/dev/null 2>&1; then
	    RSH=ssh
	else
	    PRINT cannot remote command with $host
	    RSH="NoRemoteCommand"
	fi
    else
	RSH="NotReachable"
    fi
}


# Wait for a file for at most number of minutes
# $1--the file
# $2--number of minutes
# WAIT_STATUS set to:
#	-1 if errors encountered
#	0  if file found within time limit
#	1  if file not found within time limit
WAITFOR()
{
    wait_file=$1
    nminutes=$2
    if [ -z "$wait_file" -o ! "$nminutes" -ge 0 ]
    then
	PRINT "errors in argument of WAITFOR(): wait_file($1) or nminutes($2)"
	WAIT_STATUS=-1
	return
    fi
    while [ ! -f $wait_file ]; do
	if [ $nminutes -gt 0 ]; then
	    PRINT "Wait For $wait_file to appear"
	    sleep 60			#sleep 1 minute
	else
	    WAIT_STATUS=1
	    return
	fi
	nminutes=`expr $nminutes - 1`
    done
    WAIT_STATUS=0
    return
}


# Wait till a file disappears for at most number of minutes.
# Useful to wait till a lock is removed by another process.
# $1--the file
# $2--number of minutes
# WAIT_STATUS set to:
#	-1 if errors encountered
#	0  if file disappears within time limit
#	1  if file has not disappeared within time limit
WAITTILL()
{
    wait_file=$1
    nminutes=$2
    if [ -z "$wait_file" -o ! "$nminutes" -ge 0 ]
    then
	PRINT "errors in argument of WAITTILL(): wait_file($1) or nminutes($2)"
	WAIT_STATUS=-1
	return
    fi
    while [ -f $wait_file ]; do
	if [ $nminutes -gt 0 ]; then
	    PRINT "Wait till $wait_file has disappeared"
	    sleep 60			#sleep 1 minute
	else
	    WAIT_STATUS=1
	    return
	fi
	nminutes=`expr $nminutes - 1`
    done
    WAIT_STATUS=0
    return
}


# Run one snapshot test
# $*--Types of test being run
RUNSNAPTEST()
{
    SNAPCMD_OPT="$STANDARD_OPT"		# snapshot test option
    SRCDIRNAME=${HOSTNAME}
    # restore CC, PATH in case they were changed in the last test.
    CC="$CC_SAVED"
    PATH=$PATH_SAVED
    export PATH			# DEC OSF1 needs to export PATH explicitly
    TEST_TYPE=$*
    retcode=0
    skiptest=no
    date
    PRINT "*** starting $TEST_TYPE tests in $HOSTNAME ***"
    PRINT "Uname -a: `uname -a`"

    # Parse the test type and set options accordingly.
    # See comments of SNAPTEST_CONFIG_PARSE().
    while [ $# -gt 0 ]; do
	case $1 in
	    -n32) # want -n32 option
		SRCDIRNAME=${SRCDIRNAME}-n32
		CC="cc -n32"
		export CC
		;;
	    parallel) # want parallel test
		SNAPCMD_OPT="$SNAPCMD_OPT $ENABLE_PARALLEL"
		SRCDIRNAME=${SRCDIRNAME}-pp
		;;
	    standard) # standard test
		;;
	    --*)
		# option for configure
		SNAPCMD_OPT="$SNAPCMD_OPT $1"
		;;
	    op-configure)
		# option for configure
		SNAPCMD_OPT="$SNAPCMD_OPT $1 $2"
		shift
		;;
	    op-snapshot)
		# option for snapshot
		shift
		SNAPCMD_OPT="$SNAPCMD_OPT $1"
		;;
	    setenv)
		# pass them along to snapshot set environment variable
		shift
		SNAPCMD_OPT="$SNAPCMD_OPT setenv $1 $2"
		shift
		;;
	    setenvN)
		# set environment variable with $1 values
		# e.g., setenvN 3 x a b c is same as setenv x="a b c".
		# pass them along to snapshot set environment variable
		shift
		envN=$1
		shift
		envname=$1
		SNAPCMD_OPT="$SNAPCMD_OPT setenvN $envN $envname"
		envalue=
		while test $envN -gt 0; do
		    shift
		    envalue="$envalue $1"
		    envN=`expr $envN - 1`
		done
		SNAPCMD_OPT="$SNAPCMD_OPT $envalue"
		;;
	    skip)
		# skip this test
		skiptest=yes
		;;
	    srcdirname)
		# Use this before using parallel and -n32 since this overrides
		# the others.
		shift
		SRCDIRNAME=$1
		;;
	    deploy)
		# deploy the built binary.
		shift
		SNAPCMD_OPT="$SNAPCMD_OPT deploy $1"
		;;
	    deploydir)
		# default directory for deployment.
		shift
		SNAPCMD_OPT="$SNAPCMD_OPT deploydir $1"
		;;
	    *) # unknown test
		PRINT "$0: unknown type of test ($1)"
		retcode=1
		;;
	esac
	shift
    done

    if [ $retcode -ne 0 -o $skiptest = yes ]; then
	errcode=$retcode 
	return $retcode
    fi

    # Track down the zlib software
    ans=`$SNAPYARD/current/bin/locate_sw zlib`
    if [ $? = 0 ]; then
	Z_INC=`echo $ans | cut -f1 -d,`
	Z_LIB=`echo $ans | cut -f2 -d,`
	SNAPCMD_OPT="$SNAPCMD_OPT zlib $Z_INC,$Z_LIB"
    else
	# cannot locate zlib software.
	# continue the test, maybe configure can find it.
	:
    fi

    if [ -n "${SRCDIRNAME}" ]; then
	SNAPCMD_OPT="$SNAPCMD_OPT srcdirname ${SRCDIRNAME}"
    fi

    # Setup log file name to save test output
    THIS_MINUTE=`date +%H%M`
    LOGFILE=${LOGBASENAME}/${SRCDIRNAME}_${TODAY}_${THIS_MINUTE}
    PRINT "Running snapshot with output saved in"
    PRINT "   $LOGFILE"
    (date; PRINT Hostname=$HOSTNAME) >> $LOGFILE

    (
    cd $SNAPYARD/current
    $SNAPSHOT $SNAPCMD $SNAPCMD_OPT
    ) >> $LOGFILE 2>&1
    retcode=$?
    [ $retcode -ne 0 ] && errcode=$retcode

    date >> $LOGFILE
    if [ $retcode -ne 0 ]; then
	echo FAILED in $LOGFILE >> $FAILEDDETAIL
    else
	echo PASSED in $LOGFILE >> $FAILEDDETAIL
    fi
}


# configuration parsing.
# Taking configuration from input.
# This should be invoke with configure file as stdin.
# Syntax of the configure file:
#    All lines started with the # are comment lines and are ignored.
#    Blank lines are ignored too.
#    Each config line starts with a "Scope" followed by test types.
#
# Scope can be:
#    standard ...          # what the standard test types are.
#    <host>: <test>        Do <test> for <host> 
#    all: <test>           Do <test> for all hosts. 
#    <weekday>/...         Use this scope if the <weekday> matches.
#                          <weekday> can be {Mon,Tue,Wed,Thu,Fri,Sat,Sun}
#    If no <host>: input for a <host>, the standard test is used.
#
# Test types:
#    standard		tests defined in standard scope.
#    -n32		-n32 mode.  Apply to 64/32 bit OS such as IRIX64.
#    parallel		parallel mode.
#    op-configure <option>	configure option
#    op-snapshot <option>	snapshot option
#    --*		configure option
#    setenv <name> <value>	set environment variable <name> to <value>
#                       Pass along to snapshot
#    setenvN <N> <name> <value> ...
#		        set environment variable with <N> values
#		        e.g., setenvN 3 x a b c is same as setenv x="a b c".
#                       Pass along to snapshot.
#    skip		skip this test
#    srcdirname <name>	use <name> as the build-directory.
#    deploy <name>	deploy the built binary at directory <name>.
#    deploydir <name>   use <name> as the default directory for deployment.
SNAPTEST_CONFIG_PARSE()
{
    while read x y ; do
	# Scan for entry for this weekday.
	xd=`echo $x | cut -f1 -d/`
	if [ "$xd" = ${WEEKDAY} ]; then
	    # strip away the weekday/ part.
	    x=`echo $x | cut -f2 -d/`
	fi
	case "$x" in
	    '' | '#'*)
		# blank or comment lines.  Continue.
	    	;;
	    ???/*)
		# Ignore any entry not of this weekday.
		;;
	    standard)
		#standard configuration
		STANDARD_OPT="$y"
		;;
	    all: | ${CONFIGNAME}:)
		# types of test for all hosts or this host
		if [ -n "$TEST_TYPES" ]; then
		    TEST_TYPES="$TEST_TYPES ; $y"
		else
		    TEST_TYPES="$y"
		fi
		;;
	    *:)	# ignore types of test for other hosts
		;;
	    *)	# unknown configuration option
		PRINT $x $y
		PRINT "***Unknown configuration option. Ignored.***"
		;;
	esac
    done
}

# Snap Test configuration parsing.
# If TEST_TYPES is not set, set it to do the "standard" test.
SNAPTEST_CONFIG()
{
    TEST_TYPES=
    STANDARD_OPT=
    if [ -f $SNAPTESTCFG ]; then
	SNAPTEST_CONFIG_PARSE < $SNAPTESTCFG
    fi
    TEST_TYPES=${TEST_TYPES:-'standard'}
}


# Show usage page
USAGE()
{
cat <<EOF
Usage: runtest [-h] [-debug] [-r<version>] [-all] [-nocvs] [-nodiff] [<host> ...]
    -h
	print this help page
    -debug
	turn on debug mode
    -r<version>
	do runtest for <version>
    -all
	launch tests for all pre-defined testing hosts
    -nocvs
	do not do cvs commands
    -nodiff
	do not do diff commands
    -setup
        setup the directory structure for snapshot test
    -configname <name>
	use <name> as hostname in the parsing of the snaptest configure file
    <host>
	launch tests for <host>
    
-all and <host> are contradictory and whichever is specified last, is
the one to take effect.  If neither are given, do the test for the
local host.
EOF
}


# Verify if directory ($1) exists.  If not, create it.
CHECK_DIR()
{
   dir=$1
   if test ! -e $1; then
      echo mkdir $1
      mkdir $1
      errcode=$?
   elif test ! -d $1; then
      echo $1 is not a directory
      errcode=1
   fi
}


#################################
# Main
#################################
#################################
# Set up global variables
#################################
retcode=0			# error code of individula task
errcode=0			# error code of the whole test
skiptest=no			# if test is skipped
CC_SAVED="$CC"			# CC & PATH maybe changed within a test.
PATH_SAVED=$PATH		# These save the original values.
timelimit=300			# time limit (minutes) for the timekeeper

#################################
# Parse options
#################################
while [ $# -gt 0 ]; do
    case "$1" in
	-h) # help--show usage
	    USAGE
	    exit 0
	    ;;
	-debug*)
	    # set debug mode
	    DEBUGMODE="$1"
	    SNAPSHOT="echo bin/snapshot"
	    PROGNAME="$PROGNAME $DEBUGMODE"
	    PRINT "******** DEBUGMODE is $DEBUGMODE ************"
	    ;;
	-r*)
	    # version string
	    H5VER="$1"
	    ;;
	-all)
	    # Test all hosts.
	    TESTHOST=-all
	    ;;
	-nocvs)
	    # do not do cvs commands.
	    NOCVS=nocvs
	    ;;
	-nodiff)
	    # do not do diff commands.
	    NODIFF=nodiff
	    ;;
	-configname)
	    # use <name> as hostname in the parsing of the snaptest configure file.
	    shift
	    CONFIGNAME=$1
	    ;;
	-setup)
	    # setup the directory structure for snapshot test.
	    CMD=setup
	    ;;
	-*) # Unknow option
	    PRINT "Unknown option ($1)"
	    USAGE
	    exit 1
	    ;;
	*)
	    TESTHOST=$*
	    break
	    ;;
    esac
    shift
done

# setup H5VER if not set yet
if [ -z "$H5VER" -a -f bin/snapshot_version ]
then
    . bin/snapshot_version
fi

if [ -n "$H5VER" ]
then
    H5VERSION=hdf5_`echo $H5VER | sed -e s/-r// -e s/\\\./_/g`
    PROGNAME="$PROGNAME $H5VER"
else
    H5VERSION=hdf5
fi

#################################
# Setup snapshot test directories
#################################
BASEDIR=${HOME}/snapshots-${H5VERSION}
# initial processing of setup option if requested
if test x-$CMD = x-setup; then
    CHECK_DIR $BASEDIR
    test $errcode -ne 0 && exit 1
elif [ ! -d ${BASEDIR} ]; then
    echo "BASEDIR ($BASEDIR) does not exist"
    exit 1
fi
# Show the real physical path rather than the symbolic path
SNAPYARD=`cd $BASEDIR && /bin/pwd`
# Log file basename
LOGDIR=${SNAPYARD}/log
LOGBASENAME=${LOGDIR}
PASSEDLOG=${LOGDIR}/PASSED_LOG_${TODAY}
FAILEDLOG=${LOGDIR}/FAILED_LOG_${TODAY}
FAILEDDETAIL=${LOGDIR}/FAILED_DETAIL_${TODAY}
SKIPPEDLOG=${LOGDIR}/SKIPPED_LOG_${TODAY}
TIMELOG=${LOGDIR}/TIME_LOG_${TODAY}
TIMEKEEPERLOG=${LOGDIR}/TIMEKEEPER_LOG_${TODAY}
CVSLOG=${LOGDIR}/CVS_LOG_${TODAY}
CVSLOG_LOCK=${LOGDIR}/CVS_LOG_LOCK_${TODAY}
DIFFLOG=${LOGDIR}/DIFF_LOG_${TODAY}
# Snap Test hosts and Configuration files
ALLHOSTSFILE=${SNAPYARD}/allhostfile
SNAPTESTCFG=${SNAPYARD}/snaptest.cfg

# more processing of setup option if requested
if test x-$CMD = x-setup; then
    CHECK_DIR $LOGDIR
    test $errcode -ne 0 && exit 1
    CHECK_DIR $LOGDIR/OLD
    test $errcode -ne 0 && exit 1
    CHECK_DIR $SNAPYARD/TestDir
    test $errcode -ne 0 && exit 1
    # create empty test hosts or configure files if non-existing
    for f in $ALLHOSTSFILE $SNAPTESTCFG; do
	if test ! -f $f; then
	    echo Creating $f
	    touch $f
	fi
    done
    # create or update the current source.
    echo update current source
    $SNAPSHOT checkout 
    # setup completed.  Exit.
    exit 0
fi

#################################
# Setup test host(s)
#################################
if [ "$TESTHOST" = -all ]; then
    if [ -f $ALLHOSTSFILE ]; then
	TESTHOST=`sed -e s/#.*// $ALLHOSTSFILE`
    else
	PRINT "could not access the all-hosts-file ($ALLHOSTSFILE)"
	USAGE
	exit 1
    fi
fi


#################################
# Setup to print a trailer summary when exiting not via
# the normal end of the script.
#################################
trap PRINT_TRAILER 0

#
TotalStartTime=`SecOfDay`

# Process the configuration
SNAPTEST_CONFIG
PRINT STANDARD_OPT=$STANDARD_OPT
PRINT TEST_TYPES=$TEST_TYPES
PRINT_BLANK

# Do a checkout if one has not been done today
# Also check MANIFEST file
if [ -z "$NOCVS" ]; then
    PRINT "Running CVS checkout with output saved in"
    PRINT "   $CVSLOG"
    # Set CVS lock first
    touch $CVSLOG_LOCK
    ($SNAPSHOT checkout ) >> $CVSLOG 2>&1
    # Save error code and remove the lock
    errcode=$?
    rm -f $CVSLOG_LOCK
    if [ $errcode -ne 0 ]; then
	# test failed.
	REPORT_ERR "****FAILED ${HOSTNAME}: CVS checkout****"
	exit $errcode
    fi
    PRINT Checking MAINFEST file ...
    (cd $SNAPYARD/current; bin/chkmanifest)
    errcode=$?
    if [ $errcode -ne 0 ]; then
	# test failed.
	REPORT_ERR "****FAILED ${HOSTNAME}: MANIFEST check****"
    fi
    PRINT_BLANK
else
    # make sure the cvs update, if done by another host, has completed.
    # First wait for the presence of $CVSLOG which signals some host
    # has started the cvs update.  Then wait for the absense of $CVSLOG_LOCK
    # which signals the host has completed the cvs update.
    WAITFOR $CVSLOG 90
    if [ $WAIT_STATUS -ne 0 ]; then
	errcode=$WAIT_STATUS
	REPORT_ERR "****FAILED ${HOSTNAME}: Time expired waiting CVS update to start****"
	exit $errcode
    fi
    WAITTILL $CVSLOG_LOCK 10
    if [ $WAIT_STATUS -ne 0 ]; then
	errcode=$WAIT_STATUS
	REPORT_ERR "****FAILED ${HOSTNAME}: Time expired waiting CVS update to finish****"
	exit $errcode
    fi
fi

# run a snapshot diff to see if any significant differences between
# the current and previous versions
if [ -z "$NODIFF" ]; then
    $SNAPSHOT diff >> $DIFFLOG 2>&1
    errcode=$?
    # check the errcode only if NOT in DEBUG MODE
    if [ -z "$DEBUGMODE" -a $errcode -eq 0 ]; then
	# no need to run test
	PRINT "NO TEST: no significant differences between current and previous versions" |
	    tee -a $PASSEDLOG
	exit 0
    fi
fi

# we can use the version of script in SNAPYARD/current now.
# Don't do the diff or cvs update any more.
PROGNAME="$SNAPYARD/current/$PROGNAME -nodiff -nocvs"

# Decide to do test for the local host or for remote hosts
if [ -n "$TESTHOST" -a $HOSTNAME != "$TESTHOST" ]; then
    date
    PRINT "*** launching tests from $HOSTNAME ***"
    PRINT_BLANK
    TEST_TYPE="launching"
    cd ${SNAPYARD}/log
    # Fork off timekeeper if concurrent tests will be used.
    if [ -n "$SRCDIR" ]; then
	($SNAPYARD/current/bin/timekeeper $timelimit > $TIMEKEEPERLOG 2>&1 &)
	PRINT "    Fork off timekeeper $timelimit"
    fi
    n_test=0
    runtest_type="hosts"
    for h in $TESTHOST; do
	# Must do CONFIGNAME before $h got changed by the second cut.
	# cut returns the whole string if there is no / in the string
	# at all.  But that works okay for the CONFIGNAME too.
	CONFIGNAME=`echo $h | cut -f2 -d/`
	h=`echo $h | cut -f1 -d/`
	n_test=`expr $n_test + 1`
	TMP_OUTPUT="#$h.out"
	(PRINT "=============="
	 PRINT "Testing $h"
	 PRINT "==============") > $TMP_OUTPUT
	CHECK_RSH $h
	# run the remote shell command with output to $TMP_OUTPUT
	case "$RSH" in
	    rsh|ssh)
		CMD="$RSH $h -n $PROGNAME -configname $CONFIGNAME"
		PRINT $CMD

		# launch concurrent tests only if srcdir is used
		if [ -n "$SRCDIR" ]; then
		    $CMD &
		    echo $! > PID.$h
		else
		    $CMD
		fi
		;;
	    NoRemoteCommand)
		PRINT $h does not accept Remote Command "(`date`)"
		;;
	    NotReachable)
		PRINT $h is not reachable  "(`date`)"
		;;
	    *)
		PRINT "CHECK_RSH for $h returned unknow result ($RSH)"
		;;
	esac >> $TMP_OUTPUT 2>&1
    done
    # wait for all launched tests to finish, then cat them back out.
    wait
    # Pause a moment in case the timekeeper is terminating processes.
    wait 30
    for h in $TESTHOST; do
	h=`echo $h | cut -f1 -d/`
	TMP_OUTPUT="#$h.out"
	cat $TMP_OUTPUT
	# Verify test script did complete by checking the last lines
	(tail -2 $TMP_OUTPUT | grep -s 'Grand total' > /dev/null 2>&1) ||
	    (REPORT_ERR "****FAILED ${h}: snaptest did not complete****" &&
		PRINT_BLANK)
	rm -f $TMP_OUTPUT PID.$h
    done
    exit 0
fi

# run the test(s)
# Note that first field is cut without -s but all subsequent cut
# must use -s.  If -s is not used at all, a $TEST_TYPES that has
# no ';' (only 1 test), will pass through intact in all cut. That
# results in infinite looping.
# If -s is used with the first field, it will suppress completely
# a $TYPE_TYPES that has no ';' (only 1 tst ).  That results in no
# test at all.
# Note that n_test must start as 1.
# 
n_test=1
runtest_type="tests"
TEST="`echo $TEST_TYPES | cut -f$n_test -d';'`"
while [ -n "$TEST" ]; do
    StartTime=`SecOfDay`
    RUNSNAPTEST $TEST
    REPORT_RESULT
    PRINT_TEST_TRAILER

    n_test=`expr $n_test + 1`
    TEST="`echo $TEST_TYPES | cut -f$n_test -s -d';'`"
done
# dec n_test to show the actual number of tests ran.
n_test=`expr $n_test - 1`

PRINT_TRAILER

# disable trailer summary printing since all trailers have been
# printed and we are exiting normally.
trap 0
exit $errcode