Fix for HDFFV-10166 h5watch tests fail randomly to match expected output files

(1) Re-structure tests to do one "extend" action at a time
(2) Re-try test till max limit before final failure

1 files changed, 89 insertions, 74 deletions

diff --git a/hl/tools/h5watch/testh5watch.sh.in b/hl/tools/h5watch/testh5watch.sh.in
index 02f155f..33660b4 100644
--- a/hl/tools/h5watch/testh5watch.sh.in
+++ b/hl/tools/h5watch/testh5watch.sh.in
@@ -46,6 +46,7 @@ GEN_TEST=h5watchgentest	      	# Generate HDF5 file with various datasets
 GEN_TEST_BIN=`pwd`/$GEN_TEST 	# Path of the binary GEN_TEST
 WATCHFILE=`pwd`/WATCH.h5        # The HDF5 file generated to test h5watch
 TESTFILE=TEST.h5                # The temporary file (a copy of WATCH.h5) used by tests
+TRY_MAX=10                      # Try running the test again 
 #
 # These 3 defines should be the same as the defines in ./extend_dset.c
 WRITER_MESSAGE=writer_message   # The message file created by the "extend" process
@@ -118,29 +119,30 @@ TOOLTEST() {
     exitcode=$?
     cat $actual_err >> $actual
     if [ $exitcode -ne $retvalexpect ]; then
-	$ECHO "*FAILED*"
-	nerrors="`expr $nerrors + 1`"
-	if [ yes = "$verbose" ]; then
-	    $ECHO "test returned with exit code $exitcode"
-	    $ECHO "test output: (up to $NLINES lines)"
-	    head -$NLINES $actual
-	    $ECHO "***end of test output***"
-	    $ECHO ""
-	fi
-    elif $CMP $expect $actual; then
-        $ECHO " PASSED"
-    else
         $ECHO "*FAILED*"
-	$ECHO "    Expected result differs from actual result"
-	nerrors="`expr $nerrors + 1`"
-	test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/    /'
-    fi
+        nerrors="`expr $nerrors + 1`"
+        if [ yes = "$verbose" ]; then
+            $ECHO "test returned with exit code $exitcode"
+            $ECHO "test output: (up to $NLINES lines)"
+            head -$NLINES $actual
+            $ECHO "***end of test output***"
+            $ECHO ""
+        fi
+        elif $CMP $expect $actual; then
+            $ECHO " PASSED"
+        else
+            $ECHO "*FAILED*"
+            $ECHO "    Expected result differs from actual result"
+            nerrors="`expr $nerrors + 1`"
+            test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/    /'
+        fi
 
     # Clean up output file
     if test -z "$HDF5_NOCLEANUP"; then
 	rm -f $actual $actual_err
     fi
 }
+
 #
 #
 #
@@ -150,8 +152,13 @@ TOOLTEST() {
 #       $1 -- the specified dataset to watch and to extend
 #       $2 -- the options to h5watch (can be NULL)
 #       $3 -- expected output from watching the object
+#       $4 -- action 1
+#       $5 -- action 2
 #
 TEST_WATCH() {
+try=0
+while [ $try -lt $TRY_MAX ] ;
+do
     cp $WATCHFILE $TESTFILE                             # Copy the file being watched/extended to a temporary file
     actual="../testfiles/`basename $3 .ddl`.out"        # The actual output
     expect="$srcdir/../testfiles/$3"                    # The expected output
@@ -165,7 +172,7 @@ TEST_WATCH() {
     rm -f $WRITER_MESSAGE                               # Remove the file just to be sure
     rm -f $READER_MESSAGE                               # Remove the file just to be sure
     #
-    $EXTEND_BIN $TESTFILE $1 &                          # Extend the dataset; put in background
+    $EXTEND_BIN $TESTFILE $1 $4 $5&                     # Extend the dataset; put in background
     extend_pid=$!                                       # Get "extend" process ID
     #
     # Wait for message from "extend_dset" process to start h5watch--
@@ -178,10 +185,10 @@ TEST_WATCH() {
     do
         t1=`date +%s`                           	# Get current time in seconds
         difft=`expr $t1 - $t0`                  	# Calculate the time difference
-        if [ -e $WRITER_MESSAGE ]; then  		# If message file is found:
+        if [ -e $WRITER_MESSAGE ]; then             # If message file is found:
             mexist=1                            	#       indicate the message file is found
-            rm $WRITER_MESSAGE				#       remove the message file
-            break                               	#       get out of the while loop
+            rm $WRITER_MESSAGE                      #       remove the message file
+            break                                   #       get out of the while loop
         fi
     done;
     #
@@ -207,7 +214,7 @@ TEST_WATCH() {
         extend_exit=$?                                  # Collect "extend" process' exit code
         sleep 1                                         # Sleep to make sure output is flushed
         kill $watch_pid                              	# Kill h5watch
-	wait $watch_pid					# Wait for "h5watch" process to complete
+        wait $watch_pid					                # Wait for "h5watch" process to complete
         #
         if [ $extend_exit -ne 0 ]; then                 # Error returned from "extend" process
             $ECHO "*FAILED*"
@@ -220,13 +227,21 @@ TEST_WATCH() {
                 $ECHO ""
             fi
         elif $CMP $expect $actual; then                 # Compare actual output with expected output
+            try=$TRY_MAX
             $ECHO " PASSED"
         else
-            $ECHO "*FAILED*"                            # Actual and expected outputs are different
-            $ECHO "    Expected result differs from actual result"
-            nerrors="`expr $nerrors + 1`"
-            if test yes = "$verbose"; then
-                        $DIFF $expect $actual |sed 's/^/    /'
+            try="`expr $try + 1`"
+            if [ $try -lt $TRY_MAX ]; then
+                $ECHO "*RETRY"
+                rm -f $actual
+                rm -f $TESTFILE
+            else
+                $ECHO "*FAILED*"                            # Actual and expected outputs are different
+                $ECHO "    Expected result differs from actual result"
+                nerrors="`expr $nerrors + 1`"
+                if test yes = "$verbose"; then
+                    $DIFF $expect $actual |sed 's/^/    /'
+                fi
             fi
         fi
         #
@@ -236,45 +251,45 @@ TEST_WATCH() {
             rm -f $actual
         fi
     fi
+done;
 }
-
 ##############################################################################
 ##############################################################################
-###			  T H E   T E S T S                                ###
+###                     T H E   T E S T S                                  ###
 ##############################################################################
 ##############################################################################
 #
 #
 #################################################################################################
-#												#
-# WATCH.h5: file with various types of datasets for testing--					#
-#   The following datasets are chunked, H5D_ALLOC_TIME_INCR, max. dimensional setting:		#
-#   	DSET_ONE: one-dimensional dataset							#
-#   	DSET_TWO: two-dimensional dataset							#
-#   	DSET_CMPD: one-dimensional dataset with compound type 					#
-#   	DSET_CMPD_ESC: one-dimensional dataset with compound type & escape/separator characters	#
-#   	DSET_CMPD_TWO: two-dimensional dataset with compound type 				#
-#												#
-#   The following datasets are one-dimensional, chunked, max. dimension setting:		#
-#   	DSET_ALLOC_EARLY: dataset with H5D_ALLOC_TIME_EARLY					#
-#   	DSET_ALLOC_LATE: dataset H5D_ALLOC_TIME_LATE						#
-#												#
-#   The following datasets are one-dimensional:							#
-#	DSET_NONE: fixed dimension setting, contiguous, H5D_ALLOC_TIME_LATE			#
-#	DSET_NOMAX: fixed dimension setting, chunked, H5D_ALLOC_TIME_INCR			#
-#											 	#
+#                                                                                               #
+# WATCH.h5: file with various types of datasets for testing--                                   #
+#   The following datasets are chunked, H5D_ALLOC_TIME_INCR, max. dimensional setting:          #
+#       DSET_ONE: one-dimensional dataset                                                       #
+#       DSET_TWO: two-dimensional dataset                                                       #
+#       DSET_CMPD: one-dimensional dataset with compound type                                   #
+#       DSET_CMPD_ESC: one-dimensional dataset with compound type & escape/separator characters	#
+#       DSET_CMPD_TWO: two-dimensional dataset with compound type                               #
+#                                                                                               #
+#   The following datasets are one-dimensional, chunked, max. dimension setting:                #
+#       DSET_ALLOC_EARLY: dataset with H5D_ALLOC_TIME_EARLY                                     #
+#       DSET_ALLOC_LATE: dataset H5D_ALLOC_TIME_LATE                                            #
+#                                                                                               #
+#   The following datasets are one-dimensional:                                                 #
+#       DSET_NONE: fixed dimension setting, contiguous, H5D_ALLOC_TIME_LATE                     #
+#       DSET_NOMAX: fixed dimension setting, chunked, H5D_ALLOC_TIME_INCR                       #
+#                                                                                               #
 #################################################################################################
 #
 #
 #################################################################################################
-#												#
-# Tests on expected failures:									#
-#	Invalid file name									#
-#	Unable to find dataset, invalid dataset							#
-#	  DSET_NONE and DSET_NOMAX								#
-#	Invalid input to options --width and --polling						#
-#	Invalid field names for -f option							#
-#												#
+#                                                                                               #
+# Tests on expected failures:                                                                   #
+#	Invalid file name                                                                           #
+#	Unable to find dataset, invalid dataset                                                     #
+#	  DSET_NONE and DSET_NOMAX                                                                  #
+#	Invalid input to options --width and --polling                                              #
+#	Invalid field names for -f option                                                           #
+#                                                                                               #
 #################################################################################################
 #
 # Generate file with various types of datasets
@@ -319,13 +334,13 @@ $GEN_TEST_BIN
 #       TEST.h5/DSET_CMPD_TWO
 #       TEST.h5/DSET_CMPD_ESC
 #
-TEST_WATCH DSET_ONE '' w-ext-one.ddl
-TEST_WATCH DSET_ALLOC_EARLY '' w-ext-early.ddl
-TEST_WATCH DSET_ALLOC_LATE '' w-ext-late.ddl
-TEST_WATCH DSET_CMPD '' w-ext-cmpd.ddl
-TEST_WATCH DSET_TWO '' w-ext-two.ddl
-TEST_WATCH DSET_CMPD_TWO '' w-ext-cmpd-two.ddl
-TEST_WATCH DSET_CMPD_ESC '' w-ext-cmpd-esc.ddl
+TEST_WATCH DSET_ONE '' w-ext-one.ddl 3 0            #Increase
+TEST_WATCH DSET_ALLOC_EARLY '' w-ext-early.ddl -1 0 #Decrease
+TEST_WATCH DSET_ALLOC_LATE '' w-ext-late.ddl 0 0    #Same
+TEST_WATCH DSET_CMPD '' w-ext-cmpd.ddl 3 0          #Increase
+TEST_WATCH DSET_CMPD_ESC '' w-ext-cmpd-esc.ddl -1 0 #Decrease
+TEST_WATCH DSET_TWO '' w-ext-two.ddl 2 2            #Increase, Increase
+TEST_WATCH DSET_CMPD_TWO '' w-ext-cmpd-two.ddl 2 -9 #Increase, Decrease
 #
 echo "DONE WITH 2nd SET OF TESTS"
 #
@@ -339,25 +354,25 @@ echo "DONE WITH 2nd SET OF TESTS"
 #       TEST.h5/DSET_CMPD with --fields=field1,field2
 #       TEST.h5/DSET_CMPD with --fields=field2.b,field4
 #       TEST.h5/DSET_CMPD with --fields=field2.b.a --fields=field2.c
-TEST_WATCH DSET_CMPD --fields=field1,field2 w-ext-cmpd-f1.ddl
-TEST_WATCH DSET_CMPD --fields=field2.b,field4 w-ext-cmpd-f2.ddl
-TEST_WATCH DSET_CMPD '--fields=field2.b.a --fields=field2.c' w-ext-cmpd-ff3.ddl
+TEST_WATCH DSET_CMPD --fields=field1,field2 w-ext-cmpd-f1.ddl -9 0                  #Decrease
+TEST_WATCH DSET_CMPD --fields=field2.b,field4 w-ext-cmpd-f2.ddl 3 0                 #Increase
+TEST_WATCH DSET_CMPD '--fields=field2.b.a --fields=field2.c' w-ext-cmpd-ff3.ddl 0 0 #Same
 #
 #
 #       TEST.h5/DSET_CMP_TWO with --fields=field1,field2
 #       TEST.h5/DSET_CMPD_TWO with --fields=field2.b --fields=field4
 #       TEST.h5/DSET_CMPD_TWO with --fields=field2.b.a,field2.c
-TEST_WATCH DSET_CMPD_TWO --fields=field1,field2 w-ext-cmpd-two-f1.ddl
-TEST_WATCH DSET_CMPD_TWO '--fields=field2.b --fields=field4' w-ext-cmpd-two-ff2.ddl
-TEST_WATCH DSET_CMPD_TWO --fields=field2.b.a,field2.c w-ext-cmpd-two-f3.ddl
+TEST_WATCH DSET_CMPD_TWO --fields=field1,field2 w-ext-cmpd-two-f1.ddl   2 0                 #Increase, Same
+TEST_WATCH DSET_CMPD_TWO '--fields=field2.b --fields=field4' w-ext-cmpd-two-ff2.ddl -1 2    #Decrease, Increase
+TEST_WATCH DSET_CMPD_TWO --fields=field2.b.a,field2.c w-ext-cmpd-two-f3.ddl -1 -3           #Decrease, Decrease
 #
 #
 #       TEST.h5/DSET_CMPD_ESC with --fields=field\,1,field2\.
 #       TEST.h5/DSET_CMPD_ESC with --fields=field2\..\,b --fields=field4\,
 #       TEST.h5/DSET_CMPD_ESC with --fields=field2\..\,b.a,field2\..\\K
-TEST_WATCH DSET_CMPD_ESC '--fields=field\,1,field2\.' w-ext-cmpd-esc-f1.ddl
-TEST_WATCH DSET_CMPD_ESC '--fields=field2\..\,b --fields=field4\,' w-ext-cmpd-esc-ff2.ddl
-TEST_WATCH DSET_CMPD_ESC '--fields=field2\..\,b.a,field2\..\\K' w-ext-cmpd-esc-f3.ddl
+TEST_WATCH DSET_CMPD_ESC '--fields=field\,1,field2\.' w-ext-cmpd-esc-f1.ddl 3 0                 #Increase
+TEST_WATCH DSET_CMPD_ESC '--fields=field2\..\,b --fields=field4\,' w-ext-cmpd-esc-ff2.ddl -1 0  #Decrease
+TEST_WATCH DSET_CMPD_ESC '--fields=field2\..\,b.a,field2\..\\K' w-ext-cmpd-esc-f3.ddl 3 0       #Increase
 #
 #
 echo "DONE WITH 3rd SET OF TESTS"
@@ -376,11 +391,11 @@ echo "DONE WITH 3rd SET OF TESTS"
 #       TEST.h5/DSET_TWO with --width=60 option
 #       TEST.h5/DSET_CMPD with --label option
 #       TEST.h5/DSET_ONE with --simple option
-TEST_WATCH DSET_ONE --dim w-ext-one-d.ddl
-TEST_WATCH DSET_TWO --dim w-ext-two-d.ddl
-TEST_WATCH DSET_TWO --width=30 w-ext-two-width.ddl
-TEST_WATCH DSET_CMPD --label w-ext-cmpd-label.ddl
-TEST_WATCH DSET_ONE --simple w-ext-one-simple.ddl
+TEST_WATCH DSET_ONE --dim w-ext-one-d.ddl 3 0           #Increase
+TEST_WATCH DSET_TWO --dim w-ext-two-d.ddl -2 0          #Decrease, Same
+TEST_WATCH DSET_TWO --width=30 w-ext-two-width.ddl 0 2  #Same, Increase
+TEST_WATCH DSET_CMPD --label w-ext-cmpd-label.ddl 3 0   #Increase
+TEST_WATCH DSET_ONE --simple w-ext-one-simple.ddl 2 0   #I
 #
 echo "DONE WITH 4th SET OF TESTS"
 #