HDFFV-10632 separate stderr and stdout in tests

13 files changed, 132 insertions, 57 deletions

diff --git a/hl/tools/h5watch/CMakeTests.cmake b/hl/tools/h5watch/CMakeTests.cmake
index 0b7b4d4..13d5e47 100644
--- a/hl/tools/h5watch/CMakeTests.cmake
+++ b/hl/tools/h5watch/CMakeTests.cmake
@@ -18,16 +18,16 @@
 
 set (H5WATCH_TEST_FILES
     w-help1.ddl
-    w-err-cmpd1.ddl
-    w-err-cmpd2.ddl
-    w-err-cmpd3.ddl
-    w-err-cmpd4.ddl
-    w-err-cmpd5.ddl
-    w-err-dset1.ddl
-    w-err-dset2.ddl
-    w-err-dset-nomax.ddl
-    w-err-dset-none.ddl
-    w-err-file.ddl
+    w-err-cmpd1.err
+    w-err-cmpd2.err
+    w-err-cmpd3.err
+    w-err-cmpd4.err
+    w-err-cmpd5.err
+    w-err-dset1.err
+    w-err-dset2.err
+    w-err-dset-nomax.err
+    w-err-dset-none.err
+    w-err-file.err
     w-err-poll.ddl
     w-err-poll0.ddl
     w-err-width.ddl
@@ -86,6 +86,25 @@ add_custom_target(H5WATCH_files ALL COMMENT "Copying files needed by H5WATCH tes
     endif ()
   endmacro ()
 
+  macro (ADD_H5_ERR_TEST resultfile resultcode)
+    if (NOT HDF5_ENABLE_USING_MEMCHECKER)
+      add_test (
+          NAME H5WATCH_ARGS-h5watch-${resultfile}
+          COMMAND "${CMAKE_COMMAND}"
+              -D "TEST_PROGRAM=$<TARGET_FILE:h5watch>"
+              -D "TEST_ARGS:STRING=${ARGN}"
+              -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
+              -D "TEST_OUTPUT=${resultfile}.out"
+              -D "TEST_EXPECT=${resultcode}"
+              -D "TEST_REFERENCE=${resultfile}.mty"
+              -D "TEST_ERRREF=${resultfile}.err"
+              -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake"
+      )
+      set_tests_properties (H5WATCH_ARGS-h5watch-${resultfile} PROPERTIES DEPENDS ${last_test})
+      set (last_test "H5WATCH_ARGS-h5watch-${resultfile}")
+    endif ()
+  endmacro ()
+
   macro (ADD_H5_WATCH resultfile resultcode)
     if (NOT HDF5_ENABLE_USING_MEMCHECKER)
       add_test (
@@ -162,20 +181,20 @@ if (NOT SWMR_INCOMPAT)
   ADD_H5_TEST (w-help1 0 --help)
 #
 # Tests on expected failures
-  ADD_H5_TEST (w-err-dset1 1 WATCH.h5)
-  ADD_H5_TEST (w-err-dset2 1 WATCH.h5/group/DSET_CMPD)
-  ADD_H5_TEST (w-err-dset-none 1 WATCH.h5/DSET_NONE)
-  ADD_H5_TEST (w-err-dset-nomax 1 WATCH.h5/DSET_NOMAX)
-  ADD_H5_TEST (w-err-file 1 ../WATCH.h5/DSET_CMPD)
+  ADD_H5_ERR_TEST (w-err-dset1 1 WATCH.h5)
+  ADD_H5_ERR_TEST (w-err-dset2 1 WATCH.h5/group/DSET_CMPD)
+  ADD_H5_ERR_TEST (w-err-dset-none 1 WATCH.h5/DSET_NONE)
+  ADD_H5_ERR_TEST (w-err-dset-nomax 1 WATCH.h5/DSET_NOMAX)
+  ADD_H5_ERR_TEST (w-err-file 1 ../WATCH.h5/DSET_CMPD)
   ADD_H5_TEST (w-err-width 1 --width=-8 WATCH.h5/DSET_ONE)
   ADD_H5_TEST (w-err-poll 1 --polling=-8 WATCH.h5/DSET_ONE)
   ADD_H5_TEST (w-err-poll0 1 --polling=0 WATCH.h5/DSET_ONE)
 #
 # Tests on invalid field names via --fields option for a compound typed dataset: DSET_CMPD
-  ADD_H5_TEST (w-err-cmpd1 1 --fields=fieldx WATCH.h5/DSET_CMPD)
-  ADD_H5_TEST (w-err-cmpd2 1 --fields=field1,field2. WATCH.h5/DSET_CMPD)
-  ADD_H5_TEST (w-err-cmpd3 1 --fields=field1,field2, WATCH.h5/DSET_CMPD)
-  ADD_H5_TEST (w-err-cmpd4 1 --fields=field1,field2.b.k WATCH.h5/DSET_CMPD)
-  ADD_H5_TEST (w-err-cmpd5 1 --fields=field1 --fields=field2.b.k WATCH.h5/DSET_CMPD)
+  ADD_H5_ERR_TEST (w-err-cmpd1 1 --fields=fieldx WATCH.h5/DSET_CMPD)
+  ADD_H5_ERR_TEST (w-err-cmpd2 1 --fields=field1,field2. WATCH.h5/DSET_CMPD)
+  ADD_H5_ERR_TEST (w-err-cmpd3 1 --fields=field1,field2, WATCH.h5/DSET_CMPD)
+  ADD_H5_ERR_TEST (w-err-cmpd4 1 --fields=field1,field2.b.k WATCH.h5/DSET_CMPD)
+  ADD_H5_ERR_TEST (w-err-cmpd5 1 --fields=field1 --fields=field2.b.k WATCH.h5/DSET_CMPD)
 #
 endif ()
diff --git a/hl/tools/h5watch/testh5watch.sh.in b/hl/tools/h5watch/testh5watch.sh.in
index 3f8f5ae..d31bcc3 100644
--- a/hl/tools/h5watch/testh5watch.sh.in
+++ b/hl/tools/h5watch/testh5watch.sh.in
@@ -32,19 +32,19 @@ fi
 #echo
 #exit 0
 
-H5WATCH=h5watch               	# The tool name
-H5WATCH_BIN=`pwd`/$H5WATCH    	# The path of H5WATCH
-EXTEND_DSET=extend_dset 	# Routine to extend the dataset when watching
-EXTEND_BIN=`pwd`/$EXTEND_DSET 	# The path of EXTEND_DSET
+H5WATCH=h5watch                   # The tool name
+H5WATCH_BIN=`pwd`/$H5WATCH        # The path of H5WATCH
+EXTEND_DSET=extend_dset     # Routine to extend the dataset when watching
+EXTEND_BIN=`pwd`/$EXTEND_DSET     # The path of EXTEND_DSET
 #
 EXIT_SUCCESS=0
 EXIT_FAILURE=1
 #
-GEN_TEST=h5watchgentest	      	# Generate HDF5 file with various datasets
-GEN_TEST_BIN=`pwd`/$GEN_TEST 	# Path of the binary GEN_TEST
+GEN_TEST=h5watchgentest              # Generate HDF5 file with various datasets
+GEN_TEST_BIN=`pwd`/$GEN_TEST     # Path of the binary GEN_TEST
 WATCHFILE=`pwd`/WATCH.h5        # The HDF5 file generated to test h5watch
 TESTFILE=TEST.h5                # The temporary file (a copy of WATCH.h5) used by tests
-TRY_MAX=30                      # Try running the test again 
+TRY_MAX=30                      # Try running the test again
 #
 # These 3 defines should be the same as the defines in ./extend_dset.c
 WRITER_MESSAGE=writer_message   # The message file created by the "extend" process
@@ -53,7 +53,7 @@ MESSAGE_TIMEOUT=300             # Message timeout length in secs
 #
 CMP='cmp -s'
 DIFF='diff -c'
-NLINES=20			# Max. lines of output to display if test fails
+NLINES=20            # Max. lines of output to display if test fails
 #
 # Mac OS: just to make sure echo "backslash backslash" behaves properly
 if test `uname -s` = 'Darwin'; then
@@ -83,9 +83,9 @@ TESTING() {
 # Overall algorithm:
 #
 # Run a test and print PASSED or FAILED
-# If a test did not return with the expected return code, 
+# If a test did not return with the expected return code,
 # increment the `nerrors' global variable and (if $verbose is set) display up to $NLINES
-# lines of the actual output from the test.  
+# lines of the actual output from the test.
 # If the test did return the expected code,
 # compare the actual output with the expected output;
 # If the outputs are the same, print PASSED,
@@ -137,7 +137,54 @@ TOOLTEST() {
 
     # Clean up output file
     if test -z "$HDF5_NOCLEANUP"; then
-	rm -f $actual $actual_err
+    rm -f $actual $actual_err
+    fi
+}
+#
+# TOOLTEST_ERR():
+# same as toolset only compare error file
+#
+# Arguments:
+#
+# $1 -- expected output
+# $2 -- expected return code
+# $3 and on -- arguments for h5watch
+TOOLTEST_ERR() {
+    expect="$srcdir/../testfiles/$1"
+    expect_err="$srcdir/../testfiles/`basename $1 .ddl`.err"
+    actual="../testfiles/`basename $1 .ddl`.out"
+    actual_err="../testfiles/`basename $1 .ddl`.out.err"
+    shift
+    retvalexpect=$1
+    shift
+    # Run test.
+    TESTING $H5WATCH $@
+    (
+        $RUNSERIAL $H5WATCH_BIN "$@"
+    ) > $actual 2>$actual_err
+    exitcode=$?
+    if [ $exitcode -ne $retvalexpect ]; then
+        $ECHO "*FAILED*"
+        nerrors="`expr $nerrors + 1`"
+        if [ yes = "$verbose" ]; then
+            $ECHO "test returned with exit code $exitcode"
+            $ECHO "test output: (up to $NLINES lines)"
+            head -$NLINES $actual
+            $ECHO "***end of test output***"
+            $ECHO ""
+        fi
+        elif $CMP $expect_err $actual_err; then
+            $ECHO " PASSED"
+        else
+            $ECHO "*FAILED*"
+            $ECHO "    Expected result differs from actual result"
+            nerrors="`expr $nerrors + 1`"
+            test yes = "$verbose" && $DIFF $expect_err $actual_err |sed 's/^/    /'
+        fi
+
+    # Clean up output file
+    if test -z "$HDF5_NOCLEANUP"; then
+    rm -f $actual $actual_err
     fi
 }
 
@@ -176,15 +223,15 @@ do
     # Wait for message from "extend_dset" process to start h5watch--
     # To wait for the writer message file or till the maximum # of seconds is reached
     # This performs similar function as the routine h5_wait_message() in test/h5test.c
-    mexist=0                                    	# Indicate whether the message file is found
-    t0=`date +%s`                               	# Get current time in seconds
-    difft=0                                     	# Initialize the time difference
-    while [ $difft -lt $MESSAGE_TIMEOUT ] ;     	# Loop till message times out
+    mexist=0                                        # Indicate whether the message file is found
+    t0=`date +%s`                                   # Get current time in seconds
+    difft=0                                         # Initialize the time difference
+    while [ $difft -lt $MESSAGE_TIMEOUT ] ;         # Loop till message times out
     do
-        t1=`date +%s`                           	# Get current time in seconds
-        difft=`expr $t1 - $t0`                  	# Calculate the time difference
+        t1=`date +%s`                               # Get current time in seconds
+        difft=`expr $t1 - $t0`                      # Calculate the time difference
         if [ -e $WRITER_MESSAGE ]; then             # If message file is found:
-            mexist=1                            	#       indicate the message file is found
+            mexist=1                                #       indicate the message file is found
             rm $WRITER_MESSAGE                      #       remove the message file
             break                                   #       get out of the while loop
         fi
@@ -211,8 +258,8 @@ do
         wait $extend_pid                                # Wait for "extend" process to complete
         extend_exit=$?                                  # Collect "extend" process' exit code
         sleep 1                                         # Sleep to make sure output is flushed
-        kill $watch_pid                              	# Kill h5watch
-        wait $watch_pid					                # Wait for "h5watch" process to complete
+        kill $watch_pid                                  # Kill h5watch
+        wait $watch_pid                                    # Wait for "h5watch" process to complete
         #
         if [ $extend_exit -ne 0 ]; then                 # Error returned from "extend" process
             $ECHO "*FAILED*"
@@ -265,7 +312,7 @@ done;
 #       DSET_ONE: one-dimensional dataset                                                       #
 #       DSET_TWO: two-dimensional dataset                                                       #
 #       DSET_CMPD: one-dimensional dataset with compound type                                   #
-#       DSET_CMPD_ESC: one-dimensional dataset with compound type & escape/separator characters	#
+#       DSET_CMPD_ESC: one-dimensional dataset with compound type & escape/separator characters    #
 #       DSET_CMPD_TWO: two-dimensional dataset with compound type                               #
 #                                                                                               #
 #   The following datasets are one-dimensional, chunked, max. dimension setting:                #
@@ -282,11 +329,11 @@ done;
 #################################################################################################
 #                                                                                               #
 # Tests on expected failures:                                                                   #
-#	Invalid file name                                                                           #
-#	Unable to find dataset, invalid dataset                                                     #
-#	  DSET_NONE and DSET_NOMAX                                                                  #
-#	Invalid input to options --width and --polling                                              #
-#	Invalid field names for -f option                                                           #
+#    Invalid file name                                                                           #
+#    Unable to find dataset, invalid dataset                                                     #
+#      DSET_NONE and DSET_NOMAX                                                                  #
+#    Invalid input to options --width and --polling                                              #
+#    Invalid field names for -f option                                                           #
 #                                                                                               #
 #################################################################################################
 #
@@ -296,21 +343,21 @@ $GEN_TEST_BIN
 TOOLTEST w-help1.ddl 0 --help
 #
 # Tests on expected failures
-TOOLTEST w-err-dset1.ddl 1 WATCH.h5
-TOOLTEST w-err-dset2.ddl 1 WATCH.h5/group/DSET_CMPD
-TOOLTEST w-err-dset-none.ddl 1 WATCH.h5/DSET_NONE
-TOOLTEST w-err-dset-nomax.ddl 1 WATCH.h5/DSET_NOMAX
-TOOLTEST w-err-file.ddl 1 ../WATCH.h5/DSET_CMPD
+TOOLTEST_ERR w-err-dset1.ddl 1 WATCH.h5
+TOOLTEST_ERR w-err-dset2.ddl 1 WATCH.h5/group/DSET_CMPD
+TOOLTEST_ERR w-err-dset-none.ddl 1 WATCH.h5/DSET_NONE
+TOOLTEST_ERR w-err-dset-nomax.ddl 1 WATCH.h5/DSET_NOMAX
+TOOLTEST_ERR w-err-file.ddl 1 ../WATCH.h5/DSET_CMPD
 TOOLTEST w-err-width.ddl 1 --width=-8 WATCH.h5/DSET_ONE
 TOOLTEST w-err-poll.ddl 1 --polling=-8 WATCH.h5/DSET_ONE
 TOOLTEST w-err-poll0.ddl 1 --polling=0 WATCH.h5/DSET_ONE
 #
 # Tests on invalid field names via --fields option for a compound typed dataset: DSET_CMPD
-TOOLTEST w-err-cmpd1.ddl 1 --fields=fieldx WATCH.h5/DSET_CMPD
-TOOLTEST w-err-cmpd2.ddl 1 --fields=field1,field2. WATCH.h5/DSET_CMPD
-TOOLTEST w-err-cmpd3.ddl 1 --fields=field1,field2, WATCH.h5/DSET_CMPD
-TOOLTEST w-err-cmpd4.ddl 1 --fields=field1,field2.b.k WATCH.h5/DSET_CMPD
-TOOLTEST w-err-cmpd5.ddl 1 --fields=field1 --fields=field2.b.k WATCH.h5/DSET_CMPD
+TOOLTEST_ERR w-err-cmpd1.ddl 1 --fields=fieldx WATCH.h5/DSET_CMPD
+TOOLTEST_ERR w-err-cmpd2.ddl 1 --fields=field1,field2. WATCH.h5/DSET_CMPD
+TOOLTEST_ERR w-err-cmpd3.ddl 1 --fields=field1,field2, WATCH.h5/DSET_CMPD
+TOOLTEST_ERR w-err-cmpd4.ddl 1 --fields=field1,field2.b.k WATCH.h5/DSET_CMPD
+TOOLTEST_ERR w-err-cmpd5.ddl 1 --fields=field1 --fields=field2.b.k WATCH.h5/DSET_CMPD
 #
 echo "DONE WITH 1st SET OF TESTS"
 #
diff --git a/hl/tools/testfiles/w-err-cmpd1.err b/hl/tools/testfiles/w-err-cmpd1.err
new file mode 100644
index 0000000..9c165fb
--- /dev/null
+++ b/hl/tools/testfiles/w-err-cmpd1.err
@@ -0,0 +1 @@
+h5watch error: error in processing <list_of_fields>
diff --git a/hl/tools/testfiles/w-err-cmpd2.err b/hl/tools/testfiles/w-err-cmpd2.err
new file mode 100644
index 0000000..9c165fb
--- /dev/null
+++ b/hl/tools/testfiles/w-err-cmpd2.err
@@ -0,0 +1 @@
+h5watch error: error in processing <list_of_fields>
diff --git a/hl/tools/testfiles/w-err-cmpd3.err b/hl/tools/testfiles/w-err-cmpd3.err
new file mode 100644
index 0000000..9c165fb
--- /dev/null
+++ b/hl/tools/testfiles/w-err-cmpd3.err
@@ -0,0 +1 @@
+h5watch error: error in processing <list_of_fields>
diff --git a/hl/tools/testfiles/w-err-cmpd4.err b/hl/tools/testfiles/w-err-cmpd4.err
new file mode 100644
index 0000000..9c165fb
--- /dev/null
+++ b/hl/tools/testfiles/w-err-cmpd4.err
@@ -0,0 +1 @@
+h5watch error: error in processing <list_of_fields>
diff --git a/hl/tools/testfiles/w-err-cmpd5.err b/hl/tools/testfiles/w-err-cmpd5.err
new file mode 100644
index 0000000..9c165fb
--- /dev/null
+++ b/hl/tools/testfiles/w-err-cmpd5.err
@@ -0,0 +1 @@
+h5watch error: error in processing <list_of_fields>
diff --git a/hl/tools/testfiles/w-err-dset-nomax.err b/hl/tools/testfiles/w-err-dset-nomax.err
new file mode 100644
index 0000000..fd5ca7e
--- /dev/null
+++ b/hl/tools/testfiles/w-err-dset-nomax.err
@@ -0,0 +1 @@
+h5watch error: "/DSET_NOMAX" should have unlimited or max. dimension setting
diff --git a/hl/tools/testfiles/w-err-dset-none.err b/hl/tools/testfiles/w-err-dset-none.err
new file mode 100644
index 0000000..db781c3
--- /dev/null
+++ b/hl/tools/testfiles/w-err-dset-none.err
@@ -0,0 +1 @@
+h5watch error: "/DSET_NONE" should be a chunked or virtual dataset
diff --git a/hl/tools/testfiles/w-err-dset1.err b/hl/tools/testfiles/w-err-dset1.err
new file mode 100644
index 0000000..d6b2a6e
--- /dev/null
+++ b/hl/tools/testfiles/w-err-dset1.err
@@ -0,0 +1 @@
+h5watch error: no dataset specified
diff --git a/hl/tools/testfiles/w-err-dset2.ddl b/hl/tools/testfiles/w-err-dset2.err
index 81b52de..3f77ec8 100644
--- a/hl/tools/testfiles/w-err-dset2.ddl
+++ b/hl/tools/testfiles/w-err-dset2.err
@@ -1,2 +1 @@
-Opened "WATCH.h5" with sec2 driver.
 h5watch error: unable to open dataset "/group/DSET_CMPD"
diff --git a/hl/tools/testfiles/w-err-file.ddl b/hl/tools/testfiles/w-err-file.err
index 015661f..015661f 100644
--- a/hl/tools/testfiles/w-err-file.ddl
+++ b/hl/tools/testfiles/w-err-file.err
diff --git a/hl/tools/testfiles/w-err-std.ddl b/hl/tools/testfiles/w-err-std.ddl
new file mode 100644
index 0000000..775d6a0
--- /dev/null
+++ b/hl/tools/testfiles/w-err-std.ddl
@@ -0,0 +1,2 @@
+Opened "WATCH.h5" with sec2 driver.
+h5watch error: error in processing <list_of_fields>