[svn-r20771] Problem:

Test scripts sometimes need to filter some system-specific messages from the
actual output so that it can match the correct expected output. These filtering
functions, ususally called "STDOUT_FILTER()" and "STDERR_FILTER()" were being
repeated in individual test scripts.  This becomes a maintenance problem and
is error prone.

Solution:
Extract the two filter functions code to bin/output_filter.sh and then each
test script sources it in. This allows reuse of coding and is much easier to
maintain and to add new filtering.

Tested:
LLNL Zeus (linux64 cluster) and Dawndev (Blue-Gene cluster), both for serial
mode only.
 --This line, and those below, will be ignored--

M    tools/misc/testh5mkgrp.sh
M    tools/h5dump/testh5dump.sh.in
M    tools/h5diff/testh5diff.sh
M    tools/h5copy/testh5copy.sh
M    tools/h5ls/testh5ls.sh.in
M    MANIFEST
A    bin/output_filter.sh

7 files changed, 184 insertions, 98 deletions

diff --git a/MANIFEST b/MANIFEST
index 547e593..d2a7b12 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -70,6 +70,7 @@
 ./bin/mkinstalldirs
 ./bin/newer
 ./bin/reconfigure      _DO_NOT_DISTRIBUTE_
+./bin/output_filter.sh
 ./bin/README        _DO_NOT_DISTRIBUTE_
 ./bin/release
 ./bin/runtest        _DO_NOT_DISTRIBUTE_
diff --git a/bin/output_filter.sh b/bin/output_filter.sh
new file mode 100644
index 0000000..58f1c48
--- /dev/null
+++ b/bin/output_filter.sh
@@ -0,0 +1,101 @@
+## Copyright by The HDF Group.
+## All rights reserved.
+##
+## This file is part of HDF5.  The full HDF5 copyright notice, including
+## terms governing use, modification, and redistribution, is contained in
+## the files COPYING and Copyright.html.  COPYING can be found at the root
+## of the source code distribution tree; Copyright.html can be found at the
+## root level of an installed copy of the electronic HDF5 document set and
+## is linked from the top-level documents page.  It can also be found at
+## http://hdfgroup.org/HDF5/doc/Copyright.html.  If you do not have
+## access to either file, you may request a copy from help@hdfgroup.org.
+
+# This contains function definitions of output filtering.
+# This file should only be sourced in by another shell script.
+#
+# Programmer: Albert Cheng
+# Created Date: 2011/5/3
+
+
+# Some systems will dump some messages to stdout for various reasons.
+# Remove them from the stdout result file.
+# $1 is the file name of the file to be filtered.
+# Cases of filter needed:
+# 1. Sandia Red-Storm
+#    yod always prints these two lines at the beginning.
+#    LibLustre: NAL NID: 0004a605 (5)
+#    Lustre: OBD class driver Build Version: 1, info@clusterfs.com
+# 2. LANL Lambda
+#    mpijob mirun -np always add an extra line at the end like:
+#    P4 procgroup file is /users/acheng/.lsbatch/host10524.l82
+STDOUT_FILTER() {
+    result_file=$1
+    tmp_file=/tmp/h5test_tmp_$$
+    # Filter Sandia Red-Storm yod messages.
+    cp $result_file $tmp_file
+    sed -e '/^LibLustre:/d' -e '/^Lustre:/d' \
+	< $tmp_file > $result_file
+    # Filter LANL Lambda mpirun message.
+    cp $result_file $tmp_file
+    sed -e '/^P4 procgroup file is/d' \
+	< $tmp_file > $result_file
+    # cleanup
+    rm -f $tmp_file
+}
+
+
+# Some systems will dump some messages to stderr for various reasons.
+# Remove them from the stderr result file.
+# $1 is the file name of the file to be filtered.
+# Cases of filter needed:
+# 1. MPE:
+# In parallel mode and if MPE library is used, it prints the following
+# two message lines whether the MPE tracing is used or not.
+#    Writing logfile.
+#    Finished writing logfile.
+# 2. LANL MPI:
+# The LANL MPI will print some messages like the following,
+#    LA-MPI: *** mpirun (1.5.10)
+#    LA-MPI: *** 3 process(es) on 2 host(s): 2*fln21 1*fln22
+#    LA-MPI: *** libmpi (1.5.10)
+#    LA-MPI: *** Copyright 2001-2004, ACL, Los Alamos National Laboratory
+# 3. h5diff debug output:
+#    Debug output all have prefix "h5diff debug: ".
+# 4. AIX system prints messages like these when it is aborting:
+#    ERROR: 0031-300  Forcing all remote tasks to exit due to exit code 1 in task 0
+#    ERROR: 0031-250  task 4: Terminated
+#    ERROR: 0031-250  task 3: Terminated
+#    ERROR: 0031-250  task 2: Terminated
+#    ERROR: 0031-250  task 1: Terminated
+# 5. LLNL Blue-Gene mpirun prints messages like there when it exit non-zero:
+#    <Apr 12 15:01:49.075658> BE_MPI (ERROR): The error message in the job record is as follows:
+#    <Apr 12 15:01:49.075736> BE_MPI (ERROR):   "killed by exit(1) on node 0"
+STDERR_FILTER() {
+    result_file=$1
+    tmp_file=/tmp/h5test_tmp_$$
+    # Filter LLNL Blue-Gene error messages in both serial and parallel modes
+    # since mpirun is used in both modes.
+    cp $result_file $tmp_file
+    sed -e '/ BE_MPI (ERROR): /d' \
+	< $tmp_file > $result_file
+    # Filter MPE messages
+    if test -n "$pmode"; then
+	cp $result_file $tmp_file
+	sed -e '/^Writing logfile./d' -e '/^Finished writing logfile./d' \
+	    < $tmp_file > $result_file
+    fi
+    # Filter LANL MPI messages
+    # and LLNL srun messages
+    # and AIX error messages
+    if test -n "$pmode"; then
+	cp $result_file $tmp_file
+	sed -e '/^LA-MPI:/d' -e '/^srun:/d' -e '/^ERROR:/d' \
+	    < $tmp_file > $result_file
+    fi
+    # Filter h5diff debug output
+	cp $result_file $tmp_file
+	sed -e '/^h5diff debug: /d' \
+	    < $tmp_file > $result_file
+    # clean up temporary files.
+    rm -f $tmp_file
+}
diff --git a/tools/h5copy/testh5copy.sh b/tools/h5copy/testh5copy.sh
index 3f28559..1dbb03c 100644
--- a/tools/h5copy/testh5copy.sh
+++ b/tools/h5copy/testh5copy.sh
@@ -100,6 +100,9 @@ VERIFY_OUTPUT()
     echo "Verifying output files $* $SPACES" | cut -c1-70 | tr -d '\012'
 }
 
+# Source in the output filter function definitions.
+. $srcdir/../../bin/output_filter.sh
+
 # Run a test and print PASS or *FAIL*. If h5copy can complete
 # with exit status 0, consider it pass. If a test fails then increment
 # the `nerrors' global variable.
@@ -184,6 +187,8 @@ TOOLTEST_FAIL()
     expectout="$INDIR/$1"
     actualout="$OUTDIR/$1.out"
     actualerr="$OUTDIR/$1.err"
+    actualout_sav=${actualout}-sav
+    actualerr_sav=${actualerr}-sav
     shift
     if [ "$1" = -i ]; then
       inputfile=$2
@@ -199,7 +204,13 @@ TOOLTEST_FAIL()
     #echo "#############################"
     $RUNSERIAL $H5COPY_BIN $@
     ) > $actualout 2> $actualerr
+
     RET=$?
+    # save actualout and actualerr in case they are needed later.
+    cp $actualout $actualout_sav
+    STDOUT_FILTER $actualout
+    cp $actualerr $actualerr_sav
+    STDERR_FILTER $actualerr
     if [ $RET != 0 ]; then
         echo " PASSED"
         # Verifying output text from h5copy
@@ -219,7 +230,7 @@ TOOLTEST_FAIL()
 
     # Clean up output file
     if test -z "$HDF5_NOCLEANUP"; then
-       rm -f $actualout $actualerr
+       rm -f $actualout $actualerr $actualout_sav $actualerr_sav
     fi
 }
 
diff --git a/tools/h5diff/testh5diff.sh b/tools/h5diff/testh5diff.sh
index 48dc6a9..7803022 100755
--- a/tools/h5diff/testh5diff.sh
+++ b/tools/h5diff/testh5diff.sh
@@ -143,80 +143,8 @@ TESTING() {
    echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
 }
 
-# Some systems will dump some messages to stdout for various reasons.
-# Remove them from the stdout result file.
-# $1 is the file name of the file to be filtered.
-# Cases of filter needed:
-# 1. Sandia Red-Storm
-#    yod always prints these two lines at the beginning.
-#    LibLustre: NAL NID: 0004a605 (5)
-#    Lustre: OBD class driver Build Version: 1, info@clusterfs.com
-# 2. LANL Lambda
-#    mpijob mirun -np always add an extra line at the end like:
-#    P4 procgroup file is /users/acheng/.lsbatch/host10524.l82
-STDOUT_FILTER() {
-    result_file=$1
-    tmp_file=/tmp/h5test_tmp_$$
-    # Filter Sandia Red-Storm yod messages.
-    cp $result_file $tmp_file
-    sed -e '/^LibLustre:/d' -e '/^Lustre:/d' \
-	< $tmp_file > $result_file
-    # Filter LANL Lambda mpirun message.
-    cp $result_file $tmp_file
-    sed -e '/^P4 procgroup file is/d' \
-	< $tmp_file > $result_file
-    # cleanup
-    rm -f $tmp_file
-}
-
-# Some systems will dump some messages to stderr for various reasons.
-# Remove them from the stderr result file.
-# $1 is the file name of the file to be filtered.
-# Cases of filter needed:
-# 1. MPE:
-# In parallel mode and if MPE library is used, it prints the following
-# two message lines whether the MPE tracing is used or not.
-#    Writing logfile.
-#    Finished writing logfile.
-# 2. LANL MPI:
-# The LANL MPI will print some messages like the following,
-#    LA-MPI: *** mpirun (1.5.10)
-#    LA-MPI: *** 3 process(es) on 2 host(s): 2*fln21 1*fln22
-#    LA-MPI: *** libmpi (1.5.10)
-#    LA-MPI: *** Copyright 2001-2004, ACL, Los Alamos National Laboratory
-# 3. h5diff debug output:
-#    Debug output all have prefix "h5diff debug: ".
-# 4. AIX system prints messages like these when it is aborting:
-#    ERROR: 0031-300  Forcing all remote tasks to exit due to exit code 1 in task 0
-#    ERROR: 0031-250  task 4: Terminated
-#    ERROR: 0031-250  task 3: Terminated
-#    ERROR: 0031-250  task 2: Terminated
-#    ERROR: 0031-250  task 1: Terminated
-
-STDERR_FILTER() {
-    result_file=$1
-    tmp_file=/tmp/h5test_tmp_$$
-    # Filter MPE messages
-    if test -n "$pmode"; then
-	cp $result_file $tmp_file
-	sed -e '/^Writing logfile./d' -e '/^Finished writing logfile./d' \
-	    < $tmp_file > $result_file
-    fi
-    # Filter LANL MPI messages
-    # and LLNL srun messages
-    # and AIX error messages
-    if test -n "$pmode"; then
-	cp $result_file $tmp_file
-	sed -e '/^LA-MPI:/d' -e '/^srun:/d' -e '/^ERROR:/d' \
-	    < $tmp_file > $result_file
-    fi
-    # Filter h5diff debug output
-	cp $result_file $tmp_file
-	sed -e '/^h5diff debug: /d' \
-	    < $tmp_file > $result_file
-    # clean up temporary files.
-    rm -f $tmp_file
-}
+# Source in the output filter function definitions.
+. $srcdir/../../bin/output_filter.sh
 
 # Run a test and print PASS or *FAIL*.  If a test fails then increment
 # the `nerrors' global variable and (if $verbose is set) display the
diff --git a/tools/h5dump/testh5dump.sh.in b/tools/h5dump/testh5dump.sh.in
index 24847d7..5c0503f 100644
--- a/tools/h5dump/testh5dump.sh.in
+++ b/tools/h5dump/testh5dump.sh.in
@@ -59,6 +59,9 @@ TESTING() {
    echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
 }
 
+# Source in the output filter function definitions.
+. $srcdir/../../bin/output_filter.sh
+
 # Run a test and print PASS or *FAIL*.  If a test fails then increment
 # the `nerrors' global variable and (if $verbose is set) display the
 # difference between the actual output and the expected output. The
@@ -71,6 +74,8 @@ TOOLTEST() {
     expect="$srcdir/../testfiles/$1"
     actual="../testfiles/`basename $1 .ddl`.out"
     actual_err="../testfiles/`basename $1 .ddl`.err"
+    actual_sav=${actual}-sav
+    actual_err_sav=${actual_err}-sav
     shift
 
     # Run test.
@@ -79,12 +84,18 @@ TOOLTEST() {
       echo "#############################"
       echo "Expected output for '$DUMPER $@'" 
       echo "#############################"
-    cd $srcdir/../testfiles
-      $RUNSERIAL $DUMPER_BIN $@
+	  cd $srcdir/../testfiles
+  	  $RUNSERIAL $DUMPER_BIN $@
     ) >$actual 2>$actual_err
+
+    # save actual and actual_err in case they are needed later.
+    cp $actual $actual_sav
+    STDOUT_FILTER $actual
+    cp $actual_err $actual_err_sav
+    STDERR_FILTER $actual_err
     cat $actual_err >> $actual
 
-  if [ ! -f $expect ]; then
+	if [ ! -f $expect ]; then
     # Create the expect file if it doesn't yet exist.
      echo " CREATED"
      cp $actual $expect
@@ -93,13 +104,13 @@ TOOLTEST() {
     else
      echo "*FAILED*"
      echo "    Expected result (*.ddl) differs from actual result (*.out)"
-   nerrors="`expr $nerrors + 1`"
-   test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/    /'
+	 nerrors="`expr $nerrors + 1`"
+	 test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/    /'
     fi
 
     # Clean up output file
     if test -z "$HDF5_NOCLEANUP"; then
-   rm -f $actual $actual_err
+	 rm -f $actual $actual_err $actual_sav $actual_err_sav
     fi
 
 }
@@ -112,18 +123,26 @@ TOOLTEST1() {
     expect="$srcdir/../testfiles/$1"
     actual="../testfiles/`basename $1 .ddl`.out"
     actual_err="../testfiles/`basename $1 .ddl`.err"
+    actual_sav=${actual}-sav
+    actual_err_sav=${actual_err}-sav
     shift
 
     # Run test.
     TESTING $DUMPER $@
     (
       
-    cd $srcdir/../testfiles
-      $RUNSERIAL $DUMPER_BIN $@
+	  cd $srcdir/../testfiles
+  	  $RUNSERIAL $DUMPER_BIN $@
     ) >$actual 2>$actual_err
+
+    # save actual and actual_err in case they are needed later.
+    cp $actual $actual_sav
+    STDOUT_FILTER $actual
+    cp $actual_err $actual_err_sav
+    STDERR_FILTER $actual_err
     cat $actual_err >> $actual
 
-  if [ ! -f $expect ]; then
+	if [ ! -f $expect ]; then
     # Create the expect file if it doesn't yet exist.
      echo " CREATED"
      cp $actual $expect
@@ -132,13 +151,13 @@ TOOLTEST1() {
     else
      echo "*FAILED*"
      echo "    Expected result (*.ddl) differs from actual result (*.out)"
-   nerrors="`expr $nerrors + 1`"
-   test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/    /'
+	 nerrors="`expr $nerrors + 1`"
+	 test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/    /'
     fi
 
     # Clean up output file
     if test -z "$HDF5_NOCLEANUP"; then
-   rm -f $actual $actual_err
+	 rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext
     fi
 
 }
@@ -203,6 +222,8 @@ TOOLTEST3() {
     actual="../testfiles/`basename $1 .ddl`.out"
     actual_err="../testfiles/`basename $1 .ddl`.err"
     actual_ext="../testfiles/`basename $1 .ddl`.ext"
+    actual_sav=${actual}-sav
+    actual_err_sav=${actual_err}-sav
     shift
 
     # Run test.
@@ -215,6 +236,12 @@ TOOLTEST3() {
       $RUNSERIAL $DUMPER_BIN $@
     ) >$actual 2>$actual_err
 
+    # save actual and actual_err in case they are needed later.
+    cp $actual $actual_sav
+    STDOUT_FILTER $actual
+    cp $actual_err $actual_err_sav
+    STDERR_FILTER $actual_err
+
     # Extract file name, line number, version and thread IDs because they may be different
     sed -e 's/thread [0-9]*/thread (IDs)/' -e 's/: .*\.c /: (file name) /' \
         -e 's/line [0-9]*/line (number)/' \
@@ -240,15 +267,15 @@ TOOLTEST3() {
 
     # Clean up output file
     if test -z "$HDF5_NOCLEANUP"; then
-     rm -f $actual $actual_err
+	 rm -f $actual $actual_err $actual_sav $actual_err_sav
     fi
 
 }
 
 # Print a "SKIP" message
 SKIP() {
-   TESTING $DUMPER $@
-    echo  " -SKIP-"
+	 TESTING $DUMPER $@
+	  echo  " -SKIP-"
 }
   
 # Print a line-line message left justified in a field of 70 characters
@@ -265,8 +292,8 @@ DIFFTEST()
 {
     PRINT_H5DIFF  $@
     (
-  cd $srcdir/../testfiles
-  $RUNSERIAL $H5DIFF_BIN "$@" -q
+	cd $srcdir/../testfiles
+	$RUNSERIAL $H5DIFF_BIN "$@" -q
     )
     RET=$?
     if [ $RET != 0 ] ; then
@@ -298,8 +325,8 @@ IMPORTTEST()
 
     PRINT_H5IMPORT  $@
     (
-  cd $srcdir/../testfiles
-  $RUNSERIAL $H5IMPORT_BIN "$@" 
+	cd $srcdir/../testfiles
+	$RUNSERIAL $H5IMPORT_BIN "$@" 
     )
     RET=$?
     if [ $RET != 0 ] ; then
@@ -314,7 +341,7 @@ IMPORTTEST()
 
 ##############################################################################
 ##############################################################################
-###        T H E   T E S T S                                            ###
+###			  T H E   T E S T S                                            ###
 ##############################################################################
 ##############################################################################
 
@@ -433,7 +460,7 @@ TOOLTEST tchar1.ddl -r tchar.h5
 
 # test failure handling
 # Missing file name
-TOOLTEST tnofilename-with-packed-bits.ddl
+    TOOLTEST tnofilename-with-packed-bits.ddl
 
 # rev. 2004
 
diff --git a/tools/h5ls/testh5ls.sh.in b/tools/h5ls/testh5ls.sh.in
index 3083028..224e668 100644
--- a/tools/h5ls/testh5ls.sh.in
+++ b/tools/h5ls/testh5ls.sh.in
@@ -52,6 +52,9 @@ TESTING() {
     echo "Testing $* $SPACES" |cut -c1-70 |tr -d '\012'
 }
 
+# Source in the output filter function definitions.
+. $srcdir/../../bin/output_filter.sh
+
 # Run a test and print PASS or *FAIL*. For now, if h5ls can complete
 # with exit status 0, consider it pass. If a test fails then increment
 # the `nerrors' global variable and (if $verbose is set) display up to $NLINS
@@ -64,6 +67,8 @@ TOOLTEST() {
     expect="$srcdir/../testfiles/$1"
     actual="../testfiles/`basename $1 .ls`.out"
     actual_err="../testfiles/`basename $1 .ls`.err"
+    actual_sav=${actual}-sav
+    actual_err_sav=${actual_err}-sav
     shift
     retvalexpect=$1
     shift
@@ -81,6 +86,11 @@ TOOLTEST() {
     ) >$actual 2>$actual_err 
     
     exitcode=$?
+    # save actual and actual_err in case they are needed later.
+    cp $actual $actual_sav
+    STDOUT_FILTER $actual
+    cp $actual_err $actual_err_sav
+    STDERR_FILTER $actual_err
     cat $actual_err >> $actual
     if [ $h5haveexitcode = 'yes' -a $exitcode -ne $retvalexpect ]; then
 	echo "*FAILED*"
@@ -107,7 +117,7 @@ TOOLTEST() {
 
     # Clean up output file
     if test -z "$HDF5_NOCLEANUP"; then
-	rm -f $actual $actual_err
+	rm -f $actual $actual_err $actual_sav $actual_err_sav
     fi
 }
 
diff --git a/tools/misc/testh5mkgrp.sh b/tools/misc/testh5mkgrp.sh
index 46264c2..7bb98cb 100644
--- a/tools/misc/testh5mkgrp.sh
+++ b/tools/misc/testh5mkgrp.sh
@@ -51,6 +51,9 @@ TESTING()
     echo "Testing $* $SPACES" |cut -c1-70 |tr -d '\012'
 }
 
+# Source in the output filter function definitions.
+. $srcdir/../../bin/output_filter.sh 
+
 # Print a line-line message left justified in a field of 70 characters
 # beginning with the word "Verifying".
 #
@@ -97,6 +100,7 @@ H5LSTEST()
 {
     expect="$INDIR/`basename $1 .h5`.ls"
     actual="$OUTDIR/`basename $1 .h5`.out"
+    actual_sav=${actual}-sav
 
     # Stderr is included in stdout so that the diff can detect
     # any unexpected output from that stream too.
@@ -108,6 +112,10 @@ H5LSTEST()
       $RUNSERIAL $H5LS_BIN $H5LS_ARGS $@
     ) 2>&1 |sed 's/Modified:.*/Modified:  XXXX-XX-XX XX:XX:XX XXX/' >$actual
 
+    # save actual in case it is needed later.
+    cp $actual $actual_sav
+    STDOUT_FILTER $actual
+    STDERR_FILTER $actual
 
    if [ ! -f $expect ]; then
     # Create the expect file if it doesn't yet exist.
@@ -124,7 +132,7 @@ H5LSTEST()
 
    # Clean up output file
    if test -z "$HDF5_NOCLEANUP"; then
-      rm -f $actual $actual_err
+      rm -f $actual $actual_sav
    fi
 }