HDFFV-10632 separate stderr and stdout in tests

5 files changed, 233 insertions, 186 deletions

diff --git a/config/cmake/jrunTest.cmake b/config/cmake/jrunTest.cmake
index b66f3a8..43aef22 100644
--- a/config/cmake/jrunTest.cmake
+++ b/config/cmake/jrunTest.cmake
@@ -32,9 +32,6 @@ endif ()
 if (NOT TEST_CLASSPATH)
   message (STATUS "Require TEST_CLASSPATH to be defined")
 endif ()
-if (NOT TEST_REFERENCE)
-  message (FATAL_ERROR "Require TEST_REFERENCE to be defined")
-endif ()
 
 if (EXISTS ${TEST_FOLDER}/${TEST_OUTPUT})
   file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT})
@@ -44,14 +41,6 @@ if (EXISTS ${TEST_FOLDER}/${TEST_OUTPUT}.err)
   file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}.err)
 endif ()
 
-# if there is not an error reference file add the error output to the stdout file
-if (NOT TEST_ERRREF)
-  if (NOT SKIP_APPEND)
-    # append error file since skip was not defined
-    set (ERROR_APPEND 1)
-  endif ()
-endif ()
-
 if (NOT TEST_LOG_LEVEL)
   set (LOG_LEVEL "info")
 else ()
@@ -131,52 +120,54 @@ message (STATUS "COMMAND Error: ${TEST_ERROR}")
 
 # compare output files to references unless this must be skipped
 if (NOT TEST_SKIP_COMPARE)
-  if (WIN32 AND NOT MINGW)
-    file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM)
-    file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}")
-  endif ()
+  if (EXISTS ${TEST_FOLDER}/${TEST_REFERENCE})
+    if (WIN32 AND NOT MINGW)
+      file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM)
+      file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}")
+    endif ()
 
-  # now compare the output with the reference
-  execute_process (
-      COMMAND ${CMAKE_COMMAND} -E compare_files ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_FOLDER}/${TEST_REFERENCE}
-      RESULT_VARIABLE TEST_RESULT
-  )
-  if (NOT "${TEST_RESULT}" STREQUAL "0")
-    set (TEST_RESULT 0)
-    file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} test_act)
-    list (LENGTH test_act len_act)
-    file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref)
-    list (LENGTH test_ref len_ref)
-    if (NOT "${len_act}" STREQUAL "0" AND NOT "${len_ref}" STREQUAL "0")
-      math (EXPR _FP_LEN "${len_ref} - 1")
-      foreach (line RANGE 0 ${_FP_LEN})
-        list (GET test_act ${line} str_act)
-        list (GET test_ref ${line} str_ref)
-        if (NOT "${str_act}" STREQUAL "${str_ref}")
-          if (NOT "${str_act}" STREQUAL "")
-            set (TEST_RESULT 1)
-            message ("line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n")
+    # now compare the output with the reference
+    execute_process (
+        COMMAND ${CMAKE_COMMAND} -E compare_files ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_FOLDER}/${TEST_REFERENCE}
+        RESULT_VARIABLE TEST_RESULT
+    )
+    if (NOT "${TEST_RESULT}" STREQUAL "0")
+      set (TEST_RESULT 0)
+      file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} test_act)
+      list (LENGTH test_act len_act)
+      file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref)
+      list (LENGTH test_ref len_ref)
+      if (NOT "${len_act}" STREQUAL "0" AND NOT "${len_ref}" STREQUAL "0")
+        math (EXPR _FP_LEN "${len_ref} - 1")
+        foreach (line RANGE 0 ${_FP_LEN})
+          list (GET test_act ${line} str_act)
+          list (GET test_ref ${line} str_ref)
+          if (NOT "${str_act}" STREQUAL "${str_ref}")
+            if (NOT "${str_act}" STREQUAL "")
+              set (TEST_RESULT 1)
+              message ("line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n")
+            endif ()
           endif ()
+        endforeach ()
+      else ()
+        if ("${len_act}" STREQUAL "0")
+          message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT} is empty")
+        endif ()
+        if ("${len_ref}" STREQUAL "0")
+          message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty")
         endif ()
-      endforeach ()
-    else ()
-      if ("${len_act}" STREQUAL "0")
-        message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT} is empty")
       endif ()
-      if ("${len_ref}" STREQUAL "0")
-        message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty")
+      if (NOT "${len_act}" STREQUAL "${len_ref}")
+        set (TEST_RESULT 1)
       endif ()
     endif ()
-    if (NOT "${len_act}" STREQUAL "${len_ref}")
-      set (TEST_RESULT 1)
-    endif ()
-  endif ()
 
-  message (STATUS "COMPARE Result: ${TEST_RESULT}")
+    message (STATUS "COMPARE Result: ${TEST_RESULT}")
 
-  # again, if return value is !=0 scream and shout
-  if (NOT "${TEST_RESULT}" STREQUAL "0")
-    message (FATAL_ERROR "Failed: The output of ${TEST_OUTPUT} did not match ${TEST_REFERENCE}")
+    # again, if return value is !=0 scream and shout
+    if (NOT "${TEST_RESULT}" STREQUAL "0")
+      message (FATAL_ERROR "Failed: The output of ${TEST_OUTPUT} did not match ${TEST_REFERENCE}")
+    endif ()
   endif ()
 
   # now compare the .err file with the error reference, if supplied
@@ -217,7 +208,7 @@ if (NOT TEST_SKIP_COMPARE)
         if ("${len_ref}" STREQUAL "0")
           message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_ERRREF} is empty")
         endif ()
-      endif()
+      endif ()
       if (NOT "${len_act}" STREQUAL "${len_ref}")
         set (TEST_RESULT 1)
       endif ()
@@ -226,7 +217,7 @@ if (NOT TEST_SKIP_COMPARE)
     message (STATUS "COMPARE Result: ${TEST_RESULT}")
 
     # again, if return value is !=0 scream and shout
-    if (NOT ${TEST_RESULT} STREQUAL 0)
+    if (NOT "${TEST_RESULT}" STREQUAL "0")
       message (FATAL_ERROR "Failed: The error output of ${TEST_OUTPUT}.err did not match ${TEST_ERRREF}")
     endif ()
   endif ()
diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake
index 43cd48d..9534b1e 100755
--- a/config/cmake/scripts/CTestScript.cmake
+++ b/config/cmake/scripts/CTestScript.cmake
@@ -36,10 +36,6 @@ if (NOT SITE_OS_NAME)
   message (STATUS "Dashboard script uname output: ${osname}-${osrel}-${cpu}\n")

 

   set (CTEST_BUILD_NAME  "${osname}-${osrel}-${cpu}")

-  if (SITE_BUILDNAME_SUFFIX)

-    set (CTEST_BUILD_NAME  "${SITE_BUILDNAME_SUFFIX}-${CTEST_BUILD_NAME}")

-  endif ()

-  set (BUILD_OPTIONS "${ADD_BUILD_OPTIONS}")

 else ()

   ## machine name provided

   ## --------------------------

@@ -48,11 +44,11 @@ else ()
   else ()

     set (CTEST_BUILD_NAME "${SITE_OS_NAME}-${SITE_OS_VERSION}-${SITE_COMPILER_NAME}")

   endif ()

-  if (SITE_BUILDNAME_SUFFIX)

-    set (CTEST_BUILD_NAME "${CTEST_BUILD_NAME}-${SITE_BUILDNAME_SUFFIX}")

-  endif ()

-  set (BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDNAME:STRING=${CTEST_BUILD_NAME}")

 endif ()

+if (SITE_BUILDNAME_SUFFIX)
+  set (CTEST_BUILD_NAME  "${SITE_BUILDNAME_SUFFIX}-${CTEST_BUILD_NAME}")
+endif ()
+set (BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDNAME:STRING=${CTEST_BUILD_NAME}")
 

 #-----------------------------------------------------------------------------

 # MAC machines need special option

@@ -158,7 +154,12 @@ endif ()
 # Use multiple CPU cores to build

 include (ProcessorCount)

 ProcessorCount (N)

-if (NOT N EQUAL 0)

+if (NOT N EQUAL 0)
+  if (MAX_PROC_COUNT)
+    if (N GREATER MAX_PROC_COUNT)
+      set (N ${MAX_PROC_COUNT})
+    endif ()
+  endif ()
   if (NOT WIN32)

     set (CTEST_BUILD_FLAGS -j${N})

   endif ()

@@ -189,10 +190,10 @@ endforeach ()
 #-----------------------------------------------------------------------------

 # Initialize the CTEST commands

 #------------------------------

-if(CMAKE_GENERATOR_TOOLSET)

-  set(CTEST_CONFIGURE_TOOLSET  "-T${CMAKE_GENERATOR_TOOLSET}")

+if (CMAKE_GENERATOR_TOOLSET)

+  set (CTEST_CONFIGURE_TOOLSET  "-T${CMAKE_GENERATOR_TOOLSET}")

 else ()

-  set(CTEST_CONFIGURE_TOOLSET  "")

+  set (CTEST_CONFIGURE_TOOLSET  "")

 endif()

 if (LOCAL_MEMCHECK_TEST)

   find_program (CTEST_MEMORYCHECK_COMMAND NAMES valgrind)

@@ -266,41 +267,64 @@ message (STATUS "Dashboard script configuration:\n${vars}\n")
       if (NOT LOCAL_BATCH_TEST)
         ctest_test (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args} RETURN_VALUE res)
       else ()
-        execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_NAME})
+        file(STRINGS ${CTEST_BINARY_DIRECTORY}/Testing/TAG TAG_CONTENTS REGEX "^2([0-9]+)[-]([0-9]+)$")
+        execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${CTEST_SOURCE_DIRECTORY}/bin/batch/${LOCAL_BATCH_SCRIPT_NAME})
+        message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
+        execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+        while(result)
+          ctest_sleep(60)
+          execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+        endwhile(result)
+        if (LOCAL_BATCH_SCRIPT_PARALLEL_NAME)
+          if (LOCAL_SUBMIT)
+            ctest_submit (PARTS Test)
+          endif ()
+          message(STATUS "Found ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml for serial tests. Renaming to SerialTest.xml")
+          file (RENAME ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/SerialTest.xml)
+          unset(result CACHE)
+          execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${CTEST_SOURCE_DIRECTORY}/bin/batch/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
+          message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
+          execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+          while(result)
+            ctest_sleep(60)
+            execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+          endwhile(result)
+          message(STATUS "Found ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml for parallel tests.")
+        endif()
       endif ()
-      if (LOCAL_SUBMIT)

-        ctest_submit (PARTS Test)

-      endif ()

-      if (${res} LESS 0 OR ${res} GREATER 0)

-        file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n")

-      endif ()

-    else ()

-      ctest_memcheck (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args})

-      if (LOCAL_SUBMIT)

-        ctest_submit (PARTS MemCheck)

-      endif ()

-    endif ()

-    if (LOCAL_COVERAGE_TEST)

-      ctest_coverage (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND)

-      if (LOCAL_SUBMIT)

-        ctest_submit (PARTS Coverage)

-      endif ()

-    endif ()

-  endif ()

-

-  if (NOT LOCAL_MEMCHECK_TEST AND NOT LOCAL_NO_PACKAGE AND NOT LOCAL_SKIP_BUILD)

-    ##-----------------------------------------------

-    ## Package the product

-    ##-----------------------------------------------

-    execute_process (COMMAND cpack -C ${CTEST_CONFIGURATION_TYPE} -V

-      WORKING_DIRECTORY ${CTEST_BINARY_DIRECTORY}

-      RESULT_VARIABLE cpackResult

-      OUTPUT_VARIABLE cpackLog

-      ERROR_VARIABLE cpackLog.err

-    )

-    file (WRITE ${CTEST_BINARY_DIRECTORY}/cpack.log "${cpackLog.err}" "${cpackLog}")

-    if (cpackResult GREATER 0)

-      file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed packaging: ${cpackResult}:${cpackLog.err} \n")

-    endif ()

-  endif ()

-#-----------------------------------------------------------------------------

+      if (LOCAL_SUBMIT)
+        ctest_submit (PARTS Test)
+      endif ()
+      if (${res} LESS 0 OR ${res} GREATER 0)
+        file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n")
+      endif ()
+    else ()
+      ctest_memcheck (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args})
+      if (LOCAL_SUBMIT)
+        ctest_submit (PARTS MemCheck)
+      endif ()
+    endif ()
+    if (LOCAL_COVERAGE_TEST)
+      ctest_coverage (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND)
+      if (LOCAL_SUBMIT)
+        ctest_submit (PARTS Coverage)
+      endif ()
+    endif ()
+  endif ()
+
+  if (NOT LOCAL_MEMCHECK_TEST AND NOT LOCAL_NO_PACKAGE AND NOT LOCAL_SKIP_BUILD)
+    ##-----------------------------------------------
+    ## Package the product
+    ##-----------------------------------------------
+    execute_process (COMMAND cpack -C ${CTEST_CONFIGURATION_TYPE} -V
+      WORKING_DIRECTORY ${CTEST_BINARY_DIRECTORY}
+      RESULT_VARIABLE cpackResult
+      OUTPUT_VARIABLE cpackLog
+      ERROR_VARIABLE cpackLog.err
+    )
+    file (WRITE ${CTEST_BINARY_DIRECTORY}/cpack.log "${cpackLog.err}" "${cpackLog}")
+    if (cpackResult GREATER 0)
+      file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed packaging: ${cpackResult}:${cpackLog.err} \n")
+    endif ()
+  endif ()
+#-----------------------------------------------------------------------------
diff --git a/config/cmake/scripts/HDF5options.cmake b/config/cmake/scripts/HDF5options.cmake
index 629d277..d4cc996 100755
--- a/config/cmake/scripts/HDF5options.cmake
+++ b/config/cmake/scripts/HDF5options.cmake
@@ -17,8 +17,12 @@
 ### uncomment/comment and change the following lines for other configuration options
 
 #############################################################################################
+####      maximum parallel processor count for build and test       ####
+#set (MAX_PROC_COUNT 8)
+
+#############################################################################################
 ####      alternate toolsets       ####
-#set(CMAKE_GENERATOR_TOOLSET "Intel C++ Compiler 17.0")
+#set (CMAKE_GENERATOR_TOOLSET "Intel C++ Compiler 17.0")
 
 #############################################################################################
 ####      Only build static libraries       ####
@@ -50,54 +54,55 @@ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_INSTALL_PREFIX:PATH=${INSTA
 ####      ext libraries       ####
 
 ### ext libs from tgz
-set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRING=TGZ -DTGZPATH:PATH=${CTEST_SCRIPT_DIRECTORY}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRING=TGZ -DTGZPATH:PATH=${CTEST_SCRIPT_DIRECTORY}")
 ### ext libs from git
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRING=GIT")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRING=GIT")
 ### ext libs on system
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_LIBRARY:FILEPATH=some_location/lib/zlib.lib -DZLIB_INCLUDE_DIR:PATH=some_location/include")
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSZIP_LIBRARY:FILEPATH=some_location/lib/szlib.lib -DSZIP_INCLUDE_DIR:PATH=some_location/include")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRING=NO")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_LIBRARY:FILEPATH=some_location/lib/zlib.lib -DZLIB_INCLUDE_DIR:PATH=some_location/include")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSZIP_LIBRARY:FILEPATH=some_location/lib/szlib.lib -DSZIP_INCLUDE_DIR:PATH=some_location/include")
 
-### disable ext zlib building
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=OFF")
-### disable ext szip building
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF")
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_ENCODING:BOOL=OFF")
+### disable using ext zlib
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=OFF")
+### disable using ext szip
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_ENCODING:BOOL=OFF")
 
 ####      package examples       ####
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACK_EXAMPLES:BOOL=ON -DHDF5_EXAMPLES_COMPRESSED:STRING=HDF5Examples-1.10.9-Source.tar.gz -DHDF5_EXAMPLES_COMPRESSED_DIR:PATH=${CTEST_SCRIPT_DIRECTORY}")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACK_EXAMPLES:BOOL=ON -DHDF5_EXAMPLES_COMPRESSED:STRING=HDF5Examples-1.10.9-Source.tar.gz -DHDF5_EXAMPLES_COMPRESSED_DIR:PATH=${CTEST_SCRIPT_DIRECTORY}")
 
 #############################################################################################
 ### enable parallel builds
 
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PARALLEL:BOOL=ON")
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_CPP_LIB:BOOL=OFF")
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_JAVA:BOOL=OFF")
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_THREADSAFE:BOOL=OFF")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PARALLEL:BOOL=ON")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_CPP_LIB:BOOL=OFF")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_JAVA:BOOL=OFF")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_THREADSAFE:BOOL=OFF")
 
 #############################################################################################
 ### enable thread-safety builds
 
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_THREADSAFE:BOOL=ON")
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PARALLEL:BOOL=OFF")
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_CPP_LIB:BOOL=OFF")
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=OFF")
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_HL_LIB:BOOL=OFF")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_THREADSAFE:BOOL=ON")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PARALLEL:BOOL=OFF")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_CPP_LIB:BOOL=OFF")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=OFF")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_HL_LIB:BOOL=OFF")
 
 #############################################################################################
 ### disable test program builds
 
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DBUILD_TESTING:BOOL=OFF")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DBUILD_TESTING:BOOL=OFF")
 
 #############################################################################################
 ### disable packaging
 
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_NO_PACKAGES:BOOL=ON")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_NO_PACKAGES:BOOL=ON")
 ### Create install package with external libraries (szip, zlib)
-set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACKAGE_EXTLIBS:BOOL=ON")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACKAGE_EXTLIBS:BOOL=ON")
 
 #############################################################################################
 ### use a toolchain file
 
-#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/intel.cmake")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/intel.cmake")
 
 #############################################################################################
diff --git a/config/cmake/vfdTest.cmake b/config/cmake/vfdTest.cmake
index 66a97c0..95a4c40 100644
--- a/config/cmake/vfdTest.cmake
+++ b/config/cmake/vfdTest.cmake
@@ -32,9 +32,9 @@ if (EXISTS ${TEST_FOLDER}/${TEST_OUTPUT}.err)
 endif ()
 
 # if there is not an error reference file add the error output to the stdout file
-if (NOT TEST_ERRREF)
-  set (ERROR_APPEND 1)
-endif ()
+#if (NOT TEST_ERRREF)
+#  set (ERROR_APPEND 1)
+#endif ()
 
 message (STATUS "USING ${TEST_VFD} ON COMMAND: ${TEST_PROGRAM} ${TEST_ARGS}")
 
diff --git a/config/cmake_ext_mod/runTest.cmake b/config/cmake_ext_mod/runTest.cmake
index 5569a73..e8033c5 100644
--- a/config/cmake_ext_mod/runTest.cmake
+++ b/config/cmake_ext_mod/runTest.cmake
@@ -26,9 +26,6 @@ endif ()
 if (NOT TEST_EXPECT)
   message (STATUS "Require TEST_EXPECT to be defined")
 endif ()
-if (NOT TEST_SKIP_COMPARE AND NOT TEST_REFERENCE)
-  message (FATAL_ERROR "Require TEST_REFERENCE to be defined")
-endif ()
 
 if (EXISTS ${TEST_FOLDER}/${TEST_OUTPUT})
   file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT})
@@ -38,11 +35,6 @@ if (EXISTS ${TEST_FOLDER}/${TEST_OUTPUT}.err)
   file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}.err)
 endif ()
 
-# if there is not an error reference file add the error output to the stdout file
-if (NOT TEST_ERRREF)
-  set (ERROR_APPEND 1)
-endif ()
-
 message (STATUS "COMMAND: ${TEST_PROGRAM} ${TEST_ARGS}")
 
 if (TEST_LIBRARY_DIRECTORY)
@@ -96,9 +88,19 @@ endif ()
 message (STATUS "COMMAND Result: ${TEST_RESULT}")
 
 # if the .err file exists and ERRROR_APPEND is enabled
-if (ERROR_APPEND AND EXISTS ${TEST_FOLDER}/${TEST_OUTPUT}.err)
+if (EXISTS ${TEST_FOLDER}/${TEST_OUTPUT}.err)
   file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_STREAM)
-  file (APPEND ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}")
+  if (TEST_MASK_FILE)
+    STRING(REGEX REPLACE "CurrentDir is [^\n]+\n" "CurrentDir is (dir name)\n" TEST_STREAM "${TEST_STREAM}")
+  endif ()
+
+  if (NOT ERROR_APPEND)
+    # append error output to the stdout output file
+    file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT}.err "${TEST_STREAM}")
+  else ()
+    # write back to original .err file
+    file (APPEND ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}")
+  endif ()
 endif ()
 
 # append the test result status with a predefined text
@@ -142,6 +144,7 @@ if (TEST_MASK_ERROR)
     # the error stack remains in the .err file
     file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_STREAM)
   endif ()
+  string (REGEX REPLACE "[.]*_pmi_alps[.]*\n" "" TEST_STREAM "${TEST_STREAM}")
   string (REGEX REPLACE "thread [0-9]*:" "thread (IDs):" TEST_STREAM "${TEST_STREAM}")
   string (REGEX REPLACE ": ([^\n]*)[.]c " ": (file name) " TEST_STREAM "${TEST_STREAM}")
   string (REGEX REPLACE " line [0-9]*" " line (number)" TEST_STREAM "${TEST_STREAM}")
@@ -173,63 +176,65 @@ endif ()
 
 # compare output files to references unless this must be skipped
 if (NOT TEST_SKIP_COMPARE)
-  if (WIN32 AND NOT MINGW)
-    file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM)
-    file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}")
-  endif ()
+  if (EXISTS ${TEST_FOLDER}/${TEST_REFERENCE})
+    if (WIN32 AND NOT MINGW)
+      file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM)
+      file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}")
+    endif ()
 
-  if (NOT TEST_SORT_COMPARE)
-    # now compare the output with the reference
-    execute_process (
-        COMMAND ${CMAKE_COMMAND} -E compare_files ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_FOLDER}/${TEST_REFERENCE}
-        RESULT_VARIABLE TEST_RESULT
-    )
-  else ()
-    file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} v1)
-    file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} v2)
-    list (SORT v1)
-    list (SORT v2)
-    if (NOT v1 STREQUAL v2)
-      set(TEST_RESULT 1)
+    if (NOT TEST_SORT_COMPARE)
+      # now compare the output with the reference
+      execute_process (
+          COMMAND ${CMAKE_COMMAND} -E compare_files ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_FOLDER}/${TEST_REFERENCE}
+          RESULT_VARIABLE TEST_RESULT
+      )
+    else ()
+      file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} v1)
+      file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} v2)
+      list (SORT v1)
+      list (SORT v2)
+      if (NOT v1 STREQUAL v2)
+        set(TEST_RESULT 1)
+      endif ()
     endif ()
-  endif ()
 
-  if (NOT "${TEST_RESULT}" STREQUAL "0")
-    set (TEST_RESULT 0)
-    file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} test_act)
-    list (LENGTH test_act len_act)
-    file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref)
-    list (LENGTH test_ref len_ref)
-    if (NOT "${len_act}" STREQUAL "0" AND NOT "${len_ref}" STREQUAL "0")
-      math (EXPR _FP_LEN "${len_ref} - 1")
-      foreach (line RANGE 0 ${_FP_LEN})
-        list (GET test_act ${line} str_act)
-        list (GET test_ref ${line} str_ref)
-        if (NOT "${str_act}" STREQUAL "${str_ref}")
-          if (NOT "${str_act}" STREQUAL "")
-            set (TEST_RESULT 1)
-            message ("line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n")
+    if (NOT "${TEST_RESULT}" STREQUAL "0")
+      set (TEST_RESULT 0)
+      file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} test_act)
+      list (LENGTH test_act len_act)
+      file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref)
+      list (LENGTH test_ref len_ref)
+      if (NOT "${len_act}" STREQUAL "0" AND NOT "${len_ref}" STREQUAL "0")
+        math (EXPR _FP_LEN "${len_ref} - 1")
+        foreach (line RANGE 0 ${_FP_LEN})
+          list (GET test_act ${line} str_act)
+          list (GET test_ref ${line} str_ref)
+          if (NOT "${str_act}" STREQUAL "${str_ref}")
+            if (NOT "${str_act}" STREQUAL "")
+              set (TEST_RESULT 1)
+              message ("line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n")
+            endif ()
           endif ()
+        endforeach ()
+      else ()
+        if ("${len_act}" STREQUAL "0")
+          message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT} is empty")
+        endif ()
+        if ("${len_ref}" STREQUAL "0")
+          message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty")
         endif ()
-      endforeach ()
-    else ()
-      if ("${len_act}" STREQUAL "0")
-        message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT} is empty")
       endif ()
-      if ("${len_ref}" STREQUAL "0")
-        message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty")
+      if (NOT "${len_act}" STREQUAL "${len_ref}")
+        set (TEST_RESULT 1)
       endif ()
     endif ()
-    if (NOT "${len_act}" STREQUAL "${len_ref}")
-      set (TEST_RESULT 1)
-    endif ()
-  endif ()
 
-  message (STATUS "COMPARE Result: ${TEST_RESULT}")
+    message (STATUS "COMPARE Result: ${TEST_RESULT}")
 
-  # again, if return value is !=0 scream and shout
-  if (NOT "${TEST_RESULT}" STREQUAL "0")
-    message (FATAL_ERROR "Failed: The output of ${TEST_OUTPUT} did not match ${TEST_REFERENCE}")
+    # again, if return value is !=0 scream and shout
+    if (NOT "${TEST_RESULT}" STREQUAL "0")
+      message (FATAL_ERROR "Failed: The output of ${TEST_OUTPUT} did not match ${TEST_REFERENCE}")
+    endif ()
   endif ()
 
   # now compare the .err file with the error reference, if supplied
@@ -260,7 +265,7 @@ if (NOT TEST_SKIP_COMPARE)
             if (NOT "${str_act}" STREQUAL "")
               set (TEST_RESULT 1)
               message ("line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n")
-             endif ()
+            endif ()
           endif ()
         endforeach ()
       else ()
@@ -285,5 +290,27 @@ if (NOT TEST_SKIP_COMPARE)
   endif ()
 endif ()
 
+if (TEST_GREP_COMPARE)
+  # now grep the output with the reference
+  file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM)
+
+  # TEST_REFERENCE should always be matched
+  string (REGEX MATCH "${TEST_REFERENCE}" TEST_MATCH ${TEST_STREAM})
+  string (COMPARE EQUAL "${TEST_REFERENCE}" "${TEST_MATCH}" TEST_RESULT)
+  if ("${TEST_RESULT}" STREQUAL "0")
+    message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did not contain ${TEST_REFERENCE}")
+  endif ()
+
+  string (REGEX MATCH "${TEST_FILTER}" TEST_MATCH ${TEST_STREAM})
+  if ("${TEST_EXPECT}" STREQUAL "1")
+    # TEST_EXPECT (1) interperts TEST_FILTER as NOT to match
+    string (LENGTH "${TEST_MATCH}" TEST_RESULT)
+    if (NOT "${TEST_RESULT}" STREQUAL "0")
+      message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did contain ${TEST_FILTER}")
+    endif ()
+  endif ()
+endif ()
+
 # everything went fine...
-message ("Passed: The output of ${TEST_PROGRAM} matches ${TEST_REFERENCE}")
+message ("${TEST_PROGRAM} Passed")
+