Use ctest_test and APPEND to correctly report passed and failed test numbers in CDash (#2208)

* Change how HPC tests are executed

* Move ctest_test commands for serial and parallel tests into cmake files.

* Update checks for uploading Test.xml to CDash.

* Correct cmake scripots.

* Remove extraneous characters.

* Add ctest_start (APPEND).

* Add binary directories to scripts running ctest_test.

* Correct parameters for appending Test.xml.

* Set CTEST_SITE and CTEST_BUILD_NAME.

* Try to pass site and build_name to ctest_test sripts.

* Add site name and build name for ctest_test commands vi environment.

* Revert "Try to pass site and build_name to ctest_test sripts."

This reverts commit d364aaf0fa65ee7f9e222b2633ee916d50afbc42.

* Correct typo in src/CMakeLists.txt.

* Update batch scripts to run ctest_test scripts.

Co-authored-by: Allen Byrne <byrn@hdfgroup.org>

13 files changed, 83 insertions, 43 deletions

diff --git a/bin/batch/ctest.qsub.in.cmake b/bin/batch/ctest.qsub.in.cmake
index 702fca7..da3ae2f 100755
--- a/bin/batch/ctest.qsub.in.cmake
+++ b/bin/batch/ctest.qsub.in.cmake
@@ -11,11 +11,13 @@ CTEST_CMD=`which ctest`
 
 cd @HDF5_BINARY_DIR@
 if [[ $SUMMARY_FILE == *"ctestS"* ]]; then
-  CMD="${CTEST_CMD} . -E MPI_TEST_ -C Release -j 32 -T test"
+  CMD="${CTEST_CMD} -S ctest_serial.cmake"
+  qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
+  echo "Done running ctest serial command."
+  touch ctestS.done
 else
-  CMD="${CTEST_CMD} . -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test"
+  CMD="${CTEST_CMD} -S ctest_parallel.cmake"
+  qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
+  echo "Done running ctest parallel command."
+  touch ctestP.done
 fi
-
-qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
-
-echo "Done running ctest parallel command."
diff --git a/bin/batch/ctestP.lsf.in.cmake b/bin/batch/ctestP.lsf.in.cmake
index 3fdd5ce..8b89c9c 100644
--- a/bin/batch/ctestP.lsf.in.cmake
+++ b/bin/batch/ctestP.lsf.in.cmake
@@ -14,6 +14,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run parallel test command. Test output will be in build/ctestP.out"
-ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out
+ctest -S ctest_parallel.cmake >& ctestP.out
 
 echo "Done running ctest parallel command."
+touch ctestP.done
diff --git a/bin/batch/ctestP.sl.in.cmake b/bin/batch/ctestP.sl.in.cmake
index 1069fa9..599fee8 100644
--- a/bin/batch/ctestP.sl.in.cmake
+++ b/bin/batch/ctestP.sl.in.cmake
@@ -8,7 +8,8 @@
 #SBATCH --job-name=h5_ctestP
 
 cd @HDF5_BINARY_DIR@
-ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out
-
-echo "Done running ctestP.sl"
+echo "Run parallel test command. Test output will be in build/ctestP.out"
+ctest -S ctest_parallel.cmake >& ctestP.out
 
+echo "Done running ctest parallel command."
+touch ctestP.done
diff --git a/bin/batch/ctestS.lsf.in.cmake b/bin/batch/ctestS.lsf.in.cmake
index a01d39b..73b17c0 100644
--- a/bin/batch/ctestS.lsf.in.cmake
+++ b/bin/batch/ctestS.lsf.in.cmake
@@ -11,8 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -E MPI_TEST_ -C Release -j 32 -T test >& ctestS.out
+ctest -S ctest_serial.cmake >& ctestS.out
 
-##$CMD  >& ctestS.out
 echo "Done running command."
-
+touch ctestS.done
diff --git a/bin/batch/ctestS.sl.in.cmake b/bin/batch/ctestS.sl.in.cmake
index 4f96bb9..bf0d6ae 100644
--- a/bin/batch/ctestS.sl.in.cmake
+++ b/bin/batch/ctestS.sl.in.cmake
@@ -8,8 +8,8 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
+echo "Run command. Test output will be in build/ctestS.out"
+ctest -S ctest_serial.cmake >& ctestS.out
 
-echo "Run $CMD. Test output will be in build/ctestS.out"
-$CMD  >& ctestS.out
-echo "Done running $CMD"
+echo "Done running command."
+touch ctestS.done
diff --git a/bin/batch/ctest_parallel.cmake.in b/bin/batch/ctest_parallel.cmake.in
new file mode 100644
index 0000000..f35a772
--- /dev/null
+++ b/bin/batch/ctest_parallel.cmake.in
@@ -0,0 +1,12 @@
+if(NOT "$ENV{CI_SITE_NAME}" STREQUAL "")
+  set(CTEST_SITE "$ENV{CI_SITE_NAME}")
+endif()
+if(NOT "$ENV{CI_BUILD_NAME}" STREQUAL "")
+  set(CTEST_BUILD_NAME "$ENV{CI_BUILD_NAME}")
+endif()
+
+ctest_start ("$ENV{CI_MODEL}" "@HDF5_SOURCE_DIR@" "@HDF5_BINARY_DIR@" APPEND)
+ctest_test (BUILD "@HDF5_BINARY_DIR@" APPEND INCLUDE MPI_TEST_ RETURN_VALUE res)
+if (${res} LESS 0 OR ${res} GREATER 0)
+      file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n")
+      endif ()
diff --git a/bin/batch/ctest_serial.cmake.in b/bin/batch/ctest_serial.cmake.in
new file mode 100644
index 0000000..e54421b
--- /dev/null
+++ b/bin/batch/ctest_serial.cmake.in
@@ -0,0 +1,12 @@
+if(NOT "$ENV{CI_SITE_NAME}" STREQUAL "")
+  set(CTEST_SITE "$ENV{CI_SITE_NAME}")
+endif()
+if(NOT "$ENV{CI_BUILD_NAME}" STREQUAL "")
+  set(CTEST_BUILD_NAME "$ENV{CI_BUILD_NAME}")
+endif()
+
+ctest_start ("$ENV{CI_MODEL}" "@HDF5_SOURCE_DIR@" "@HDF5_BINARY_DIR@" APPEND)
+ctest_test (BUILD "@HDF5_BINARY_DIR@" APPEND EXCLUDE MPI_TEST_ PARALLEL_LEVEL 32 RETURN_VALUE res)
+if (${res} LESS 0 OR ${res} GREATER 0)
+      file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n")
+      endif ()
diff --git a/bin/batch/knl_ctestP.sl.in.cmake b/bin/batch/knl_ctestP.sl.in.cmake
index f985fbb..97bd2ad 100644
--- a/bin/batch/knl_ctestP.sl.in.cmake
+++ b/bin/batch/knl_ctestP.sl.in.cmake
@@ -9,8 +9,8 @@
 #SBATCH --job-name=h5_ctestP
 
 cd @HDF5_BINARY_DIR@
-#run parallel tests except t_cache_image test
-ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out
-
-echo "Done running $CMD"
+echo "Run parallel test command. Test output will be in build/ctestP.out"
+ctest -S ctest_parallel.cmake >& ctestP.out
 
+echo "Done running ctest parallel command."
+touch ctestP.done
diff --git a/bin/batch/knl_ctestS.sl.in.cmake b/bin/batch/knl_ctestS.sl.in.cmake
index af6353b..87c4a48 100644
--- a/bin/batch/knl_ctestS.sl.in.cmake
+++ b/bin/batch/knl_ctestS.sl.in.cmake
@@ -9,9 +9,8 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
-
-echo "Run $CMD. Test output will be in build/ctestS.out"
-$CMD  >& ctestS.out
-echo "Done running $CMD"
+echo "Run command. Test output will be in build/ctestS.out"
+ctest -S ctest_serial.cmake >& ctestS.out
 
+echo "Done running command."
+touch ctestS.done
diff --git a/bin/batch/ray_ctestP.lsf.in.cmake b/bin/batch/ray_ctestP.lsf.in.cmake
index 7067a65..51425b5 100644
--- a/bin/batch/ray_ctestP.lsf.in.cmake
+++ b/bin/batch/ray_ctestP.lsf.in.cmake
@@ -15,6 +15,8 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run parallel test command. Test output will be in build/ctestP.out"
-ctest . -R 'MPI_TEST_' -C Release -T test >& ctestP.out
+ctest -S ctest_parallel.cmake >& ctestP.out
 
 echo "Done running ctest parallel command."
+touch ctestP.done
+~                
diff --git a/bin/batch/ray_ctestS.lsf.in.cmake b/bin/batch/ray_ctestS.lsf.in.cmake
index da20438..7f585b4 100644
--- a/bin/batch/ray_ctestS.lsf.in.cmake
+++ b/bin/batch/ray_ctestS.lsf.in.cmake
@@ -11,8 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -E 'MPI_TEST_' -C Release -j 32 -T test >& ctestS.out
+ctest -S ctest_serial.cmake >& ctestS.out
 
-##$CMD  >& ctestS.out
 echo "Done running command."
-
+touch ctestS.done
diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake
index 3882bbe..f277864 100644
--- a/config/cmake/scripts/CTestScript.cmake
+++ b/config/cmake/scripts/CTestScript.cmake
@@ -261,6 +261,10 @@ if (NOT DEFINED MODEL)
   set (MODEL "Experimental")
 endif ()
 
+set (ENV{CI_SITE_NAME} ${CTEST_SITE})
+set (ENV{CI_BUILD_NAME} ${CTEST_BUILD_NAME})
+set (ENV{CI_MODEL} ${MODEL})
+
 #-----------------------------------------------------------------------------
   ## NORMAL process
   ## -- LOCAL_UPDATE updates the source folder from svn
@@ -315,19 +319,17 @@ endif ()
             execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
           endif()
         endif ()
-        message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
-        execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+        message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/ctestS.done")
+        execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/ctestS.done RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
         while(result)
           ctest_sleep(60)
-          execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+          execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/ctestS.done RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
         endwhile(result)
+        message(STATUS "Serial tests completed.")
+        if (LOCAL_SUBMIT)
+          ctest_submit (PARTS Test)
+        endif ()
         if (LOCAL_BATCH_SCRIPT_PARALLEL_NAME)
-          if (LOCAL_SUBMIT)
-            ctest_submit (PARTS Test)
-          endif ()
-          message(STATUS "Found ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml for serial tests. Renaming to SerialTest.xml")
-          file (RENAME ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/SerialTest.xml)
-          file (RENAME ${CTEST_BINARY_DIRECTORY}/Testing/Temporary/LastTest_${TAG_CONTENTS}.log ${CTEST_BINARY_DIRECTORY}/Testing/Temporary/LastTest_${TAG_CONTENTS}_Serial.log)
           unset(result CACHE)
           if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
             execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
@@ -338,13 +340,13 @@ endif ()
               execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
             endif ()
           endif ()
-          message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
-          execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+          message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/ctestP.done")
+          execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/ctestP.done RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
           while(result)
             ctest_sleep(60)
-            execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+            execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/ctestP.done RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
           endwhile(result)
-          message(STATUS "Found ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml for parallel tests.")
+          message(STATUS "parallel tests completed.")
         endif()
       endif ()
       if (LOCAL_SUBMIT)
@@ -365,6 +367,9 @@ endif ()
         ctest_submit (PARTS Coverage)
       endif ()
     endif ()
+    if (LOCAL_SUBMIT)
+      ctest_submit (PARTS Done)
+    endif ()
   endif ()
 
   if (NOT LOCAL_MEMCHECK_TEST AND NOT LOCAL_NO_PACKAGE AND NOT LOCAL_SKIP_BUILD)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fe96011..355881d 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1047,6 +1047,14 @@ if (BUILD_SHARED_LIBS)
 endif ()
 
 if (LOCAL_BATCH_TEST)
+    configure_file (
+      ${HDF5_SOURCE_DIR}/bin/batch/ctest_serial.cmake.in
+      ${HDF5_BINARY_DIR}/ctest_serial.cmake ESCAPE_QUOTES @ONLY
+    )
+    configure_file (
+      ${HDF5_SOURCE_DIR}/bin/batch/ctest_parallel.cmake.in
+      ${HDF5_BINARY_DIR}/ctest_parallel.cmake ESCAPE_QUOTES @ONLY
+    )
   if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
     configure_file (
         ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_COMMAND}