Use ctest_test and APPEND to correctly report passed and failed test numbers in CDash (#2208)

* Change how HPC tests are executed

* Move ctest_test commands for serial and parallel tests into cmake files.

* Update checks for uploading Test.xml to CDash.

* Correct cmake scripots.

* Remove extraneous characters.

* Add ctest_start (APPEND).

* Add binary directories to scripts running ctest_test.

* Correct parameters for appending Test.xml.

* Set CTEST_SITE and CTEST_BUILD_NAME.

* Try to pass site and build_name to ctest_test sripts.

* Add site name and build name for ctest_test commands vi environment.

* Revert "Try to pass site and build_name to ctest_test sripts."

This reverts commit d364aaf0fa65ee7f9e222b2633ee916d50afbc42.

* Correct typo in src/CMakeLists.txt.

* Update batch scripts to run ctest_test scripts.

Co-authored-by: Allen Byrne <byrn@hdfgroup.org>

11 files changed, 57 insertions, 30 deletions

diff --git a/bin/batch/ctest.qsub.in.cmake b/bin/batch/ctest.qsub.in.cmake
index 702fca7..da3ae2f 100755
--- a/bin/batch/ctest.qsub.in.cmake
+++ b/bin/batch/ctest.qsub.in.cmake
@@ -11,11 +11,13 @@ CTEST_CMD=`which ctest`
 
 cd @HDF5_BINARY_DIR@
 if [[ $SUMMARY_FILE == *"ctestS"* ]]; then
-  CMD="${CTEST_CMD} . -E MPI_TEST_ -C Release -j 32 -T test"
+  CMD="${CTEST_CMD} -S ctest_serial.cmake"
+  qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
+  echo "Done running ctest serial command."
+  touch ctestS.done
 else
-  CMD="${CTEST_CMD} . -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test"
+  CMD="${CTEST_CMD} -S ctest_parallel.cmake"
+  qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
+  echo "Done running ctest parallel command."
+  touch ctestP.done
 fi
-
-qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
-
-echo "Done running ctest parallel command."
diff --git a/bin/batch/ctestP.lsf.in.cmake b/bin/batch/ctestP.lsf.in.cmake
index 3fdd5ce..8b89c9c 100644
--- a/bin/batch/ctestP.lsf.in.cmake
+++ b/bin/batch/ctestP.lsf.in.cmake
@@ -14,6 +14,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run parallel test command. Test output will be in build/ctestP.out"
-ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out
+ctest -S ctest_parallel.cmake >& ctestP.out
 
 echo "Done running ctest parallel command."
+touch ctestP.done
diff --git a/bin/batch/ctestP.sl.in.cmake b/bin/batch/ctestP.sl.in.cmake
index 1069fa9..599fee8 100644
--- a/bin/batch/ctestP.sl.in.cmake
+++ b/bin/batch/ctestP.sl.in.cmake
@@ -8,7 +8,8 @@
 #SBATCH --job-name=h5_ctestP
 
 cd @HDF5_BINARY_DIR@
-ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out
-
-echo "Done running ctestP.sl"
+echo "Run parallel test command. Test output will be in build/ctestP.out"
+ctest -S ctest_parallel.cmake >& ctestP.out
 
+echo "Done running ctest parallel command."
+touch ctestP.done
diff --git a/bin/batch/ctestS.lsf.in.cmake b/bin/batch/ctestS.lsf.in.cmake
index a01d39b..73b17c0 100644
--- a/bin/batch/ctestS.lsf.in.cmake
+++ b/bin/batch/ctestS.lsf.in.cmake
@@ -11,8 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -E MPI_TEST_ -C Release -j 32 -T test >& ctestS.out
+ctest -S ctest_serial.cmake >& ctestS.out
 
-##$CMD  >& ctestS.out
 echo "Done running command."
-
+touch ctestS.done
diff --git a/bin/batch/ctestS.sl.in.cmake b/bin/batch/ctestS.sl.in.cmake
index 4f96bb9..bf0d6ae 100644
--- a/bin/batch/ctestS.sl.in.cmake
+++ b/bin/batch/ctestS.sl.in.cmake
@@ -8,8 +8,8 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
+echo "Run command. Test output will be in build/ctestS.out"
+ctest -S ctest_serial.cmake >& ctestS.out
 
-echo "Run $CMD. Test output will be in build/ctestS.out"
-$CMD  >& ctestS.out
-echo "Done running $CMD"
+echo "Done running command."
+touch ctestS.done
diff --git a/bin/batch/ctest_parallel.cmake.in b/bin/batch/ctest_parallel.cmake.in
new file mode 100644
index 0000000..f35a772
--- /dev/null
+++ b/bin/batch/ctest_parallel.cmake.in
@@ -0,0 +1,12 @@
+if(NOT "$ENV{CI_SITE_NAME}" STREQUAL "")
+  set(CTEST_SITE "$ENV{CI_SITE_NAME}")
+endif()
+if(NOT "$ENV{CI_BUILD_NAME}" STREQUAL "")
+  set(CTEST_BUILD_NAME "$ENV{CI_BUILD_NAME}")
+endif()
+
+ctest_start ("$ENV{CI_MODEL}" "@HDF5_SOURCE_DIR@" "@HDF5_BINARY_DIR@" APPEND)
+ctest_test (BUILD "@HDF5_BINARY_DIR@" APPEND INCLUDE MPI_TEST_ RETURN_VALUE res)
+if (${res} LESS 0 OR ${res} GREATER 0)
+      file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n")
+      endif ()
diff --git a/bin/batch/ctest_serial.cmake.in b/bin/batch/ctest_serial.cmake.in
new file mode 100644
index 0000000..e54421b
--- /dev/null
+++ b/bin/batch/ctest_serial.cmake.in
@@ -0,0 +1,12 @@
+if(NOT "$ENV{CI_SITE_NAME}" STREQUAL "")
+  set(CTEST_SITE "$ENV{CI_SITE_NAME}")
+endif()
+if(NOT "$ENV{CI_BUILD_NAME}" STREQUAL "")
+  set(CTEST_BUILD_NAME "$ENV{CI_BUILD_NAME}")
+endif()
+
+ctest_start ("$ENV{CI_MODEL}" "@HDF5_SOURCE_DIR@" "@HDF5_BINARY_DIR@" APPEND)
+ctest_test (BUILD "@HDF5_BINARY_DIR@" APPEND EXCLUDE MPI_TEST_ PARALLEL_LEVEL 32 RETURN_VALUE res)
+if (${res} LESS 0 OR ${res} GREATER 0)
+      file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n")
+      endif ()
diff --git a/bin/batch/knl_ctestP.sl.in.cmake b/bin/batch/knl_ctestP.sl.in.cmake
index f985fbb..97bd2ad 100644
--- a/bin/batch/knl_ctestP.sl.in.cmake
+++ b/bin/batch/knl_ctestP.sl.in.cmake
@@ -9,8 +9,8 @@
 #SBATCH --job-name=h5_ctestP
 
 cd @HDF5_BINARY_DIR@
-#run parallel tests except t_cache_image test
-ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out
-
-echo "Done running $CMD"
+echo "Run parallel test command. Test output will be in build/ctestP.out"
+ctest -S ctest_parallel.cmake >& ctestP.out
 
+echo "Done running ctest parallel command."
+touch ctestP.done
diff --git a/bin/batch/knl_ctestS.sl.in.cmake b/bin/batch/knl_ctestS.sl.in.cmake
index af6353b..87c4a48 100644
--- a/bin/batch/knl_ctestS.sl.in.cmake
+++ b/bin/batch/knl_ctestS.sl.in.cmake
@@ -9,9 +9,8 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
-
-echo "Run $CMD. Test output will be in build/ctestS.out"
-$CMD  >& ctestS.out
-echo "Done running $CMD"
+echo "Run command. Test output will be in build/ctestS.out"
+ctest -S ctest_serial.cmake >& ctestS.out
 
+echo "Done running command."
+touch ctestS.done
diff --git a/bin/batch/ray_ctestP.lsf.in.cmake b/bin/batch/ray_ctestP.lsf.in.cmake
index 7067a65..51425b5 100644
--- a/bin/batch/ray_ctestP.lsf.in.cmake
+++ b/bin/batch/ray_ctestP.lsf.in.cmake
@@ -15,6 +15,8 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run parallel test command. Test output will be in build/ctestP.out"
-ctest . -R 'MPI_TEST_' -C Release -T test >& ctestP.out
+ctest -S ctest_parallel.cmake >& ctestP.out
 
 echo "Done running ctest parallel command."
+touch ctestP.done
+~                
diff --git a/bin/batch/ray_ctestS.lsf.in.cmake b/bin/batch/ray_ctestS.lsf.in.cmake
index da20438..7f585b4 100644
--- a/bin/batch/ray_ctestS.lsf.in.cmake
+++ b/bin/batch/ray_ctestS.lsf.in.cmake
@@ -11,8 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -E 'MPI_TEST_' -C Release -j 32 -T test >& ctestS.out
+ctest -S ctest_serial.cmake >& ctestS.out
 
-##$CMD  >& ctestS.out
 echo "Done running command."
-
+touch ctestS.done