ctest: Add --repeat-until-pass option

Add an option to re-run tests if they fail.  This will help tolerate
sporadic failures.

Issue: #17010
Co-Author: Ben Boeckel <ben.boeckel@kitware.com>
Co-Author: Chuck Atkins <chuck.atkins@kitware.com>

13 files changed, 81 insertions, 4 deletions

diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index fd2c97f..15a90fc 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -4,6 +4,16 @@ set(RunCMake_TEST_TIMEOUT 60)
 unset(ENV{CTEST_PARALLEL_LEVEL})
 unset(ENV{CTEST_OUTPUT_ON_FAILURE})
 
+run_cmake_command(repeat-until-pass-bad1
+  ${CMAKE_CTEST_COMMAND} --repeat-until-pass
+  )
+run_cmake_command(repeat-until-pass-bad2
+  ${CMAKE_CTEST_COMMAND} --repeat-until-pass foo
+  )
+run_cmake_command(repeat-until-pass-good
+  ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2
+  )
+
 run_cmake_command(repeat-until-fail-bad1
   ${CMAKE_CTEST_COMMAND} --repeat-until-fail
   )
@@ -14,14 +24,29 @@ run_cmake_command(repeat-until-fail-good
   ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
   )
 
-function(run_repeat_until_fail_tests)
+run_cmake_command(repeat-until-pass-and-fail
+  ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
+  )
+run_cmake_command(repeat-until-fail-and-pass
+  ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
+  )
+
+function(run_repeat_until_pass_tests)
   # Use a single build tree for a few tests without cleaning.
-  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-pass-build)
+  run_cmake(repeat-until-pass-cmake)
   set(RunCMake_TEST_NO_CLEAN 1)
-  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
-  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  run_cmake_command(repeat-until-pass-ctest
+    ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-pass 3
+    )
+endfunction()
+run_repeat_until_pass_tests()
 
+function(run_repeat_until_fail_tests)
+  # Use a single build tree for a few tests without cleaning.
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
   run_cmake(repeat-until-fail-cmake)
+  set(RunCMake_TEST_NO_CLEAN 1)
   run_cmake_command(repeat-until-fail-ctest
     ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3
     )
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt
new file mode 100644
index 0000000..15ee3a9
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: At most one '--repeat-\*' option may be used\.$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt
new file mode 100644
index 0000000..15ee3a9
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: At most one '--repeat-\*' option may be used\.$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt
new file mode 100644
index 0000000..c6afb1d
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-until-pass' requires an argument$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt
new file mode 100644
index 0000000..cc3aed5
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-until-pass' given non-integer value 'foo'$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake
new file mode 100644
index 0000000..d109551
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake
@@ -0,0 +1,15 @@
+enable_testing()
+
+set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
+add_test(NAME initialization
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
+add_test(NAME test1
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-pass.cmake")
+set_tests_properties(test1 PROPERTIES DEPENDS "initialization")
+
+add_test(hello ${CMAKE_COMMAND} -E echo hello)
+add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt
new file mode 100644
index 0000000..3745dc2
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt
@@ -0,0 +1,15 @@
+^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-until-pass-build
+    Start 1: initialization
+1/4 Test #1: initialization ...................   Passed +[0-9.]+ sec
+    Start 2: test1
+2/4 Test #2: test1 ............................\*\*\*Failed +[0-9.]+ sec
+    Start 2: test1
+    Test #2: test1 ............................   Passed +[0-9.]+ sec
+    Start 3: hello
+3/4 Test #3: hello ............................   Passed +[0-9.]+ sec
+    Start 4: goodbye
+4/4 Test #4: goodbye ..........................   Passed +[0-9.]+ sec
+
+100% tests passed, 0 tests failed out of 4
+
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt
new file mode 100644
index 0000000..a7c4b11
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$
diff --git a/Tests/RunCMake/CTestCommandLine/test1-pass.cmake b/Tests/RunCMake/CTestCommandLine/test1-pass.cmake
new file mode 100644
index 0000000..dda8dea
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test1-pass.cmake
@@ -0,0 +1,13 @@
+# This is run by test test1 in repeat-until-pass-cmake.cmake with cmake -P.
+# It reads the file TEST_OUTPUT_FILE and increments the number
+# found in the file by 1.  Unless the number is 2, then the
+# code sends out a cmake error causing the test to pass only on
+# the second time it is run.
+message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
+file(READ "${TEST_OUTPUT_FILE}" COUNT)
+message("COUNT= ${COUNT}")
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
+if(NOT COUNT EQUAL 2)
+  message(FATAL_ERROR "this test passes only on the 2nd run")
+endif()