ctest: Add --repeat-after-timeout option

Add an option to re-run tests if they timeout.  This will help tolerate
sporadic timeouts on busy machines.

11 files changed, 75 insertions, 0 deletions

diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index 15a90fc..d7f4133 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -24,12 +24,25 @@ run_cmake_command(repeat-until-fail-good
   ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
   )
 
+run_cmake_command(repeat-after-timeout-bad1
+  ${CMAKE_CTEST_COMMAND} --repeat-after-timeout
+  )
+run_cmake_command(repeat-after-timeout-bad2
+  ${CMAKE_CTEST_COMMAND} --repeat-after-timeout foo
+  )
+run_cmake_command(repeat-after-timeout-good
+  ${CMAKE_CTEST_COMMAND} --repeat-after-timeout 2
+  )
+
 run_cmake_command(repeat-until-pass-and-fail
   ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
   )
 run_cmake_command(repeat-until-fail-and-pass
   ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
   )
+run_cmake_command(repeat-until-fail-and-timeout
+  ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-after-timeout 2
+  )
 
 function(run_repeat_until_pass_tests)
   # Use a single build tree for a few tests without cleaning.
@@ -42,6 +55,17 @@ function(run_repeat_until_pass_tests)
 endfunction()
 run_repeat_until_pass_tests()
 
+function(run_repeat_after_timeout_tests)
+  # Use a single build tree for a few tests without cleaning.
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-after-timeout-build)
+  run_cmake(repeat-after-timeout-cmake)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_cmake_command(repeat-after-timeout-ctest
+    ${CMAKE_CTEST_COMMAND} -C Debug --repeat-after-timeout 3
+    )
+endfunction()
+run_repeat_after_timeout_tests()
+
 function(run_repeat_until_fail_tests)
   # Use a single build tree for a few tests without cleaning.
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt
new file mode 100644
index 0000000..aea92b8
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-after-timeout' requires an argument$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt
new file mode 100644
index 0000000..c5db55b
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--repeat-after-timeout' given non-integer value 'foo'$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake
new file mode 100644
index 0000000..4dc9c36
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake
@@ -0,0 +1,15 @@
+enable_testing()
+
+set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
+add_test(NAME initialization
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
+add_test(NAME test1
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-timeout.cmake")
+set_tests_properties(test1 PROPERTIES DEPENDS "initialization" TIMEOUT 2)
+
+add_test(hello ${CMAKE_COMMAND} -E echo hello)
+add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt
new file mode 100644
index 0000000..d0a5487
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt
@@ -0,0 +1,15 @@
+^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-build
+    Start 1: initialization
+1/4 Test #1: initialization ...................   Passed +[0-9.]+ sec
+    Start 2: test1
+2/4 Test #2: test1 ............................\*\*\*Timeout +[0-9.]+ sec
+    Start 2: test1
+    Test #2: test1 ............................   Passed +[0-9.]+ sec
+    Start 3: hello
+3/4 Test #3: hello ............................   Passed +[0-9.]+ sec
+    Start 4: goodbye
+4/4 Test #4: goodbye ..........................   Passed +[0-9.]+ sec
+
+100% tests passed, 0 tests failed out of 4
+
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt
new file mode 100644
index 0000000..a7c4b11
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt
new file mode 100644
index 0000000..15ee3a9
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: At most one '--repeat-\*' option may be used\.$
diff --git a/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake b/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake
new file mode 100644
index 0000000..fbf2ccc
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake
@@ -0,0 +1,14 @@
+# This is run by test test1 in repeat-after-timeout-cmake.cmake with cmake -P.
+# It reads the file TEST_OUTPUT_FILE and increments the number
+# found in the file by 1.  Unless the number is 2, then the
+# code sends out a cmake error causing the test to not timeout only on
+# the second time it is run.
+message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
+file(READ "${TEST_OUTPUT_FILE}" COUNT)
+message("COUNT= ${COUNT}")
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
+if(NOT COUNT EQUAL 2)
+  message("this test times out except on the 2nd run")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
+endif()