ctest_test: Add option to REPEAT tests

11 files changed, 86 insertions, 1 deletions

diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
index 6877e6a..84d1d66 100644
--- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
+++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
@@ -1,6 +1,9 @@
 include(RunCTest)
 set(RunCMake_TEST_TIMEOUT 60)
 
+unset(ENV{CTEST_PARALLEL_LEVEL})
+unset(ENV{CTEST_OUTPUT_ON_FAILURE})
+
 set(CASE_CTEST_TEST_ARGS "")
 set(CASE_CTEST_TEST_LOAD "")
 
@@ -71,7 +74,24 @@ add_test(NAME PassingTest COMMAND ${CMAKE_COMMAND} -E echo PassingTestOutput)
 add_test(NAME FailingTest COMMAND ${CMAKE_COMMAND} -E no_such_command)
   ]])
 
-  unset(ENV{CTEST_PARALLEL_LEVEL})
   run_ctest(TestOutputSize)
 endfunction()
 run_TestOutputSize()
+
+run_ctest_test(TestRepeatBad1 REPEAT UNKNOWN:3)
+run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1)
+
+function(run_TestRepeat case)
+  set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion ${ARGN})
+  string(CONCAT CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME testRepeat
+  COMMAND ${CMAKE_COMMAND} -D COUNT_FILE=${CMAKE_CURRENT_BINARY_DIR}/count.cmake
+                           -P "]] "${RunCMake_SOURCE_DIR}/TestRepeat${case}" [[.cmake")
+set_property(TEST testRepeat PROPERTY TIMEOUT 5)
+  ]])
+
+  run_ctest(TestRepeat${case})
+endfunction()
+run_TestRepeat(UntilFail REPEAT UNTIL_FAIL:3)
+run_TestRepeat(UntilPass REPEAT UNTIL_PASS:3)
+run_TestRepeat(AfterTimeout REPEAT AFTER_TIMEOUT:3)
diff --git a/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-stdout.txt b/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-stdout.txt
new file mode 100644
index 0000000..17657c5
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-stdout.txt
@@ -0,0 +1,10 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-build
+    Start 1: testRepeat
+1/1 Test #1: testRepeat .......................\*\*\*Timeout +[0-9.]+ sec
+    Start 1: testRepeat
+    Test #1: testRepeat .......................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout.cmake b/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout.cmake
new file mode 100644
index 0000000..abde4f0
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout.cmake
@@ -0,0 +1,10 @@
+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(NOT COUNT EQUAL 2)
+  message("this test times out except on the 2nd run")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
+endif()
diff --git a/Tests/RunCMake/ctest_test/TestRepeatBad1-result.txt b/Tests/RunCMake/ctest_test/TestRepeatBad1-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatBad1-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_test/TestRepeatBad1-stderr.txt b/Tests/RunCMake/ctest_test/TestRepeatBad1-stderr.txt
new file mode 100644
index 0000000..37cffbf
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatBad1-stderr.txt
@@ -0,0 +1 @@
+Repeat option invalid value: UNKNOWN:3
diff --git a/Tests/RunCMake/ctest_test/TestRepeatBad2-result.txt b/Tests/RunCMake/ctest_test/TestRepeatBad2-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatBad2-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_test/TestRepeatBad2-stderr.txt b/Tests/RunCMake/ctest_test/TestRepeatBad2-stderr.txt
new file mode 100644
index 0000000..ca5cef7
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatBad2-stderr.txt
@@ -0,0 +1 @@
+Repeat option invalid value: UNTIL_FAIL:-1
diff --git a/Tests/RunCMake/ctest_test/TestRepeatUntilFail-stdout.txt b/Tests/RunCMake/ctest_test/TestRepeatUntilFail-stdout.txt
new file mode 100644
index 0000000..5f91a67
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatUntilFail-stdout.txt
@@ -0,0 +1,13 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestRepeatUntilFail-build
+    Start 1: testRepeat
+    Test #1: testRepeat .......................   Passed +[0-9.]+ sec
+    Start 1: testRepeat
+    Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec
++
+0% tests passed, 1 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests FAILED:
+[	 ]+1 - testRepeat \(Failed\)$
diff --git a/Tests/RunCMake/ctest_test/TestRepeatUntilFail.cmake b/Tests/RunCMake/ctest_test/TestRepeatUntilFail.cmake
new file mode 100644
index 0000000..5eb0d8a
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatUntilFail.cmake
@@ -0,0 +1,9 @@
+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(COUNT EQUAL 2)
+  message(FATAL_ERROR "this test fails on the 2nd run")
+endif()
diff --git a/Tests/RunCMake/ctest_test/TestRepeatUntilPass-stdout.txt b/Tests/RunCMake/ctest_test/TestRepeatUntilPass-stdout.txt
new file mode 100644
index 0000000..bc6939a
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatUntilPass-stdout.txt
@@ -0,0 +1,10 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestRepeatUntilPass-build
+    Start 1: testRepeat
+1/1 Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec
+    Start 1: testRepeat
+    Test #1: testRepeat .......................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/TestRepeatUntilPass.cmake b/Tests/RunCMake/ctest_test/TestRepeatUntilPass.cmake
new file mode 100644
index 0000000..0662522
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestRepeatUntilPass.cmake
@@ -0,0 +1,9 @@
+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(NOT COUNT EQUAL 2)
+  message(FATAL_ERROR "this test passes only on the 2nd run")
+endif()