From 0187e522448c8fe92c15d4a983afac36950d1d59 Mon Sep 17 00:00:00 2001 From: Brad King Date: Tue, 29 Oct 2019 13:41:52 -0400 Subject: cmCTestRunTest: Use inline member initializers --- Source/CTest/cmCTestRunTest.cxx | 3 --- Source/CTest/cmCTestRunTest.h | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 7f7f736..2e7836a 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -34,9 +34,6 @@ cmCTestRunTest::cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler) this->TestResult.Status = cmCTestTestHandler::NOT_RUN; this->TestResult.TestCount = 0; this->TestResult.Properties = nullptr; - this->NumberOfRunsLeft = 1; // default to 1 run of the test - this->RunUntilFail = false; // default to run the test once - this->RunAgain = false; // default to not having to run again } void cmCTestRunTest::CheckOutput(std::string const& line) diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index 085a6b8..7b202d1 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -129,9 +129,9 @@ private: std::vector>> AllocatedHardware; - bool RunUntilFail; - int NumberOfRunsLeft; - bool RunAgain; + bool RunUntilFail = false; // default to run the test once + int NumberOfRunsLeft = 1; // default to 1 run of the test + bool RunAgain = false; // default to not having to run again size_t TotalNumberOfTests; }; -- cgit v0.12 From 80c2c9d14cf1c1a8f162e119bd00d5f483a94af2 Mon Sep 17 00:00:00 2001 From: Brad King Date: Tue, 29 Oct 2019 14:45:21 -0400 Subject: ctest: Add --repeat-until-pass option Add an option to re-run tests if they fail. This will help tolerate sporadic failures. Issue: #17010 Co-Author: Ben Boeckel Co-Author: Chuck Atkins --- Help/manual/ctest.1.rst | 5 ++++ Help/release/dev/ctest-repeat-until-pass.rst | 5 ++++ Source/CTest/cmCTestMultiProcessHandler.cxx | 4 +-- Source/CTest/cmCTestRunTest.cxx | 14 ++++++--- Source/CTest/cmCTestRunTest.h | 15 ++++++---- Source/cmCTest.cxx | 35 +++++++++++++++++++--- Source/cmCTest.h | 9 ++++-- Source/ctest.cxx | 5 ++-- Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 33 +++++++++++++++++--- .../repeat-until-fail-and-pass-result.txt | 1 + .../repeat-until-fail-and-pass-stderr.txt | 1 + .../repeat-until-pass-and-fail-result.txt | 1 + .../repeat-until-pass-and-fail-stderr.txt | 1 + .../repeat-until-pass-bad1-result.txt | 1 + .../repeat-until-pass-bad1-stderr.txt | 1 + .../repeat-until-pass-bad2-result.txt | 1 + .../repeat-until-pass-bad2-stderr.txt | 1 + .../CTestCommandLine/repeat-until-pass-cmake.cmake | 15 ++++++++++ .../repeat-until-pass-ctest-stdout.txt | 15 ++++++++++ .../repeat-until-pass-good-stderr.txt | 1 + Tests/RunCMake/CTestCommandLine/test1-pass.cmake | 13 ++++++++ 21 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 Help/release/dev/ctest-repeat-until-pass.rst create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/test1-pass.cmake diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index a18d43f..031b9a7 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -266,6 +266,11 @@ Options This is useful in finding sporadic failures in test cases. +``--repeat-until-pass `` + Allow each test to run up to ```` times in order to pass. + + This is useful in tolerating sporadic failures in test cases. + ``--max-width `` Set the max width for a test name to output. diff --git a/Help/release/dev/ctest-repeat-until-pass.rst b/Help/release/dev/ctest-repeat-until-pass.rst new file mode 100644 index 0000000..003cdf2 --- /dev/null +++ b/Help/release/dev/ctest-repeat-until-pass.rst @@ -0,0 +1,5 @@ +ctest-repeat-until-pass +----------------------- + +* The :manual:`ctest(1)` tool learned a new ``--repeat-until-pass `` + option to help tolerate sporadic test failures. diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 7e8d548..4812f30 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -171,8 +171,8 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test) this->RunningCount += GetProcessorsUsed(test); cmCTestRunTest* testRun = new cmCTestRunTest(*this); - if (this->CTest->GetRepeatUntilFail()) { - testRun->SetRunUntilFailOn(); + if (this->CTest->GetRerunMode() != cmCTest::Rerun::Never) { + testRun->SetRerunMode(this->CTest->GetRerunMode()); testRun->SetNumberOfRuns(this->CTest->GetTestRepeat()); } testRun->SetIndex(test); diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 2e7836a..ce9e13b 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -340,10 +340,12 @@ bool cmCTestRunTest::NeedsToRerun() return false; } // if number of runs left is not 0, and we are running until - // we find a failed test, then return true so the test can be + // we find a failed (or passed) test, then return true so the test can be // restarted - if (this->RunUntilFail && - this->TestResult.Status == cmCTestTestHandler::COMPLETED) { + if ((this->RerunMode == cmCTest::Rerun::UntilFail && + this->TestResult.Status == cmCTestTestHandler::COMPLETED) || + (this->RerunMode == cmCTest::Rerun::UntilPass && + this->TestResult.Status != cmCTestTestHandler::COMPLETED)) { this->RunAgain = true; return true; } @@ -743,7 +745,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) // then it will never print out the completed / total, same would // got for run until pass. Trick is when this is called we don't // yet know if we are passing or failing. - if (this->NumberOfRunsLeft == 1 || this->CTest->GetTestProgressOutput()) { + if ((this->RerunMode != cmCTest::Rerun::UntilPass && + this->NumberOfRunsLeft == 1) || + (this->RerunMode == cmCTest::Rerun::UntilPass && + this->NumberOfRunsLeft == this->NumberOfRunsTotal) || + this->CTest->GetTestProgressOutput()) { outputStream << std::setw(getNumWidth(total)) << completed << "/"; outputStream << std::setw(getNumWidth(total)) << total << " "; } diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index 7b202d1..881cbb6 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -13,13 +13,12 @@ #include +#include "cmCTest.h" #include "cmCTestMultiProcessHandler.h" #include "cmCTestTestHandler.h" #include "cmDuration.h" #include "cmProcess.h" -class cmCTest; - /** \class cmRunTest * \brief represents a single test to be run * @@ -30,8 +29,13 @@ class cmCTestRunTest public: explicit cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler); - void SetNumberOfRuns(int n) { this->NumberOfRunsLeft = n; } - void SetRunUntilFailOn() { this->RunUntilFail = true; } + void SetNumberOfRuns(int n) + { + this->NumberOfRunsLeft = n; + this->NumberOfRunsTotal = n; + } + + void SetRerunMode(cmCTest::Rerun r) { this->RerunMode = r; } void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties* prop) { this->TestProperties = prop; @@ -129,8 +133,9 @@ private: std::vector>> AllocatedHardware; - bool RunUntilFail = false; // default to run the test once + cmCTest::Rerun RerunMode = cmCTest::Rerun::Never; int NumberOfRunsLeft = 1; // default to 1 run of the test + int NumberOfRunsTotal = 1; // default to 1 run of the test bool RunAgain = false; // default to not having to run again size_t TotalNumberOfTests; }; diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 10b7646..7276d98 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -84,7 +84,7 @@ struct cmCTest::Private }; int RepeatTests = 1; // default to run each test once - bool RepeatUntilFail = false; + cmCTest::Rerun RerunMode = cmCTest::Rerun::Never; std::string ConfigType; std::string ScheduleType; std::chrono::system_clock::time_point StopTime; @@ -1839,11 +1839,16 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, this->SetParallelLevel(plevel); this->Impl->ParallelLevelSetInCli = true; } + if (this->CheckArgument(arg, "--repeat-until-fail")) { if (i >= args.size() - 1) { errormsg = "'--repeat-until-fail' requires an argument"; return false; } + if (this->Impl->RerunMode != cmCTest::Rerun::Never) { + errormsg = "At most one '--repeat-*' option may be used."; + return false; + } i++; long repeat = 1; if (!cmStrToLong(args[i], &repeat)) { @@ -1853,7 +1858,29 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, } this->Impl->RepeatTests = static_cast(repeat); if (repeat > 1) { - this->Impl->RepeatUntilFail = true; + this->Impl->RerunMode = cmCTest::Rerun::UntilFail; + } + } + + if (this->CheckArgument(arg, "--repeat-until-pass")) { + if (i >= args.size() - 1) { + errormsg = "'--repeat-until-pass' requires an argument"; + return false; + } + if (this->Impl->RerunMode != cmCTest::Rerun::Never) { + errormsg = "At most one '--repeat-*' option may be used."; + return false; + } + i++; + long repeat = 1; + if (!cmStrToLong(args[i], &repeat)) { + errormsg = + "'--repeat-until-pass' given non-integer value '" + args[i] + "'"; + return false; + } + this->Impl->RepeatTests = static_cast(repeat); + if (repeat > 1) { + this->Impl->RerunMode = cmCTest::Rerun::UntilPass; } } @@ -2852,9 +2879,9 @@ int cmCTest::GetTestRepeat() const return this->Impl->RepeatTests; } -bool cmCTest::GetRepeatUntilFail() const +cmCTest::Rerun cmCTest::GetRerunMode() const { - return this->Impl->RepeatUntilFail; + return this->Impl->RerunMode; } void cmCTest::SetBuildID(const std::string& id) diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 82a6f4c..c6b8928 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -433,8 +433,13 @@ public: /** Return the number of times a test should be run */ int GetTestRepeat() const; - /** Return true if test should run until fail */ - bool GetRepeatUntilFail() const; + enum class Rerun + { + Never, + UntilFail, + UntilPass, + }; + Rerun GetRerunMode() const; void GenerateSubprojectsOutput(cmXMLWriter& xml); std::vector GetLabelsForSubprojects(); diff --git a/Source/ctest.cxx b/Source/ctest.cxx index 91ee598..f716d7a 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -99,8 +99,9 @@ static const char* cmDocumentationOptions[][2] = { { "-U, --union", "Take the Union of -I and -R" }, { "--rerun-failed", "Run only the tests that failed previously" }, { "--repeat-until-fail ", - "Require each test to run " - "times without failing in order to pass" }, + "Require each test to run times without failing in order to pass" }, + { "--repeat-until-pass ", + "Allow each test to run up to times in order to pass" }, { "--max-width ", "Set the max width for a test name to output" }, { "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." }, { "--hardware-spec-file ", "Set the hardware spec file to use." }, diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index fd2c97f..15a90fc 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -4,6 +4,16 @@ set(RunCMake_TEST_TIMEOUT 60) unset(ENV{CTEST_PARALLEL_LEVEL}) unset(ENV{CTEST_OUTPUT_ON_FAILURE}) +run_cmake_command(repeat-until-pass-bad1 + ${CMAKE_CTEST_COMMAND} --repeat-until-pass + ) +run_cmake_command(repeat-until-pass-bad2 + ${CMAKE_CTEST_COMMAND} --repeat-until-pass foo + ) +run_cmake_command(repeat-until-pass-good + ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 + ) + run_cmake_command(repeat-until-fail-bad1 ${CMAKE_CTEST_COMMAND} --repeat-until-fail ) @@ -14,14 +24,29 @@ run_cmake_command(repeat-until-fail-good ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 ) -function(run_repeat_until_fail_tests) +run_cmake_command(repeat-until-pass-and-fail + ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2 + ) +run_cmake_command(repeat-until-fail-and-pass + ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2 + ) + +function(run_repeat_until_pass_tests) # Use a single build tree for a few tests without cleaning. - set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-pass-build) + run_cmake(repeat-until-pass-cmake) set(RunCMake_TEST_NO_CLEAN 1) - file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") - file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + run_cmake_command(repeat-until-pass-ctest + ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-pass 3 + ) +endfunction() +run_repeat_until_pass_tests() +function(run_repeat_until_fail_tests) + # Use a single build tree for a few tests without cleaning. + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build) run_cmake(repeat-until-fail-cmake) + set(RunCMake_TEST_NO_CLEAN 1) run_cmake_command(repeat-until-fail-ctest ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3 ) diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt new file mode 100644 index 0000000..15ee3a9 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt @@ -0,0 +1 @@ +^CMake Error: At most one '--repeat-\*' option may be used\.$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt new file mode 100644 index 0000000..15ee3a9 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt @@ -0,0 +1 @@ +^CMake Error: At most one '--repeat-\*' option may be used\.$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt new file mode 100644 index 0000000..c6afb1d --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-until-pass' requires an argument$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt new file mode 100644 index 0000000..cc3aed5 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-until-pass' given non-integer value 'foo'$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake new file mode 100644 index 0000000..d109551 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake @@ -0,0 +1,15 @@ +enable_testing() + +set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt") +add_test(NAME initialization + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake") +add_test(NAME test1 + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-pass.cmake") +set_tests_properties(test1 PROPERTIES DEPENDS "initialization") + +add_test(hello ${CMAKE_COMMAND} -E echo hello) +add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye) diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt new file mode 100644 index 0000000..3745dc2 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt @@ -0,0 +1,15 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-until-pass-build + Start 1: initialization +1/4 Test #1: initialization ................... Passed +[0-9.]+ sec + Start 2: test1 +2/4 Test #2: test1 ............................\*\*\*Failed +[0-9.]+ sec + Start 2: test1 + Test #2: test1 ............................ Passed +[0-9.]+ sec + Start 3: hello +3/4 Test #3: hello ............................ Passed +[0-9.]+ sec + Start 4: goodbye +4/4 Test #4: goodbye .......................... Passed +[0-9.]+ sec + +100% tests passed, 0 tests failed out of 4 + +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt new file mode 100644 index 0000000..a7c4b11 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$ diff --git a/Tests/RunCMake/CTestCommandLine/test1-pass.cmake b/Tests/RunCMake/CTestCommandLine/test1-pass.cmake new file mode 100644 index 0000000..dda8dea --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test1-pass.cmake @@ -0,0 +1,13 @@ +# This is run by test test1 in repeat-until-pass-cmake.cmake with cmake -P. +# It reads the file TEST_OUTPUT_FILE and increments the number +# found in the file by 1. Unless the number is 2, then the +# code sends out a cmake error causing the test to pass only on +# the second time it is run. +message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}") +file(READ "${TEST_OUTPUT_FILE}" COUNT) +message("COUNT= ${COUNT}") +math(EXPR COUNT "${COUNT} + 1") +file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}") +if(NOT COUNT EQUAL 2) + message(FATAL_ERROR "this test passes only on the 2nd run") +endif() -- cgit v0.12 From 39ac8b4eb5c5ea99cf1053bd37e76d811f5122fc Mon Sep 17 00:00:00 2001 From: Brad King Date: Tue, 29 Oct 2019 14:21:38 -0400 Subject: ctest: Add --repeat-after-timeout option Add an option to re-run tests if they timeout. This will help tolerate sporadic timeouts on busy machines. --- Help/manual/ctest.1.rst | 8 ++++++++ Help/release/dev/ctest-repeat-until-pass.rst | 5 +++-- Source/CTest/cmCTestRunTest.cxx | 13 +++++++----- Source/cmCTest.cxx | 22 ++++++++++++++++++++ Source/cmCTest.h | 1 + Source/ctest.cxx | 2 ++ Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 24 ++++++++++++++++++++++ .../repeat-after-timeout-bad1-result.txt | 1 + .../repeat-after-timeout-bad1-stderr.txt | 1 + .../repeat-after-timeout-bad2-result.txt | 1 + .../repeat-after-timeout-bad2-stderr.txt | 1 + .../repeat-after-timeout-cmake.cmake | 15 ++++++++++++++ .../repeat-after-timeout-ctest-stdout.txt | 15 ++++++++++++++ .../repeat-after-timeout-good-stderr.txt | 1 + .../repeat-until-fail-and-timeout-result.txt | 1 + .../repeat-until-fail-and-timeout-stderr.txt | 1 + .../RunCMake/CTestCommandLine/test1-timeout.cmake | 14 +++++++++++++ 17 files changed, 119 insertions(+), 7 deletions(-) create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/test1-timeout.cmake diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 031b9a7..0097cee 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -268,9 +268,17 @@ Options ``--repeat-until-pass `` Allow each test to run up to ```` times in order to pass. + Repeats tests if they fail for any reason. This is useful in tolerating sporadic failures in test cases. +``--repeat-after-timeout `` + Allow each test to run up to ```` times in order to pass. + Repeats tests only if they timeout. + + This is useful in tolerating sporadic timeouts in test cases + on busy machines. + ``--max-width `` Set the max width for a test name to output. diff --git a/Help/release/dev/ctest-repeat-until-pass.rst b/Help/release/dev/ctest-repeat-until-pass.rst index 003cdf2..d177247 100644 --- a/Help/release/dev/ctest-repeat-until-pass.rst +++ b/Help/release/dev/ctest-repeat-until-pass.rst @@ -1,5 +1,6 @@ ctest-repeat-until-pass ----------------------- -* The :manual:`ctest(1)` tool learned a new ``--repeat-until-pass `` - option to help tolerate sporadic test failures. +* The :manual:`ctest(1)` tool learned new ``--repeat-until-pass `` + and ``--repeat-after-timeout `` options to help tolerate sporadic + test failures. diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index ce9e13b..ba14653 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -345,7 +345,9 @@ bool cmCTestRunTest::NeedsToRerun() if ((this->RerunMode == cmCTest::Rerun::UntilFail && this->TestResult.Status == cmCTestTestHandler::COMPLETED) || (this->RerunMode == cmCTest::Rerun::UntilPass && - this->TestResult.Status != cmCTestTestHandler::COMPLETED)) { + this->TestResult.Status != cmCTestTestHandler::COMPLETED) || + (this->RerunMode == cmCTest::Rerun::AfterTimeout && + this->TestResult.Status == cmCTestTestHandler::TIMEOUT)) { this->RunAgain = true; return true; } @@ -745,10 +747,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) // then it will never print out the completed / total, same would // got for run until pass. Trick is when this is called we don't // yet know if we are passing or failing. - if ((this->RerunMode != cmCTest::Rerun::UntilPass && - this->NumberOfRunsLeft == 1) || - (this->RerunMode == cmCTest::Rerun::UntilPass && - this->NumberOfRunsLeft == this->NumberOfRunsTotal) || + bool const progressOnLast = + (this->RerunMode != cmCTest::Rerun::UntilPass && + this->RerunMode != cmCTest::Rerun::AfterTimeout); + if ((progressOnLast && this->NumberOfRunsLeft == 1) || + (!progressOnLast && this->NumberOfRunsLeft == this->NumberOfRunsTotal) || this->CTest->GetTestProgressOutput()) { outputStream << std::setw(getNumWidth(total)) << completed << "/"; outputStream << std::setw(getNumWidth(total)) << total << " "; diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 7276d98..20445b0 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -1884,6 +1884,28 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, } } + if (this->CheckArgument(arg, "--repeat-after-timeout")) { + if (i >= args.size() - 1) { + errormsg = "'--repeat-after-timeout' requires an argument"; + return false; + } + if (this->Impl->RerunMode != cmCTest::Rerun::Never) { + errormsg = "At most one '--repeat-*' option may be used."; + return false; + } + i++; + long repeat = 1; + if (!cmStrToLong(args[i], &repeat)) { + errormsg = + "'--repeat-after-timeout' given non-integer value '" + args[i] + "'"; + return false; + } + this->Impl->RepeatTests = static_cast(repeat); + if (repeat > 1) { + this->Impl->RerunMode = cmCTest::Rerun::AfterTimeout; + } + } + if (this->CheckArgument(arg, "--test-load") && i < args.size() - 1) { i++; unsigned long load; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index c6b8928..bef0f8d 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -438,6 +438,7 @@ public: Never, UntilFail, UntilPass, + AfterTimeout, }; Rerun GetRerunMode() const; diff --git a/Source/ctest.cxx b/Source/ctest.cxx index f716d7a..2659e30 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -102,6 +102,8 @@ static const char* cmDocumentationOptions[][2] = { "Require each test to run times without failing in order to pass" }, { "--repeat-until-pass ", "Allow each test to run up to times in order to pass" }, + { "--repeat-after-timeout ", + "Allow each test to run up to times if it times out" }, { "--max-width ", "Set the max width for a test name to output" }, { "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." }, { "--hardware-spec-file ", "Set the hardware spec file to use." }, diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 15a90fc..d7f4133 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -24,12 +24,25 @@ run_cmake_command(repeat-until-fail-good ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 ) +run_cmake_command(repeat-after-timeout-bad1 + ${CMAKE_CTEST_COMMAND} --repeat-after-timeout + ) +run_cmake_command(repeat-after-timeout-bad2 + ${CMAKE_CTEST_COMMAND} --repeat-after-timeout foo + ) +run_cmake_command(repeat-after-timeout-good + ${CMAKE_CTEST_COMMAND} --repeat-after-timeout 2 + ) + run_cmake_command(repeat-until-pass-and-fail ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2 ) run_cmake_command(repeat-until-fail-and-pass ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2 ) +run_cmake_command(repeat-until-fail-and-timeout + ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-after-timeout 2 + ) function(run_repeat_until_pass_tests) # Use a single build tree for a few tests without cleaning. @@ -42,6 +55,17 @@ function(run_repeat_until_pass_tests) endfunction() run_repeat_until_pass_tests() +function(run_repeat_after_timeout_tests) + # Use a single build tree for a few tests without cleaning. + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-after-timeout-build) + run_cmake(repeat-after-timeout-cmake) + set(RunCMake_TEST_NO_CLEAN 1) + run_cmake_command(repeat-after-timeout-ctest + ${CMAKE_CTEST_COMMAND} -C Debug --repeat-after-timeout 3 + ) +endfunction() +run_repeat_after_timeout_tests() + function(run_repeat_until_fail_tests) # Use a single build tree for a few tests without cleaning. set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build) diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt new file mode 100644 index 0000000..aea92b8 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-after-timeout' requires an argument$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt new file mode 100644 index 0000000..c5db55b --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--repeat-after-timeout' given non-integer value 'foo'$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake new file mode 100644 index 0000000..4dc9c36 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake @@ -0,0 +1,15 @@ +enable_testing() + +set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt") +add_test(NAME initialization + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake") +add_test(NAME test1 + COMMAND ${CMAKE_COMMAND} + "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-timeout.cmake") +set_tests_properties(test1 PROPERTIES DEPENDS "initialization" TIMEOUT 2) + +add_test(hello ${CMAKE_COMMAND} -E echo hello) +add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye) diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt new file mode 100644 index 0000000..d0a5487 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt @@ -0,0 +1,15 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-build + Start 1: initialization +1/4 Test #1: initialization ................... Passed +[0-9.]+ sec + Start 2: test1 +2/4 Test #2: test1 ............................\*\*\*Timeout +[0-9.]+ sec + Start 2: test1 + Test #2: test1 ............................ Passed +[0-9.]+ sec + Start 3: hello +3/4 Test #3: hello ............................ Passed +[0-9.]+ sec + Start 4: goodbye +4/4 Test #4: goodbye .......................... Passed +[0-9.]+ sec + +100% tests passed, 0 tests failed out of 4 + +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt new file mode 100644 index 0000000..a7c4b11 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$ diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt new file mode 100644 index 0000000..15ee3a9 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt @@ -0,0 +1 @@ +^CMake Error: At most one '--repeat-\*' option may be used\.$ diff --git a/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake b/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake new file mode 100644 index 0000000..fbf2ccc --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test1-timeout.cmake @@ -0,0 +1,14 @@ +# This is run by test test1 in repeat-after-timeout-cmake.cmake with cmake -P. +# It reads the file TEST_OUTPUT_FILE and increments the number +# found in the file by 1. Unless the number is 2, then the +# code sends out a cmake error causing the test to not timeout only on +# the second time it is run. +message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}") +file(READ "${TEST_OUTPUT_FILE}" COUNT) +message("COUNT= ${COUNT}") +math(EXPR COUNT "${COUNT} + 1") +file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}") +if(NOT COUNT EQUAL 2) + message("this test times out except on the 2nd run") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10) +endif() -- cgit v0.12