From ed65b3e984ccce9b915a5a71e5a8d13b892b63d8 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 7 Nov 2019 11:21:03 -0500 Subject: CTest: Rename internal APIs for --repeat options Replace use of the term "rerun" with "repeat" to match the public names. --- Source/CTest/cmCTestMultiProcessHandler.cxx | 6 +++--- Source/CTest/cmCTestRunTest.cxx | 14 +++++++------- Source/CTest/cmCTestRunTest.h | 6 +++--- Source/cmCTest.cxx | 30 ++++++++++++++--------------- Source/cmCTest.h | 6 +++--- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 9ab3c2e..992a2ef 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -171,9 +171,9 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test) this->RunningCount += GetProcessorsUsed(test); cmCTestRunTest* testRun = new cmCTestRunTest(*this); - if (this->CTest->GetRerunMode() != cmCTest::Rerun::Never) { - testRun->SetRerunMode(this->CTest->GetRerunMode()); - testRun->SetNumberOfRuns(this->CTest->GetTestRepeat()); + if (this->CTest->GetRepeatMode() != cmCTest::Repeat::Never) { + testRun->SetRepeatMode(this->CTest->GetRepeatMode()); + testRun->SetNumberOfRuns(this->CTest->GetRepeatCount()); } testRun->SetIndex(test); testRun->SetTestProperties(this->Properties[test]); diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 317f0b4..cc5de43 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -307,7 +307,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) } // If the test does not need to rerun push the current TestResult onto the // TestHandler vector - if (!this->NeedsToRerun()) { + if (!this->NeedsToRepeat()) { this->TestHandler->TestResults.push_back(this->TestResult); } this->TestProcess.reset(); @@ -333,7 +333,7 @@ bool cmCTestRunTest::StartAgain(size_t completed) return true; } -bool cmCTestRunTest::NeedsToRerun() +bool cmCTestRunTest::NeedsToRepeat() { this->NumberOfRunsLeft--; if (this->NumberOfRunsLeft == 0) { @@ -342,11 +342,11 @@ bool cmCTestRunTest::NeedsToRerun() // if number of runs left is not 0, and we are running until // we find a failed (or passed) test, then return true so the test can be // restarted - if ((this->RerunMode == cmCTest::Rerun::UntilFail && + if ((this->RepeatMode == cmCTest::Repeat::UntilFail && this->TestResult.Status == cmCTestTestHandler::COMPLETED) || - (this->RerunMode == cmCTest::Rerun::UntilPass && + (this->RepeatMode == cmCTest::Repeat::UntilPass && this->TestResult.Status != cmCTestTestHandler::COMPLETED) || - (this->RerunMode == cmCTest::Rerun::AfterTimeout && + (this->RepeatMode == cmCTest::Repeat::AfterTimeout && this->TestResult.Status == cmCTestTestHandler::TIMEOUT)) { this->RunAgain = true; return true; @@ -748,8 +748,8 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) // got for run until pass. Trick is when this is called we don't // yet know if we are passing or failing. bool const progressOnLast = - (this->RerunMode != cmCTest::Rerun::UntilPass && - this->RerunMode != cmCTest::Rerun::AfterTimeout); + (this->RepeatMode != cmCTest::Repeat::UntilPass && + this->RepeatMode != cmCTest::Repeat::AfterTimeout); if ((progressOnLast && this->NumberOfRunsLeft == 1) || (!progressOnLast && this->NumberOfRunsLeft == this->NumberOfRunsTotal) || this->CTest->GetTestProgressOutput()) { diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index d5e83d9..7eeaebd 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -35,7 +35,7 @@ public: this->NumberOfRunsTotal = n; } - void SetRerunMode(cmCTest::Rerun r) { this->RerunMode = r; } + void SetRepeatMode(cmCTest::Repeat r) { this->RepeatMode = r; } void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties* prop) { this->TestProperties = prop; @@ -102,7 +102,7 @@ public: } private: - bool NeedsToRerun(); + bool NeedsToRepeat(); void DartProcessing(); void ExeNotFound(std::string exe); bool ForkProcess(cmDuration testTimeOut, bool explicitTimeout, @@ -136,7 +136,7 @@ private: std::vector>> AllocatedResources; - cmCTest::Rerun RerunMode = cmCTest::Rerun::Never; + cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never; int NumberOfRunsLeft = 1; // default to 1 run of the test int NumberOfRunsTotal = 1; // default to 1 run of the test bool RunAgain = false; // default to not having to run again diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 34e9479..a49866c 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -83,8 +83,8 @@ struct cmCTest::Private std::string Name; }; - int RepeatTests = 1; // default to run each test once - cmCTest::Rerun RerunMode = cmCTest::Rerun::Never; + int RepeatCount = 1; // default to run each test once + cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never; std::string ConfigType; std::string ScheduleType; std::chrono::system_clock::time_point StopTime; @@ -1845,7 +1845,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, errormsg = "'--repeat-until-fail' requires an argument"; return false; } - if (this->Impl->RerunMode != cmCTest::Rerun::Never) { + if (this->Impl->RepeatMode != cmCTest::Repeat::Never) { errormsg = "At most one '--repeat-*' option may be used."; return false; } @@ -1856,9 +1856,9 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, "'--repeat-until-fail' given non-integer value '" + args[i] + "'"; return false; } - this->Impl->RepeatTests = static_cast(repeat); + this->Impl->RepeatCount = static_cast(repeat); if (repeat > 1) { - this->Impl->RerunMode = cmCTest::Rerun::UntilFail; + this->Impl->RepeatMode = cmCTest::Repeat::UntilFail; } } @@ -1867,7 +1867,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, errormsg = "'--repeat-until-pass' requires an argument"; return false; } - if (this->Impl->RerunMode != cmCTest::Rerun::Never) { + if (this->Impl->RepeatMode != cmCTest::Repeat::Never) { errormsg = "At most one '--repeat-*' option may be used."; return false; } @@ -1878,9 +1878,9 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, "'--repeat-until-pass' given non-integer value '" + args[i] + "'"; return false; } - this->Impl->RepeatTests = static_cast(repeat); + this->Impl->RepeatCount = static_cast(repeat); if (repeat > 1) { - this->Impl->RerunMode = cmCTest::Rerun::UntilPass; + this->Impl->RepeatMode = cmCTest::Repeat::UntilPass; } } @@ -1889,7 +1889,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, errormsg = "'--repeat-after-timeout' requires an argument"; return false; } - if (this->Impl->RerunMode != cmCTest::Rerun::Never) { + if (this->Impl->RepeatMode != cmCTest::Repeat::Never) { errormsg = "At most one '--repeat-*' option may be used."; return false; } @@ -1900,9 +1900,9 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, "'--repeat-after-timeout' given non-integer value '" + args[i] + "'"; return false; } - this->Impl->RepeatTests = static_cast(repeat); + this->Impl->RepeatCount = static_cast(repeat); if (repeat > 1) { - this->Impl->RerunMode = cmCTest::Rerun::AfterTimeout; + this->Impl->RepeatMode = cmCTest::Repeat::AfterTimeout; } } @@ -2896,14 +2896,14 @@ const std::map& cmCTest::GetDefinitions() const return this->Impl->Definitions; } -int cmCTest::GetTestRepeat() const +int cmCTest::GetRepeatCount() const { - return this->Impl->RepeatTests; + return this->Impl->RepeatCount; } -cmCTest::Rerun cmCTest::GetRerunMode() const +cmCTest::Repeat cmCTest::GetRepeatMode() const { - return this->Impl->RerunMode; + return this->Impl->RepeatMode; } void cmCTest::SetBuildID(const std::string& id) diff --git a/Source/cmCTest.h b/Source/cmCTest.h index bef0f8d..e0ae100 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -431,16 +431,16 @@ public: const std::map& GetDefinitions() const; /** Return the number of times a test should be run */ - int GetTestRepeat() const; + int GetRepeatCount() const; - enum class Rerun + enum class Repeat { Never, UntilFail, UntilPass, AfterTimeout, }; - Rerun GetRerunMode() const; + Repeat GetRepeatMode() const; void GenerateSubprojectsOutput(cmXMLWriter& xml); std::vector GetLabelsForSubprojects(); -- cgit v0.12 From 42d5d8f425e655a07d3dcb0802d90cb5c167b164 Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 7 Nov 2019 11:10:06 -0500 Subject: cmCTestMultiProcessHandler: Hold repeat mode as a member --- Source/CTest/cmCTestMultiProcessHandler.cxx | 6 +++--- Source/CTest/cmCTestMultiProcessHandler.h | 10 +++++++++- Source/CTest/cmCTestTestHandler.cxx | 2 ++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 992a2ef..04abf98 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -171,9 +171,9 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test) this->RunningCount += GetProcessorsUsed(test); cmCTestRunTest* testRun = new cmCTestRunTest(*this); - if (this->CTest->GetRepeatMode() != cmCTest::Repeat::Never) { - testRun->SetRepeatMode(this->CTest->GetRepeatMode()); - testRun->SetNumberOfRuns(this->CTest->GetRepeatCount()); + if (this->RepeatMode != cmCTest::Repeat::Never) { + testRun->SetRepeatMode(this->RepeatMode); + testRun->SetNumberOfRuns(this->RepeatCount); } testRun->SetIndex(test); testRun->SetTestProperties(this->Properties[test]); diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h index 1db4bfd..4837401 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.h +++ b/Source/CTest/cmCTestMultiProcessHandler.h @@ -14,11 +14,11 @@ #include "cm_uv.h" +#include "cmCTest.h" #include "cmCTestResourceAllocator.h" #include "cmCTestTestHandler.h" #include "cmUVHandlePtr.h" -class cmCTest; struct cmCTestBinPackerAllocation; class cmCTestResourceSpec; class cmCTestRunTest; @@ -85,6 +85,12 @@ public: cmCTestTestHandler* GetTestHandler() { return this->TestHandler; } + void SetRepeatMode(cmCTest::Repeat mode, int count) + { + this->RepeatMode = mode; + this->RepeatCount = count; + } + void SetQuiet(bool b) { this->Quiet = b; } void InitResourceAllocator(const cmCTestResourceSpec& spec) @@ -179,6 +185,8 @@ protected: cmCTestTestHandler* TestHandler; cmCTest* CTest; bool HasCycles; + cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never; + int RepeatCount = 1; bool Quiet; bool SerialTestRunning; }; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 8e3ac22..9a06083 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -1231,6 +1231,8 @@ void cmCTestTestHandler::ProcessDirectory(std::vector& passed, parallel->SetCTest(this->CTest); parallel->SetParallelLevel(this->CTest->GetParallelLevel()); parallel->SetTestHandler(this); + parallel->SetRepeatMode(this->CTest->GetRepeatMode(), + this->CTest->GetRepeatCount()); parallel->SetQuiet(this->Quiet); if (this->TestLoad > 0) { parallel->SetTestLoad(this->TestLoad); -- cgit v0.12 From 28994115e83b568ccba546db26a55381cbad244d Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 7 Nov 2019 11:27:48 -0500 Subject: ctest_test: Add option to REPEAT tests --- Help/command/ctest_test.rst | 20 ++++++++++++++ Help/release/dev/ctest-repeat-until-pass.rst | 3 ++ Source/CTest/cmCTestTestCommand.cxx | 4 +++ Source/CTest/cmCTestTestCommand.h | 1 + Source/CTest/cmCTestTestHandler.cxx | 32 ++++++++++++++++++++-- Source/CTest/cmCTestTestHandler.h | 4 ++- Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 22 ++++++++++++++- .../ctest_test/TestRepeatAfterTimeout-stdout.txt | 10 +++++++ .../ctest_test/TestRepeatAfterTimeout.cmake | 10 +++++++ .../RunCMake/ctest_test/TestRepeatBad1-result.txt | 1 + .../RunCMake/ctest_test/TestRepeatBad1-stderr.txt | 1 + .../RunCMake/ctest_test/TestRepeatBad2-result.txt | 1 + .../RunCMake/ctest_test/TestRepeatBad2-stderr.txt | 1 + .../ctest_test/TestRepeatUntilFail-stdout.txt | 13 +++++++++ .../RunCMake/ctest_test/TestRepeatUntilFail.cmake | 9 ++++++ .../ctest_test/TestRepeatUntilPass-stdout.txt | 10 +++++++ .../RunCMake/ctest_test/TestRepeatUntilPass.cmake | 9 ++++++ 17 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatAfterTimeout.cmake create mode 100644 Tests/RunCMake/ctest_test/TestRepeatBad1-result.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatBad1-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatBad2-result.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatBad2-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatUntilFail-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatUntilFail.cmake create mode 100644 Tests/RunCMake/ctest_test/TestRepeatUntilPass-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatUntilPass.cmake diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index 7a3393b..5c67b2c 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -23,6 +23,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`. [STOP_TIME ] [RETURN_VALUE ] [CAPTURE_CMAKE_ERROR ] + [REPEAT :] [QUIET] ) @@ -95,6 +96,25 @@ The options are: and then the ``--test-load`` command-line argument to :manual:`ctest(1)`. See also the ``TestLoad`` setting in the :ref:`CTest Test Step`. +``REPEAT :`` + Run tests repeatedly based on the given ```` up to ```` times. + The modes are: + + ``UNTIL_FAIL`` + Require each test to run ```` times without failing in order to pass. + This is useful in finding sporadic failures in test cases. + + ``UNTIL_PASS`` + Allow each test to run up to ```` times in order to pass. + Repeats tests if they fail for any reason. + This is useful in tolerating sporadic failures in test cases. + + ``AFTER_TIMEOUT`` + Allow each test to run up to ```` times in order to pass. + Repeats tests only if they timeout. + This is useful in tolerating sporadic timeouts in test cases + on busy machines. + ``SCHEDULE_RANDOM `` Launch tests in a random order. This may be useful for detecting implicit test dependencies. diff --git a/Help/release/dev/ctest-repeat-until-pass.rst b/Help/release/dev/ctest-repeat-until-pass.rst index d177247..b5f6c26 100644 --- a/Help/release/dev/ctest-repeat-until-pass.rst +++ b/Help/release/dev/ctest-repeat-until-pass.rst @@ -4,3 +4,6 @@ ctest-repeat-until-pass * The :manual:`ctest(1)` tool learned new ``--repeat-until-pass `` and ``--repeat-after-timeout `` options to help tolerate sporadic test failures. + +* The :command:`ctest_test` command gained a ``REPEAT :`` option + to specify conditions in which to repeat tests. diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index 9784214..0f9b695 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -29,6 +29,7 @@ void cmCTestTestCommand::BindArguments() this->Bind("EXCLUDE_FIXTURE_SETUP"_s, this->ExcludeFixtureSetup); this->Bind("EXCLUDE_FIXTURE_CLEANUP"_s, this->ExcludeFixtureCleanup); this->Bind("PARALLEL_LEVEL"_s, this->ParallelLevel); + this->Bind("REPEAT"_s, this->Repeat); this->Bind("SCHEDULE_RANDOM"_s, this->ScheduleRandom); this->Bind("STOP_TIME"_s, this->StopTime); this->Bind("TEST_LOAD"_s, this->TestLoad); @@ -85,6 +86,9 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() if (!this->ParallelLevel.empty()) { handler->SetOption("ParallelLevel", this->ParallelLevel.c_str()); } + if (!this->Repeat.empty()) { + handler->SetOption("Repeat", this->Repeat.c_str()); + } if (!this->ScheduleRandom.empty()) { handler->SetOption("ScheduleRandom", this->ScheduleRandom.c_str()); } diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index 4019694..2345afb 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -55,6 +55,7 @@ protected: std::string ExcludeFixtureSetup; std::string ExcludeFixtureCleanup; std::string ParallelLevel; + std::string Repeat; std::string ScheduleRandom; std::string StopTime; std::string TestLoad; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 9a06083..37c7154 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -471,6 +471,30 @@ bool cmCTestTestHandler::ProcessOptions() if (cmIsOn(this->GetOption("ScheduleRandom"))) { this->CTest->SetScheduleType("Random"); } + if (const char* repeat = this->GetOption("Repeat")) { + cmsys::RegularExpression repeatRegex( + "^(UNTIL_FAIL|UNTIL_PASS|AFTER_TIMEOUT):([0-9]+)$"); + if (repeatRegex.find(repeat)) { + std::string const& count = repeatRegex.match(2); + unsigned long n = 1; + cmStrToULong(count, &n); // regex guarantees success + this->RepeatCount = static_cast(n); + if (this->RepeatCount > 1) { + std::string const& mode = repeatRegex.match(1); + if (mode == "UNTIL_FAIL") { + this->RepeatMode = cmCTest::Repeat::UntilFail; + } else if (mode == "UNTIL_PASS") { + this->RepeatMode = cmCTest::Repeat::UntilPass; + } else if (mode == "AFTER_TIMEOUT") { + this->RepeatMode = cmCTest::Repeat::AfterTimeout; + } + } + } else { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Repeat option invalid value: " << repeat << std::endl); + return false; + } + } if (this->GetOption("ParallelLevel")) { this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel"))); } @@ -1231,8 +1255,12 @@ void cmCTestTestHandler::ProcessDirectory(std::vector& passed, parallel->SetCTest(this->CTest); parallel->SetParallelLevel(this->CTest->GetParallelLevel()); parallel->SetTestHandler(this); - parallel->SetRepeatMode(this->CTest->GetRepeatMode(), - this->CTest->GetRepeatCount()); + if (this->RepeatMode != cmCTest::Repeat::Never) { + parallel->SetRepeatMode(this->RepeatMode, this->RepeatCount); + } else { + parallel->SetRepeatMode(this->CTest->GetRepeatMode(), + this->CTest->GetRepeatCount()); + } parallel->SetQuiet(this->Quiet); if (this->TestLoad > 0) { parallel->SetTestLoad(this->TestLoad); diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index eab75d0..55237f9 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -18,12 +18,12 @@ #include "cmsys/RegularExpression.hxx" +#include "cmCTest.h" #include "cmCTestGenericHandler.h" #include "cmCTestResourceSpec.h" #include "cmDuration.h" #include "cmListFileCache.h" -class cmCTest; class cmMakefile; class cmXMLWriter; @@ -353,6 +353,8 @@ private: std::ostream* LogFile; + cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never; + int RepeatCount = 1; bool RerunFailed; }; diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 6877e6a..84d1d66 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -1,6 +1,9 @@ include(RunCTest) set(RunCMake_TEST_TIMEOUT 60) +unset(ENV{CTEST_PARALLEL_LEVEL}) +unset(ENV{CTEST_OUTPUT_ON_FAILURE}) + set(CASE_CTEST_TEST_ARGS "") set(CASE_CTEST_TEST_LOAD "") @@ -71,7 +74,24 @@ add_test(NAME PassingTest COMMAND ${CMAKE_COMMAND} -E echo PassingTestOutput) add_test(NAME FailingTest COMMAND ${CMAKE_COMMAND} -E no_such_command) ]]) - unset(ENV{CTEST_PARALLEL_LEVEL}) run_ctest(TestOutputSize) endfunction() run_TestOutputSize() + +run_ctest_test(TestRepeatBad1 REPEAT UNKNOWN:3) +run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1) + +function(run_TestRepeat case) + set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion ${ARGN}) + string(CONCAT CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME testRepeat + COMMAND ${CMAKE_COMMAND} -D COUNT_FILE=${CMAKE_CURRENT_BINARY_DIR}/count.cmake + -P "]] "${RunCMake_SOURCE_DIR}/TestRepeat${case}" [[.cmake") +set_property(TEST testRepeat PROPERTY TIMEOUT 5) + ]]) + + run_ctest(TestRepeat${case}) +endfunction() +run_TestRepeat(UntilFail REPEAT UNTIL_FAIL:3) +run_TestRepeat(UntilPass REPEAT UNTIL_PASS:3) +run_TestRepeat(AfterTimeout REPEAT AFTER_TIMEOUT:3) diff --git a/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-stdout.txt b/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-stdout.txt new file mode 100644 index 0000000..17657c5 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-stdout.txt @@ -0,0 +1,10 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-build + Start 1: testRepeat +1/1 Test #1: testRepeat .......................\*\*\*Timeout +[0-9.]+ sec + Start 1: testRepeat + Test #1: testRepeat ....................... Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout.cmake b/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout.cmake new file mode 100644 index 0000000..abde4f0 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout.cmake @@ -0,0 +1,10 @@ +include("${COUNT_FILE}" OPTIONAL) +if(NOT COUNT) + set(COUNT 0) +endif() +math(EXPR COUNT "${COUNT} + 1") +file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n") +if(NOT COUNT EQUAL 2) + message("this test times out except on the 2nd run") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10) +endif() diff --git a/Tests/RunCMake/ctest_test/TestRepeatBad1-result.txt b/Tests/RunCMake/ctest_test/TestRepeatBad1-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatBad1-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/TestRepeatBad1-stderr.txt b/Tests/RunCMake/ctest_test/TestRepeatBad1-stderr.txt new file mode 100644 index 0000000..37cffbf --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatBad1-stderr.txt @@ -0,0 +1 @@ +Repeat option invalid value: UNKNOWN:3 diff --git a/Tests/RunCMake/ctest_test/TestRepeatBad2-result.txt b/Tests/RunCMake/ctest_test/TestRepeatBad2-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatBad2-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/TestRepeatBad2-stderr.txt b/Tests/RunCMake/ctest_test/TestRepeatBad2-stderr.txt new file mode 100644 index 0000000..ca5cef7 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatBad2-stderr.txt @@ -0,0 +1 @@ +Repeat option invalid value: UNTIL_FAIL:-1 diff --git a/Tests/RunCMake/ctest_test/TestRepeatUntilFail-stdout.txt b/Tests/RunCMake/ctest_test/TestRepeatUntilFail-stdout.txt new file mode 100644 index 0000000..5f91a67 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatUntilFail-stdout.txt @@ -0,0 +1,13 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/TestRepeatUntilFail-build + Start 1: testRepeat + Test #1: testRepeat ....................... Passed +[0-9.]+ sec + Start 1: testRepeat + Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec ++ +0% tests passed, 1 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests FAILED: +[ ]+1 - testRepeat \(Failed\)$ diff --git a/Tests/RunCMake/ctest_test/TestRepeatUntilFail.cmake b/Tests/RunCMake/ctest_test/TestRepeatUntilFail.cmake new file mode 100644 index 0000000..5eb0d8a --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatUntilFail.cmake @@ -0,0 +1,9 @@ +include("${COUNT_FILE}" OPTIONAL) +if(NOT COUNT) + set(COUNT 0) +endif() +math(EXPR COUNT "${COUNT} + 1") +file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n") +if(COUNT EQUAL 2) + message(FATAL_ERROR "this test fails on the 2nd run") +endif() diff --git a/Tests/RunCMake/ctest_test/TestRepeatUntilPass-stdout.txt b/Tests/RunCMake/ctest_test/TestRepeatUntilPass-stdout.txt new file mode 100644 index 0000000..bc6939a --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatUntilPass-stdout.txt @@ -0,0 +1,10 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/TestRepeatUntilPass-build + Start 1: testRepeat +1/1 Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec + Start 1: testRepeat + Test #1: testRepeat ....................... Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/TestRepeatUntilPass.cmake b/Tests/RunCMake/ctest_test/TestRepeatUntilPass.cmake new file mode 100644 index 0000000..0662522 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatUntilPass.cmake @@ -0,0 +1,9 @@ +include("${COUNT_FILE}" OPTIONAL) +if(NOT COUNT) + set(COUNT 0) +endif() +math(EXPR COUNT "${COUNT} + 1") +file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n") +if(NOT COUNT EQUAL 2) + message(FATAL_ERROR "this test passes only on the 2nd run") +endif() -- cgit v0.12