From b070947dffd141d50227c59feed4c3d784b39157 Mon Sep 17 00:00:00 2001 From: Betsy McPhail Date: Mon, 27 Mar 2017 14:08:50 -0400 Subject: Add 'DISABLED' test property When this property is set, the test is skipped and its status is automatically set to 'Not Run'. A disabled test will not be counted in the total number of tests and its completion status will be 'Disabled'. --- Help/manual/cmake-properties.7.rst | 1 + Help/prop_test/DISABLED.rst | 15 ++++ Help/release/dev/ctest-disable-tests.rst | 5 ++ Source/CTest/cmCTestMultiProcessHandler.cxx | 4 +- Source/CTest/cmCTestRunTest.cxx | 20 +++++ Source/CTest/cmCTestTestHandler.cxx | 50 ++++++++++-- Source/CTest/cmCTestTestHandler.h | 1 + Tests/RunCMake/CMakeLists.txt | 1 + .../RunCMake/ctest_disabled_test/CMakeLists.txt.in | 6 ++ .../ctest_disabled_test/CTestConfig.cmake.in | 1 + .../ctest_disabled_test/DisableAllTests-result.txt | 1 + .../ctest_disabled_test/DisableAllTests-stderr.txt | 1 + .../ctest_disabled_test/DisableAllTests-stdout.txt | 2 + .../DisableCleanupTest-stdout.txt | 11 +++ .../DisableFailingTest-stdout.txt | 9 +++ .../DisableNotRunTest-result.txt | 1 + .../DisableNotRunTest-stderr.txt | 1 + .../DisableNotRunTest-stdout.txt | 17 +++++ .../DisableRequiredTest-stdout.txt | 13 ++++ .../DisableSetupTest-stdout.txt | 13 ++++ .../ctest_disabled_test/DisabledTest-result.txt | 1 + .../ctest_disabled_test/DisabledTest-stderr.txt | 1 + .../ctest_disabled_test/DisabledTest-stdout.txt | 17 +++++ .../ctest_disabled_test/RunCMakeTest.cmake | 89 ++++++++++++++++++++++ Tests/RunCMake/ctest_disabled_test/test.cmake.in | 16 ++++ 25 files changed, 289 insertions(+), 8 deletions(-) create mode 100644 Help/prop_test/DISABLED.rst create mode 100644 Help/release/dev/ctest-disable-tests.rst create mode 100644 Tests/RunCMake/ctest_disabled_test/CMakeLists.txt.in create mode 100644 Tests/RunCMake/ctest_disabled_test/CTestConfig.cmake.in create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableAllTests-result.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableAllTests-stderr.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableAllTests-stdout.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-result.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stderr.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisabledTest-result.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisabledTest-stderr.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt create mode 100644 Tests/RunCMake/ctest_disabled_test/RunCMakeTest.cmake create mode 100644 Tests/RunCMake/ctest_disabled_test/test.cmake.in diff --git a/Help/manual/cmake-properties.7.rst b/Help/manual/cmake-properties.7.rst index 9967d00..072d7c5 100644 --- a/Help/manual/cmake-properties.7.rst +++ b/Help/manual/cmake-properties.7.rst @@ -325,6 +325,7 @@ Properties on Tests /prop_test/ATTACHED_FILES /prop_test/COST /prop_test/DEPENDS + /prop_test/DISABLED /prop_test/ENVIRONMENT /prop_test/FAIL_REGULAR_EXPRESSION /prop_test/FIXTURES_CLEANUP diff --git a/Help/prop_test/DISABLED.rst b/Help/prop_test/DISABLED.rst new file mode 100644 index 0000000..c18ae7f --- /dev/null +++ b/Help/prop_test/DISABLED.rst @@ -0,0 +1,15 @@ +DISABLED +-------- + +If set to true, the test will be skipped and its status will be 'Not Run'. A +DISABLED test will not be counted in the total number of tests and its +completion status will be reported to CDash as 'Disabled'. + +A DISABLED test does not participate in test fixture dependency resolution. +If a DISABLED test has fixture requirements defined in its +:prop_test:`FIXTURES_REQUIRED` property, it will not cause setup or cleanup +tests for those fixtures to be added to the test set. + +If a test with the :prop_test:`FIXTURES_SETUP` property set is DISABLED, the +fixture behavior will be as though that setup test was passing and any test +case requiring that fixture will still run. diff --git a/Help/release/dev/ctest-disable-tests.rst b/Help/release/dev/ctest-disable-tests.rst new file mode 100644 index 0000000..9208f0c --- /dev/null +++ b/Help/release/dev/ctest-disable-tests.rst @@ -0,0 +1,5 @@ +ctest-disable-tests +------------------- + +* A :prop_test:`DISABLED` test property was added to mark tests that + are configured but explicitly disabled so they do not run. diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index ff465ab..2d4726c 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -163,7 +163,9 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test) this->TestRunningMap[test] = false; this->RunningCount -= GetProcessorsUsed(test); testRun->EndTest(this->Completed, this->Total, false); - this->Failed->push_back(this->Properties[test]->Name); + if (!this->Properties[test]->Disabled) { + this->Failed->push_back(this->Properties[test]->Name); + } delete testRun; } } diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index be2b59e..94aa4bd 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -256,6 +256,8 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) cmCTestLog(this->CTest, HANDLER_OUTPUT, "Other"); this->TestResult.Status = cmCTestTestHandler::OTHER_FAULT; } + } else if ("Disabled" == this->TestResult.CompletionStatus) { + cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Not Run (Disabled) "); } else // cmsysProcess_State_Error { cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Not Run "); @@ -416,6 +418,24 @@ bool cmCTestRunTest::StartTest(size_t total) << this->TestProperties->Index << ": " << this->TestProperties->Name << std::endl); this->ProcessOutput.clear(); + + // Return immediately if test is disabled + if (this->TestProperties->Disabled) { + this->TestResult.Properties = this->TestProperties; + this->TestResult.ExecutionTime = 0; + this->TestResult.CompressOutput = false; + this->TestResult.ReturnValue = -1; + this->TestResult.CompletionStatus = "Disabled"; + this->TestResult.Status = cmCTestTestHandler::NOT_RUN; + this->TestResult.TestCount = this->TestProperties->Index; + this->TestResult.Name = this->TestProperties->Name; + this->TestResult.Path = this->TestProperties->Directory; + this->TestProcess = new cmProcess; + this->TestResult.Output = "Disabled"; + this->TestResult.FullCommandLine = ""; + return false; + } + this->ComputeArguments(); std::vector& args = this->TestProperties->Args; this->TestResult.Properties = this->TestProperties; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index f377de1..814b310 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -487,6 +487,19 @@ int cmCTestTestHandler::ProcessHandler() } } + typedef std::set + SetOfTests; + SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end()); + std::vector disabledTests; + + for (SetOfTests::iterator ftit = resultsSet.begin(); + ftit != resultsSet.end(); ++ftit) { + if (ftit->CompletionStatus == "Disabled") { + disabledTests.push_back(*ftit); + } + } + float percent = float(passed.size()) * 100.0f / float(total); if (!failed.empty() && percent > 99) { percent = 99; @@ -505,21 +518,33 @@ int cmCTestTestHandler::ProcessHandler() "\nTotal Test time (real) = " << realBuf << "\n", this->Quiet); + if (!disabledTests.empty()) { + cmGeneratedFileStream ofs; + cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl + << "The following tests are disabled and did not run:" + << std::endl); + this->StartLogFile("TestsDisabled", ofs); + + for (std::vector::iterator dtit = + disabledTests.begin(); + dtit != disabledTests.end(); ++dtit) { + ofs << dtit->TestCount << ":" << dtit->Name << std::endl; + cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t" + << std::setw(3) << dtit->TestCount << " - " << dtit->Name + << std::endl); + } + } + if (!failed.empty()) { cmGeneratedFileStream ofs; cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl << "The following tests FAILED:" << std::endl); this->StartLogFile("TestsFailed", ofs); - typedef std::set - SetOfTests; - SetOfTests resultsSet(this->TestResults.begin(), - this->TestResults.end()); - for (SetOfTests::iterator ftit = resultsSet.begin(); ftit != resultsSet.end(); ++ftit) { - if (ftit->Status != cmCTestTestHandler::COMPLETED) { + if (ftit->Status != cmCTestTestHandler::COMPLETED && + ftit->CompletionStatus != "Disabled") { ofs << ftit->TestCount << ":" << ftit->Name << std::endl; cmCTestLog( this->CTest, HANDLER_OUTPUT, "\t" @@ -841,6 +866,11 @@ void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const size_t fixtureTestsAdded = 0; std::set addedFixtures; for (size_t i = 0; i < tests.size(); ++i) { + // Skip disabled tests + if (tests[i].Disabled) { + continue; + } + // There are two things to do for each test: // 1. For every fixture required by this test, record that fixture as // being required and create dependencies on that fixture's setup @@ -1200,6 +1230,7 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml) cmCTestTestResult* result = &this->TestResults[cc]; this->WriteTestResultHeader(xml, result); xml.StartElement("Results"); + if (result->Status != cmCTestTestHandler::NOT_RUN) { if (result->Status != cmCTestTestHandler::COMPLETED || result->ReturnValue) { @@ -1208,6 +1239,7 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml) xml.Attribute("name", "Exit Code"); xml.Element("Value", this->GetTestStatus(result->Status)); xml.EndElement(); // NamedMeasurement + xml.StartElement("NamedMeasurement"); xml.Attribute("type", "text/string"); xml.Attribute("name", "Exit Value"); @@ -2001,6 +2033,9 @@ bool cmCTestTestHandler::SetTestsProperties( if (key == "WILL_FAIL") { rtit->WillFail = cmSystemTools::IsOn(val.c_str()); } + if (key == "DISABLED") { + rtit->Disabled = cmSystemTools::IsOn(val.c_str()); + } if (key == "ATTACHED_FILES") { cmSystemTools::ExpandListArgument(val, rtit->AttachedFiles); } @@ -2179,6 +2214,7 @@ bool cmCTestTestHandler::AddTest(const std::vector& args) test.IsInBasedOnREOptions = true; test.WillFail = false; + test.Disabled = false; test.RunSerial = false; test.Timeout = 0; test.ExplicitTimeout = false; diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index 5b07e98..a95f088 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -114,6 +114,7 @@ public: std::map Measurements; bool IsInBasedOnREOptions; bool WillFail; + bool Disabled; float Cost; int PreviousRuns; bool RunSerial; diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index 217f0d5..e2dc128 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -202,6 +202,7 @@ endif() add_RunCMake_test(ctest_start) add_RunCMake_test(ctest_submit) add_RunCMake_test(ctest_test) +add_RunCMake_test(ctest_disabled_test) add_RunCMake_test(ctest_upload) add_RunCMake_test(ctest_fixtures) add_RunCMake_test(file) diff --git a/Tests/RunCMake/ctest_disabled_test/CMakeLists.txt.in b/Tests/RunCMake/ctest_disabled_test/CMakeLists.txt.in new file mode 100644 index 0000000..d34fcac --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/CMakeLists.txt.in @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.7) +project(@CASE_NAME@ NONE) +include(CTest) + +add_test(NAME SuccessfulTest COMMAND "${CMAKE_COMMAND}" --version) +@CASE_CMAKELISTS_SUFFIX_CODE@ diff --git a/Tests/RunCMake/ctest_disabled_test/CTestConfig.cmake.in b/Tests/RunCMake/ctest_disabled_test/CTestConfig.cmake.in new file mode 100644 index 0000000..c0d7e42 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/CTestConfig.cmake.in @@ -0,0 +1 @@ +set(CTEST_PROJECT_NAME "@CASE_NAME@") diff --git a/Tests/RunCMake/ctest_disabled_test/DisableAllTests-result.txt b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stderr.txt b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stderr.txt new file mode 100644 index 0000000..eafba1c --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stderr.txt @@ -0,0 +1 @@ +No tests were found!!! diff --git a/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stdout.txt new file mode 100644 index 0000000..6c824f6 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableAllTests-stdout.txt @@ -0,0 +1,2 @@ + Start 1: SuccessfulTest +1/1 Test #1: SuccessfulTest ...................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt new file mode 100644 index 0000000..ee0dc51 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt @@ -0,0 +1,11 @@ + Start 1: SuccessfulTest +1/2 Test #1: SuccessfulTest ................... Passed +[0-9.]+ sec + Start 2: CleanupTest +2/2 Test #2: CleanupTest ......................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests are disabled and did not run: +.*2 \- CleanupTest diff --git a/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt new file mode 100644 index 0000000..e2c9f92 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt @@ -0,0 +1,9 @@ +50% tests passed, 1 tests failed out of 2 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests are disabled and did not run: +.*3 \- DisabledFailingTest ++ +The following tests FAILED: +.*2 \- FailingTest \(Failed\) diff --git a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-result.txt b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stderr.txt b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stderr.txt new file mode 100644 index 0000000..83c332b --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stderr.txt @@ -0,0 +1 @@ +Unable to find executable: invalidCommand diff --git a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt new file mode 100644 index 0000000..d8bf966 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt @@ -0,0 +1,17 @@ + Start 1: SuccessfulTest +1/3 Test #1: SuccessfulTest ................... Passed +[0-9.]+ sec + Start 2: DisabledTest +2/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec + Start 3: NotRunTest +.* +3/3 Test #3: NotRunTest .......................\*\*\*\Not Run +[0-9.]+ sec ++ +50% tests passed, 1 tests failed out of 2 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests are disabled and did not run: +.*2 \- DisabledTest ++ +The following tests FAILED: +.*3 - NotRunTest \(Not Run\) diff --git a/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt new file mode 100644 index 0000000..886efb8 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt @@ -0,0 +1,13 @@ + Start 1: SuccessfulTest +1/3 Test #1: SuccessfulTest ................... Passed +[0-9.]+ sec + Start 2: DisabledTest +2/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec + Start 3: SuccessfulCleanupTest +3/3 Test #3: SuccessfulCleanupTest ............ Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 2 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests are disabled and did not run: +.*2 \- DisabledTest diff --git a/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt new file mode 100644 index 0000000..dc27950 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt @@ -0,0 +1,13 @@ + Start 2: DisabledTest +1/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec + Start 1: SuccessfulTest +2/3 Test #1: SuccessfulTest ................... Passed +[0-9.]+ sec + Start 3: SuccessfulCleanupTest +3/3 Test #3: SuccessfulCleanupTest ............ Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 2 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests are disabled and did not run: +.*2 \- DisabledTest diff --git a/Tests/RunCMake/ctest_disabled_test/DisabledTest-result.txt b/Tests/RunCMake/ctest_disabled_test/DisabledTest-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisabledTest-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_disabled_test/DisabledTest-stderr.txt b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stderr.txt new file mode 100644 index 0000000..83c332b --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stderr.txt @@ -0,0 +1 @@ +Unable to find executable: invalidCommand diff --git a/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt new file mode 100644 index 0000000..d8bf966 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt @@ -0,0 +1,17 @@ + Start 1: SuccessfulTest +1/3 Test #1: SuccessfulTest ................... Passed +[0-9.]+ sec + Start 2: DisabledTest +2/3 Test #2: DisabledTest .....................\*\*\*\Not Run \(Disabled\) +[0-9.]+ sec + Start 3: NotRunTest +.* +3/3 Test #3: NotRunTest .......................\*\*\*\Not Run +[0-9.]+ sec ++ +50% tests passed, 1 tests failed out of 2 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests are disabled and did not run: +.*2 \- DisabledTest ++ +The following tests FAILED: +.*3 - NotRunTest \(Not Run\) diff --git a/Tests/RunCMake/ctest_disabled_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_disabled_test/RunCMakeTest.cmake new file mode 100644 index 0000000..12541c4 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/RunCMakeTest.cmake @@ -0,0 +1,89 @@ +include(RunCTest) + +function(run_DisableNotRunTest) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME DisabledTest COMMAND notACommand --version) +add_test(NAME NotRunTest COMMAND invalidCommand --version) + +set_tests_properties(SuccessfulTest PROPERTIES DISABLED false) +set_tests_properties(DisabledTest PROPERTIES DISABLED true) + ]]) + run_ctest(DisableNotRunTest) +endfunction() +run_DisableNotRunTest() + +function(run_DisableFailingTest) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +set(someFile "${CMAKE_CURRENT_SOURCE_DIR}/test.cmake") +add_test(NAME FailingTest + COMMAND ${CMAKE_COMMAND} -E compare_files "${someFile}" "${someFile}xxx") +add_test(NAME DisabledFailingTest + COMMAND ${CMAKE_COMMAND} -E compare_files "${someFile}" "${someFile}xxx") + +set_tests_properties(FailingTest PROPERTIES DISABLED false) +set_tests_properties(DisabledFailingTest PROPERTIES DISABLED true) + ]]) + run_ctest(DisableFailingTest) +endfunction() +run_DisableFailingTest() + +function(run_DisableSetupTest) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME DisabledTest COMMAND "${CMAKE_COMMAND}" --version) +add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version) + +set_tests_properties(DisabledTest PROPERTIES DISABLED true + FIXTURES_SETUP "Foo") +set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo") +set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo") + ]]) +run_ctest(DisableSetupTest) +endfunction() +run_DisableSetupTest() + +function(run_DisableRequiredTest) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME DisabledTest COMMAND "${CMAKE_COMMAND}" --version) +add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version) + +set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_SETUP "Foo") +set_tests_properties(DisabledTest PROPERTIES DISABLED true + FIXTURES_REQUIRED "Foo") +set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo") + ]]) +run_ctest(DisableRequiredTest) +endfunction() +run_DisableRequiredTest() + +function(run_DisableCleanupTest) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME CleanupTest COMMAND "${CMAKE_COMMAND}" --version) + +set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo") +set_tests_properties(CleanupTest PROPERTIES DISABLED true + FIXTURES_CLEANUP "Foo") + ]]) +run_ctest(DisableCleanupTest) +endfunction() +run_DisableCleanupTest() + +# Consider a fixture that has a setup test, a cleanup test and a disabled test +# which requires that fixture. Limit the test list with a regular expression +# that matches the disabled test but not the setup or cleanup tests, so the +# initial set of tests to be executed contains just the disabled test. Since +# the only test requiring the fixture is disabled, CTest should not +# automatically add in the setup and cleanup tests for the fixture, since no +# enabled test requires them. +function(run_DisableAllTests) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME SetupTest COMMAND "${CMAKE_COMMAND}" --version) +add_test(NAME CleanupTest COMMAND "${CMAKE_COMMAND}" --version) + +set_tests_properties(SetupTest PROPERTIES FIXTURES_SETUP "Foo") +set_tests_properties(SuccessfulTest PROPERTIES DISABLED true + FIXTURES_REQUIRED "Foo") +set_tests_properties(CleanupTest PROPERTIES FIXTURES_CLEANUP "Foo") + ]]) +run_ctest(DisableAllTests -R Successful) +endfunction() +run_DisableAllTests() diff --git a/Tests/RunCMake/ctest_disabled_test/test.cmake.in b/Tests/RunCMake/ctest_disabled_test/test.cmake.in new file mode 100644 index 0000000..ca23c83 --- /dev/null +++ b/Tests/RunCMake/ctest_disabled_test/test.cmake.in @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.7) + +set(CTEST_SITE "test-site") +set(CTEST_BUILD_NAME "test-build-name") +set(CTEST_SOURCE_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@") +set(CTEST_BINARY_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@-build") +set(CTEST_CMAKE_GENERATOR "@RunCMake_GENERATOR@") +set(CTEST_CMAKE_GENERATOR_PLATFORM "@RunCMake_GENERATOR_PLATFORM@") +set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@") +set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}") + +set(ctest_test_args "@CASE_CTEST_TEST_ARGS@") +ctest_start(Experimental) +ctest_configure() +ctest_build() +ctest_test(${ctest_test_args}) -- cgit v0.12