diff options
20 files changed, 170 insertions, 18 deletions
diff --git a/Help/release/dev/ctest_test-ignore-skipped-tests.rst b/Help/release/dev/ctest_test-ignore-skipped-tests.rst new file mode 100644 index 0000000..1e2486c --- /dev/null +++ b/Help/release/dev/ctest_test-ignore-skipped-tests.rst @@ -0,0 +1,7 @@ +ctest_test-ignore-skipped-tests +------------------------------- + +* When running tests, CTest learned to treat skipped tests (using the + :prop_test:`SKIP_RETURN_CODE` property) the same as tests with the + :prop_test:`DISABLED` property. Due to this change, CTest will not indicate + failure when all tests are either skipped or pass. diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index a4853b7..fe23075 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -167,6 +167,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) std::vector<std::pair<cmsys::RegularExpression, std::string> >::iterator passIt; bool forceFail = false; + bool skipped = false; bool outputTestErrorsToConsole = false; if (!this->TestProperties->RequiredRegularExpressions.empty() && this->FailedDependencies.empty()) { @@ -219,6 +220,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) s << "SKIP_RETURN_CODE=" << this->TestProperties->SkipReturnCode; this->TestResult.CompletionStatus = s.str(); cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Skipped "); + skipped = true; } else if ((success && !this->TestProperties->WillFail) || (!success && this->TestProperties->WillFail)) { this->TestResult.Status = cmCTestTestHandler::COMPLETED; @@ -338,7 +340,9 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) compress ? this->CompressedOutput : this->ProcessOutput; this->TestResult.CompressOutput = compress; this->TestResult.ReturnValue = this->TestProcess->GetExitValue(); - this->TestResult.CompletionStatus = "Completed"; + if (!skipped) { + this->TestResult.CompletionStatus = "Completed"; + } this->TestResult.ExecutionTime = this->TestProcess->GetTotalTime(); this->MemCheckPostProcess(); this->ComputeWeightedCost(); @@ -349,7 +353,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) this->TestHandler->TestResults.push_back(this->TestResult); } delete this->TestProcess; - return passed; + return passed || skipped; } bool cmCTestRunTest::StartAgain() diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 167fecf..c24aecb 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -17,6 +17,7 @@ #include <string.h> #include <time.h> +#include "cmAlgorithms.h" #include "cmCTest.h" #include "cmCTestBatchTestHandler.h" #include "cmCTestMultiProcessHandler.h" @@ -495,7 +496,8 @@ int cmCTestTestHandler::ProcessHandler() for (SetOfTests::iterator ftit = resultsSet.begin(); ftit != resultsSet.end(); ++ftit) { - if (ftit->CompletionStatus == "Disabled") { + if (cmHasLiteralPrefix(ftit->CompletionStatus, "SKIP_RETURN_CODE=") || + ftit->CompletionStatus == "Disabled") { disabledTests.push_back(*ftit); } } @@ -521,17 +523,22 @@ int cmCTestTestHandler::ProcessHandler() if (!disabledTests.empty()) { cmGeneratedFileStream ofs; cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl - << "The following tests are disabled and did not run:" - << std::endl); + << "The following tests did not run:" << std::endl); this->StartLogFile("TestsDisabled", ofs); + const char* disabled_reason; for (std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator dtit = disabledTests.begin(); dtit != disabledTests.end(); ++dtit) { ofs << dtit->TestCount << ":" << dtit->Name << std::endl; + if (dtit->CompletionStatus == "Disabled") { + disabled_reason = "Disabled"; + } else { + disabled_reason = "Skipped"; + } cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t" << std::setw(3) << dtit->TestCount << " - " << dtit->Name - << std::endl); + << " (" << disabled_reason << ")" << std::endl); } } @@ -544,6 +551,7 @@ int cmCTestTestHandler::ProcessHandler() for (SetOfTests::iterator ftit = resultsSet.begin(); ftit != resultsSet.end(); ++ftit) { if (ftit->Status != cmCTestTestHandler::COMPLETED && + !cmHasLiteralPrefix(ftit->CompletionStatus, "SKIP_RETURN_CODE=") && ftit->CompletionStatus != "Disabled") { ofs << ftit->TestCount << ":" << ftit->Name << std::endl; cmCTestLog( diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index 7db5243..cf778b6 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -199,6 +199,7 @@ add_RunCMake_test(ctest_start) add_RunCMake_test(ctest_submit) add_RunCMake_test(ctest_test) add_RunCMake_test(ctest_disabled_test) +add_RunCMake_test(ctest_skipped_test) add_RunCMake_test(ctest_upload) add_RunCMake_test(ctest_fixtures) add_RunCMake_test(file) diff --git a/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt index ee0dc51..9449e65 100644 --- a/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt +++ b/Tests/RunCMake/ctest_disabled_test/DisableCleanupTest-stdout.txt @@ -7,5 +7,5 @@ + Total Test time \(real\) = +[0-9.]+ sec + -The following tests are disabled and did not run: -.*2 \- CleanupTest +The following tests did not run: +.*2 \- CleanupTest \(Disabled\) diff --git a/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt index e2c9f92..486722e 100644 --- a/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt +++ b/Tests/RunCMake/ctest_disabled_test/DisableFailingTest-stdout.txt @@ -2,8 +2,8 @@ + Total Test time \(real\) = +[0-9.]+ sec + -The following tests are disabled and did not run: -.*3 \- DisabledFailingTest +The following tests did not run: +.*3 \- DisabledFailingTest \(Disabled\) + The following tests FAILED: .*2 \- FailingTest \(Failed\) diff --git a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt index d8bf966..9078aeb 100644 --- a/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt +++ b/Tests/RunCMake/ctest_disabled_test/DisableNotRunTest-stdout.txt @@ -10,8 +10,8 @@ + Total Test time \(real\) = +[0-9.]+ sec + -The following tests are disabled and did not run: -.*2 \- DisabledTest +The following tests did not run: +.*2 \- DisabledTest \(Disabled\) + The following tests FAILED: .*3 - NotRunTest \(Not Run\) diff --git a/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt index 886efb8..10d385e 100644 --- a/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt +++ b/Tests/RunCMake/ctest_disabled_test/DisableRequiredTest-stdout.txt @@ -9,5 +9,5 @@ + Total Test time \(real\) = +[0-9.]+ sec + -The following tests are disabled and did not run: -.*2 \- DisabledTest +The following tests did not run: +.*2 \- DisabledTest \(Disabled\) diff --git a/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt index dc27950..2dfd10d 100644 --- a/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt +++ b/Tests/RunCMake/ctest_disabled_test/DisableSetupTest-stdout.txt @@ -9,5 +9,5 @@ + Total Test time \(real\) = +[0-9.]+ sec + -The following tests are disabled and did not run: -.*2 \- DisabledTest +The following tests did not run: +.*2 \- DisabledTest \(Disabled\) diff --git a/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt index d8bf966..9078aeb 100644 --- a/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt +++ b/Tests/RunCMake/ctest_disabled_test/DisabledTest-stdout.txt @@ -10,8 +10,8 @@ + Total Test time \(real\) = +[0-9.]+ sec + -The following tests are disabled and did not run: -.*2 \- DisabledTest +The following tests did not run: +.*2 \- DisabledTest \(Disabled\) + The following tests FAILED: .*3 - NotRunTest \(Not Run\) diff --git a/Tests/RunCMake/ctest_skipped_test/CMakeLists.txt.in b/Tests/RunCMake/ctest_skipped_test/CMakeLists.txt.in new file mode 100644 index 0000000..cc4b8ed --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/CMakeLists.txt.in @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.8) +project(@CASE_NAME@ C) +include(CTest) + +if (WIN32) + set(skip_command "@CMAKE_CURRENT_LIST_DIR@/skip.bat") +else () + set(skip_command "@CMAKE_CURRENT_LIST_DIR@/skip.sh") +endif () + +add_test(NAME SuccessfulTest COMMAND "${CMAKE_COMMAND}" --version) +@CASE_CMAKELISTS_SUFFIX_CODE@ diff --git a/Tests/RunCMake/ctest_skipped_test/CTestConfig.cmake.in b/Tests/RunCMake/ctest_skipped_test/CTestConfig.cmake.in new file mode 100644 index 0000000..c0d7e42 --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/CTestConfig.cmake.in @@ -0,0 +1 @@ +set(CTEST_PROJECT_NAME "@CASE_NAME@") diff --git a/Tests/RunCMake/ctest_skipped_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_skipped_test/RunCMakeTest.cmake new file mode 100644 index 0000000..dcf5cd4 --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/RunCMakeTest.cmake @@ -0,0 +1,51 @@ +include(RunCTest) + +function(run_SkipTest) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME SkipTest COMMAND ${skip_command}) + +set_tests_properties(SkipTest PROPERTIES SKIP_RETURN_CODE 125) + ]]) + run_ctest(SkipTest) +endfunction() +run_SkipTest() + +function(run_SkipSetupTest) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME SkipTest COMMAND ${skip_command}) +add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version) + +set_tests_properties(SkipTest PROPERTIES SKIP_RETURN_CODE 125 + FIXTURES_SETUP "Foo") +set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo") +set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo") + ]]) + run_ctest(SkipSetupTest) +endfunction() +run_SkipSetupTest() + +function(run_SkipRequiredTest) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME SkipTest COMMAND ${skip_command}) +add_test(NAME SuccessfulCleanupTest COMMAND "${CMAKE_COMMAND}" --version) + +set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_SETUP "Foo") +set_tests_properties(SkipTest PROPERTIES SKIP_RETURN_CODE 125 + FIXTURES_REQUIRED "Foo") +set_tests_properties(SuccessfulCleanupTest PROPERTIES FIXTURES_CLEANUP "Foo") + ]]) + run_ctest(SkipRequiredTest) +endfunction() +run_SkipRequiredTest() + +function(run_SkipCleanupTest) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME CleanupTest COMMAND ${skip_command}) + +set_tests_properties(SuccessfulTest PROPERTIES FIXTURES_REQUIRED "Foo") +set_tests_properties(CleanupTest PROPERTIES SKIP_RETURN_CODE 125 + FIXTURES_CLEANUP "Foo") + ]]) + run_ctest(SkipCleanupTest) +endfunction() +run_SkipCleanupTest() diff --git a/Tests/RunCMake/ctest_skipped_test/SkipCleanupTest-stdout.txt b/Tests/RunCMake/ctest_skipped_test/SkipCleanupTest-stdout.txt new file mode 100644 index 0000000..3b14b7a --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/SkipCleanupTest-stdout.txt @@ -0,0 +1,11 @@ + Start 1: SuccessfulTest +1/2 Test #1: SuccessfulTest ................... Passed +[0-9.]+ sec + Start 2: CleanupTest +2/2 Test #2: CleanupTest ......................\*\*\*\Skipped +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 2 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests did not run: +.*2 \- CleanupTest \(Skipped\) diff --git a/Tests/RunCMake/ctest_skipped_test/SkipRequiredTest-stdout.txt b/Tests/RunCMake/ctest_skipped_test/SkipRequiredTest-stdout.txt new file mode 100644 index 0000000..8ecc6e3 --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/SkipRequiredTest-stdout.txt @@ -0,0 +1,13 @@ + Start 1: SuccessfulTest +1/3 Test #1: SuccessfulTest ................... Passed +[0-9.]+ sec + Start 2: SkipTest +2/3 Test #2: SkipTest .........................\*\*\*\Skipped +[0-9.]+ sec + Start 3: SuccessfulCleanupTest +3/3 Test #3: SuccessfulCleanupTest ............ Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 3 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests did not run: +.*2 \- SkipTest \(Skipped\) diff --git a/Tests/RunCMake/ctest_skipped_test/SkipSetupTest-stdout.txt b/Tests/RunCMake/ctest_skipped_test/SkipSetupTest-stdout.txt new file mode 100644 index 0000000..fe9bf34 --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/SkipSetupTest-stdout.txt @@ -0,0 +1,13 @@ + Start 2: SkipTest +1/3 Test #2: SkipTest .........................\*\*\*\Skipped +[0-9.]+ sec + Start 1: SuccessfulTest +2/3 Test #1: SuccessfulTest ................... Passed +[0-9.]+ sec + Start 3: SuccessfulCleanupTest +3/3 Test #3: SuccessfulCleanupTest ............ Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 3 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests did not run: +.*2 \- SkipTest \(Skipped\) diff --git a/Tests/RunCMake/ctest_skipped_test/SkipTest-stdout.txt b/Tests/RunCMake/ctest_skipped_test/SkipTest-stdout.txt new file mode 100644 index 0000000..52e7a0b --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/SkipTest-stdout.txt @@ -0,0 +1,11 @@ + Start 1: SuccessfulTest +1/2 Test #1: SuccessfulTest ................... Passed +[0-9.]+ sec + Start 2: SkipTest +2/2 Test #2: SkipTest .........................\*\*\*\Skipped +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 2 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests did not run: +.*2 \- SkipTest \(Skipped\) diff --git a/Tests/RunCMake/ctest_skipped_test/skip.bat b/Tests/RunCMake/ctest_skipped_test/skip.bat new file mode 100755 index 0000000..80e1290 --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/skip.bat @@ -0,0 +1 @@ +EXIT 125 diff --git a/Tests/RunCMake/ctest_skipped_test/skip.sh b/Tests/RunCMake/ctest_skipped_test/skip.sh new file mode 100755 index 0000000..f9c4603 --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/skip.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exit 125 diff --git a/Tests/RunCMake/ctest_skipped_test/test.cmake.in b/Tests/RunCMake/ctest_skipped_test/test.cmake.in new file mode 100644 index 0000000..ca23c83 --- /dev/null +++ b/Tests/RunCMake/ctest_skipped_test/test.cmake.in @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.7) + +set(CTEST_SITE "test-site") +set(CTEST_BUILD_NAME "test-build-name") +set(CTEST_SOURCE_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@") +set(CTEST_BINARY_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@-build") +set(CTEST_CMAKE_GENERATOR "@RunCMake_GENERATOR@") +set(CTEST_CMAKE_GENERATOR_PLATFORM "@RunCMake_GENERATOR_PLATFORM@") +set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@") +set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}") + +set(ctest_test_args "@CASE_CTEST_TEST_ARGS@") +ctest_start(Experimental) +ctest_configure() +ctest_build() +ctest_test(${ctest_test_args}) |