From 4f1dec86a53b789d164827eb6e89696f9e59f1c0 Mon Sep 17 00:00:00 2001 From: Gabor Bencze Date: Thu, 18 Jul 2019 16:56:16 +0200 Subject: CTest: Add SKIP_REGULAR_EXPRESSION test property --- Source/CTest/cmCTestMultiProcessHandler.cxx | 5 +++++ Source/CTest/cmCTestRunTest.cxx | 25 ++++++++++++++++++++++--- Source/CTest/cmCTestTestHandler.cxx | 11 +++++++++-- Source/CTest/cmCTestTestHandler.h | 2 ++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 37a8abf..42534f7 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -823,6 +823,11 @@ static Json::Value DumpCTestProperties( "FAIL_REGULAR_EXPRESSION", DumpRegExToJsonArray(testProperties.ErrorRegularExpressions))); } + if (!testProperties.SkipRegularExpressions.empty()) { + properties.append(DumpCTestProperty( + "SKIP_REGULAR_EXPRESSION", + DumpRegExToJsonArray(testProperties.SkipRegularExpressions))); + } if (!testProperties.FixturesCleanup.empty()) { properties.append(DumpCTestProperty( "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup))); diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index f9ac1eb..65cf646 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -77,6 +77,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) } std::int64_t retVal = this->TestProcess->GetExitValue(); bool forceFail = false; + bool forceSkip = false; bool skipped = false; bool outputTestErrorsToConsole = false; if (!this->TestProperties->RequiredRegularExpressions.empty() && @@ -116,16 +117,34 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) } } } + if (!this->TestProperties->SkipRegularExpressions.empty() && + this->FailedDependencies.empty()) { + for (auto& skip : this->TestProperties->SkipRegularExpressions) { + if (skip.first.find(this->ProcessOutput)) { + reason = "Skip regular expression found in output."; + reason += " Regex=["; + reason += skip.second; + reason += "]"; + forceSkip = true; + break; + } + } + } std::ostringstream outputStream; if (res == cmProcess::State::Exited) { bool success = !forceFail && (retVal == 0 || !this->TestProperties->RequiredRegularExpressions.empty()); - if (this->TestProperties->SkipReturnCode >= 0 && - this->TestProperties->SkipReturnCode == retVal) { + if ((this->TestProperties->SkipReturnCode >= 0 && + this->TestProperties->SkipReturnCode == retVal) || + forceSkip) { this->TestResult.Status = cmCTestTestHandler::NOT_RUN; std::ostringstream s; - s << "SKIP_RETURN_CODE=" << this->TestProperties->SkipReturnCode; + if (forceSkip) { + s << "SKIP_REGULAR_EXPRESSION_MATCHED"; + } else { + s << "SKIP_RETURN_CODE=" << this->TestProperties->SkipReturnCode; + } this->TestResult.CompletionStatus = s.str(); cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Skipped "); skipped = true; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 9916ca3..c2748e1 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -526,7 +526,7 @@ int cmCTestTestHandler::ProcessHandler() std::vector disabledTests; for (cmCTestTestResult const& ft : resultsSet) { - if (cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_RETURN_CODE=") || + if (cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_") || ft.CompletionStatus == "Disabled") { disabledTests.push_back(ft); } @@ -599,7 +599,7 @@ int cmCTestTestHandler::ProcessHandler() for (cmCTestTestResult const& ft : resultsSet) { if (ft.Status != cmCTestTestHandler::COMPLETED && - !cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_RETURN_CODE=") && + !cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_") && ft.CompletionStatus != "Disabled") { ofs << ft.TestCount << ":" << ft.Name << std::endl; auto testColor = cmCTest::Color::RED; @@ -2229,6 +2229,13 @@ bool cmCTestTestHandler::SetTestsProperties( rt.ErrorRegularExpressions.emplace_back(cr, cr); } } + if (key == "SKIP_REGULAR_EXPRESSION") { + std::vector lval; + cmSystemTools::ExpandListArgument(val, lval); + for (std::string const& cr : lval) { + rt.SkipRegularExpressions.emplace_back(cr, cr); + } + } if (key == "PROCESSORS") { rt.Processors = atoi(val.c_str()); if (rt.Processors < 1) { diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index 7f3f5e4..5bbc68e 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -118,6 +118,8 @@ public: std::vector> RequiredRegularExpressions; std::vector> + SkipRegularExpressions; + std::vector> TimeoutRegularExpressions; std::map Measurements; bool IsInBasedOnREOptions; -- cgit v0.12 From d7955d4e5d3038930f333bf12e15a09c7ae6088c Mon Sep 17 00:00:00 2001 From: Gabor Bencze Date: Thu, 18 Jul 2019 16:58:14 +0200 Subject: Tests: Create test for SKIP_REGULAR_EXPRESSION test property --- Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 14 ++++++++++++++ Tests/RunCMake/CTestCommandLine/SkipRegexFound-check.cmake | 13 +++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 Tests/RunCMake/CTestCommandLine/SkipRegexFound-check.cmake diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index f417db0..3fee79c 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -120,6 +120,20 @@ set_tests_properties(test1 PROPERTIES FAIL_REGULAR_EXPRESSION \"foo;test1;bar\" endfunction() run_FailRegexFoundTest() +function(run_SkipRegexFoundTest) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/SkipRegexFound) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" " +add_test(test1 \"${CMAKE_COMMAND}\" -E echo \"test1\") +set_tests_properties(test1 PROPERTIES SKIP_REGULAR_EXPRESSION \"test1\") +") + + run_cmake_command(SkipRegexFound ${CMAKE_CTEST_COMMAND} -V) +endfunction() +run_SkipRegexFoundTest() + function(run_SerialFailed) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/SerialFailed) set(RunCMake_TEST_NO_CLEAN 1) diff --git a/Tests/RunCMake/CTestCommandLine/SkipRegexFound-check.cmake b/Tests/RunCMake/CTestCommandLine/SkipRegexFound-check.cmake new file mode 100644 index 0000000..1a2dfa3 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/SkipRegexFound-check.cmake @@ -0,0 +1,13 @@ +set(last_test_log "${RunCMake_TEST_BINARY_DIR}/Testing/Temporary/LastTest.log") +if(EXISTS "${last_test_log}") + file(READ "${last_test_log}" last_test_log_content) + string(REGEX REPLACE "\n+$" "" last_test_log_content "${last_test_log_content}") + if(NOT last_test_log_content MATCHES " +Test Pass Reason: +Skip regular expression found in output. Regex=[[]test1]") + string(REPLACE "\n" "\n " last_test_log_content " ${last_test_log_content}") + set(RunCMake_TEST_FAILED "LastTest.log does not have expected content:\n${last_test_log_content}") + endif() +else() + set(RunCMake_TEST_FAILED "LastTest.log missing:\n ${last_test_log}") +endif() -- cgit v0.12 From 407dd1a910b1232cf915db9afb9c09022c6e22a9 Mon Sep 17 00:00:00 2001 From: Gabor Bencze Date: Thu, 18 Jul 2019 16:59:15 +0200 Subject: Help: Add documentation for SKIP_REGULAR_EXPRESSION test property --- Help/command/add_test.rst | 5 +++-- Help/manual/cmake-properties.7.rst | 1 + Help/prop_test/SKIP_REGULAR_EXPRESSION.rst | 17 +++++++++++++++++ Help/prop_test/SKIP_RETURN_CODE.rst | 2 ++ .../dev/add_skip_regular_expression_test_property.rst | 10 ++++++++++ 5 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 Help/prop_test/SKIP_REGULAR_EXPRESSION.rst create mode 100644 Help/release/dev/add_skip_regular_expression_test_property.rst diff --git a/Help/command/add_test.rst b/Help/command/add_test.rst index 884b2ee..a77ba37 100644 --- a/Help/command/add_test.rst +++ b/Help/command/add_test.rst @@ -38,8 +38,9 @@ The given test command is expected to exit with code ``0`` to pass and non-zero to fail, or vice-versa if the :prop_test:`WILL_FAIL` test property is set. Any output written to stdout or stderr will be captured by :manual:`ctest(1)` but does not affect the pass/fail status -unless the :prop_test:`PASS_REGULAR_EXPRESSION` or -:prop_test:`FAIL_REGULAR_EXPRESSION` test property is used. +unless the :prop_test:`PASS_REGULAR_EXPRESSION`, +:prop_test:`FAIL_REGULAR_EXPRESSION` or +:prop_test:`SKIP_REGULAR_EXPRESSION` test property is used. The ``COMMAND`` and ``WORKING_DIRECTORY`` options may use "generator expressions" with the syntax ``$<...>``. See the diff --git a/Help/manual/cmake-properties.7.rst b/Help/manual/cmake-properties.7.rst index 0beca82..ae4be3e 100644 --- a/Help/manual/cmake-properties.7.rst +++ b/Help/manual/cmake-properties.7.rst @@ -407,6 +407,7 @@ Properties on Tests /prop_test/REQUIRED_FILES /prop_test/RESOURCE_LOCK /prop_test/RUN_SERIAL + /prop_test/SKIP_REGULAR_EXPRESSION /prop_test/SKIP_RETURN_CODE /prop_test/TIMEOUT /prop_test/TIMEOUT_AFTER_MATCH diff --git a/Help/prop_test/SKIP_REGULAR_EXPRESSION.rst b/Help/prop_test/SKIP_REGULAR_EXPRESSION.rst new file mode 100644 index 0000000..2c6d980 --- /dev/null +++ b/Help/prop_test/SKIP_REGULAR_EXPRESSION.rst @@ -0,0 +1,17 @@ +SKIP_REGULAR_EXPRESSION +----------------------- + +If the output matches this regular expression the test will be marked as skipped. + +If set, if the output matches one of specified regular expressions, +the test will be marked as skipped. Example: + +.. code-block:: cmake + + set_property(TEST mytest PROPERTY + SKIP_REGULAR_EXPRESSION "[^a-z]Skip" "SKIP" "Skipped" + ) + +``SKIP_REGULAR_EXPRESSION`` expects a list of regular expressions. + +See also the :prop_test:`SKIP_RETURN_CODE` property. diff --git a/Help/prop_test/SKIP_RETURN_CODE.rst b/Help/prop_test/SKIP_RETURN_CODE.rst index 3d48c2f..23c4c62 100644 --- a/Help/prop_test/SKIP_RETURN_CODE.rst +++ b/Help/prop_test/SKIP_RETURN_CODE.rst @@ -8,3 +8,5 @@ test are met. If such a situation should not be considered a hard failure a return code of the process can be specified that will mark the test as ``Not Run`` if it is encountered. Valid values are in the range of 0 to 255, inclusive. + +See also the :prop_test:`SKIP_REGULAR_EXPRESSION` property. diff --git a/Help/release/dev/add_skip_regular_expression_test_property.rst b/Help/release/dev/add_skip_regular_expression_test_property.rst new file mode 100644 index 0000000..20ef214 --- /dev/null +++ b/Help/release/dev/add_skip_regular_expression_test_property.rst @@ -0,0 +1,10 @@ +add_skip_regular_expression_test_property +----------------------------------------- + +* A new test property, :prop_test:`SKIP_REGULAR_EXPRESSION`, has been added. + This property is similar to :prop_test:`FAIL_REGULAR_EXPRESSION` and + :prop_test:`PASS_REGULAR_EXPRESSION`, but with the same meaning as + :prop_test:`SKIP_RETURN_CODE`. This is useful, for example, in cases where + the user has no control over the return code of the test. For example, in + Catch2, the return value is the number of assertion failed, therefore it is + impossible to use it for :prop_test:`SKIP_RETURN_CODE`. -- cgit v0.12 From 5dbd9c8583f44d0d9abe2ec95b2ddfd4ac65dc0d Mon Sep 17 00:00:00 2001 From: Gabor Bencze Date: Thu, 18 Jul 2019 17:00:08 +0200 Subject: Vim: Add SKIP_REGULAR_EXPRESSION test property to vim syntax file --- Auxiliary/vim/syntax/cmake.vim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Auxiliary/vim/syntax/cmake.vim b/Auxiliary/vim/syntax/cmake.vim index 8712fb9..5de117b 100644 --- a/Auxiliary/vim/syntax/cmake.vim +++ b/Auxiliary/vim/syntax/cmake.vim @@ -288,6 +288,7 @@ syn keyword cmakeProperty contained \ SKIP_AUTORCC \ SKIP_AUTOUIC \ SKIP_BUILD_RPATH + \ SKIP_REGULAR_EXPRESSION \ SKIP_RETURN_CODE \ SOURCES \ SOURCE_DIR @@ -1772,6 +1773,7 @@ syn keyword cmakeKWadd_test contained \ NAME \ OFF \ PASS_REGULAR_EXPRESSION + \ SKIP_REGULAR_EXPRESSION \ TARGET_FILE \ WILL_FAIL \ WORKING_DIRECTORY -- cgit v0.12