From c5032519974ae8649036f0b85a7100632dc064a5 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 26 May 2020 11:36:05 -0400 Subject: Tests: Add coverage of ctest_test RETURN_VALUE and REPEAT --- Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 84d1d66..71d6b54 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -81,8 +81,8 @@ run_TestOutputSize() run_ctest_test(TestRepeatBad1 REPEAT UNKNOWN:3) run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1) -function(run_TestRepeat case) - set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion ${ARGN}) +function(run_TestRepeat case return_value ) + set(CASE_CTEST_TEST_ARGS RETURN_VALUE result EXCLUDE RunCMakeVersion ${ARGN}) string(CONCAT CASE_CMAKELISTS_SUFFIX_CODE [[ add_test(NAME testRepeat COMMAND ${CMAKE_COMMAND} -D COUNT_FILE=${CMAKE_CURRENT_BINARY_DIR}/count.cmake @@ -91,7 +91,23 @@ set_property(TEST testRepeat PROPERTY TIMEOUT 5) ]]) run_ctest(TestRepeat${case}) + + #write to end of the test file logic to Verify we get the expected + #return code + string(REPLACE "RETURN_VALUE:" "" return_value "${return_value}" ) + file(APPEND "${RunCMake_BINARY_DIR}/TestRepeat${case}/test.cmake" +" + + set(expected_result ${return_value}) + message(STATUS \${result}) + if(NOT result EQUAL expected_result) + message(FATAL_ERROR \"expected a return value of: \${expected_result}, + instead got: \${result}\") + endif() +" + ) endfunction() -run_TestRepeat(UntilFail REPEAT UNTIL_FAIL:3) -run_TestRepeat(UntilPass REPEAT UNTIL_PASS:3) -run_TestRepeat(AfterTimeout REPEAT AFTER_TIMEOUT:3) + +run_TestRepeat(UntilFail RETURN_VALUE:1 REPEAT UNTIL_FAIL:3) +run_TestRepeat(UntilPass RETURN_VALUE:0 REPEAT UNTIL_PASS:3) +run_TestRepeat(AfterTimeout RETURN_VALUE:0 REPEAT AFTER_TIMEOUT:3) -- cgit v0.12 From bbb62dcc72394316f151ead984a9f7cdc258e98f Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 26 May 2020 11:48:41 -0400 Subject: CTest: Make sure NOT_RUN tests show up in the failed test log Issue: #20543 --- Source/CTest/cmCTestRunTest.cxx | 5 +++++ Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 12 +++++++++++- Tests/RunCMake/ctest_test/TestRepeatNotRun-result.txt | 1 + Tests/RunCMake/ctest_test/TestRepeatNotRun-stderr.txt | 1 + Tests/RunCMake/ctest_test/TestRepeatNotRun-stdout.txt | 5 +++++ Tests/RunCMake/ctest_test/TestRepeatNotRun.cmake | 10 ++++++++++ 6 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 Tests/RunCMake/ctest_test/TestRepeatNotRun-result.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatNotRun-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatNotRun-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/TestRepeatNotRun.cmake diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index ec54960..58289ea 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -340,6 +340,11 @@ bool cmCTestRunTest::NeedsToRepeat() if (this->NumberOfRunsLeft == 0) { return false; } + // If a test is marked as NOT_RUN it will not be repeated + // no matter the repeat settings, so just record it as-is. + if (this->TestResult.Status == cmCTestTestHandler::NOT_RUN) { + return false; + } // if number of runs left is not 0, and we are running until // we find a failed (or passed) test, then return true so the test can be // restarted diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 71d6b54..d0a6e5f 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -83,12 +83,13 @@ run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1) function(run_TestRepeat case return_value ) set(CASE_CTEST_TEST_ARGS RETURN_VALUE result EXCLUDE RunCMakeVersion ${ARGN}) - string(CONCAT CASE_CMAKELISTS_SUFFIX_CODE [[ + string(CONCAT suffix_code [[ add_test(NAME testRepeat COMMAND ${CMAKE_COMMAND} -D COUNT_FILE=${CMAKE_CURRENT_BINARY_DIR}/count.cmake -P "]] "${RunCMake_SOURCE_DIR}/TestRepeat${case}" [[.cmake") set_property(TEST testRepeat PROPERTY TIMEOUT 5) ]]) + string(APPEND CASE_CMAKELISTS_SUFFIX_CODE "${suffix_code}") run_ctest(TestRepeat${case}) @@ -111,3 +112,12 @@ endfunction() run_TestRepeat(UntilFail RETURN_VALUE:1 REPEAT UNTIL_FAIL:3) run_TestRepeat(UntilPass RETURN_VALUE:0 REPEAT UNTIL_PASS:3) run_TestRepeat(AfterTimeout RETURN_VALUE:0 REPEAT AFTER_TIMEOUT:3) + +# test repeat and not run tests interact correctly +set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME testNotRun + COMMAND ${CMAKE_COMMAND}/doesnt_exist) + set_property(TEST testNotRun PROPERTY TIMEOUT 5) + ]]) +run_TestRepeat(NotRun RETURN_VALUE:1 REPEAT UNTIL_PASS:3) +unset(CASE_CMAKELISTS_SUFFIX_CODE) diff --git a/Tests/RunCMake/ctest_test/TestRepeatNotRun-result.txt b/Tests/RunCMake/ctest_test/TestRepeatNotRun-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatNotRun-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/TestRepeatNotRun-stderr.txt b/Tests/RunCMake/ctest_test/TestRepeatNotRun-stderr.txt new file mode 100644 index 0000000..a69932d --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatNotRun-stderr.txt @@ -0,0 +1 @@ +.*Unable to find executable.* diff --git a/Tests/RunCMake/ctest_test/TestRepeatNotRun-stdout.txt b/Tests/RunCMake/ctest_test/TestRepeatNotRun-stdout.txt new file mode 100644 index 0000000..72c98bc --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatNotRun-stdout.txt @@ -0,0 +1,5 @@ +.* +50% tests passed, 1 tests failed out of 2 +.* +The following tests FAILED: +.*testNotRun.*Not Run.* diff --git a/Tests/RunCMake/ctest_test/TestRepeatNotRun.cmake b/Tests/RunCMake/ctest_test/TestRepeatNotRun.cmake new file mode 100644 index 0000000..abde4f0 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestRepeatNotRun.cmake @@ -0,0 +1,10 @@ +include("${COUNT_FILE}" OPTIONAL) +if(NOT COUNT) + set(COUNT 0) +endif() +math(EXPR COUNT "${COUNT} + 1") +file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n") +if(NOT COUNT EQUAL 2) + message("this test times out except on the 2nd run") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10) +endif() -- cgit v0.12