From 8673264e25b4abfb3b328131d983e7668a8c5e1a Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 8 Mar 2024 16:03:00 -0500 Subject: Tests: Make ctest tests-from-file expected output more precise --- Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 2 +- .../CTestCommandLine/TestsFromFile-exclude-missing-stdout.txt | 2 +- Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-stdout.txt | 2 ++ .../CTestCommandLine/TestsFromFile-include-missing-stdout.txt | 2 +- Tests/RunCMake/CTestCommandLine/TestsFromFile-include-stdout.txt | 2 ++ Tests/RunCMake/ctest_test/TestsFromFile-exclude-stdout.txt | 4 ++++ Tests/RunCMake/ctest_test/TestsFromFile-include-stdout.txt | 2 ++ 7 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 715a5fa..8c3065e 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -209,7 +209,7 @@ run_SkipRegexFoundTest() function(run_TestsFromFileTest case) - set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestsFromFile) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestsFromFile-${case}) set(RunCMake_TEST_NO_CLEAN 1) file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stdout.txt index 1e4e805..2448284 100644 --- a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stdout.txt +++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stdout.txt @@ -1,2 +1,2 @@ Test project [^ -]*/Tests/RunCMake/CTestCommandLine/TestsFromFile$ +]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing$ diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-stdout.txt index 96d8c3b..6db9b50 100644 --- a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-stdout.txt +++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-stdout.txt @@ -1,3 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude +Start 2: Test2 1/2 Test #2: Test2 ............................ Passed +[0-9.]+ sec +Start 3: Test11 diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stdout.txt index 1e4e805..1f87a44 100644 --- a/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stdout.txt +++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stdout.txt @@ -1,2 +1,2 @@ Test project [^ -]*/Tests/RunCMake/CTestCommandLine/TestsFromFile$ +]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing$ diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-stdout.txt index 653d56c..d07400d 100644 --- a/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-stdout.txt +++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-stdout.txt @@ -1,3 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-include +Start 1: Test1 1/1 Test #1: Test1 ............................ Passed +[0-9.]+ sec + diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-exclude-stdout.txt b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-stdout.txt index 29c2b7f..f401861 100644 --- a/Tests/RunCMake/ctest_test/TestsFromFile-exclude-stdout.txt +++ b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-stdout.txt @@ -1,3 +1,7 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/TestsFromFile-exclude-build + +Start 1: RunCMakeVersion +1/3 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec +Start 3: Test2 2/3 Test #3: Test2 ............................ Passed +[0-9.]+ sec +Start 4: Test11 diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-include-stdout.txt b/Tests/RunCMake/ctest_test/TestsFromFile-include-stdout.txt index e3ca719..8e7093b 100644 --- a/Tests/RunCMake/ctest_test/TestsFromFile-include-stdout.txt +++ b/Tests/RunCMake/ctest_test/TestsFromFile-include-stdout.txt @@ -1,3 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/TestsFromFile-include-build +Start 2: Test1 1/1 Test #2: Test1 ............................ Passed +[0-9.]+ sec + -- cgit v0.12 From d52c66bfb3ec6bbf79fab9ef6ff4cdc9ba27a45b Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 8 Mar 2024 16:05:34 -0500 Subject: ctest: Honor tests-from-file options with empty input If the `--tests-from-file` input file is empty, no tests should run. --- Source/CTest/cmCTestTestHandler.cxx | 38 ++++++++++++---------- Source/CTest/cmCTestTestHandler.h | 8 ++--- Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 2 ++ .../TestsFromFile-TestList-empty.txt | 0 .../TestsFromFile-exclude-empty-stdout.txt | 10 ++++++ .../TestsFromFile-include-empty-stderr.txt | 1 + Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 2 ++ .../ctest_test/TestsFromFile-TestList-empty.txt | 0 .../TestsFromFile-exclude-empty-stdout.txt | 12 +++++++ .../TestsFromFile-include-empty-result.txt | 1 + .../TestsFromFile-include-empty-stderr.txt | 1 + 11 files changed, 53 insertions(+), 22 deletions(-) create mode 100644 Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList-empty.txt create mode 100644 Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-empty-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/TestsFromFile-include-empty-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/TestsFromFile-TestList-empty.txt create mode 100644 Tests/RunCMake/ctest_test/TestsFromFile-exclude-empty-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/TestsFromFile-include-empty-result.txt create mode 100644 Tests/RunCMake/ctest_test/TestsFromFile-include-empty-stderr.txt diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index ebb5073..63ff8d1 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -347,6 +347,8 @@ void cmCTestTestHandler::Initialize() this->ExcludeFixtureCleanupRegExp.clear(); this->TestListFile.clear(); this->ExcludeTestListFile.clear(); + this->TestsToRunByName.reset(); + this->TestsToExcludeByName.reset(); this->TestsToRunString.clear(); this->UseUnion = false; @@ -944,16 +946,16 @@ bool cmCTestTestHandler::ComputeTestList() } } - if (!this->TestsToRunByName.empty()) { - if (this->TestsToRunByName.find(tp.Name) == - this->TestsToRunByName.end()) { + if (this->TestsToRunByName) { + if (this->TestsToRunByName->find(tp.Name) == + this->TestsToRunByName->end()) { continue; } } - if (!this->TestsToExcludeByName.empty()) { - if (this->TestsToExcludeByName.find(tp.Name) != - this->TestsToExcludeByName.end()) { + if (this->TestsToExcludeByName) { + if (this->TestsToExcludeByName->find(tp.Name) != + this->TestsToExcludeByName->end()) { continue; } } @@ -1845,13 +1847,15 @@ bool cmCTestTestHandler::GetListOfTests() } if (!this->TestListFile.empty()) { - if (!this->ReadTestListFile(this->TestListFile, this->TestsToRunByName)) { + this->TestsToRunByName = this->ReadTestListFile(this->TestListFile); + if (!this->TestsToRunByName) { return false; } } if (!this->ExcludeTestListFile.empty()) { - if (!this->ReadTestListFile(this->ExcludeTestListFile, - this->TestsToExcludeByName)) { + this->TestsToExcludeByName = + this->ReadTestListFile(this->ExcludeTestListFile); + if (!this->TestsToExcludeByName) { return false; } } @@ -2024,12 +2028,13 @@ void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed() } } -bool cmCTestTestHandler::ReadTestListFile( - std::string const& testListFileName, std::set& testNames) const +cm::optional> cmCTestTestHandler::ReadTestListFile( + std::string const& testListFileName) const { - testNames.clear(); + cm::optional> result; cmsys::ifstream ifs(testListFileName.c_str()); if (ifs) { + std::set testNames; std::string line; while (cmSystemTools::GetLineFromStream(ifs, line)) { std::string trimmed = cmTrimWhitespace(line); @@ -2039,17 +2044,14 @@ bool cmCTestTestHandler::ReadTestListFile( testNames.insert(trimmed); } - ifs.close(); - } else if (!this->CTest->GetShowOnly() && - !this->CTest->ShouldPrintLabels()) { + result = std::move(testNames); + } else { cmCTestLog(this->CTest, ERROR_MESSAGE, "Problem reading test list file: " << testListFileName << " while generating list of tests to run." << std::endl); - return false; } - - return true; + return result; } void cmCTestTestHandler::RecordCustomTestMeasurements(cmXMLWriter& xml, diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index e9c964b..6932800 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -341,8 +341,8 @@ private: std::string GetTestStatus(cmCTestTestResult const&); void ExpandTestsToRunInformation(size_t numPossibleTests); void ExpandTestsToRunInformationForRerunFailed(); - bool ReadTestListFile(std::string const& testListFileName, - std::set& testNames) const; + cm::optional> ReadTestListFile( + std::string const& testListFileName) const; std::vector CustomPreTest; std::vector CustomPostTest; @@ -363,8 +363,8 @@ private: cmsys::RegularExpression ExcludeTestsRegularExpression; std::string TestListFile; std::string ExcludeTestListFile; - std::set TestsToRunByName; - std::set TestsToExcludeByName; + cm::optional> TestsToRunByName; + cm::optional> TestsToExcludeByName; std::string ResourceSpecFile; diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index 8c3065e..a5e8037 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -223,6 +223,8 @@ add_test(Test11 \"${CMAKE_COMMAND}\" -E echo \"test11\") endfunction() run_TestsFromFileTest(include --tests-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt) run_TestsFromFileTest(exclude --exclude-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt) +run_TestsFromFileTest(include-empty --tests-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-empty.txt) +run_TestsFromFileTest(exclude-empty --exclude-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-empty.txt) run_TestsFromFileTest(include-missing --tests-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt) run_TestsFromFileTest(exclude-missing --exclude-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt) diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList-empty.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList-empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-empty-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-empty-stdout.txt new file mode 100644 index 0000000..64a5473 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-empty-stdout.txt @@ -0,0 +1,10 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-empty + +Start 1: Test1 +1/3 Test #1: Test1 ............................ Passed +[0-9.]+ sec + +Start 2: Test2 +2/3 Test #2: Test2 ............................ Passed +[0-9.]+ sec + +Start 3: Test11 +3/3 Test #3: Test11 ........................... Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 3 diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-empty-stderr.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-empty-stderr.txt new file mode 100644 index 0000000..a7c4b11 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-empty-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$ diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index e2c41a0..768b0be 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -177,6 +177,8 @@ add_test(NAME Test11 COMMAND ${CMAKE_COMMAND} -E true) endfunction() run_tests_from_file(include INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt) run_tests_from_file(exclude EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt) +run_tests_from_file(include-empty INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-empty.txt) +run_tests_from_file(exclude-empty EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-empty.txt) run_tests_from_file(include-missing INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt) run_tests_from_file(exclude-missing EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt) diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-TestList-empty.txt b/Tests/RunCMake/ctest_test/TestsFromFile-TestList-empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-exclude-empty-stdout.txt b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-empty-stdout.txt new file mode 100644 index 0000000..a0faf57 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-empty-stdout.txt @@ -0,0 +1,12 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/TestsFromFile-exclude-empty-build + +Start 1: RunCMakeVersion +1/4 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec + +Start 2: Test1 +2/4 Test #2: Test1 ............................ Passed +[0-9.]+ sec + +Start 3: Test2 +3/4 Test #3: Test2 ............................ Passed +[0-9.]+ sec + +Start 4: Test11 +4/4 Test #4: Test11 ........................... Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 4 diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-include-empty-result.txt b/Tests/RunCMake/ctest_test/TestsFromFile-include-empty-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestsFromFile-include-empty-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-include-empty-stderr.txt b/Tests/RunCMake/ctest_test/TestsFromFile-include-empty-stderr.txt new file mode 100644 index 0000000..a7c4b11 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestsFromFile-include-empty-stderr.txt @@ -0,0 +1 @@ +^No tests were found!!!$ -- cgit v0.12 From 1a4837641e8fad5d8a0c611ca1c70543c5748fe2 Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 8 Mar 2024 16:07:36 -0500 Subject: ctest: Remove unnecessary and ambiguous tests-from-file comment syntax Test names can contain `#`. Since we ignore lines that do not match any test names anyway, "commenting" can still work without explicit syntax. Also drop whitespace trimming for similar reasons. Fixes: #25741 --- Help/manual/ctest.1.rst | 4 ++-- Source/CTest/cmCTestTestHandler.cxx | 7 ++----- Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList.txt | 3 +++ Tests/RunCMake/ctest_test/TestsFromFile-TestList.txt | 3 +++ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 602a82d..aa95ac6 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -242,7 +242,7 @@ Run Tests This option tells CTest to run the tests which are listed in the given file. The file must contain one exact test name per line. - Lines can be commented out using a ``#``. + Lines that do not exactly match any test names are ignored. This option can be combined with the other options like ``-R``, ``-E``, ``-L`` or ``-LE``. @@ -254,7 +254,7 @@ Run Tests This option tells CTest to NOT run the tests which are listed in the given file. The file must contain one exact test name per line. - Lines can be commented out using a ``#``. + Lines that do not exactly match any test names are ignored. This option can be combined with the other options like ``-R``, ``-E``, ``-L`` or ``-LE``. diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 63ff8d1..dae205c 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -2037,12 +2037,9 @@ cm::optional> cmCTestTestHandler::ReadTestListFile( std::set testNames; std::string line; while (cmSystemTools::GetLineFromStream(ifs, line)) { - std::string trimmed = cmTrimWhitespace(line); - if (trimmed.empty() || (trimmed[0] == '#')) { - continue; + if (!line.empty()) { + testNames.insert(line); } - - testNames.insert(trimmed); } result = std::move(testNames); } else { diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList.txt index 91f6504..975a21c 100644 --- a/Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList.txt +++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList.txt @@ -1,2 +1,5 @@ Test1 + +est + Test11 # Test11 diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-TestList.txt b/Tests/RunCMake/ctest_test/TestsFromFile-TestList.txt index 91f6504..975a21c 100644 --- a/Tests/RunCMake/ctest_test/TestsFromFile-TestList.txt +++ b/Tests/RunCMake/ctest_test/TestsFromFile-TestList.txt @@ -1,2 +1,5 @@ Test1 + +est + Test11 # Test11 -- cgit v0.12 From 170ec486014958c372df4c38e940d50b2c0dc0af Mon Sep 17 00:00:00 2001 From: Brad King Date: Fri, 8 Mar 2024 16:16:26 -0500 Subject: Help: Improve ctest tests-from-file documentation wording and wrapping --- Help/manual/ctest.1.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index aa95ac6..9f2eb13 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -240,8 +240,8 @@ Run Tests Run tests listed in the given file. - This option tells CTest to run the tests which are listed in the given - file. The file must contain one exact test name per line. + This option tells CTest to run tests that are listed in the given file. + The file must contain one exact test name per line. Lines that do not exactly match any test names are ignored. This option can be combined with the other options like ``-R``, ``-E``, ``-L`` or ``-LE``. @@ -252,8 +252,8 @@ Run Tests Exclude tests listed in the given file. - This option tells CTest to NOT run the tests which are listed in the given - file. The file must contain one exact test name per line. + This option tells CTest to NOT run tests that are listed in the given file. + The file must contain one exact test name per line. Lines that do not exactly match any test names are ignored. This option can be combined with the other options like ``-R``, ``-E``, ``-L`` or ``-LE``. -- cgit v0.12