Merge topic 'ctest-tests-from-file' into release-3.29

ca0a9def2e ctest: Exit with failure when tests-from-file input is missing
60433fc6e8 Tests: Generalize ctest tests-from-file test cases

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !9317

20 files changed, 47 insertions, 19 deletions

diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index dc16f66..ebb5073 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -1845,11 +1845,15 @@ bool cmCTestTestHandler::GetListOfTests()
   }
 
   if (!this->TestListFile.empty()) {
-    this->TestsToRunByName = this->ReadTestListFile(this->TestListFile);
+    if (!this->ReadTestListFile(this->TestListFile, this->TestsToRunByName)) {
+      return false;
+    }
   }
   if (!this->ExcludeTestListFile.empty()) {
-    this->TestsToExcludeByName =
-      this->ReadTestListFile(this->ExcludeTestListFile);
+    if (!this->ReadTestListFile(this->ExcludeTestListFile,
+                                this->TestsToExcludeByName)) {
+      return false;
+    }
   }
 
   cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
@@ -2020,11 +2024,10 @@ void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed()
   }
 }
 
-std::set<std::string> cmCTestTestHandler::ReadTestListFile(
-  const std::string& testListFileName) const
+bool cmCTestTestHandler::ReadTestListFile(
+  std::string const& testListFileName, std::set<std::string>& testNames) const
 {
-  std::set<std::string> testNames;
-
+  testNames.clear();
   cmsys::ifstream ifs(testListFileName.c_str());
   if (ifs) {
     std::string line;
@@ -2043,9 +2046,10 @@ std::set<std::string> cmCTestTestHandler::ReadTestListFile(
                "Problem reading test list file: "
                  << testListFileName
                  << " while generating list of tests to run." << std::endl);
+    return false;
   }
 
-  return testNames;
+  return true;
 }
 
 void cmCTestTestHandler::RecordCustomTestMeasurements(cmXMLWriter& xml,
diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h
index 84e6098..e9c964b 100644
--- a/Source/CTest/cmCTestTestHandler.h
+++ b/Source/CTest/cmCTestTestHandler.h
@@ -341,8 +341,8 @@ private:
   std::string GetTestStatus(cmCTestTestResult const&);
   void ExpandTestsToRunInformation(size_t numPossibleTests);
   void ExpandTestsToRunInformationForRerunFailed();
-  std::set<std::string> ReadTestListFile(
-    const std::string& testListFileName) const;
+  bool ReadTestListFile(std::string const& testListFileName,
+                        std::set<std::string>& testNames) const;
 
   std::vector<std::string> CustomPreTest;
   std::vector<std::string> CustomPostTest;
diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index c09fff8..715a5fa 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -208,7 +208,7 @@ endfunction()
 run_SkipRegexFoundTest()
 
 
-function(run_TestsFromFileTest arg)
+function(run_TestsFromFileTest case)
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestsFromFile)
   set(RunCMake_TEST_NO_CLEAN 1)
   file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
@@ -219,10 +219,12 @@ add_test(Test1 \"${CMAKE_COMMAND}\" -E echo \"test1\")
 add_test(Test2 \"${CMAKE_COMMAND}\" -E echo \"test2\")
 add_test(Test11 \"${CMAKE_COMMAND}\" -E echo \"test11\")
 ")
-  run_cmake_command(TestsFromFile-${arg} ${CMAKE_CTEST_COMMAND} --${arg} ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt )
+  run_cmake_command(TestsFromFile-${case} ${CMAKE_CTEST_COMMAND} ${ARGN})
 endfunction()
-run_TestsFromFileTest(tests-from-file)
-run_TestsFromFileTest(exclude-from-file)
+run_TestsFromFileTest(include --tests-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
+run_TestsFromFileTest(exclude --exclude-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
+run_TestsFromFileTest(include-missing --tests-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)
+run_TestsFromFileTest(exclude-missing --exclude-from-file ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)
 
 
 function(run_SerialFailed)
diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-result.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-result.txt
new file mode 100644
index 0000000..d197c91
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-result.txt
@@ -0,0 +1 @@
+[^0]
diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stderr.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stderr.txt
new file mode 100644
index 0000000..1d22abc
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stderr.txt
@@ -0,0 +1,2 @@
+Problem reading test list file: [^
+]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList-missing\.txt while generating list of tests to run\.
diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stdout.txt
new file mode 100644
index 0000000..1e4e805
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-missing-stdout.txt
@@ -0,0 +1,2 @@
+Test project [^
+]*/Tests/RunCMake/CTestCommandLine/TestsFromFile$
diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-from-file-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-stdout.txt
index 96d8c3b..96d8c3b 100644
--- a/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-from-file-stdout.txt
+++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-exclude-stdout.txt
diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-result.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-result.txt
new file mode 100644
index 0000000..d197c91
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-result.txt
@@ -0,0 +1 @@
+[^0]
diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stderr.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stderr.txt
new file mode 100644
index 0000000..1d22abc
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stderr.txt
@@ -0,0 +1,2 @@
+Problem reading test list file: [^
+]*/Tests/RunCMake/CTestCommandLine/TestsFromFile-TestList-missing\.txt while generating list of tests to run\.
diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stdout.txt
new file mode 100644
index 0000000..1e4e805
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-missing-stdout.txt
@@ -0,0 +1,2 @@
+Test project [^
+]*/Tests/RunCMake/CTestCommandLine/TestsFromFile$
diff --git a/Tests/RunCMake/CTestCommandLine/TestsFromFile-tests-from-file-stdout.txt b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-stdout.txt
index 653d56c..653d56c 100644
--- a/Tests/RunCMake/CTestCommandLine/TestsFromFile-tests-from-file-stdout.txt
+++ b/Tests/RunCMake/CTestCommandLine/TestsFromFile-include-stdout.txt
diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
index 4413a4d..e2c41a0 100644
--- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
+++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
@@ -165,18 +165,20 @@ run_stop_on_failure()
 
 
 # test include/exclude tests from file
-function(run_tests_from_file mode)
-  set(CASE_CTEST_TEST_ARGS ${mode} ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
+function(run_tests_from_file case)
+  set(CASE_CTEST_TEST_ARGS ${ARGN})
   set(CASE_CMAKELISTS_SUFFIX_CODE [[
 add_test(NAME Test1 COMMAND ${CMAKE_COMMAND} -E true)
 add_test(NAME Test2 COMMAND ${CMAKE_COMMAND} -E true)
 add_test(NAME Test11 COMMAND ${CMAKE_COMMAND} -E true)
   ]])
 
-  run_ctest(TestsFromFile-${mode})
+  run_ctest(TestsFromFile-${case})
 endfunction()
-run_tests_from_file(INCLUDE_FROM_FILE)
-run_tests_from_file(EXCLUDE_FROM_FILE)
+run_tests_from_file(include INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
+run_tests_from_file(exclude EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
+run_tests_from_file(include-missing INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)
+run_tests_from_file(exclude-missing EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)
 
 
 # Make sure environment gets logged
diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-result.txt b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-stderr.txt b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-stderr.txt
new file mode 100644
index 0000000..a5b3e46
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-stderr.txt
@@ -0,0 +1,2 @@
+Problem reading test list file: [^
+]*/Tests/RunCMake/ctest_test/TestsFromFile-TestList-missing\.txt while generating list of tests to run\.
diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-stdout.txt b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-stdout.txt
new file mode 100644
index 0000000..ac5a727
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-stdout.txt
@@ -0,0 +1,2 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestsFromFile-exclude-missing-build$
diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-EXCLUDE_FROM_FILE-stdout.txt b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-stdout.txt
index 29c2b7f..29c2b7f 100644
--- a/Tests/RunCMake/ctest_test/TestsFromFile-EXCLUDE_FROM_FILE-stdout.txt
+++ b/Tests/RunCMake/ctest_test/TestsFromFile-exclude-stdout.txt
diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-result.txt b/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-stderr.txt b/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-stderr.txt
new file mode 100644
index 0000000..a5b3e46
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-stderr.txt
@@ -0,0 +1,2 @@
+Problem reading test list file: [^
+]*/Tests/RunCMake/ctest_test/TestsFromFile-TestList-missing\.txt while generating list of tests to run\.
diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-stdout.txt b/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-stdout.txt
new file mode 100644
index 0000000..4a9447c
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-stdout.txt
@@ -0,0 +1,2 @@
+Test project [^
+]*/Tests/RunCMake/ctest_test/TestsFromFile-include-missing-build$
diff --git a/Tests/RunCMake/ctest_test/TestsFromFile-INCLUDE_FROM_FILE-stdout.txt b/Tests/RunCMake/ctest_test/TestsFromFile-include-stdout.txt
index e3ca719..e3ca719 100644
--- a/Tests/RunCMake/ctest_test/TestsFromFile-INCLUDE_FROM_FILE-stdout.txt
+++ b/Tests/RunCMake/ctest_test/TestsFromFile-include-stdout.txt