Merge topic 'ctest-add-ignore-no-tests-option'

a39d4139d0 Add --no-tests=<[error|ignore]> option to CTest

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !4258

16 files changed, 94 insertions, 1 deletions

diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst
index 25cb639..2bfaafe 100644
--- a/Help/manual/ctest.1.rst
+++ b/Help/manual/ctest.1.rst
@@ -370,6 +370,14 @@ See `Build and Test Mode`_.
  This option will not run any tests, it will simply print the list of
  all labels associated with the test set.
 
+``--no-tests=<[error|ignore]>``
+ Regard no tests found either as error or ignore it.
+
+ If no tests were found, the default behavior of CTest is to always log an
+ error message but to return an error code in script mode only.  This option
+ unifies the behavior of CTest by either returning an error code if no tests
+ were found or by ignoring it.
+
 .. include:: OPTIONS_HELP.txt
 
 .. _`Label and Subproject Summary`:
diff --git a/Help/release/dev/ctest-no-tests.rst b/Help/release/dev/ctest-no-tests.rst
new file mode 100644
index 0000000..be0f28b
--- /dev/null
+++ b/Help/release/dev/ctest-no-tests.rst
@@ -0,0 +1,6 @@
+ctest-no-tests
+--------------
+
+* The :manual:`ctest(1)` tool gained a ``--no-tests=<[error|ignore]>`` option
+  to explicitly set and unify the behavior between direct invocation and
+  script mode if no tests were found.
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index e70bc5a..78c68be 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -410,10 +410,15 @@ int cmCTestTestHandler::ProcessHandler()
 
   auto clock_finish = std::chrono::steady_clock::now();
 
+  bool noTestsFoundError = false;
   if (passed.size() + failed.size() == 0) {
-    if (!this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels()) {
+    if (!this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels() &&
+        this->CTest->GetNoTestsMode() != cmCTest::NoTests::Ignore) {
       cmCTestLog(this->CTest, ERROR_MESSAGE,
                  "No tests were found!!!" << std::endl);
+      if (this->CTest->GetNoTestsMode() == cmCTest::NoTests::Error) {
+        noTestsFoundError = true;
+      }
     }
   } else {
     if (this->HandlerVerbose && !passed.empty() &&
@@ -459,6 +464,12 @@ int cmCTestTestHandler::ProcessHandler()
     this->LogFile = nullptr;
     return -1;
   }
+
+  if (noTestsFoundError) {
+    this->LogFile = nullptr;
+    return -1;
+  }
+
   this->LogFile = nullptr;
   return 0;
 }
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 22a6e38..04f75bd 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -208,6 +208,8 @@ struct cmCTest::Private
   bool OutputColorCode = cmCTest::ColoredOutputSupportedByConsole();
 
   std::map<std::string, std::string> Definitions;
+
+  cmCTest::NoTests NoTestsMode = cmCTest::NoTests::Legacy;
 };
 
 struct tm* cmCTest::GetNightlyTime(std::string const& str, bool tomorrowtag)
@@ -2059,6 +2061,19 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     this->SetNotesFiles(args[i].c_str());
   }
 
+  const std::string noTestsPrefix = "--no-tests=";
+  if (cmHasPrefix(arg, noTestsPrefix)) {
+    const std::string noTestsMode = arg.substr(noTestsPrefix.length());
+    if (noTestsMode == "error") {
+      this->Impl->NoTestsMode = cmCTest::NoTests::Error;
+    } else if (noTestsMode != "ignore") {
+      errormsg = "'--no-tests=' given unknown value '" + noTestsMode + "'";
+      return false;
+    } else {
+      this->Impl->NoTestsMode = cmCTest::NoTests::Ignore;
+    }
+  }
+
   // options that control what tests are run
   if (this->CheckArgument(arg, "-I", "--tests-information") &&
       i < args.size() - 1) {
@@ -2896,6 +2911,11 @@ cmCTest::Repeat cmCTest::GetRepeatMode() const
   return this->Impl->RepeatMode;
 }
 
+cmCTest::NoTests cmCTest::GetNoTestsMode() const
+{
+  return this->Impl->NoTestsMode;
+}
+
 void cmCTest::SetBuildID(const std::string& id)
 {
   this->Impl->BuildID = id;
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index e0ae100..7f8f913 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -442,6 +442,14 @@ public:
   };
   Repeat GetRepeatMode() const;
 
+  enum class NoTests
+  {
+    Legacy,
+    Error,
+    Ignore
+  };
+  NoTests GetNoTestsMode() const;
+
   void GenerateSubprojectsOutput(cmXMLWriter& xml);
   std::vector<std::string> GetLabelsForSubprojects();
 
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index 9b45bb0..fbdf75a 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -144,6 +144,8 @@ static const char* cmDocumentationOptions[][2] = {
   { "--http1.0", "Submit using HTTP 1.0." },
   { "--no-compress-output", "Do not compress test output when submitting." },
   { "--print-labels", "Print all available test labels." },
+  { "--no-tests=<[error|ignore]>",
+    "Regard no tests found either as 'error' or 'ignore' it." },
   { nullptr, nullptr }
 };
 
diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index b2de596..9b9ae65 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -314,5 +314,34 @@ function(run_ShowOnly)
 endfunction()
 run_ShowOnly()
 
+function(run_NoTests)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/NoTests)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "")
+  run_cmake_command(no-tests_ignore ${CMAKE_CTEST_COMMAND} --no-tests=ignore)
+  run_cmake_command(no-tests_error ${CMAKE_CTEST_COMMAND} --no-tests=error)
+  run_cmake_command(no-tests_bad ${CMAKE_CTEST_COMMAND} --no-tests=bad)
+  run_cmake_command(no-tests_legacy ${CMAKE_CTEST_COMMAND})
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake" "
+    set(CTEST_COMMAND \"${CMAKE_CTEST_COMMAND}\")
+    set(CTEST_SOURCE_DIRECTORY \"${RunCMake_SOURCE_DIR}\")
+    set(CTEST_BINARY_DIRECTORY \"${RunCMake_TEST_BINARY_DIR}\")
+    ctest_start(Experimental)
+    ctest_test()
+")
+  run_cmake_command(
+    no-tests-script_ignore ${CMAKE_CTEST_COMMAND} --no-tests=ignore
+    -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake")
+  run_cmake_command(
+    no-tests-script_error ${CMAKE_CTEST_COMMAND} --no-tests=error
+    -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake")
+  run_cmake_command(
+    no-tests-script_legacy ${CMAKE_CTEST_COMMAND}
+    -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake")
+endfunction()
+run_NoTests()
+
 # Check the configuration type variable is passed
 run_ctest(check-configuration-type)
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt
new file mode 100644
index 0000000..a7c4b11
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt
new file mode 100644
index 0000000..a7c4b11
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt
new file mode 100644
index 0000000..1703539
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--no-tests=' given unknown value 'bad'$
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt
new file mode 100644
index 0000000..45a4fb7
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt
@@ -0,0 +1 @@
+8
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt
new file mode 100644
index 0000000..eafba1c
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt
@@ -0,0 +1 @@
+No tests were found!!!
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt
new file mode 100644
index 0000000..a7c4b11
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$