Add --no-tests=<[error|ignore]> option to CTest

If no tests were found, the default behavior of CTest is to always log an
error message but to return an error code in script mode only. This option
unifies the behavior of CTest by either returning an error code if no tests
were found or by ignoring it.

Signed-off-by: Stefan Dinkelacker <s.dinkelacker@dkfz-heidelberg.de>

16 files changed, 94 insertions, 1 deletions

diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst
index 25cb639..2bfaafe 100644
--- a/Help/manual/ctest.1.rst
+++ b/Help/manual/ctest.1.rst
@@ -370,6 +370,14 @@ See `Build and Test Mode`_.
  This option will not run any tests, it will simply print the list of
  all labels associated with the test set.
 
+``--no-tests=<[error|ignore]>``
+ Regard no tests found either as error or ignore it.
+
+ If no tests were found, the default behavior of CTest is to always log an
+ error message but to return an error code in script mode only.  This option
+ unifies the behavior of CTest by either returning an error code if no tests
+ were found or by ignoring it.
+
 .. include:: OPTIONS_HELP.txt
 
 .. _`Label and Subproject Summary`:
diff --git a/Help/release/dev/ctest-no-tests.rst b/Help/release/dev/ctest-no-tests.rst
new file mode 100644
index 0000000..be0f28b
--- /dev/null
+++ b/Help/release/dev/ctest-no-tests.rst
@@ -0,0 +1,6 @@
+ctest-no-tests
+--------------
+
+* The :manual:`ctest(1)` tool gained a ``--no-tests=<[error|ignore]>`` option
+  to explicitly set and unify the behavior between direct invocation and
+  script mode if no tests were found.
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index e70bc5a..78c68be 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -410,10 +410,15 @@ int cmCTestTestHandler::ProcessHandler()
 
   auto clock_finish = std::chrono::steady_clock::now();
 
+  bool noTestsFoundError = false;
   if (passed.size() + failed.size() == 0) {
-    if (!this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels()) {
+    if (!this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels() &&
+        this->CTest->GetNoTestsMode() != cmCTest::NoTests::Ignore) {
       cmCTestLog(this->CTest, ERROR_MESSAGE,
                  "No tests were found!!!" << std::endl);
+      if (this->CTest->GetNoTestsMode() == cmCTest::NoTests::Error) {
+        noTestsFoundError = true;
+      }
     }
   } else {
     if (this->HandlerVerbose && !passed.empty() &&
@@ -459,6 +464,12 @@ int cmCTestTestHandler::ProcessHandler()
     this->LogFile = nullptr;
     return -1;
   }
+
+  if (noTestsFoundError) {
+    this->LogFile = nullptr;
+    return -1;
+  }
+
   this->LogFile = nullptr;
   return 0;
 }
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 22a6e38..04f75bd 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -208,6 +208,8 @@ struct cmCTest::Private
   bool OutputColorCode = cmCTest::ColoredOutputSupportedByConsole();
 
   std::map<std::string, std::string> Definitions;
+
+  cmCTest::NoTests NoTestsMode = cmCTest::NoTests::Legacy;
 };
 
 struct tm* cmCTest::GetNightlyTime(std::string const& str, bool tomorrowtag)
@@ -2059,6 +2061,19 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     this->SetNotesFiles(args[i].c_str());
   }
 
+  const std::string noTestsPrefix = "--no-tests=";
+  if (cmHasPrefix(arg, noTestsPrefix)) {
+    const std::string noTestsMode = arg.substr(noTestsPrefix.length());
+    if (noTestsMode == "error") {
+      this->Impl->NoTestsMode = cmCTest::NoTests::Error;
+    } else if (noTestsMode != "ignore") {
+      errormsg = "'--no-tests=' given unknown value '" + noTestsMode + "'";
+      return false;
+    } else {
+      this->Impl->NoTestsMode = cmCTest::NoTests::Ignore;
+    }
+  }
+
   // options that control what tests are run
   if (this->CheckArgument(arg, "-I", "--tests-information") &&
       i < args.size() - 1) {
@@ -2896,6 +2911,11 @@ cmCTest::Repeat cmCTest::GetRepeatMode() const
   return this->Impl->RepeatMode;
 }
 
+cmCTest::NoTests cmCTest::GetNoTestsMode() const
+{
+  return this->Impl->NoTestsMode;
+}
+
 void cmCTest::SetBuildID(const std::string& id)
 {
   this->Impl->BuildID = id;
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index e0ae100..7f8f913 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -442,6 +442,14 @@ public:
   };
   Repeat GetRepeatMode() const;
 
+  enum class NoTests
+  {
+    Legacy,
+    Error,
+    Ignore
+  };
+  NoTests GetNoTestsMode() const;
+
   void GenerateSubprojectsOutput(cmXMLWriter& xml);
   std::vector<std::string> GetLabelsForSubprojects();
 
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index 9b45bb0..fbdf75a 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -144,6 +144,8 @@ static const char* cmDocumentationOptions[][2] = {
   { "--http1.0", "Submit using HTTP 1.0." },
   { "--no-compress-output", "Do not compress test output when submitting." },
   { "--print-labels", "Print all available test labels." },
+  { "--no-tests=<[error|ignore]>",
+    "Regard no tests found either as 'error' or 'ignore' it." },
   { nullptr, nullptr }
 };
 
diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index b2de596..9b9ae65 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -314,5 +314,34 @@ function(run_ShowOnly)
 endfunction()
 run_ShowOnly()
 
+function(run_NoTests)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/NoTests)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "")
+  run_cmake_command(no-tests_ignore ${CMAKE_CTEST_COMMAND} --no-tests=ignore)
+  run_cmake_command(no-tests_error ${CMAKE_CTEST_COMMAND} --no-tests=error)
+  run_cmake_command(no-tests_bad ${CMAKE_CTEST_COMMAND} --no-tests=bad)
+  run_cmake_command(no-tests_legacy ${CMAKE_CTEST_COMMAND})
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake" "
+    set(CTEST_COMMAND \"${CMAKE_CTEST_COMMAND}\")
+    set(CTEST_SOURCE_DIRECTORY \"${RunCMake_SOURCE_DIR}\")
+    set(CTEST_BINARY_DIRECTORY \"${RunCMake_TEST_BINARY_DIR}\")
+    ctest_start(Experimental)
+    ctest_test()
+")
+  run_cmake_command(
+    no-tests-script_ignore ${CMAKE_CTEST_COMMAND} --no-tests=ignore
+    -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake")
+  run_cmake_command(
+    no-tests-script_error ${CMAKE_CTEST_COMMAND} --no-tests=error
+    -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake")
+  run_cmake_command(
+    no-tests-script_legacy ${CMAKE_CTEST_COMMAND}
+    -S "${RunCMake_TEST_BINARY_DIR}/NoTestsScript.cmake")
+endfunction()
+run_NoTests()
+
 # Check the configuration type variable is passed
 run_ctest(check-configuration-type)
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt
new file mode 100644
index 0000000..a7c4b11
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_error-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt
new file mode 100644
index 0000000..a7c4b11
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests-script_legacy-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_bad-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt
new file mode 100644
index 0000000..1703539
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_bad-stderr.txt
@@ -0,0 +1 @@
+^CMake Error: '--no-tests=' given unknown value 'bad'$
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt b/Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt
new file mode 100644
index 0000000..45a4fb7
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_error-result.txt
@@ -0,0 +1 @@
+8
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt
new file mode 100644
index 0000000..eafba1c
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_error-stderr.txt
@@ -0,0 +1 @@
+No tests were found!!!
diff --git a/Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt b/Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt
new file mode 100644
index 0000000..a7c4b11
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/no-tests_legacy-stderr.txt
@@ -0,0 +1 @@
+^No tests were found!!!$