Merge topic 'ctest-disable-tests'

b070947d Add 'DISABLED' test property
42c68aad CTest: Improve 'Completion Status' reported to CDash for 'Not Run' tests

Acked-by: Kitware Robot <kwrobot@kitware.com>
Reviewed-by: Brad King <brad.king@kitware.com>
Merge-request: !571

4 files changed, 82 insertions, 19 deletions

diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index ff465ab..2d4726c 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -163,7 +163,9 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
     this->TestRunningMap[test] = false;
     this->RunningCount -= GetProcessorsUsed(test);
     testRun->EndTest(this->Completed, this->Total, false);
-    this->Failed->push_back(this->Properties[test]->Name);
+    if (!this->Properties[test]->Disabled) {
+      this->Failed->push_back(this->Properties[test]->Name);
+    }
     delete testRun;
   }
 }
diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx
index f148f30..94aa4bd 100644
--- a/Source/CTest/cmCTestRunTest.cxx
+++ b/Source/CTest/cmCTestRunTest.cxx
@@ -215,6 +215,9 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
     if (this->TestProperties->SkipReturnCode >= 0 &&
         this->TestProperties->SkipReturnCode == retVal) {
       this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
+      std::ostringstream s;
+      s << "SKIP_RETURN_CODE=" << this->TestProperties->SkipReturnCode;
+      this->TestResult.CompletionStatus = s.str();
       cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Skipped ");
     } else if ((success && !this->TestProperties->WillFail) ||
                (!success && this->TestProperties->WillFail)) {
@@ -253,6 +256,8 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
         cmCTestLog(this->CTest, HANDLER_OUTPUT, "Other");
         this->TestResult.Status = cmCTestTestHandler::OTHER_FAULT;
     }
+  } else if ("Disabled" == this->TestResult.CompletionStatus) {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Not Run (Disabled) ");
   } else // cmsysProcess_State_Error
   {
     cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Not Run ");
@@ -413,6 +418,24 @@ bool cmCTestRunTest::StartTest(size_t total)
                << this->TestProperties->Index << ": "
                << this->TestProperties->Name << std::endl);
   this->ProcessOutput.clear();
+
+  // Return immediately if test is disabled
+  if (this->TestProperties->Disabled) {
+    this->TestResult.Properties = this->TestProperties;
+    this->TestResult.ExecutionTime = 0;
+    this->TestResult.CompressOutput = false;
+    this->TestResult.ReturnValue = -1;
+    this->TestResult.CompletionStatus = "Disabled";
+    this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
+    this->TestResult.TestCount = this->TestProperties->Index;
+    this->TestResult.Name = this->TestProperties->Name;
+    this->TestResult.Path = this->TestProperties->Directory;
+    this->TestProcess = new cmProcess;
+    this->TestResult.Output = "Disabled";
+    this->TestResult.FullCommandLine = "";
+    return false;
+  }
+
   this->ComputeArguments();
   std::vector<std::string>& args = this->TestProperties->Args;
   this->TestResult.Properties = this->TestProperties;
@@ -437,7 +460,7 @@ bool cmCTestRunTest::StartTest(size_t total)
     cmCTestLog(this->CTest, HANDLER_OUTPUT, msg << std::endl);
     this->TestResult.Output = msg;
     this->TestResult.FullCommandLine = "";
-    this->TestResult.CompletionStatus = "Not Run";
+    this->TestResult.CompletionStatus = "Fixture dependency failed";
     this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
     return false;
   }
@@ -457,7 +480,7 @@ bool cmCTestRunTest::StartTest(size_t total)
     cmCTestLog(this->CTest, ERROR_MESSAGE, msg << std::endl);
     this->TestResult.Output = msg;
     this->TestResult.FullCommandLine = "";
-    this->TestResult.CompletionStatus = "Not Run";
+    this->TestResult.CompletionStatus = "Missing Configuration";
     this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
     return false;
   }
@@ -477,7 +500,7 @@ bool cmCTestRunTest::StartTest(size_t total)
                  "Unable to find required file: " << file << std::endl);
       this->TestResult.Output = "Unable to find required file: " + file;
       this->TestResult.FullCommandLine = "";
-      this->TestResult.CompletionStatus = "Not Run";
+      this->TestResult.CompletionStatus = "Required Files Missing";
       this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
       return false;
     }
@@ -493,7 +516,7 @@ bool cmCTestRunTest::StartTest(size_t total)
                "Unable to find executable: " << args[1] << std::endl);
     this->TestResult.Output = "Unable to find executable: " + args[1];
     this->TestResult.FullCommandLine = "";
-    this->TestResult.CompletionStatus = "Not Run";
+    this->TestResult.CompletionStatus = "Unable to find executable";
     this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
     return false;
   }
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index 9d22065..814b310 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -487,6 +487,19 @@ int cmCTestTestHandler::ProcessHandler()
       }
     }
 
+    typedef std::set<cmCTestTestHandler::cmCTestTestResult,
+                     cmCTestTestResultLess>
+      SetOfTests;
+    SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end());
+    std::vector<cmCTestTestHandler::cmCTestTestResult> disabledTests;
+
+    for (SetOfTests::iterator ftit = resultsSet.begin();
+         ftit != resultsSet.end(); ++ftit) {
+      if (ftit->CompletionStatus == "Disabled") {
+        disabledTests.push_back(*ftit);
+      }
+    }
+
     float percent = float(passed.size()) * 100.0f / float(total);
     if (!failed.empty() && percent > 99) {
       percent = 99;
@@ -505,21 +518,33 @@ int cmCTestTestHandler::ProcessHandler()
                        "\nTotal Test time (real) = " << realBuf << "\n",
                        this->Quiet);
 
+    if (!disabledTests.empty()) {
+      cmGeneratedFileStream ofs;
+      cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
+                   << "The following tests are disabled and did not run:"
+                   << std::endl);
+      this->StartLogFile("TestsDisabled", ofs);
+
+      for (std::vector<cmCTestTestHandler::cmCTestTestResult>::iterator dtit =
+             disabledTests.begin();
+           dtit != disabledTests.end(); ++dtit) {
+        ofs << dtit->TestCount << ":" << dtit->Name << std::endl;
+        cmCTestLog(this->CTest, HANDLER_OUTPUT, "\t"
+                     << std::setw(3) << dtit->TestCount << " - " << dtit->Name
+                     << std::endl);
+      }
+    }
+
     if (!failed.empty()) {
       cmGeneratedFileStream ofs;
       cmCTestLog(this->CTest, HANDLER_OUTPUT, std::endl
                    << "The following tests FAILED:" << std::endl);
       this->StartLogFile("TestsFailed", ofs);
 
-      typedef std::set<cmCTestTestHandler::cmCTestTestResult,
-                       cmCTestTestResultLess>
-        SetOfTests;
-      SetOfTests resultsSet(this->TestResults.begin(),
-                            this->TestResults.end());
-
       for (SetOfTests::iterator ftit = resultsSet.begin();
            ftit != resultsSet.end(); ++ftit) {
-        if (ftit->Status != cmCTestTestHandler::COMPLETED) {
+        if (ftit->Status != cmCTestTestHandler::COMPLETED &&
+            ftit->CompletionStatus != "Disabled") {
           ofs << ftit->TestCount << ":" << ftit->Name << std::endl;
           cmCTestLog(
             this->CTest, HANDLER_OUTPUT, "\t"
@@ -841,6 +866,11 @@ void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const
   size_t fixtureTestsAdded = 0;
   std::set<std::string> addedFixtures;
   for (size_t i = 0; i < tests.size(); ++i) {
+    // Skip disabled tests
+    if (tests[i].Disabled) {
+      continue;
+    }
+
     // There are two things to do for each test:
     //   1. For every fixture required by this test, record that fixture as
     //      being required and create dependencies on that fixture's setup
@@ -1200,6 +1230,7 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml)
     cmCTestTestResult* result = &this->TestResults[cc];
     this->WriteTestResultHeader(xml, result);
     xml.StartElement("Results");
+
     if (result->Status != cmCTestTestHandler::NOT_RUN) {
       if (result->Status != cmCTestTestHandler::COMPLETED ||
           result->ReturnValue) {
@@ -1208,6 +1239,7 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml)
         xml.Attribute("name", "Exit Code");
         xml.Element("Value", this->GetTestStatus(result->Status));
         xml.EndElement(); // NamedMeasurement
+
         xml.StartElement("NamedMeasurement");
         xml.Attribute("type", "text/string");
         xml.Attribute("name", "Exit Value");
@@ -1222,8 +1254,7 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml)
       xml.EndElement(); // NamedMeasurement
       if (!result->Reason.empty()) {
         const char* reasonType = "Pass Reason";
-        if (result->Status != cmCTestTestHandler::COMPLETED &&
-            result->Status != cmCTestTestHandler::NOT_RUN) {
+        if (result->Status != cmCTestTestHandler::COMPLETED) {
           reasonType = "Fail Reason";
         }
         xml.StartElement("NamedMeasurement");
@@ -1232,12 +1263,14 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml)
         xml.Element("Value", result->Reason);
         xml.EndElement(); // NamedMeasurement
       }
-      xml.StartElement("NamedMeasurement");
-      xml.Attribute("type", "text/string");
-      xml.Attribute("name", "Completion Status");
-      xml.Element("Value", result->CompletionStatus);
-      xml.EndElement(); // NamedMeasurement
     }
+
+    xml.StartElement("NamedMeasurement");
+    xml.Attribute("type", "text/string");
+    xml.Attribute("name", "Completion Status");
+    xml.Element("Value", result->CompletionStatus);
+    xml.EndElement(); // NamedMeasurement
+
     xml.StartElement("NamedMeasurement");
     xml.Attribute("type", "text/string");
     xml.Attribute("name", "Command Line");
@@ -2000,6 +2033,9 @@ bool cmCTestTestHandler::SetTestsProperties(
           if (key == "WILL_FAIL") {
             rtit->WillFail = cmSystemTools::IsOn(val.c_str());
           }
+          if (key == "DISABLED") {
+            rtit->Disabled = cmSystemTools::IsOn(val.c_str());
+          }
           if (key == "ATTACHED_FILES") {
             cmSystemTools::ExpandListArgument(val, rtit->AttachedFiles);
           }
@@ -2178,6 +2214,7 @@ bool cmCTestTestHandler::AddTest(const std::vector<std::string>& args)
 
   test.IsInBasedOnREOptions = true;
   test.WillFail = false;
+  test.Disabled = false;
   test.RunSerial = false;
   test.Timeout = 0;
   test.ExplicitTimeout = false;
diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h
index 5b07e98..a95f088 100644
--- a/Source/CTest/cmCTestTestHandler.h
+++ b/Source/CTest/cmCTestTestHandler.h
@@ -114,6 +114,7 @@ public:
     std::map<std::string, std::string> Measurements;
     bool IsInBasedOnREOptions;
     bool WillFail;
+    bool Disabled;
     float Cost;
     int PreviousRuns;
     bool RunSerial;