Merge topic 'ctest_junit'

25bf514447 ctest: Add support for writing test results in JUnit XML format

Acked-by: Kitware Robot <kwrobot@kitware.com>
Acked-by: Zack Galbreath <zack.galbreath@kitware.com>
Acked-by: Michael Hirsch <michael@scivision.dev>
Acked-by: MvdHurk <maikelvandenhurk@hotmail.com>
Acked-by: Alexander Richardson <arichardson.kde@gmail.com>
Merge-request: !6020

8 files changed, 165 insertions, 6 deletions

diff --git a/Source/CTest/cmCTestMemCheckCommand.cxx b/Source/CTest/cmCTestMemCheckCommand.cxx
index d0e2974..37b3628 100644
--- a/Source/CTest/cmCTestMemCheckCommand.cxx
+++ b/Source/CTest/cmCTestMemCheckCommand.cxx
@@ -14,7 +14,7 @@ void cmCTestMemCheckCommand::BindArguments()
   this->Bind("DEFECT_COUNT"_s, this->DefectCount);
 }
 
-cmCTestGenericHandler* cmCTestMemCheckCommand::InitializeActualHandler()
+cmCTestTestHandler* cmCTestMemCheckCommand::InitializeActualHandler()
 {
   cmCTestMemCheckHandler* handler = this->CTest->GetMemCheckHandler();
   handler->Initialize();
diff --git a/Source/CTest/cmCTestMemCheckCommand.h b/Source/CTest/cmCTestMemCheckCommand.h
index 6544f16..ee39e49 100644
--- a/Source/CTest/cmCTestMemCheckCommand.h
+++ b/Source/CTest/cmCTestMemCheckCommand.h
@@ -13,6 +13,7 @@
 #include "cmCommand.h"
 
 class cmCTestGenericHandler;
+class cmCTestTestHandler;
 
 /** \class cmCTestMemCheck
  * \brief Run a ctest script
@@ -36,7 +37,7 @@ public:
 protected:
   void BindArguments() override;
 
-  cmCTestGenericHandler* InitializeActualHandler() override;
+  cmCTestTestHandler* InitializeActualHandler() override;
 
   void ProcessAdditionalValues(cmCTestGenericHandler* handler) override;
 
diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx
index 886c263..67f4986 100644
--- a/Source/CTest/cmCTestTestCommand.cxx
+++ b/Source/CTest/cmCTestTestCommand.cxx
@@ -9,7 +9,6 @@
 #include <cmext/string_view>
 
 #include "cmCTest.h"
-#include "cmCTestGenericHandler.h"
 #include "cmCTestTestHandler.h"
 #include "cmDuration.h"
 #include "cmMakefile.h"
@@ -36,6 +35,7 @@ void cmCTestTestCommand::BindArguments()
   this->Bind("TEST_LOAD"_s, this->TestLoad);
   this->Bind("RESOURCE_SPEC_FILE"_s, this->ResourceSpecFile);
   this->Bind("STOP_ON_FAILURE"_s, this->StopOnFailure);
+  this->Bind("OUTPUT_JUNIT"_s, this->OutputJUnit);
 }
 
 cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
@@ -60,7 +60,7 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
     this->ResourceSpecFile = *resourceSpecFile;
   }
 
-  cmCTestGenericHandler* handler = this->InitializeActualHandler();
+  cmCTestTestHandler* handler = this->InitializeActualHandler();
   if (!this->Start.empty() || !this->End.empty() || !this->Stride.empty()) {
     handler->SetOption(
       "TestsToRunInformation",
@@ -140,11 +140,15 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
                                        *labelsForSubprojects, this->Quiet);
   }
 
+  if (!this->OutputJUnit.empty()) {
+    handler->SetJUnitXMLFileName(this->OutputJUnit);
+  }
+
   handler->SetQuiet(this->Quiet);
   return handler;
 }
 
-cmCTestGenericHandler* cmCTestTestCommand::InitializeActualHandler()
+cmCTestTestHandler* cmCTestTestCommand::InitializeActualHandler()
 {
   cmCTestTestHandler* handler = this->CTest->GetTestHandler();
   handler->Initialize();
diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h
index 624cd91..24e74e2 100644
--- a/Source/CTest/cmCTestTestCommand.h
+++ b/Source/CTest/cmCTestTestCommand.h
@@ -13,6 +13,7 @@
 #include "cmCommand.h"
 
 class cmCTestGenericHandler;
+class cmCTestTestHandler;
 
 /** \class cmCTestTest
  * \brief Run a ctest script
@@ -40,7 +41,7 @@ public:
 
 protected:
   void BindArguments() override;
-  virtual cmCTestGenericHandler* InitializeActualHandler();
+  virtual cmCTestTestHandler* InitializeActualHandler();
   cmCTestGenericHandler* InitializeHandler() override;
 
   std::string Start;
@@ -59,5 +60,6 @@ protected:
   std::string StopTime;
   std::string TestLoad;
   std::string ResourceSpecFile;
+  std::string OutputJUnit;
   bool StopOnFailure = false;
 };
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index db5cb9c..1596d4a 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -42,6 +42,7 @@
 #include "cmStateSnapshot.h"
 #include "cmStringAlgorithms.h"
 #include "cmSystemTools.h"
+#include "cmTimestamp.h"
 #include "cmWorkingDirectory.h"
 #include "cmXMLWriter.h"
 #include "cmake.h"
@@ -299,6 +300,9 @@ cmCTestTestHandler::cmCTestTestHandler()
 
   this->LogFile = nullptr;
 
+  // Support for JUnit XML output.
+  this->JUnitXMLFileName = "";
+
   // regex to detect <DartMeasurement>...</DartMeasurement>
   this->DartStuff.compile("(<DartMeasurement.*/DartMeasurement[a-zA-Z]*>)");
   // regex to detect each individual <DartMeasurement>...</DartMeasurement>
@@ -456,6 +460,10 @@ int cmCTestTestHandler::ProcessHandler()
     return 1;
   }
 
+  if (!this->WriteJUnitXML()) {
+    return 1;
+  }
+
   if (!this->PostProcessHandler()) {
     this->LogFile = nullptr;
     return -1;
@@ -2457,3 +2465,125 @@ bool cmCTestTestHandler::cmCTestTestResourceRequirement::operator!=(
 {
   return !(*this == other);
 }
+
+void cmCTestTestHandler::SetJUnitXMLFileName(const std::string& filename)
+{
+  this->JUnitXMLFileName = filename;
+}
+
+bool cmCTestTestHandler::WriteJUnitXML()
+{
+  if (this->JUnitXMLFileName.empty()) {
+    return true;
+  }
+
+  // Open new XML file for writing.
+  cmGeneratedFileStream xmlfile;
+  xmlfile.SetTempExt("tmp");
+  xmlfile.Open(this->JUnitXMLFileName);
+  if (!xmlfile) {
+    cmCTestLog(this->CTest, ERROR_MESSAGE,
+               "Problem opening file: " << this->JUnitXMLFileName
+                                        << std::endl);
+    return false;
+  }
+  cmXMLWriter xml(xmlfile);
+
+  // Iterate over the test results to get the number of tests that
+  // passed, failed, etc.
+  auto num_tests = 0;
+  auto num_passed = 0;
+  auto num_failed = 0;
+  auto num_notrun = 0;
+  auto num_disabled = 0;
+  SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end());
+  for (cmCTestTestResult const& result : resultsSet) {
+    num_tests++;
+    if (result.Status == cmCTestTestHandler::COMPLETED) {
+      num_passed++;
+    } else if (result.Status == cmCTestTestHandler::NOT_RUN) {
+      if (result.CompletionStatus == "Disabled") {
+        num_disabled++;
+      } else {
+        num_notrun++;
+      }
+    } else {
+      num_failed++;
+    }
+  }
+
+  // Write <testsuite> element.
+  xml.StartDocument();
+  xml.StartElement("testsuite");
+
+  xml.Attribute("name",
+                cmCTest::SafeBuildIdField(
+                  this->CTest->GetCTestConfiguration("BuildName")));
+  xml.BreakAttributes();
+
+  xml.Attribute("tests", num_tests);
+  xml.Attribute("failures", num_failed);
+
+  // CTest disabled => JUnit disabled
+  xml.Attribute("disabled", num_disabled);
+
+  // Otherwise, CTest notrun => JUnit skipped.
+  // The distinction between JUnit disabled vs. skipped is that
+  // skipped tests can have a message associated with them
+  // (why the test was skipped).
+  xml.Attribute("skipped", num_notrun);
+
+  xml.Attribute("hostname", this->CTest->GetCTestConfiguration("Site"));
+  xml.Attribute(
+    "time",
+    std::chrono::duration_cast<std::chrono::seconds>(this->ElapsedTestingTime)
+      .count());
+  const std::time_t start_test_time_t =
+    std::chrono::system_clock::to_time_t(this->StartTestTime);
+  cmTimestamp cmts;
+  xml.Attribute("timestamp",
+                cmts.CreateTimestampFromTimeT(start_test_time_t,
+                                              "%Y-%m-%dT%H:%M:%S", false));
+
+  // Write <testcase> elements.
+  for (cmCTestTestResult const& result : resultsSet) {
+    xml.StartElement("testcase");
+    xml.Attribute("name", result.Name);
+    xml.Attribute("classname", result.Name);
+    xml.Attribute("time", result.ExecutionTime.count());
+
+    std::string status;
+    if (result.Status == cmCTestTestHandler::COMPLETED) {
+      status = "run";
+    } else if (result.Status == cmCTestTestHandler::NOT_RUN) {
+      if (result.CompletionStatus == "Disabled") {
+        status = "disabled";
+      } else {
+        status = "notrun";
+      }
+    } else {
+      status = "fail";
+    }
+    xml.Attribute("status", status);
+
+    if (status == "notrun") {
+      xml.StartElement("skipped");
+      xml.Attribute("message", result.CompletionStatus);
+      xml.EndElement(); // </skipped>
+    } else if (status == "fail") {
+      xml.StartElement("failure");
+      xml.Attribute("message", result.Reason);
+      xml.EndElement(); // </failure>
+    }
+
+    // Note: compressed test output is unconditionally disabled when
+    // --output-junit is specified.
+    xml.Element("system-out", result.Output);
+    xml.EndElement(); // </testcase>
+  }
+
+  xml.EndElement(); // </testsuite>
+  xml.EndDocument();
+
+  return true;
+}
diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h
index 6fa18a9..6841624 100644
--- a/Source/CTest/cmCTestTestHandler.h
+++ b/Source/CTest/cmCTestTestHandler.h
@@ -209,6 +209,9 @@ public:
 
   using ListOfTests = std::vector<cmCTestTestProperties>;
 
+  // Support for writing test results in JUnit XML format.
+  void SetJUnitXMLFileName(const std::string& id);
+
 protected:
   using SetOfTests =
     std::set<cmCTestTestHandler::cmCTestTestResult, cmCTestTestResultLess>;
@@ -274,6 +277,11 @@ private:
    */
   virtual void GenerateDartOutput(cmXMLWriter& xml);
 
+  /**
+   * Write test results in JUnit XML format
+   */
+  bool WriteJUnitXML();
+
   void PrintLabelOrSubprojectSummary(bool isSubProject);
 
   /**
@@ -354,4 +362,6 @@ private:
   cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never;
   int RepeatCount = 1;
   bool RerunFailed;
+
+  std::string JUnitXMLFileName;
 };
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 643b43f..79a3925 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -2069,6 +2069,17 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     }
     i++;
     this->Impl->TestDir = std::string(args[i]);
+  } else if (this->CheckArgument(arg, "--output-junit"_s)) {
+    if (i >= args.size() - 1) {
+      errormsg = "'--output-junit' requires an argument";
+      return false;
+    }
+    i++;
+    this->Impl->TestHandler.SetJUnitXMLFileName(std::string(args[i]));
+    // Turn test output compression off.
+    // This makes it easier to include test output in the resulting
+    // JUnit XML report.
+    this->Impl->CompressTestOutput = false;
   }
 
   cm::string_view noTestsPrefix = "--no-tests=";
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index a4b85ae..cad27fa 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -50,6 +50,7 @@ static const char* cmDocumentationOptions[][2] = {
     "given number of jobs." },
   { "-Q,--quiet", "Make ctest quiet." },
   { "-O <file>, --output-log <file>", "Output to log file" },
+  { "--output-junit <file>", "Output test results to JUnit XML file." },
   { "-N,--show-only[=format]",
     "Disable actual execution of tests. The optional 'format' defines the "
     "format of the test information and can be 'human' for the current text "