diff options
Diffstat (limited to 'Source/CTest')
-rw-r--r-- | Source/CTest/cmCTestBuildAndTestHandler.cxx | 4 | ||||
-rw-r--r-- | Source/CTest/cmCTestBuildCommand.cxx | 5 | ||||
-rw-r--r-- | Source/CTest/cmCTestBuildCommand.h | 1 | ||||
-rw-r--r-- | Source/CTest/cmCTestBuildHandler.cxx | 5 | ||||
-rw-r--r-- | Source/CTest/cmCTestCoverageHandler.cxx | 7 | ||||
-rw-r--r-- | Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx | 2 | ||||
-rw-r--r-- | Source/CTest/cmCTestGenericHandler.cxx | 52 | ||||
-rw-r--r-- | Source/CTest/cmCTestGenericHandler.h | 34 | ||||
-rw-r--r-- | Source/CTest/cmCTestMemCheckCommand.cxx | 2 | ||||
-rw-r--r-- | Source/CTest/cmCTestMemCheckCommand.h | 3 | ||||
-rw-r--r-- | Source/CTest/cmCTestMultiProcessHandler.cxx | 10 | ||||
-rw-r--r-- | Source/CTest/cmCTestRunScriptCommand.cxx | 4 | ||||
-rw-r--r-- | Source/CTest/cmCTestRunTest.cxx | 33 | ||||
-rw-r--r-- | Source/CTest/cmCTestScriptHandler.cxx | 25 | ||||
-rw-r--r-- | Source/CTest/cmCTestScriptHandler.h | 12 | ||||
-rw-r--r-- | Source/CTest/cmCTestTestCommand.cxx | 16 | ||||
-rw-r--r-- | Source/CTest/cmCTestTestCommand.h | 4 | ||||
-rw-r--r-- | Source/CTest/cmCTestTestHandler.cxx | 359 | ||||
-rw-r--r-- | Source/CTest/cmCTestTestHandler.h | 23 | ||||
-rw-r--r-- | Source/CTest/cmCTestUpdateCommand.cxx | 1 |
20 files changed, 448 insertions, 154 deletions
diff --git a/Source/CTest/cmCTestBuildAndTestHandler.cxx b/Source/CTest/cmCTestBuildAndTestHandler.cxx index a18cbb4..adfc8ef 100644 --- a/Source/CTest/cmCTestBuildAndTestHandler.cxx +++ b/Source/CTest/cmCTestBuildAndTestHandler.cxx @@ -19,6 +19,8 @@ #include "cmWorkingDirectory.h" #include "cmake.h" +struct cmMessageMetadata; + cmCTestBuildAndTestHandler::cmCTestBuildAndTestHandler() { this->BuildTwoConfig = false; @@ -125,7 +127,7 @@ public: : CM(cm) { cmSystemTools::SetMessageCallback( - [&s](const std::string& msg, const char* /*unused*/) { + [&s](const std::string& msg, const cmMessageMetadata& /* unused */) { s += msg; s += "\n"; }); diff --git a/Source/CTest/cmCTestBuildCommand.cxx b/Source/CTest/cmCTestBuildCommand.cxx index 88e2871..483c316 100644 --- a/Source/CTest/cmCTestBuildCommand.cxx +++ b/Source/CTest/cmCTestBuildCommand.cxx @@ -27,6 +27,7 @@ void cmCTestBuildCommand::BindArguments() this->Bind("CONFIGURATION"_s, this->Configuration); this->Bind("FLAGS"_s, this->Flags); this->Bind("PROJECT_NAME"_s, this->ProjectName); + this->Bind("PARALLEL_LEVEL"_s, this->ParallelLevel); } cmCTestBuildCommand::~cmCTestBuildCommand() = default; @@ -98,8 +99,8 @@ cmCTestGenericHandler* cmCTestBuildCommand::InitializeHandler() std::string dir = this->CTest->GetCTestConfiguration("BuildDirectory"); std::string buildCommand = this->GlobalGenerator->GenerateCMakeBuildCommand( - cmakeBuildTarget, cmakeBuildConfiguration, cmakeBuildAdditionalFlags, - this->Makefile->IgnoreErrorsCMP0061()); + cmakeBuildTarget, cmakeBuildConfiguration, this->ParallelLevel, + cmakeBuildAdditionalFlags, this->Makefile->IgnoreErrorsCMP0061()); cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "SetMakeCommand:" << buildCommand << "\n", this->Quiet); diff --git a/Source/CTest/cmCTestBuildCommand.h b/Source/CTest/cmCTestBuildCommand.h index 00dbcc4..1254dad 100644 --- a/Source/CTest/cmCTestBuildCommand.h +++ b/Source/CTest/cmCTestBuildCommand.h @@ -60,4 +60,5 @@ protected: std::string Configuration; std::string Flags; std::string ProjectName; + std::string ParallelLevel; }; diff --git a/Source/CTest/cmCTestBuildHandler.cxx b/Source/CTest/cmCTestBuildHandler.cxx index 103dc1e..03caa63 100644 --- a/Source/CTest/cmCTestBuildHandler.cxx +++ b/Source/CTest/cmCTestBuildHandler.cxx @@ -3,7 +3,6 @@ #include "cmCTestBuildHandler.h" #include <cstdlib> -#include <cstring> #include <set> #include <utility> @@ -657,14 +656,14 @@ bool cmCTestBuildHandler::IsLaunchedErrorFile(const char* fname) { // error-{hash}.xml return (cmHasLiteralPrefix(fname, "error-") && - strcmp(fname + strlen(fname) - 4, ".xml") == 0); + cmHasLiteralSuffix(fname, ".xml")); } bool cmCTestBuildHandler::IsLaunchedWarningFile(const char* fname) { // warning-{hash}.xml return (cmHasLiteralPrefix(fname, "warning-") && - strcmp(fname + strlen(fname) - 4, ".xml") == 0); + cmHasLiteralSuffix(fname, ".xml")); } //###################################################################### diff --git a/Source/CTest/cmCTestCoverageHandler.cxx b/Source/CTest/cmCTestCoverageHandler.cxx index 3d7d031..57b1dda 100644 --- a/Source/CTest/cmCTestCoverageHandler.cxx +++ b/Source/CTest/cmCTestCoverageHandler.cxx @@ -289,6 +289,13 @@ int cmCTestCoverageHandler::ProcessHandler() this->CTest->GetCTestConfiguration("SourceDirectory"); std::string binaryDir = this->CTest->GetCTestConfiguration("BuildDirectory"); + if (binaryDir.empty()) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Binary directory is not set. " + "No coverage checking will be performed." + << std::endl); + return 0; + } this->LoadLabels(); cmGeneratedFileStream ofs; diff --git a/Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx b/Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx index 051c117..af495bb 100644 --- a/Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx +++ b/Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx @@ -16,7 +16,7 @@ bool cmCTestEmptyBinaryDirectoryCommand::InitialPass( return false; } - if (!cmCTestScriptHandler::EmptyBinaryDirectory(args[0].c_str())) { + if (!cmCTestScriptHandler::EmptyBinaryDirectory(args[0])) { std::ostringstream ostr; ostr << "problem removing the binary directory: " << args[0]; this->SetError(ostr.str()); diff --git a/Source/CTest/cmCTestGenericHandler.cxx b/Source/CTest/cmCTestGenericHandler.cxx index 91818bb..cc756d7 100644 --- a/Source/CTest/cmCTestGenericHandler.cxx +++ b/Source/CTest/cmCTestGenericHandler.cxx @@ -21,32 +21,47 @@ cmCTestGenericHandler::cmCTestGenericHandler() cmCTestGenericHandler::~cmCTestGenericHandler() = default; -void cmCTestGenericHandler::SetOption(const std::string& op, const char* value) +/* Modify the given `map`, setting key `op` to `value` if `value` + * is non-null, otherwise removing key `op` (if it exists). + */ +static void SetMapValue(cmCTestGenericHandler::t_StringToString& map, + const std::string& op, const char* value) { if (!value) { - auto remit = this->Options.find(op); - if (remit != this->Options.end()) { - this->Options.erase(remit); - } + map.erase(op); return; } - this->Options[op] = value; + map[op] = value; +} + +void cmCTestGenericHandler::SetOption(const std::string& op, const char* value) +{ + SetMapValue(this->Options, op, value); } void cmCTestGenericHandler::SetPersistentOption(const std::string& op, const char* value) { this->SetOption(op, value); - if (!value) { - auto remit = this->PersistentOptions.find(op); - if (remit != this->PersistentOptions.end()) { - this->PersistentOptions.erase(remit); - } - return; + SetMapValue(this->PersistentOptions, op, value); +} + +void cmCTestGenericHandler::AddMultiOption(const std::string& op, + const std::string& value) +{ + if (!value.empty()) { + this->MultiOptions[op].emplace_back(value); } +} - this->PersistentOptions[op] = value; +void cmCTestGenericHandler::AddPersistentMultiOption(const std::string& op, + const std::string& value) +{ + if (!value.empty()) { + this->MultiOptions[op].emplace_back(value); + this->PersistentMultiOptions[op].emplace_back(value); + } } void cmCTestGenericHandler::Initialize() @@ -68,6 +83,17 @@ const char* cmCTestGenericHandler::GetOption(const std::string& op) return remit->second.c_str(); } +std::vector<std::string> cmCTestGenericHandler::GetMultiOption( + const std::string& optionName) const +{ + // Avoid inserting a key, which MultiOptions[op] would do. + auto remit = this->MultiOptions.find(optionName); + if (remit == this->MultiOptions.end()) { + return {}; + } + return remit->second; +} + bool cmCTestGenericHandler::StartResultingXML(cmCTest::Part part, const char* name, cmGeneratedFileStream& xofs) diff --git a/Source/CTest/cmCTestGenericHandler.h b/Source/CTest/cmCTestGenericHandler.h index 89d7596..e846fd9 100644 --- a/Source/CTest/cmCTestGenericHandler.h +++ b/Source/CTest/cmCTestGenericHandler.h @@ -13,7 +13,6 @@ #include "cmCTest.h" #include "cmSystemTools.h" -class cmCTestCommand; class cmGeneratedFileStream; class cmMakefile; @@ -72,12 +71,40 @@ public: virtual ~cmCTestGenericHandler(); using t_StringToString = std::map<std::string, std::string>; + using t_StringToMultiString = + std::map<std::string, std::vector<std::string>>; + /** + * Options collect a single value from flags; passing the + * flag multiple times on the command-line *overwrites* values, + * and only the last one specified counts. Set an option to + * nullptr to "unset" it. + * + * The value is stored as a string. The values set for single + * and multi-options (see below) live in different spaces, + * so calling a single-getter for a key that has only been set + * as a multi-value will return nullptr. + */ void SetPersistentOption(const std::string& op, const char* value); void SetOption(const std::string& op, const char* value); const char* GetOption(const std::string& op); - void SetCommand(cmCTestCommand* command) { this->Command = command; } + /** + * Multi-Options collect one or more values from flags; passing + * the flag multiple times on the command-line *adds* values, + * rather than overwriting the previous values. + * + * Adding an empty value does nothing. + * + * The value is stored as a vector of strings. The values set for single + * (see above) and multi-options live in different spaces, + * so calling a multi-getter for a key that has only been set + * as a single-value will return an empty vector. + */ + void AddPersistentMultiOption(const std::string& optionName, + const std::string& value); + void AddMultiOption(const std::string& optionName, const std::string& value); + std::vector<std::string> GetMultiOption(const std::string& op) const; void SetSubmitIndex(int idx) { this->SubmitIndex = idx; } int GetSubmitIndex() { return this->SubmitIndex; } @@ -100,8 +127,9 @@ protected: cmCTest* CTest; t_StringToString Options; t_StringToString PersistentOptions; + t_StringToMultiString MultiOptions; + t_StringToMultiString PersistentMultiOptions; t_StringToString LogFileNames; - cmCTestCommand* Command; int SubmitIndex; }; diff --git a/Source/CTest/cmCTestMemCheckCommand.cxx b/Source/CTest/cmCTestMemCheckCommand.cxx index d0e2974..37b3628 100644 --- a/Source/CTest/cmCTestMemCheckCommand.cxx +++ b/Source/CTest/cmCTestMemCheckCommand.cxx @@ -14,7 +14,7 @@ void cmCTestMemCheckCommand::BindArguments() this->Bind("DEFECT_COUNT"_s, this->DefectCount); } -cmCTestGenericHandler* cmCTestMemCheckCommand::InitializeActualHandler() +cmCTestTestHandler* cmCTestMemCheckCommand::InitializeActualHandler() { cmCTestMemCheckHandler* handler = this->CTest->GetMemCheckHandler(); handler->Initialize(); diff --git a/Source/CTest/cmCTestMemCheckCommand.h b/Source/CTest/cmCTestMemCheckCommand.h index 6544f16..ee39e49 100644 --- a/Source/CTest/cmCTestMemCheckCommand.h +++ b/Source/CTest/cmCTestMemCheckCommand.h @@ -13,6 +13,7 @@ #include "cmCommand.h" class cmCTestGenericHandler; +class cmCTestTestHandler; /** \class cmCTestMemCheck * \brief Run a ctest script @@ -36,7 +37,7 @@ public: protected: void BindArguments() override; - cmCTestGenericHandler* InitializeActualHandler() override; + cmCTestTestHandler* InitializeActualHandler() override; void ProcessAdditionalValues(cmCTestGenericHandler* handler) override; diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 852f9d9..86a8e00 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -587,24 +587,24 @@ void cmCTestMultiProcessHandler::StartNextTests() onlyRunSerialTestsLeft = false; } } - cmCTestLog(this->CTest, HANDLER_OUTPUT, "***** WAITING, "); + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "***** WAITING, "); if (this->SerialTestRunning) { - cmCTestLog(this->CTest, HANDLER_OUTPUT, + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Waiting for RUN_SERIAL test to finish."); } else if (onlyRunSerialTestsLeft) { - cmCTestLog(this->CTest, HANDLER_OUTPUT, + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Only RUN_SERIAL tests remain, awaiting available slot."); } else { /* clang-format off */ - cmCTestLog(this->CTest, HANDLER_OUTPUT, + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "System Load: " << systemLoad << ", " "Max Allowed Load: " << this->TestLoad << ", " "Smallest test " << testWithMinProcessors << " requires " << minProcessorsRequired); /* clang-format on */ } - cmCTestLog(this->CTest, HANDLER_OUTPUT, "*****" << std::endl); + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "*****" << std::endl); // Wait between 1 and 5 seconds before trying again. unsigned int milliseconds = (cmSystemTools::RandomSeed() % 5 + 1) * 1000; diff --git a/Source/CTest/cmCTestRunScriptCommand.cxx b/Source/CTest/cmCTestRunScriptCommand.cxx index f59ca57..7661d4d 100644 --- a/Source/CTest/cmCTestRunScriptCommand.cxx +++ b/Source/CTest/cmCTestRunScriptCommand.cxx @@ -37,8 +37,8 @@ bool cmCTestRunScriptCommand::InitialPass(std::vector<std::string> const& args, ++i; } else { int ret; - cmCTestScriptHandler::RunScript(this->CTest, this->Makefile, - args[i].c_str(), !np, &ret); + cmCTestScriptHandler::RunScript(this->CTest, this->Makefile, args[i], + !np, &ret); this->Makefile->AddDefinition(returnVariable, std::to_string(ret)); } } diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 5a6c775..50072c5 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -2,6 +2,7 @@ file Copyright.txt or https://cmake.org/licensing for details. */ #include "cmCTestRunTest.h" +#include <algorithm> #include <chrono> #include <cstddef> // IWYU pragma: keep #include <cstdint> @@ -40,6 +41,38 @@ void cmCTestRunTest::CheckOutput(std::string const& line) { cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->GetIndex() << ": " << line << std::endl); + + // Check for special CTest XML tags in this line of output. + // If any are found, this line is excluded from ProcessOutput. + if (!line.empty() && line.find("<CTest") != std::string::npos) { + bool ctest_tag_found = false; + if (this->TestHandler->CustomCompletionStatusRegex.find(line)) { + ctest_tag_found = true; + this->TestResult.CustomCompletionStatus = + this->TestHandler->CustomCompletionStatusRegex.match(1); + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + this->GetIndex() << ": " + << "Test Details changed to '" + << this->TestResult.CustomCompletionStatus + << "'" << std::endl); + } else if (this->TestHandler->CustomLabelRegex.find(line)) { + ctest_tag_found = true; + auto label = this->TestHandler->CustomLabelRegex.match(1); + auto& labels = this->TestProperties->Labels; + if (std::find(labels.begin(), labels.end(), label) == labels.end()) { + labels.push_back(label); + std::sort(labels.begin(), labels.end()); + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + this->GetIndex() + << ": " + << "Test Label added: '" << label << "'" << std::endl); + } + } + if (ctest_tag_found) { + return; + } + } + this->ProcessOutput += line; this->ProcessOutput += "\n"; diff --git a/Source/CTest/cmCTestScriptHandler.cxx b/Source/CTest/cmCTestScriptHandler.cxx index 4808c36..d2cad39 100644 --- a/Source/CTest/cmCTestScriptHandler.cxx +++ b/Source/CTest/cmCTestScriptHandler.cxx @@ -4,7 +4,6 @@ #include <cstdio> #include <cstdlib> -#include <cstring> #include <map> #include <ratio> #include <sstream> @@ -91,7 +90,7 @@ void cmCTestScriptHandler::Initialize() cmCTestScriptHandler::~cmCTestScriptHandler() = default; // just adds an argument to the vector -void cmCTestScriptHandler::AddConfigurationScript(const char* script, +void cmCTestScriptHandler::AddConfigurationScript(const std::string& script, bool pscope) { this->ConfigurationScripts.emplace_back(script); @@ -676,7 +675,7 @@ int cmCTestScriptHandler::RunConfigurationDashboard() // clear the binary directory? if (this->EmptyBinDir) { - if (!cmCTestScriptHandler::EmptyBinaryDirectory(this->BinaryDir.c_str())) { + if (!cmCTestScriptHandler::EmptyBinaryDirectory(this->BinaryDir)) { cmCTestLog(this->CTest, ERROR_MESSAGE, "Problem removing the binary directory" << std::endl); } @@ -724,8 +723,8 @@ int cmCTestScriptHandler::RunConfigurationDashboard() // put the initial cache into the bin dir if (!this->InitialCache.empty()) { - if (!cmCTestScriptHandler::WriteInitialCache(this->BinaryDir.c_str(), - this->InitialCache.c_str())) { + if (!cmCTestScriptHandler::WriteInitialCache(this->BinaryDir, + this->InitialCache)) { this->RestoreBackupDirectories(); return 9; } @@ -812,8 +811,8 @@ int cmCTestScriptHandler::RunConfigurationDashboard() return 0; } -bool cmCTestScriptHandler::WriteInitialCache(const char* directory, - const char* text) +bool cmCTestScriptHandler::WriteInitialCache(const std::string& directory, + const std::string& text) { std::string cacheFile = cmStrCat(directory, "/CMakeCache.txt"); cmGeneratedFileStream fout(cacheFile); @@ -821,9 +820,7 @@ bool cmCTestScriptHandler::WriteInitialCache(const char* directory, return false; } - if (text != nullptr) { - fout.write(text, strlen(text)); - } + fout.write(text.data(), text.size()); // Make sure the operating system has finished writing the file // before closing it. This will ensure the file is finished before @@ -852,7 +849,7 @@ void cmCTestScriptHandler::RestoreBackupDirectories() } bool cmCTestScriptHandler::RunScript(cmCTest* ctest, cmMakefile* mf, - const char* sname, bool InProcess, + const std::string& sname, bool InProcess, int* returnValue) { auto sh = cm::make_unique<cmCTestScriptHandler>(); @@ -866,10 +863,10 @@ bool cmCTestScriptHandler::RunScript(cmCTest* ctest, cmMakefile* mf, return true; } -bool cmCTestScriptHandler::EmptyBinaryDirectory(const char* sname) +bool cmCTestScriptHandler::EmptyBinaryDirectory(const std::string& sname) { // try to avoid deleting root - if (!sname || strlen(sname) < 2) { + if (sname.size() < 2) { return false; } @@ -924,7 +921,7 @@ bool cmCTestScriptHandler::TryToRemoveBinaryDirectoryOnce( } } - return cmSystemTools::RemoveADirectory(directoryPath); + return static_cast<bool>(cmSystemTools::RemoveADirectory(directoryPath)); } cmDuration cmCTestScriptHandler::GetRemainingTimeAllowed() diff --git a/Source/CTest/cmCTestScriptHandler.h b/Source/CTest/cmCTestScriptHandler.h index 8eb9658..b7764b2 100644 --- a/Source/CTest/cmCTestScriptHandler.h +++ b/Source/CTest/cmCTestScriptHandler.h @@ -62,7 +62,7 @@ public: /** * Add a script to run, and if is should run in the current process */ - void AddConfigurationScript(const char*, bool pscope); + void AddConfigurationScript(const std::string&, bool pscope); /** * Run a dashboard using a specified confiuration script @@ -72,19 +72,21 @@ public: /* * Run a script */ - static bool RunScript(cmCTest* ctest, cmMakefile* mf, const char* script, - bool InProcess, int* returnValue); + static bool RunScript(cmCTest* ctest, cmMakefile* mf, + const std::string& script, bool InProcess, + int* returnValue); int RunCurrentScript(); /* * Empty Binary Directory */ - static bool EmptyBinaryDirectory(const char* dir); + static bool EmptyBinaryDirectory(const std::string& dir); /* * Write an initial CMakeCache.txt from the given contents. */ - static bool WriteInitialCache(const char* directory, const char* text); + static bool WriteInitialCache(const std::string& directory, + const std::string& text); /* * Some elapsed time handling functions diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index 4403733..67f4986 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -9,7 +9,6 @@ #include <cmext/string_view> #include "cmCTest.h" -#include "cmCTestGenericHandler.h" #include "cmCTestTestHandler.h" #include "cmDuration.h" #include "cmMakefile.h" @@ -36,6 +35,7 @@ void cmCTestTestCommand::BindArguments() this->Bind("TEST_LOAD"_s, this->TestLoad); this->Bind("RESOURCE_SPEC_FILE"_s, this->ResourceSpecFile); this->Bind("STOP_ON_FAILURE"_s, this->StopOnFailure); + this->Bind("OUTPUT_JUNIT"_s, this->OutputJUnit); } cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() @@ -60,7 +60,7 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() this->ResourceSpecFile = *resourceSpecFile; } - cmCTestGenericHandler* handler = this->InitializeActualHandler(); + cmCTestTestHandler* handler = this->InitializeActualHandler(); if (!this->Start.empty() || !this->End.empty() || !this->Stride.empty()) { handler->SetOption( "TestsToRunInformation", @@ -73,11 +73,11 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() handler->SetOption("IncludeRegularExpression", this->Include.c_str()); } if (!this->ExcludeLabel.empty()) { - handler->SetOption("ExcludeLabelRegularExpression", - this->ExcludeLabel.c_str()); + handler->AddMultiOption("ExcludeLabelRegularExpression", + this->ExcludeLabel); } if (!this->IncludeLabel.empty()) { - handler->SetOption("LabelRegularExpression", this->IncludeLabel.c_str()); + handler->AddMultiOption("LabelRegularExpression", this->IncludeLabel); } if (!this->ExcludeFixture.empty()) { handler->SetOption("ExcludeFixtureRegularExpression", @@ -140,11 +140,15 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() *labelsForSubprojects, this->Quiet); } + if (!this->OutputJUnit.empty()) { + handler->SetJUnitXMLFileName(this->OutputJUnit); + } + handler->SetQuiet(this->Quiet); return handler; } -cmCTestGenericHandler* cmCTestTestCommand::InitializeActualHandler() +cmCTestTestHandler* cmCTestTestCommand::InitializeActualHandler() { cmCTestTestHandler* handler = this->CTest->GetTestHandler(); handler->Initialize(); diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index 624cd91..24e74e2 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -13,6 +13,7 @@ #include "cmCommand.h" class cmCTestGenericHandler; +class cmCTestTestHandler; /** \class cmCTestTest * \brief Run a ctest script @@ -40,7 +41,7 @@ public: protected: void BindArguments() override; - virtual cmCTestGenericHandler* InitializeActualHandler(); + virtual cmCTestTestHandler* InitializeActualHandler(); cmCTestGenericHandler* InitializeHandler() override; std::string Start; @@ -59,5 +60,6 @@ protected: std::string StopTime; std::string TestLoad; std::string ResourceSpecFile; + std::string OutputJUnit; bool StopOnFailure = false; }; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 1cb5d00..aeaf696 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -42,6 +42,7 @@ #include "cmStateSnapshot.h" #include "cmStringAlgorithms.h" #include "cmSystemTools.h" +#include "cmTimestamp.h" #include "cmWorkingDirectory.h" #include "cmXMLWriter.h" #include "cmake.h" @@ -287,8 +288,6 @@ cmCTestTestHandler::cmCTestTestHandler() { this->UseUnion = false; - this->UseIncludeLabelRegExpFlag = false; - this->UseExcludeLabelRegExpFlag = false; this->UseIncludeRegExpFlag = false; this->UseExcludeRegExpFlag = false; this->UseExcludeRegExpFirst = false; @@ -301,11 +300,20 @@ cmCTestTestHandler::cmCTestTestHandler() this->LogFile = nullptr; + // Support for JUnit XML output. + this->JUnitXMLFileName = ""; + // regex to detect <DartMeasurement>...</DartMeasurement> this->DartStuff.compile("(<DartMeasurement.*/DartMeasurement[a-zA-Z]*>)"); // regex to detect each individual <DartMeasurement>...</DartMeasurement> this->DartStuff1.compile( "(<DartMeasurement[^<]*</DartMeasurement[a-zA-Z]*>)"); + + // regex to detect <CTestDetails>...</CTestDetails> + this->CustomCompletionStatusRegex.compile( + "<CTestDetails>(.*)</CTestDetails>"); + // regex to detect <CTestLabel>...</CTestLabel> + this->CustomLabelRegex.compile("<CTestLabel>(.*)</CTestLabel>"); } void cmCTestTestHandler::Initialize() @@ -327,13 +335,11 @@ void cmCTestTestHandler::Initialize() this->TestsToRun.clear(); - this->UseIncludeLabelRegExpFlag = false; - this->UseExcludeLabelRegExpFlag = false; this->UseIncludeRegExpFlag = false; this->UseExcludeRegExpFlag = false; this->UseExcludeRegExpFirst = false; - this->IncludeLabelRegularExpression = ""; - this->ExcludeLabelRegularExpression = ""; + this->IncludeLabelRegularExpressions.clear(); + this->ExcludeLabelRegularExpressions.clear(); this->IncludeRegExp.clear(); this->ExcludeRegExp.clear(); this->ExcludeFixtureRegExp.clear(); @@ -460,6 +466,10 @@ int cmCTestTestHandler::ProcessHandler() return 1; } + if (!this->WriteJUnitXML()) { + return 1; + } + if (!this->PostProcessHandler()) { this->LogFile = nullptr; return -1; @@ -479,6 +489,22 @@ int cmCTestTestHandler::ProcessHandler() return 0; } +/* Given a multi-option value `parts`, compile those parts into + * regular expressions in `expressions`. Skip empty values. + * Returns true if there were any expressions. + */ +static bool BuildLabelRE(const std::vector<std::string>& parts, + std::vector<cmsys::RegularExpression>& expressions) +{ + expressions.clear(); + for (const auto& p : parts) { + if (!p.empty()) { + expressions.emplace_back(p); + } + } + return !expressions.empty(); +} + bool cmCTestTestHandler::ProcessOptions() { // Update internal data structure from generic one @@ -519,18 +545,11 @@ bool cmCTestTestHandler::ProcessOptions() this->CTest->SetStopOnFailure(true); } - const char* val; - val = this->GetOption("LabelRegularExpression"); - if (val) { - this->UseIncludeLabelRegExpFlag = true; - this->IncludeLabelRegExp = val; - } - val = this->GetOption("ExcludeLabelRegularExpression"); - if (val) { - this->UseExcludeLabelRegExpFlag = true; - this->ExcludeLabelRegExp = val; - } - val = this->GetOption("IncludeRegularExpression"); + BuildLabelRE(this->GetMultiOption("LabelRegularExpression"), + this->IncludeLabelRegularExpressions); + BuildLabelRE(this->GetMultiOption("ExcludeLabelRegularExpression"), + this->ExcludeLabelRegularExpressions); + const char* val = this->GetOption("IncludeRegularExpression"); if (val) { this->UseIncludeRegExp(); this->SetIncludeRegExp(val); @@ -763,10 +782,40 @@ void cmCTestTestHandler::PrintLabelOrSubprojectSummary(bool doSubProject) cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, "\n", this->Quiet); } +/** + * Check if the labels (from a test) match all the expressions. + * + * Each of the RE's must match at least one label + * (e.g. all of the REs must match **some** label, + * in order for the filter to apply to the test). + */ +static bool MatchLabelsAgainstFilterRE( + const std::vector<std::string>& labels, + const std::vector<cmsys::RegularExpression>& expressions) +{ + for (const auto& re : expressions) { + // check to see if the label regular expression matches + bool found = false; // assume it does not match + cmsys::RegularExpressionMatch match; + // loop over all labels and look for match + for (std::string const& l : labels) { + if (re.find(l.c_str(), match)) { + found = true; + break; + } + } + // if no match was found, exclude the test + if (!found) { + return false; + } + } + return true; +} + void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it) { // if not using Labels to filter then return - if (!this->UseIncludeLabelRegExpFlag) { + if (this->IncludeLabelRegularExpressions.empty()) { return; } // if there are no labels and we are filtering by labels @@ -775,16 +824,9 @@ void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it) it.IsInBasedOnREOptions = false; return; } - // check to see if the label regular expression matches - bool found = false; // assume it does not match - // loop over all labels and look for match - for (std::string const& l : it.Labels) { - if (this->IncludeLabelRegularExpression.find(l)) { - found = true; - } - } // if no match was found, exclude the test - if (!found) { + if (!MatchLabelsAgainstFilterRE(it.Labels, + this->IncludeLabelRegularExpressions)) { it.IsInBasedOnREOptions = false; } } @@ -792,7 +834,7 @@ void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it) void cmCTestTestHandler::CheckLabelFilterExclude(cmCTestTestProperties& it) { // if not using Labels to filter then return - if (!this->UseExcludeLabelRegExpFlag) { + if (this->ExcludeLabelRegularExpressions.empty()) { return; } // if there are no labels and we are excluding by labels @@ -800,16 +842,9 @@ void cmCTestTestHandler::CheckLabelFilterExclude(cmCTestTestProperties& it) if (it.Labels.empty()) { return; } - // check to see if the label regular expression matches - bool found = false; // assume it does not match - // loop over all labels and look for match - for (std::string const& l : it.Labels) { - if (this->ExcludeLabelRegularExpression.find(l)) { - found = true; - } - } // if match was found, exclude the test - if (found) { + if (MatchLabelsAgainstFilterRE(it.Labels, + this->ExcludeLabelRegularExpressions)) { it.IsInBasedOnREOptions = false; } } @@ -1431,7 +1466,11 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml) xml.StartElement("NamedMeasurement"); xml.Attribute("type", "text/string"); xml.Attribute("name", "Completion Status"); - xml.Element("Value", result.CompletionStatus); + if (result.CustomCompletionStatus.empty()) { + xml.Element("Value", result.CompletionStatus); + } else { + xml.Element("Value", result.CustomCompletionStatus); + } xml.EndElement(); // NamedMeasurement xml.StartElement("NamedMeasurement"); @@ -1521,19 +1560,29 @@ void cmCTestTestHandler::AttachFiles(cmXMLWriter& xml, result.Properties->AttachOnFail.end()); } for (std::string const& file : result.Properties->AttachedFiles) { - const std::string& base64 = this->CTest->Base64GzipEncodeFile(file); - std::string const fname = cmSystemTools::GetFilenameName(file); - xml.StartElement("NamedMeasurement"); - xml.Attribute("name", "Attached File"); - xml.Attribute("encoding", "base64"); - xml.Attribute("compression", "tar/gzip"); - xml.Attribute("filename", fname); - xml.Attribute("type", "file"); - xml.Element("Value", base64); - xml.EndElement(); // NamedMeasurement + this->AttachFile(xml, file, ""); } } +void cmCTestTestHandler::AttachFile(cmXMLWriter& xml, std::string const& file, + std::string const& name) +{ + const std::string& base64 = this->CTest->Base64GzipEncodeFile(file); + std::string const fname = cmSystemTools::GetFilenameName(file); + xml.StartElement("NamedMeasurement"); + std::string measurement_name = name; + if (measurement_name.empty()) { + measurement_name = "Attached File"; + } + xml.Attribute("name", measurement_name); + xml.Attribute("encoding", "base64"); + xml.Attribute("compression", "tar/gzip"); + xml.Attribute("filename", fname); + xml.Attribute("type", "file"); + xml.Element("Value", base64); + xml.EndElement(); // NamedMeasurement +} + int cmCTestTestHandler::ExecuteCommands(std::vector<std::string>& vec) { for (std::string const& it : vec) { @@ -1704,19 +1753,11 @@ bool cmCTestTestHandler::ParseResourceGroupsProperty( bool cmCTestTestHandler::GetListOfTests() { - if (!this->IncludeLabelRegExp.empty()) { - this->IncludeLabelRegularExpression.compile( - this->IncludeLabelRegExp.c_str()); - } - if (!this->ExcludeLabelRegExp.empty()) { - this->ExcludeLabelRegularExpression.compile( - this->ExcludeLabelRegExp.c_str()); - } if (!this->IncludeRegExp.empty()) { - this->IncludeTestsRegularExpression.compile(this->IncludeRegExp.c_str()); + this->IncludeTestsRegularExpression.compile(this->IncludeRegExp); } if (!this->ExcludeRegExp.empty()) { - this->ExcludeTestsRegularExpression.compile(this->ExcludeRegExp.c_str()); + this->ExcludeTestsRegularExpression.compile(this->ExcludeRegExp); } cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Constructing a list of tests" << std::endl, this->Quiet); @@ -1868,7 +1909,7 @@ void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed() std::string dirName = this->CTest->GetBinaryDir() + "/Testing/Temporary"; cmsys::Directory directory; - if (directory.Load(dirName) == 0) { + if (!directory.Load(dirName)) { cmCTestLog(this->CTest, ERROR_MESSAGE, "Unable to read the contents of " << dirName << std::endl); return; @@ -2020,11 +2061,11 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml, cmCTest::CleanString(measurementfile.match(5)); if (cmSystemTools::FileExists(filename)) { long len = cmSystemTools::FileLength(filename); + std::string k1 = measurementfile.match(1); + std::string v1 = measurementfile.match(2); + std::string k2 = measurementfile.match(3); + std::string v2 = measurementfile.match(4); if (len == 0) { - std::string k1 = measurementfile.match(1); - std::string v1 = measurementfile.match(2); - std::string k2 = measurementfile.match(3); - std::string v2 = measurementfile.match(4); if (cmSystemTools::LowerCase(k1) == "type") { v1 = "text/string"; } @@ -2039,35 +2080,53 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml, xml.Element("Value", "Image " + filename + " is empty"); xml.EndElement(); } else { - cmsys::ifstream ifs(filename.c_str(), - std::ios::in + std::string type; + std::string name; + if (cmSystemTools::LowerCase(k1) == "type") { + type = v1; + } else if (cmSystemTools::LowerCase(k2) == "type") { + type = v2; + } + if (cmSystemTools::LowerCase(k1) == "name") { + name = v1; + } else if (cmSystemTools::LowerCase(k2) == "name") { + name = v2; + } + if (type == "file") { + // Treat this measurement like an "ATTACHED_FILE" when the type + // is explicitly "file" (not an image). + this->AttachFile(xml, filename, name); + } else { + cmsys::ifstream ifs(filename.c_str(), + std::ios::in #ifdef _WIN32 - | std::ios::binary + | std::ios::binary #endif - ); - auto file_buffer = cm::make_unique<unsigned char[]>(len + 1); - ifs.read(reinterpret_cast<char*>(file_buffer.get()), len); - auto encoded_buffer = cm::make_unique<unsigned char[]>( - static_cast<int>(static_cast<double>(len) * 1.5 + 5.0)); - - size_t rlen = cmsysBase64_Encode(file_buffer.get(), len, - encoded_buffer.get(), 1); - - xml.StartElement("NamedMeasurement"); - xml.Attribute(measurementfile.match(1).c_str(), - measurementfile.match(2)); - xml.Attribute(measurementfile.match(3).c_str(), - measurementfile.match(4)); - xml.Attribute("encoding", "base64"); - std::ostringstream ostr; - for (size_t cc = 0; cc < rlen; cc++) { - ostr << encoded_buffer[cc]; - if (cc % 60 == 0 && cc) { - ostr << std::endl; + ); + auto file_buffer = cm::make_unique<unsigned char[]>(len + 1); + ifs.read(reinterpret_cast<char*>(file_buffer.get()), len); + auto encoded_buffer = cm::make_unique<unsigned char[]>( + static_cast<int>(static_cast<double>(len) * 1.5 + 5.0)); + + size_t rlen = cmsysBase64_Encode(file_buffer.get(), len, + encoded_buffer.get(), 1); + + xml.StartElement("NamedMeasurement"); + xml.Attribute(measurementfile.match(1).c_str(), + measurementfile.match(2)); + xml.Attribute(measurementfile.match(3).c_str(), + measurementfile.match(4)); + xml.Attribute("encoding", "base64"); + std::ostringstream ostr; + for (size_t cc = 0; cc < rlen; cc++) { + ostr << encoded_buffer[cc]; + if (cc % 60 == 0 && cc) { + ostr << std::endl; + } } + xml.Element("Value", ostr.str()); + xml.EndElement(); // NamedMeasurement } - xml.Element("Value", ostr.str()); - xml.EndElement(); // NamedMeasurement } } else { int idx = 4; @@ -2444,3 +2503,125 @@ bool cmCTestTestHandler::cmCTestTestResourceRequirement::operator!=( { return !(*this == other); } + +void cmCTestTestHandler::SetJUnitXMLFileName(const std::string& filename) +{ + this->JUnitXMLFileName = filename; +} + +bool cmCTestTestHandler::WriteJUnitXML() +{ + if (this->JUnitXMLFileName.empty()) { + return true; + } + + // Open new XML file for writing. + cmGeneratedFileStream xmlfile; + xmlfile.SetTempExt("tmp"); + xmlfile.Open(this->JUnitXMLFileName); + if (!xmlfile) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Problem opening file: " << this->JUnitXMLFileName + << std::endl); + return false; + } + cmXMLWriter xml(xmlfile); + + // Iterate over the test results to get the number of tests that + // passed, failed, etc. + auto num_tests = 0; + auto num_passed = 0; + auto num_failed = 0; + auto num_notrun = 0; + auto num_disabled = 0; + SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end()); + for (cmCTestTestResult const& result : resultsSet) { + num_tests++; + if (result.Status == cmCTestTestHandler::COMPLETED) { + num_passed++; + } else if (result.Status == cmCTestTestHandler::NOT_RUN) { + if (result.CompletionStatus == "Disabled") { + num_disabled++; + } else { + num_notrun++; + } + } else { + num_failed++; + } + } + + // Write <testsuite> element. + xml.StartDocument(); + xml.StartElement("testsuite"); + + xml.Attribute("name", + cmCTest::SafeBuildIdField( + this->CTest->GetCTestConfiguration("BuildName"))); + xml.BreakAttributes(); + + xml.Attribute("tests", num_tests); + xml.Attribute("failures", num_failed); + + // CTest disabled => JUnit disabled + xml.Attribute("disabled", num_disabled); + + // Otherwise, CTest notrun => JUnit skipped. + // The distinction between JUnit disabled vs. skipped is that + // skipped tests can have a message associated with them + // (why the test was skipped). + xml.Attribute("skipped", num_notrun); + + xml.Attribute("hostname", this->CTest->GetCTestConfiguration("Site")); + xml.Attribute( + "time", + std::chrono::duration_cast<std::chrono::seconds>(this->ElapsedTestingTime) + .count()); + const std::time_t start_test_time_t = + std::chrono::system_clock::to_time_t(this->StartTestTime); + cmTimestamp cmts; + xml.Attribute("timestamp", + cmts.CreateTimestampFromTimeT(start_test_time_t, + "%Y-%m-%dT%H:%M:%S", false)); + + // Write <testcase> elements. + for (cmCTestTestResult const& result : resultsSet) { + xml.StartElement("testcase"); + xml.Attribute("name", result.Name); + xml.Attribute("classname", result.Name); + xml.Attribute("time", result.ExecutionTime.count()); + + std::string status; + if (result.Status == cmCTestTestHandler::COMPLETED) { + status = "run"; + } else if (result.Status == cmCTestTestHandler::NOT_RUN) { + if (result.CompletionStatus == "Disabled") { + status = "disabled"; + } else { + status = "notrun"; + } + } else { + status = "fail"; + } + xml.Attribute("status", status); + + if (status == "notrun") { + xml.StartElement("skipped"); + xml.Attribute("message", result.CompletionStatus); + xml.EndElement(); // </skipped> + } else if (status == "fail") { + xml.StartElement("failure"); + xml.Attribute("message", result.Reason); + xml.EndElement(); // </failure> + } + + // Note: compressed test output is unconditionally disabled when + // --output-junit is specified. + xml.Element("system-out", result.Output); + xml.EndElement(); // </testcase> + } + + xml.EndElement(); // </testsuite> + xml.EndDocument(); + + return true; +} diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index aa29eeb..cc19984 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -175,6 +175,7 @@ public: std::string ExceptionStatus; bool CompressOutput; std::string CompletionStatus; + std::string CustomCompletionStatus; std::string Output; std::string DartString; int TestCount; @@ -209,6 +210,9 @@ public: using ListOfTests = std::vector<cmCTestTestProperties>; + // Support for writing test results in JUnit XML format. + void SetJUnitXMLFileName(const std::string& id); + protected: using SetOfTests = std::set<cmCTestTestHandler::cmCTestTestResult, cmCTestTestResultLess>; @@ -234,6 +238,8 @@ protected: cmCTestTestResult const& result); // Write attached test files into the xml void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result); + void AttachFile(cmXMLWriter& xml, std::string const& file, + std::string const& name); //! Clean test output to specified length void CleanTestOutput(std::string& output, size_t length); @@ -274,6 +280,11 @@ private: */ virtual void GenerateDartOutput(cmXMLWriter& xml); + /** + * Write test results in JUnit XML format + */ + bool WriteJUnitXML(); + void PrintLabelOrSubprojectSummary(bool isSubProject); /** @@ -320,20 +331,16 @@ private: std::vector<int> TestsToRun; - bool UseIncludeLabelRegExpFlag; - bool UseExcludeLabelRegExpFlag; bool UseIncludeRegExpFlag; bool UseExcludeRegExpFlag; bool UseExcludeRegExpFirst; - std::string IncludeLabelRegExp; - std::string ExcludeLabelRegExp; std::string IncludeRegExp; std::string ExcludeRegExp; std::string ExcludeFixtureRegExp; std::string ExcludeFixtureSetupRegExp; std::string ExcludeFixtureCleanupRegExp; - cmsys::RegularExpression IncludeLabelRegularExpression; - cmsys::RegularExpression ExcludeLabelRegularExpression; + std::vector<cmsys::RegularExpression> IncludeLabelRegularExpressions; + std::vector<cmsys::RegularExpression> ExcludeLabelRegularExpressions; cmsys::RegularExpression IncludeTestsRegularExpression; cmsys::RegularExpression ExcludeTestsRegularExpression; @@ -352,10 +359,14 @@ private: ListOfTests TestList; size_t TotalNumberOfTests; cmsys::RegularExpression DartStuff; + cmsys::RegularExpression CustomCompletionStatusRegex; + cmsys::RegularExpression CustomLabelRegex; std::ostream* LogFile; cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never; int RepeatCount = 1; bool RerunFailed; + + std::string JUnitXMLFileName; }; diff --git a/Source/CTest/cmCTestUpdateCommand.cxx b/Source/CTest/cmCTestUpdateCommand.cxx index 0ba2c41..797cb01 100644 --- a/Source/CTest/cmCTestUpdateCommand.cxx +++ b/Source/CTest/cmCTestUpdateCommand.cxx @@ -75,7 +75,6 @@ cmCTestGenericHandler* cmCTestUpdateCommand::InitializeHandler() cmCTestUpdateHandler* handler = this->CTest->GetUpdateHandler(); handler->Initialize(); - handler->SetCommand(this); if (source_dir.empty()) { this->SetError("source directory not specified. Please use SOURCE tag"); return nullptr; |