diff options
Diffstat (limited to 'Source/CTest')
30 files changed, 799 insertions, 349 deletions
diff --git a/Source/CTest/cmCTestBuildAndTestHandler.cxx b/Source/CTest/cmCTestBuildAndTestHandler.cxx index a18cbb4..adfc8ef 100644 --- a/Source/CTest/cmCTestBuildAndTestHandler.cxx +++ b/Source/CTest/cmCTestBuildAndTestHandler.cxx @@ -19,6 +19,8 @@ #include "cmWorkingDirectory.h" #include "cmake.h" +struct cmMessageMetadata; + cmCTestBuildAndTestHandler::cmCTestBuildAndTestHandler() { this->BuildTwoConfig = false; @@ -125,7 +127,7 @@ public: : CM(cm) { cmSystemTools::SetMessageCallback( - [&s](const std::string& msg, const char* /*unused*/) { + [&s](const std::string& msg, const cmMessageMetadata& /* unused */) { s += msg; s += "\n"; }); diff --git a/Source/CTest/cmCTestBuildCommand.cxx b/Source/CTest/cmCTestBuildCommand.cxx index 88e2871..483c316 100644 --- a/Source/CTest/cmCTestBuildCommand.cxx +++ b/Source/CTest/cmCTestBuildCommand.cxx @@ -27,6 +27,7 @@ void cmCTestBuildCommand::BindArguments() this->Bind("CONFIGURATION"_s, this->Configuration); this->Bind("FLAGS"_s, this->Flags); this->Bind("PROJECT_NAME"_s, this->ProjectName); + this->Bind("PARALLEL_LEVEL"_s, this->ParallelLevel); } cmCTestBuildCommand::~cmCTestBuildCommand() = default; @@ -98,8 +99,8 @@ cmCTestGenericHandler* cmCTestBuildCommand::InitializeHandler() std::string dir = this->CTest->GetCTestConfiguration("BuildDirectory"); std::string buildCommand = this->GlobalGenerator->GenerateCMakeBuildCommand( - cmakeBuildTarget, cmakeBuildConfiguration, cmakeBuildAdditionalFlags, - this->Makefile->IgnoreErrorsCMP0061()); + cmakeBuildTarget, cmakeBuildConfiguration, this->ParallelLevel, + cmakeBuildAdditionalFlags, this->Makefile->IgnoreErrorsCMP0061()); cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "SetMakeCommand:" << buildCommand << "\n", this->Quiet); diff --git a/Source/CTest/cmCTestBuildCommand.h b/Source/CTest/cmCTestBuildCommand.h index 00dbcc4..1254dad 100644 --- a/Source/CTest/cmCTestBuildCommand.h +++ b/Source/CTest/cmCTestBuildCommand.h @@ -60,4 +60,5 @@ protected: std::string Configuration; std::string Flags; std::string ProjectName; + std::string ParallelLevel; }; diff --git a/Source/CTest/cmCTestBuildHandler.cxx b/Source/CTest/cmCTestBuildHandler.cxx index 103dc1e..03caa63 100644 --- a/Source/CTest/cmCTestBuildHandler.cxx +++ b/Source/CTest/cmCTestBuildHandler.cxx @@ -3,7 +3,6 @@ #include "cmCTestBuildHandler.h" #include <cstdlib> -#include <cstring> #include <set> #include <utility> @@ -657,14 +656,14 @@ bool cmCTestBuildHandler::IsLaunchedErrorFile(const char* fname) { // error-{hash}.xml return (cmHasLiteralPrefix(fname, "error-") && - strcmp(fname + strlen(fname) - 4, ".xml") == 0); + cmHasLiteralSuffix(fname, ".xml")); } bool cmCTestBuildHandler::IsLaunchedWarningFile(const char* fname) { // warning-{hash}.xml return (cmHasLiteralPrefix(fname, "warning-") && - strcmp(fname + strlen(fname) - 4, ".xml") == 0); + cmHasLiteralSuffix(fname, ".xml")); } //###################################################################### diff --git a/Source/CTest/cmCTestCoverageHandler.cxx b/Source/CTest/cmCTestCoverageHandler.cxx index 3d7d031..57b1dda 100644 --- a/Source/CTest/cmCTestCoverageHandler.cxx +++ b/Source/CTest/cmCTestCoverageHandler.cxx @@ -289,6 +289,13 @@ int cmCTestCoverageHandler::ProcessHandler() this->CTest->GetCTestConfiguration("SourceDirectory"); std::string binaryDir = this->CTest->GetCTestConfiguration("BuildDirectory"); + if (binaryDir.empty()) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Binary directory is not set. " + "No coverage checking will be performed." + << std::endl); + return 0; + } this->LoadLabels(); cmGeneratedFileStream ofs; diff --git a/Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx b/Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx index 051c117..af495bb 100644 --- a/Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx +++ b/Source/CTest/cmCTestEmptyBinaryDirectoryCommand.cxx @@ -16,7 +16,7 @@ bool cmCTestEmptyBinaryDirectoryCommand::InitialPass( return false; } - if (!cmCTestScriptHandler::EmptyBinaryDirectory(args[0].c_str())) { + if (!cmCTestScriptHandler::EmptyBinaryDirectory(args[0])) { std::ostringstream ostr; ostr << "problem removing the binary directory: " << args[0]; this->SetError(ostr.str()); diff --git a/Source/CTest/cmCTestGenericHandler.cxx b/Source/CTest/cmCTestGenericHandler.cxx index 91818bb..dc7103a 100644 --- a/Source/CTest/cmCTestGenericHandler.cxx +++ b/Source/CTest/cmCTestGenericHandler.cxx @@ -21,51 +21,97 @@ cmCTestGenericHandler::cmCTestGenericHandler() cmCTestGenericHandler::~cmCTestGenericHandler() = default; -void cmCTestGenericHandler::SetOption(const std::string& op, const char* value) +namespace { +/* Modify the given `map`, setting key `op` to `value` if `value` + * is non-null, otherwise removing key `op` (if it exists). + */ +void SetMapValue(cmCTestGenericHandler::t_StringToString& map, + const std::string& op, const char* value) +{ + if (!value) { + map.erase(op); + return; + } + + map[op] = value; +} +void SetMapValue(cmCTestGenericHandler::t_StringToString& map, + const std::string& op, cmProp value) { if (!value) { - auto remit = this->Options.find(op); - if (remit != this->Options.end()) { - this->Options.erase(remit); - } + map.erase(op); return; } - this->Options[op] = value; + map[op] = *value; +} +} + +void cmCTestGenericHandler::SetOption(const std::string& op, const char* value) +{ + SetMapValue(this->Options, op, value); +} +void cmCTestGenericHandler::SetOption(const std::string& op, cmProp value) +{ + SetMapValue(this->Options, op, value); } void cmCTestGenericHandler::SetPersistentOption(const std::string& op, const char* value) { this->SetOption(op, value); - if (!value) { - auto remit = this->PersistentOptions.find(op); - if (remit != this->PersistentOptions.end()) { - this->PersistentOptions.erase(remit); - } - return; + SetMapValue(this->PersistentOptions, op, value); +} +void cmCTestGenericHandler::SetPersistentOption(const std::string& op, + cmProp value) +{ + this->SetOption(op, value); + SetMapValue(this->PersistentOptions, op, value); +} + +void cmCTestGenericHandler::AddMultiOption(const std::string& op, + const std::string& value) +{ + if (!value.empty()) { + this->MultiOptions[op].emplace_back(value); } +} - this->PersistentOptions[op] = value; +void cmCTestGenericHandler::AddPersistentMultiOption(const std::string& op, + const std::string& value) +{ + if (!value.empty()) { + this->MultiOptions[op].emplace_back(value); + this->PersistentMultiOptions[op].emplace_back(value); + } } void cmCTestGenericHandler::Initialize() { this->AppendXML = false; this->TestLoad = 0; - this->Options.clear(); - for (auto const& po : this->PersistentOptions) { - this->Options[po.first] = po.second; - } + this->Options = this->PersistentOptions; + this->MultiOptions = this->PersistentMultiOptions; } -const char* cmCTestGenericHandler::GetOption(const std::string& op) +cmProp cmCTestGenericHandler::GetOption(const std::string& op) { auto remit = this->Options.find(op); if (remit == this->Options.end()) { return nullptr; } - return remit->second.c_str(); + return cmProp(remit->second); +} + +std::vector<std::string> cmCTestGenericHandler::GetMultiOption( + const std::string& optionName) const +{ + // Avoid inserting a key, which MultiOptions[op] would do. + auto remit = this->MultiOptions.find(optionName); + if (remit == this->MultiOptions.end()) { + return {}; + } + return remit->second; } bool cmCTestGenericHandler::StartResultingXML(cmCTest::Part part, diff --git a/Source/CTest/cmCTestGenericHandler.h b/Source/CTest/cmCTestGenericHandler.h index 89d7596..852d4ea 100644 --- a/Source/CTest/cmCTestGenericHandler.h +++ b/Source/CTest/cmCTestGenericHandler.h @@ -11,9 +11,9 @@ #include <stddef.h> #include "cmCTest.h" +#include "cmProperty.h" #include "cmSystemTools.h" -class cmCTestCommand; class cmGeneratedFileStream; class cmMakefile; @@ -72,12 +72,50 @@ public: virtual ~cmCTestGenericHandler(); using t_StringToString = std::map<std::string, std::string>; + using t_StringToMultiString = + std::map<std::string, std::vector<std::string>>; + /** + * Options collect a single value from flags; passing the + * flag multiple times on the command-line *overwrites* values, + * and only the last one specified counts. Set an option to + * nullptr to "unset" it. + * + * The value is stored as a string. The values set for single + * and multi-options (see below) live in different spaces, + * so calling a single-getter for a key that has only been set + * as a multi-value will return nullptr. + */ void SetPersistentOption(const std::string& op, const char* value); + void SetPersistentOption(const std::string& op, const std::string& value) + { + this->SetPersistentOption(op, cmProp(value)); + } + void SetPersistentOption(const std::string& op, cmProp value); void SetOption(const std::string& op, const char* value); - const char* GetOption(const std::string& op); + void SetOption(const std::string& op, const std::string& value) + { + this->SetOption(op, cmProp(value)); + } + void SetOption(const std::string& op, cmProp value); + cmProp GetOption(const std::string& op); - void SetCommand(cmCTestCommand* command) { this->Command = command; } + /** + * Multi-Options collect one or more values from flags; passing + * the flag multiple times on the command-line *adds* values, + * rather than overwriting the previous values. + * + * Adding an empty value does nothing. + * + * The value is stored as a vector of strings. The values set for single + * (see above) and multi-options live in different spaces, + * so calling a multi-getter for a key that has only been set + * as a single-value will return an empty vector. + */ + void AddPersistentMultiOption(const std::string& optionName, + const std::string& value); + void AddMultiOption(const std::string& optionName, const std::string& value); + std::vector<std::string> GetMultiOption(const std::string& op) const; void SetSubmitIndex(int idx) { this->SubmitIndex = idx; } int GetSubmitIndex() { return this->SubmitIndex; } @@ -100,8 +138,9 @@ protected: cmCTest* CTest; t_StringToString Options; t_StringToString PersistentOptions; + t_StringToMultiString MultiOptions; + t_StringToMultiString PersistentMultiOptions; t_StringToString LogFileNames; - cmCTestCommand* Command; int SubmitIndex; }; diff --git a/Source/CTest/cmCTestMemCheckCommand.cxx b/Source/CTest/cmCTestMemCheckCommand.cxx index d0e2974..37b3628 100644 --- a/Source/CTest/cmCTestMemCheckCommand.cxx +++ b/Source/CTest/cmCTestMemCheckCommand.cxx @@ -14,7 +14,7 @@ void cmCTestMemCheckCommand::BindArguments() this->Bind("DEFECT_COUNT"_s, this->DefectCount); } -cmCTestGenericHandler* cmCTestMemCheckCommand::InitializeActualHandler() +cmCTestTestHandler* cmCTestMemCheckCommand::InitializeActualHandler() { cmCTestMemCheckHandler* handler = this->CTest->GetMemCheckHandler(); handler->Initialize(); diff --git a/Source/CTest/cmCTestMemCheckCommand.h b/Source/CTest/cmCTestMemCheckCommand.h index 6544f16..ee39e49 100644 --- a/Source/CTest/cmCTestMemCheckCommand.h +++ b/Source/CTest/cmCTestMemCheckCommand.h @@ -13,6 +13,7 @@ #include "cmCommand.h" class cmCTestGenericHandler; +class cmCTestTestHandler; /** \class cmCTestMemCheck * \brief Run a ctest script @@ -36,7 +37,7 @@ public: protected: void BindArguments() override; - cmCTestGenericHandler* InitializeActualHandler() override; + cmCTestTestHandler* InitializeActualHandler() override; void ProcessAdditionalValues(cmCTestGenericHandler* handler) override; diff --git a/Source/CTest/cmCTestMemCheckHandler.cxx b/Source/CTest/cmCTestMemCheckHandler.cxx index 125d003..6bb8e79 100644 --- a/Source/CTest/cmCTestMemCheckHandler.cxx +++ b/Source/CTest/cmCTestMemCheckHandler.cxx @@ -305,7 +305,7 @@ int cmCTestMemCheckHandler::GetDefectCount() const return this->DefectCount; } -void cmCTestMemCheckHandler::GenerateDartOutput(cmXMLWriter& xml) +void cmCTestMemCheckHandler::GenerateCTestXML(cmXMLWriter& xml) { if (!this->CTest->GetProduceXML()) { return; diff --git a/Source/CTest/cmCTestMemCheckHandler.h b/Source/CTest/cmCTestMemCheckHandler.h index b200c43..a63a24d 100644 --- a/Source/CTest/cmCTestMemCheckHandler.h +++ b/Source/CTest/cmCTestMemCheckHandler.h @@ -119,9 +119,9 @@ private: bool InitializeMemoryChecking(); /** - * Generate the Dart compatible output + * Generate CTest DynamicAnalysis.xml files */ - void GenerateDartOutput(cmXMLWriter& xml) override; + void GenerateCTestXML(cmXMLWriter& xml) override; std::vector<std::string> CustomPreMemCheck; std::vector<std::string> CustomPostMemCheck; diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 852f9d9..d90c4a6 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -587,24 +587,24 @@ void cmCTestMultiProcessHandler::StartNextTests() onlyRunSerialTestsLeft = false; } } - cmCTestLog(this->CTest, HANDLER_OUTPUT, "***** WAITING, "); + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "***** WAITING, "); if (this->SerialTestRunning) { - cmCTestLog(this->CTest, HANDLER_OUTPUT, + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Waiting for RUN_SERIAL test to finish."); } else if (onlyRunSerialTestsLeft) { - cmCTestLog(this->CTest, HANDLER_OUTPUT, + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Only RUN_SERIAL tests remain, awaiting available slot."); } else { /* clang-format off */ - cmCTestLog(this->CTest, HANDLER_OUTPUT, + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "System Load: " << systemLoad << ", " "Max Allowed Load: " << this->TestLoad << ", " "Smallest test " << testWithMinProcessors << " requires " << minProcessorsRequired); /* clang-format on */ } - cmCTestLog(this->CTest, HANDLER_OUTPUT, "*****" << std::endl); + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "*****" << std::endl); // Wait between 1 and 5 seconds before trying again. unsigned int milliseconds = (cmSystemTools::RandomSeed() % 5 + 1) * 1000; @@ -1026,6 +1026,11 @@ static Json::Value DumpCTestProperties( properties.append(DumpCTestProperty( "ENVIRONMENT", DumpToJsonArray(testProperties.Environment))); } + if (!testProperties.EnvironmentModification.empty()) { + properties.append(DumpCTestProperty( + "ENVIRONMENT_MODIFICATION", + DumpToJsonArray(testProperties.EnvironmentModification))); + } if (!testProperties.ErrorRegularExpressions.empty()) { properties.append(DumpCTestProperty( "FAIL_REGULAR_EXPRESSION", diff --git a/Source/CTest/cmCTestRunScriptCommand.cxx b/Source/CTest/cmCTestRunScriptCommand.cxx index f59ca57..7661d4d 100644 --- a/Source/CTest/cmCTestRunScriptCommand.cxx +++ b/Source/CTest/cmCTestRunScriptCommand.cxx @@ -37,8 +37,8 @@ bool cmCTestRunScriptCommand::InitialPass(std::vector<std::string> const& args, ++i; } else { int ret; - cmCTestScriptHandler::RunScript(this->CTest, this->Makefile, - args[i].c_str(), !np, &ret); + cmCTestScriptHandler::RunScript(this->CTest, this->Makefile, args[i], + !np, &ret); this->Makefile->AddDefinition(returnVariable, std::to_string(ret)); } } diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 5a6c775..20f0ed3 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -2,17 +2,22 @@ file Copyright.txt or https://cmake.org/licensing for details. */ #include "cmCTestRunTest.h" +#include <algorithm> #include <chrono> #include <cstddef> // IWYU pragma: keep #include <cstdint> #include <cstdio> #include <cstring> +#include <functional> #include <iomanip> #include <ratio> #include <sstream> #include <utility> #include <cm/memory> +#include <cm/optional> +#include <cm/string_view> +#include <cmext/string_view> #include "cmsys/RegularExpression.hxx" @@ -40,6 +45,38 @@ void cmCTestRunTest::CheckOutput(std::string const& line) { cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->GetIndex() << ": " << line << std::endl); + + // Check for special CTest XML tags in this line of output. + // If any are found, this line is excluded from ProcessOutput. + if (!line.empty() && line.find("<CTest") != std::string::npos) { + bool ctest_tag_found = false; + if (this->TestHandler->CustomCompletionStatusRegex.find(line)) { + ctest_tag_found = true; + this->TestResult.CustomCompletionStatus = + this->TestHandler->CustomCompletionStatusRegex.match(1); + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + this->GetIndex() << ": " + << "Test Details changed to '" + << this->TestResult.CustomCompletionStatus + << "'" << std::endl); + } else if (this->TestHandler->CustomLabelRegex.find(line)) { + ctest_tag_found = true; + auto label = this->TestHandler->CustomLabelRegex.match(1); + auto& labels = this->TestProperties->Labels; + if (std::find(labels.begin(), labels.end(), label) == labels.end()) { + labels.push_back(label); + std::sort(labels.begin(), labels.end()); + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + this->GetIndex() + << ": " + << "Test Label added: '" << label << "'" << std::endl); + } + } + if (ctest_tag_found) { + return; + } + } + this->ProcessOutput += line; this->ProcessOutput += "\n"; @@ -229,7 +266,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started) *this->TestHandler->LogFile << "Test time = " << buf << std::endl; } - this->DartProcessing(); + this->ParseOutputForMeasurements(); // if this is doing MemCheck then all the output needs to be put into // Output since that is what is parsed by cmCTestMemCheckHandler @@ -607,6 +644,7 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total) return this->ForkProcess(timeout, this->TestProperties->ExplicitTimeout, &this->TestProperties->Environment, + &this->TestProperties->EnvironmentModification, &this->TestProperties->Affinity); } @@ -663,28 +701,45 @@ void cmCTestRunTest::ComputeArguments() cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, this->Index << ": " << env << std::endl); } + if (!this->TestProperties->EnvironmentModification.empty()) { + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + this->Index << ": " + << "Environment variable modifications: " + << std::endl); + } + for (std::string const& envmod : + this->TestProperties->EnvironmentModification) { + cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, + this->Index << ": " << envmod << std::endl); + } } -void cmCTestRunTest::DartProcessing() +void cmCTestRunTest::ParseOutputForMeasurements() { if (!this->ProcessOutput.empty() && - this->ProcessOutput.find("<DartMeasurement") != std::string::npos) { - if (this->TestHandler->DartStuff.find(this->ProcessOutput)) { - this->TestResult.DartString = this->TestHandler->DartStuff.match(1); + (this->ProcessOutput.find("<DartMeasurement") != std::string::npos || + this->ProcessOutput.find("<CTestMeasurement") != std::string::npos)) { + if (this->TestHandler->AllTestMeasurementsRegex.find( + this->ProcessOutput)) { + this->TestResult.TestMeasurementsOutput = + this->TestHandler->AllTestMeasurementsRegex.match(1); // keep searching and replacing until none are left - while (this->TestHandler->DartStuff1.find(this->ProcessOutput)) { + while (this->TestHandler->SingleTestMeasurementRegex.find( + this->ProcessOutput)) { // replace the exact match for the string cmSystemTools::ReplaceString( - this->ProcessOutput, this->TestHandler->DartStuff1.match(1).c_str(), - ""); + this->ProcessOutput, + this->TestHandler->SingleTestMeasurementRegex.match(1).c_str(), ""); } } } } -bool cmCTestRunTest::ForkProcess(cmDuration testTimeOut, bool explicitTimeout, - std::vector<std::string>* environment, - std::vector<size_t>* affinity) +bool cmCTestRunTest::ForkProcess( + cmDuration testTimeOut, bool explicitTimeout, + std::vector<std::string>* environment, + std::vector<std::string>* environment_modification, + std::vector<size_t>* affinity) { this->TestProcess->SetId(this->Index); this->TestProcess->SetWorkingDirectory(this->TestProperties->Directory); @@ -733,6 +788,127 @@ bool cmCTestRunTest::ForkProcess(cmDuration testTimeOut, bool explicitTimeout, } } + if (environment_modification && !environment_modification->empty()) { + std::map<std::string, cm::optional<std::string>> env_application; + +#ifdef _WIN32 + char path_sep = ';'; +#else + char path_sep = ':'; +#endif + + auto apply_diff = + [&env_application](const std::string& name, + std::function<void(std::string&)> const& apply) { + auto entry = env_application.find(name); + std::string output; + if (entry != env_application.end() && entry->second) { + output = *entry->second; + } + apply(output); + entry->second = output; + }; + + bool err_occurred = false; + + for (auto const& envmod : *environment_modification) { + // Split on `=` + auto const eq_loc = envmod.find_first_of('='); + if (eq_loc == std::string::npos) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Error: Missing `=` after the variable name in: " + << envmod << std::endl); + err_occurred = true; + continue; + } + auto const name = envmod.substr(0, eq_loc); + + // Split value on `:` + auto const op_value_start = eq_loc + 1; + auto const colon_loc = envmod.find_first_of(':', op_value_start); + if (colon_loc == std::string::npos) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Error: Missing `:` after the operation in: " << envmod + << std::endl); + err_occurred = true; + continue; + } + auto const op = + envmod.substr(op_value_start, colon_loc - op_value_start); + + auto const value_start = colon_loc + 1; + auto const value = envmod.substr(value_start); + + // Determine what to do with the operation. + if (op == "reset"_s) { + auto entry = env_application.find(name); + if (entry != env_application.end()) { + env_application.erase(entry); + } + } else if (op == "set"_s) { + env_application[name] = value; + } else if (op == "unset"_s) { + env_application[name] = {}; + } else if (op == "string_append"_s) { + apply_diff(name, [&value](std::string& output) { output += value; }); + } else if (op == "string_prepend"_s) { + apply_diff(name, + [&value](std::string& output) { output.insert(0, value); }); + } else if (op == "path_list_append"_s) { + apply_diff(name, [&value, path_sep](std::string& output) { + if (!output.empty()) { + output += path_sep; + } + output += value; + }); + } else if (op == "path_list_prepend"_s) { + apply_diff(name, [&value, path_sep](std::string& output) { + if (!output.empty()) { + output.insert(output.begin(), path_sep); + } + output.insert(0, value); + }); + } else if (op == "cmake_list_append"_s) { + apply_diff(name, [&value](std::string& output) { + if (!output.empty()) { + output += ';'; + } + output += value; + }); + } else if (op == "cmake_list_prepend"_s) { + apply_diff(name, [&value](std::string& output) { + if (!output.empty()) { + output.insert(output.begin(), ';'); + } + output.insert(0, value); + }); + } else { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Error: Unrecognized environment manipulation argument: " + << op << std::endl); + err_occurred = true; + continue; + } + } + + if (err_occurred) { + return false; + } + + for (auto const& env_apply : env_application) { + if (env_apply.second) { + auto const env_update = + cmStrCat(env_apply.first, '=', *env_apply.second); + cmSystemTools::PutEnv(env_update); + envMeasurement << env_update << std::endl; + } else { + cmSystemTools::UnsetEnv(env_apply.first.c_str()); + // Signify that this variable is being actively unset + envMeasurement << "#" << env_apply.first << "=" << std::endl; + } + } + } + if (this->UseAllocatedResources) { std::vector<std::string> envLog; this->SetupResourcesEnvironment(&envLog); diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index 863ac1b..2082156 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -109,10 +109,11 @@ public: private: bool NeedsToRepeat(); - void DartProcessing(); + void ParseOutputForMeasurements(); void ExeNotFound(std::string exe); bool ForkProcess(cmDuration testTimeOut, bool explicitTimeout, std::vector<std::string>* environment, + std::vector<std::string>* environment_modification, std::vector<size_t>* affinity); void WriteLogOutputTop(size_t completed, size_t total); // Run post processing of the process output for MemCheck diff --git a/Source/CTest/cmCTestScriptHandler.cxx b/Source/CTest/cmCTestScriptHandler.cxx index 4808c36..d2cad39 100644 --- a/Source/CTest/cmCTestScriptHandler.cxx +++ b/Source/CTest/cmCTestScriptHandler.cxx @@ -4,7 +4,6 @@ #include <cstdio> #include <cstdlib> -#include <cstring> #include <map> #include <ratio> #include <sstream> @@ -91,7 +90,7 @@ void cmCTestScriptHandler::Initialize() cmCTestScriptHandler::~cmCTestScriptHandler() = default; // just adds an argument to the vector -void cmCTestScriptHandler::AddConfigurationScript(const char* script, +void cmCTestScriptHandler::AddConfigurationScript(const std::string& script, bool pscope) { this->ConfigurationScripts.emplace_back(script); @@ -676,7 +675,7 @@ int cmCTestScriptHandler::RunConfigurationDashboard() // clear the binary directory? if (this->EmptyBinDir) { - if (!cmCTestScriptHandler::EmptyBinaryDirectory(this->BinaryDir.c_str())) { + if (!cmCTestScriptHandler::EmptyBinaryDirectory(this->BinaryDir)) { cmCTestLog(this->CTest, ERROR_MESSAGE, "Problem removing the binary directory" << std::endl); } @@ -724,8 +723,8 @@ int cmCTestScriptHandler::RunConfigurationDashboard() // put the initial cache into the bin dir if (!this->InitialCache.empty()) { - if (!cmCTestScriptHandler::WriteInitialCache(this->BinaryDir.c_str(), - this->InitialCache.c_str())) { + if (!cmCTestScriptHandler::WriteInitialCache(this->BinaryDir, + this->InitialCache)) { this->RestoreBackupDirectories(); return 9; } @@ -812,8 +811,8 @@ int cmCTestScriptHandler::RunConfigurationDashboard() return 0; } -bool cmCTestScriptHandler::WriteInitialCache(const char* directory, - const char* text) +bool cmCTestScriptHandler::WriteInitialCache(const std::string& directory, + const std::string& text) { std::string cacheFile = cmStrCat(directory, "/CMakeCache.txt"); cmGeneratedFileStream fout(cacheFile); @@ -821,9 +820,7 @@ bool cmCTestScriptHandler::WriteInitialCache(const char* directory, return false; } - if (text != nullptr) { - fout.write(text, strlen(text)); - } + fout.write(text.data(), text.size()); // Make sure the operating system has finished writing the file // before closing it. This will ensure the file is finished before @@ -852,7 +849,7 @@ void cmCTestScriptHandler::RestoreBackupDirectories() } bool cmCTestScriptHandler::RunScript(cmCTest* ctest, cmMakefile* mf, - const char* sname, bool InProcess, + const std::string& sname, bool InProcess, int* returnValue) { auto sh = cm::make_unique<cmCTestScriptHandler>(); @@ -866,10 +863,10 @@ bool cmCTestScriptHandler::RunScript(cmCTest* ctest, cmMakefile* mf, return true; } -bool cmCTestScriptHandler::EmptyBinaryDirectory(const char* sname) +bool cmCTestScriptHandler::EmptyBinaryDirectory(const std::string& sname) { // try to avoid deleting root - if (!sname || strlen(sname) < 2) { + if (sname.size() < 2) { return false; } @@ -924,7 +921,7 @@ bool cmCTestScriptHandler::TryToRemoveBinaryDirectoryOnce( } } - return cmSystemTools::RemoveADirectory(directoryPath); + return static_cast<bool>(cmSystemTools::RemoveADirectory(directoryPath)); } cmDuration cmCTestScriptHandler::GetRemainingTimeAllowed() diff --git a/Source/CTest/cmCTestScriptHandler.h b/Source/CTest/cmCTestScriptHandler.h index 8eb9658..b7764b2 100644 --- a/Source/CTest/cmCTestScriptHandler.h +++ b/Source/CTest/cmCTestScriptHandler.h @@ -62,7 +62,7 @@ public: /** * Add a script to run, and if is should run in the current process */ - void AddConfigurationScript(const char*, bool pscope); + void AddConfigurationScript(const std::string&, bool pscope); /** * Run a dashboard using a specified confiuration script @@ -72,19 +72,21 @@ public: /* * Run a script */ - static bool RunScript(cmCTest* ctest, cmMakefile* mf, const char* script, - bool InProcess, int* returnValue); + static bool RunScript(cmCTest* ctest, cmMakefile* mf, + const std::string& script, bool InProcess, + int* returnValue); int RunCurrentScript(); /* * Empty Binary Directory */ - static bool EmptyBinaryDirectory(const char* dir); + static bool EmptyBinaryDirectory(const std::string& dir); /* * Write an initial CMakeCache.txt from the given contents. */ - static bool WriteInitialCache(const char* directory, const char* text); + static bool WriteInitialCache(const std::string& directory, + const std::string& text); /* * Some elapsed time handling functions diff --git a/Source/CTest/cmCTestStartCommand.cxx b/Source/CTest/cmCTestStartCommand.cxx index 53e1b2f..a8c2403 100644 --- a/Source/CTest/cmCTestStartCommand.cxx +++ b/Source/CTest/cmCTestStartCommand.cxx @@ -30,8 +30,8 @@ bool cmCTestStartCommand::InitialPass(std::vector<std::string> const& args, size_t cnt = 0; const char* smodel = nullptr; - const std::string* src_dir = nullptr; - const std::string* bld_dir = nullptr; + cmProp src_dir; + cmProp bld_dir; while (cnt < args.size()) { if (args[cnt] == "GROUP" || args[cnt] == "TRACK") { @@ -55,10 +55,10 @@ bool cmCTestStartCommand::InitialPass(std::vector<std::string> const& args, smodel = args[cnt].c_str(); cnt++; } else if (!src_dir) { - src_dir = &args[cnt]; + src_dir = cmProp(args[cnt]); cnt++; } else if (!bld_dir) { - bld_dir = &args[cnt]; + bld_dir = cmProp(args[cnt]); cnt++; } else { this->SetError("Too many arguments"); diff --git a/Source/CTest/cmCTestSubmitCommand.cxx b/Source/CTest/cmCTestSubmitCommand.cxx index bdba0e5..6824752 100644 --- a/Source/CTest/cmCTestSubmitCommand.cxx +++ b/Source/CTest/cmCTestSubmitCommand.cxx @@ -36,8 +36,8 @@ std::unique_ptr<cmCommand> cmCTestSubmitCommand::Clone() cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler() { - const std::string* submitURL = !this->SubmitURL.empty() - ? &this->SubmitURL + cmProp submitURL = !this->SubmitURL.empty() + ? cmProp(this->SubmitURL) : this->Makefile->GetDefinition("CTEST_SUBMIT_URL"); if (submitURL) { @@ -119,15 +119,15 @@ cmCTestGenericHandler* cmCTestSubmitCommand::InitializeHandler() handler->SetHttpHeaders(this->HttpHeaders); } - handler->SetOption("RetryDelay", this->RetryDelay.c_str()); - handler->SetOption("RetryCount", this->RetryCount.c_str()); + handler->SetOption("RetryDelay", this->RetryDelay); + handler->SetOption("RetryCount", this->RetryCount); handler->SetOption("InternalTest", this->InternalTest ? "ON" : "OFF"); handler->SetQuiet(this->Quiet); if (this->CDashUpload) { - handler->SetOption("CDashUploadFile", this->CDashUploadFile.c_str()); - handler->SetOption("CDashUploadType", this->CDashUploadType.c_str()); + handler->SetOption("CDashUploadFile", this->CDashUploadFile); + handler->SetOption("CDashUploadType", this->CDashUploadType); } return handler; } diff --git a/Source/CTest/cmCTestSubmitHandler.cxx b/Source/CTest/cmCTestSubmitHandler.cxx index 5b54573..9322bd5 100644 --- a/Source/CTest/cmCTestSubmitHandler.cxx +++ b/Source/CTest/cmCTestSubmitHandler.cxx @@ -357,12 +357,8 @@ bool cmCTestSubmitHandler::SubmitUsingHTTP( // If curl failed for any reason, or checksum fails, wait and retry // if (res != CURLE_OK || this->HasErrors) { - std::string retryDelay = this->GetOption("RetryDelay") == nullptr - ? "" - : this->GetOption("RetryDelay"); - std::string retryCount = this->GetOption("RetryCount") == nullptr - ? "" - : this->GetOption("RetryCount"); + std::string retryDelay = *this->GetOption("RetryDelay"); + std::string retryCount = *this->GetOption("RetryCount"); auto delay = cmDuration( retryDelay.empty() @@ -522,12 +518,8 @@ int cmCTestSubmitHandler::HandleCDashUploadFile(std::string const& file, bool internalTest = cmIsOn(this->GetOption("InternalTest")); // Get RETRY_COUNT and RETRY_DELAY values if they were set. - std::string retryDelayString = this->GetOption("RetryDelay") == nullptr - ? "" - : this->GetOption("RetryDelay"); - std::string retryCountString = this->GetOption("RetryCount") == nullptr - ? "" - : this->GetOption("RetryCount"); + std::string retryDelayString = *this->GetOption("RetryDelay"); + std::string retryCountString = *this->GetOption("RetryCount"); auto retryDelay = std::chrono::seconds(0); if (!retryDelayString.empty()) { unsigned long retryDelayValue = 0; @@ -716,8 +708,8 @@ int cmCTestSubmitHandler::HandleCDashUploadFile(std::string const& file, int cmCTestSubmitHandler::ProcessHandler() { - const char* cdashUploadFile = this->GetOption("CDashUploadFile"); - const char* cdashUploadType = this->GetOption("CDashUploadType"); + cmProp cdashUploadFile = this->GetOption("CDashUploadFile"); + cmProp cdashUploadType = this->GetOption("CDashUploadType"); if (cdashUploadFile && cdashUploadType) { return this->HandleCDashUploadFile(cdashUploadFile, cdashUploadType); } @@ -804,6 +796,7 @@ int cmCTestSubmitHandler::ProcessHandler() } } this->CTest->AddIfExists(cmCTest::PartMemCheck, "DynamicAnalysis.xml"); + this->CTest->AddIfExists(cmCTest::PartMemCheck, "DynamicAnalysis-Test.xml"); this->CTest->AddIfExists(cmCTest::PartMemCheck, "Purify.xml"); this->CTest->AddIfExists(cmCTest::PartNotes, "Notes.xml"); this->CTest->AddIfExists(cmCTest::PartUpload, "Upload.xml"); diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index 4403733..80b8d4f 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -9,7 +9,6 @@ #include <cmext/string_view> #include "cmCTest.h" -#include "cmCTestGenericHandler.h" #include "cmCTestTestHandler.h" #include "cmDuration.h" #include "cmMakefile.h" @@ -36,6 +35,7 @@ void cmCTestTestCommand::BindArguments() this->Bind("TEST_LOAD"_s, this->TestLoad); this->Bind("RESOURCE_SPEC_FILE"_s, this->ResourceSpecFile); this->Bind("STOP_ON_FAILURE"_s, this->StopOnFailure); + this->Bind("OUTPUT_JUNIT"_s, this->OutputJUnit); } cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() @@ -60,51 +60,51 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() this->ResourceSpecFile = *resourceSpecFile; } - cmCTestGenericHandler* handler = this->InitializeActualHandler(); + cmCTestTestHandler* handler = this->InitializeActualHandler(); if (!this->Start.empty() || !this->End.empty() || !this->Stride.empty()) { handler->SetOption( "TestsToRunInformation", - cmStrCat(this->Start, ',', this->End, ',', this->Stride).c_str()); + cmStrCat(this->Start, ',', this->End, ',', this->Stride)); } if (!this->Exclude.empty()) { - handler->SetOption("ExcludeRegularExpression", this->Exclude.c_str()); + handler->SetOption("ExcludeRegularExpression", this->Exclude); } if (!this->Include.empty()) { - handler->SetOption("IncludeRegularExpression", this->Include.c_str()); + handler->SetOption("IncludeRegularExpression", this->Include); } if (!this->ExcludeLabel.empty()) { - handler->SetOption("ExcludeLabelRegularExpression", - this->ExcludeLabel.c_str()); + handler->AddMultiOption("ExcludeLabelRegularExpression", + this->ExcludeLabel); } if (!this->IncludeLabel.empty()) { - handler->SetOption("LabelRegularExpression", this->IncludeLabel.c_str()); + handler->AddMultiOption("LabelRegularExpression", this->IncludeLabel); } if (!this->ExcludeFixture.empty()) { handler->SetOption("ExcludeFixtureRegularExpression", - this->ExcludeFixture.c_str()); + this->ExcludeFixture); } if (!this->ExcludeFixtureSetup.empty()) { handler->SetOption("ExcludeFixtureSetupRegularExpression", - this->ExcludeFixtureSetup.c_str()); + this->ExcludeFixtureSetup); } if (!this->ExcludeFixtureCleanup.empty()) { handler->SetOption("ExcludeFixtureCleanupRegularExpression", - this->ExcludeFixtureCleanup.c_str()); + this->ExcludeFixtureCleanup); } if (this->StopOnFailure) { handler->SetOption("StopOnFailure", "ON"); } if (!this->ParallelLevel.empty()) { - handler->SetOption("ParallelLevel", this->ParallelLevel.c_str()); + handler->SetOption("ParallelLevel", this->ParallelLevel); } if (!this->Repeat.empty()) { - handler->SetOption("Repeat", this->Repeat.c_str()); + handler->SetOption("Repeat", this->Repeat); } if (!this->ScheduleRandom.empty()) { - handler->SetOption("ScheduleRandom", this->ScheduleRandom.c_str()); + handler->SetOption("ScheduleRandom", this->ScheduleRandom); } if (!this->ResourceSpecFile.empty()) { - handler->SetOption("ResourceSpecFile", this->ResourceSpecFile.c_str()); + handler->SetOption("ResourceSpecFile", this->ResourceSpecFile); } if (!this->StopTime.empty()) { this->CTest->SetStopTime(this->StopTime); @@ -140,11 +140,15 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() *labelsForSubprojects, this->Quiet); } + if (!this->OutputJUnit.empty()) { + handler->SetJUnitXMLFileName(this->OutputJUnit); + } + handler->SetQuiet(this->Quiet); return handler; } -cmCTestGenericHandler* cmCTestTestCommand::InitializeActualHandler() +cmCTestTestHandler* cmCTestTestCommand::InitializeActualHandler() { cmCTestTestHandler* handler = this->CTest->GetTestHandler(); handler->Initialize(); diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index 624cd91..24e74e2 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -13,6 +13,7 @@ #include "cmCommand.h" class cmCTestGenericHandler; +class cmCTestTestHandler; /** \class cmCTestTest * \brief Run a ctest script @@ -40,7 +41,7 @@ public: protected: void BindArguments() override; - virtual cmCTestGenericHandler* InitializeActualHandler(); + virtual cmCTestTestHandler* InitializeActualHandler(); cmCTestGenericHandler* InitializeHandler() override; std::string Start; @@ -59,5 +60,6 @@ protected: std::string StopTime; std::string TestLoad; std::string ResourceSpecFile; + std::string OutputJUnit; bool StopOnFailure = false; }; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 1cb5d00..c41b661 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -32,6 +32,7 @@ #include "cmCTest.h" #include "cmCTestMultiProcessHandler.h" #include "cmCTestResourceGroupsLexerHelper.h" +#include "cmCTestTestMeasurementXMLParser.h" #include "cmDuration.h" #include "cmExecutionStatus.h" #include "cmGeneratedFileStream.h" @@ -42,6 +43,7 @@ #include "cmStateSnapshot.h" #include "cmStringAlgorithms.h" #include "cmSystemTools.h" +#include "cmTimestamp.h" #include "cmWorkingDirectory.h" #include "cmXMLWriter.h" #include "cmake.h" @@ -287,8 +289,6 @@ cmCTestTestHandler::cmCTestTestHandler() { this->UseUnion = false; - this->UseIncludeLabelRegExpFlag = false; - this->UseExcludeLabelRegExpFlag = false; this->UseIncludeRegExpFlag = false; this->UseExcludeRegExpFlag = false; this->UseExcludeRegExpFirst = false; @@ -301,11 +301,27 @@ cmCTestTestHandler::cmCTestTestHandler() this->LogFile = nullptr; - // regex to detect <DartMeasurement>...</DartMeasurement> - this->DartStuff.compile("(<DartMeasurement.*/DartMeasurement[a-zA-Z]*>)"); - // regex to detect each individual <DartMeasurement>...</DartMeasurement> - this->DartStuff1.compile( - "(<DartMeasurement[^<]*</DartMeasurement[a-zA-Z]*>)"); + // Support for JUnit XML output. + this->JUnitXMLFileName = ""; + + // Regular expressions to scan test output for custom measurements. + + // Capture the whole section of test output from the first opening + // <(CTest|Dart)Measurement*> tag to the last </(CTest|Dart)Measurement*> + // closing tag. + this->AllTestMeasurementsRegex.compile( + "(<(CTest|Dart)Measurement.*/(CTest|Dart)Measurement[a-zA-Z]*>)"); + + // Capture a single <(CTest|Dart)Measurement*> XML element. + this->SingleTestMeasurementRegex.compile( + "(<(CTest|Dart)Measurement[^<]*</(CTest|Dart)Measurement[a-zA-Z]*>)"); + + // Capture content from <CTestDetails>...</CTestDetails> + this->CustomCompletionStatusRegex.compile( + "<CTestDetails>(.*)</CTestDetails>"); + + // Capture content from <CTestLabel>...</CTestLabel> + this->CustomLabelRegex.compile("<CTestLabel>(.*)</CTestLabel>"); } void cmCTestTestHandler::Initialize() @@ -327,13 +343,11 @@ void cmCTestTestHandler::Initialize() this->TestsToRun.clear(); - this->UseIncludeLabelRegExpFlag = false; - this->UseExcludeLabelRegExpFlag = false; this->UseIncludeRegExpFlag = false; this->UseExcludeRegExpFlag = false; this->UseExcludeRegExpFirst = false; - this->IncludeLabelRegularExpression = ""; - this->ExcludeLabelRegularExpression = ""; + this->IncludeLabelRegularExpressions.clear(); + this->ExcludeLabelRegularExpressions.clear(); this->IncludeRegExp.clear(); this->ExcludeRegExp.clear(); this->ExcludeFixtureRegExp.clear(); @@ -460,6 +474,10 @@ int cmCTestTestHandler::ProcessHandler() return 1; } + if (!this->WriteJUnitXML()) { + return 1; + } + if (!this->PostProcessHandler()) { this->LogFile = nullptr; return -1; @@ -479,6 +497,22 @@ int cmCTestTestHandler::ProcessHandler() return 0; } +/* Given a multi-option value `parts`, compile those parts into + * regular expressions in `expressions`. Skip empty values. + * Returns true if there were any expressions. + */ +static bool BuildLabelRE(const std::vector<std::string>& parts, + std::vector<cmsys::RegularExpression>& expressions) +{ + expressions.clear(); + for (const auto& p : parts) { + if (!p.empty()) { + expressions.emplace_back(p); + } + } + return !expressions.empty(); +} + bool cmCTestTestHandler::ProcessOptions() { // Update internal data structure from generic one @@ -487,7 +521,7 @@ bool cmCTestTestHandler::ProcessOptions() if (cmIsOn(this->GetOption("ScheduleRandom"))) { this->CTest->SetScheduleType("Random"); } - if (const char* repeat = this->GetOption("Repeat")) { + if (cmProp repeat = this->GetOption("Repeat")) { cmsys::RegularExpression repeatRegex( "^(UNTIL_FAIL|UNTIL_PASS|AFTER_TIMEOUT):([0-9]+)$"); if (repeatRegex.find(repeat)) { @@ -512,25 +546,18 @@ bool cmCTestTestHandler::ProcessOptions() } } if (this->GetOption("ParallelLevel")) { - this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel"))); + this->CTest->SetParallelLevel(std::stoi(this->GetOption("ParallelLevel"))); } if (this->GetOption("StopOnFailure")) { this->CTest->SetStopOnFailure(true); } - const char* val; - val = this->GetOption("LabelRegularExpression"); - if (val) { - this->UseIncludeLabelRegExpFlag = true; - this->IncludeLabelRegExp = val; - } - val = this->GetOption("ExcludeLabelRegularExpression"); - if (val) { - this->UseExcludeLabelRegExpFlag = true; - this->ExcludeLabelRegExp = val; - } - val = this->GetOption("IncludeRegularExpression"); + BuildLabelRE(this->GetMultiOption("LabelRegularExpression"), + this->IncludeLabelRegularExpressions); + BuildLabelRE(this->GetMultiOption("ExcludeLabelRegularExpression"), + this->ExcludeLabelRegularExpressions); + cmProp val = this->GetOption("IncludeRegularExpression"); if (val) { this->UseIncludeRegExp(); this->SetIncludeRegExp(val); @@ -542,19 +569,19 @@ bool cmCTestTestHandler::ProcessOptions() } val = this->GetOption("ExcludeFixtureRegularExpression"); if (val) { - this->ExcludeFixtureRegExp = val; + this->ExcludeFixtureRegExp = *val; } val = this->GetOption("ExcludeFixtureSetupRegularExpression"); if (val) { - this->ExcludeFixtureSetupRegExp = val; + this->ExcludeFixtureSetupRegExp = *val; } val = this->GetOption("ExcludeFixtureCleanupRegularExpression"); if (val) { - this->ExcludeFixtureCleanupRegExp = val; + this->ExcludeFixtureCleanupRegExp = *val; } val = this->GetOption("ResourceSpecFile"); if (val) { - this->ResourceSpecFile = val; + this->ResourceSpecFile = *val; } this->SetRerunFailed(cmIsOn(this->GetOption("RerunFailed"))); @@ -675,7 +702,22 @@ bool cmCTestTestHandler::GenerateXML() return false; } cmXMLWriter xml(xmlfile); - this->GenerateDartOutput(xml); + this->GenerateCTestXML(xml); + } + + if (this->MemCheck) { + cmGeneratedFileStream xmlfile; + if (!this->StartResultingXML(cmCTest::PartTest, "DynamicAnalysis-Test", + xmlfile)) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Cannot create testing XML file" << std::endl); + this->LogFile = nullptr; + return false; + } + cmXMLWriter xml(xmlfile); + // Explicitly call this class' `GenerateCTestXML` method to make `Test.xml` + // as well. + this->cmCTestTestHandler::GenerateCTestXML(xml); } return true; @@ -763,10 +805,40 @@ void cmCTestTestHandler::PrintLabelOrSubprojectSummary(bool doSubProject) cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, "\n", this->Quiet); } +/** + * Check if the labels (from a test) match all the expressions. + * + * Each of the RE's must match at least one label + * (e.g. all of the REs must match **some** label, + * in order for the filter to apply to the test). + */ +static bool MatchLabelsAgainstFilterRE( + const std::vector<std::string>& labels, + const std::vector<cmsys::RegularExpression>& expressions) +{ + for (const auto& re : expressions) { + // check to see if the label regular expression matches + bool found = false; // assume it does not match + cmsys::RegularExpressionMatch match; + // loop over all labels and look for match + for (std::string const& l : labels) { + if (re.find(l.c_str(), match)) { + found = true; + break; + } + } + // if no match was found, exclude the test + if (!found) { + return false; + } + } + return true; +} + void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it) { // if not using Labels to filter then return - if (!this->UseIncludeLabelRegExpFlag) { + if (this->IncludeLabelRegularExpressions.empty()) { return; } // if there are no labels and we are filtering by labels @@ -775,16 +847,9 @@ void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it) it.IsInBasedOnREOptions = false; return; } - // check to see if the label regular expression matches - bool found = false; // assume it does not match - // loop over all labels and look for match - for (std::string const& l : it.Labels) { - if (this->IncludeLabelRegularExpression.find(l)) { - found = true; - } - } // if no match was found, exclude the test - if (!found) { + if (!MatchLabelsAgainstFilterRE(it.Labels, + this->IncludeLabelRegularExpressions)) { it.IsInBasedOnREOptions = false; } } @@ -792,7 +857,7 @@ void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it) void cmCTestTestHandler::CheckLabelFilterExclude(cmCTestTestProperties& it) { // if not using Labels to filter then return - if (!this->UseExcludeLabelRegExpFlag) { + if (this->ExcludeLabelRegularExpressions.empty()) { return; } // if there are no labels and we are excluding by labels @@ -800,16 +865,9 @@ void cmCTestTestHandler::CheckLabelFilterExclude(cmCTestTestProperties& it) if (it.Labels.empty()) { return; } - // check to see if the label regular expression matches - bool found = false; // assume it does not match - // loop over all labels and look for match - for (std::string const& l : it.Labels) { - if (this->ExcludeLabelRegularExpression.find(l)) { - found = true; - } - } // if match was found, exclude the test - if (found) { + if (MatchLabelsAgainstFilterRE(it.Labels, + this->ExcludeLabelRegularExpressions)) { it.IsInBasedOnREOptions = false; } } @@ -1367,7 +1425,7 @@ void cmCTestTestHandler::GenerateTestCommand( { } -void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml) +void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml) { if (!this->CTest->GetProduceXML()) { return; @@ -1403,7 +1461,7 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml) xml.Element("Value", result.ReturnValue); xml.EndElement(); // NamedMeasurement } - this->GenerateRegressionImages(xml, result.DartString); + this->RecordCustomTestMeasurements(xml, result.TestMeasurementsOutput); xml.StartElement("NamedMeasurement"); xml.Attribute("type", "numeric/double"); xml.Attribute("name", "Execution Time"); @@ -1431,7 +1489,11 @@ void cmCTestTestHandler::GenerateDartOutput(cmXMLWriter& xml) xml.StartElement("NamedMeasurement"); xml.Attribute("type", "text/string"); xml.Attribute("name", "Completion Status"); - xml.Element("Value", result.CompletionStatus); + if (result.CustomCompletionStatus.empty()) { + xml.Element("Value", result.CompletionStatus); + } else { + xml.Element("Value", result.CustomCompletionStatus); + } xml.EndElement(); // NamedMeasurement xml.StartElement("NamedMeasurement"); @@ -1521,19 +1583,29 @@ void cmCTestTestHandler::AttachFiles(cmXMLWriter& xml, result.Properties->AttachOnFail.end()); } for (std::string const& file : result.Properties->AttachedFiles) { - const std::string& base64 = this->CTest->Base64GzipEncodeFile(file); - std::string const fname = cmSystemTools::GetFilenameName(file); - xml.StartElement("NamedMeasurement"); - xml.Attribute("name", "Attached File"); - xml.Attribute("encoding", "base64"); - xml.Attribute("compression", "tar/gzip"); - xml.Attribute("filename", fname); - xml.Attribute("type", "file"); - xml.Element("Value", base64); - xml.EndElement(); // NamedMeasurement + this->AttachFile(xml, file, ""); } } +void cmCTestTestHandler::AttachFile(cmXMLWriter& xml, std::string const& file, + std::string const& name) +{ + const std::string& base64 = this->CTest->Base64GzipEncodeFile(file); + std::string const fname = cmSystemTools::GetFilenameName(file); + xml.StartElement("NamedMeasurement"); + std::string measurement_name = name; + if (measurement_name.empty()) { + measurement_name = "Attached File"; + } + xml.Attribute("name", measurement_name); + xml.Attribute("encoding", "base64"); + xml.Attribute("compression", "tar/gzip"); + xml.Attribute("filename", fname); + xml.Attribute("type", "file"); + xml.Element("Value", base64); + xml.EndElement(); // NamedMeasurement +} + int cmCTestTestHandler::ExecuteCommands(std::vector<std::string>& vec) { for (std::string const& it : vec) { @@ -1704,19 +1776,11 @@ bool cmCTestTestHandler::ParseResourceGroupsProperty( bool cmCTestTestHandler::GetListOfTests() { - if (!this->IncludeLabelRegExp.empty()) { - this->IncludeLabelRegularExpression.compile( - this->IncludeLabelRegExp.c_str()); - } - if (!this->ExcludeLabelRegExp.empty()) { - this->ExcludeLabelRegularExpression.compile( - this->ExcludeLabelRegExp.c_str()); - } if (!this->IncludeRegExp.empty()) { - this->IncludeTestsRegularExpression.compile(this->IncludeRegExp.c_str()); + this->IncludeTestsRegularExpression.compile(this->IncludeRegExp); } if (!this->ExcludeRegExp.empty()) { - this->ExcludeTestsRegularExpression.compile(this->ExcludeRegExp.c_str()); + this->ExcludeTestsRegularExpression.compile(this->ExcludeRegExp); } cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Constructing a list of tests" << std::endl, this->Quiet); @@ -1868,7 +1932,7 @@ void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed() std::string dirName = this->CTest->GetBinaryDir() + "/Testing/Temporary"; cmsys::Directory directory; - if (directory.Load(dirName) == 0) { + if (!directory.Load(dirName)) { cmCTestLog(this->CTest, ERROR_MESSAGE, "Unable to read the contents of " << dirName << std::endl); return; @@ -1937,180 +2001,106 @@ void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed() } } -// Just for convenience -#define SPACE_REGEX "[ \t\r\n]" -void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml, - const std::string& dart) -{ - cmsys::RegularExpression twoattributes( - "<DartMeasurement" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*>([^<]*)</DartMeasurement>"); - cmsys::RegularExpression threeattributes( - "<DartMeasurement" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*>([^<]*)</DartMeasurement>"); - cmsys::RegularExpression fourattributes( - "<DartMeasurement" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*>([^<]*)</DartMeasurement>"); - cmsys::RegularExpression cdatastart( - "<DartMeasurement" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*>" SPACE_REGEX "*<!\\[CDATA\\["); - cmsys::RegularExpression cdataend("]]>" SPACE_REGEX "*</DartMeasurement>"); - cmsys::RegularExpression measurementfile( - "<DartMeasurementFile" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*(name|type|encoding|compression)=\"([^\"]*)\"" SPACE_REGEX - "*>([^<]*)</DartMeasurementFile>"); - - bool done = false; - std::string cxml = dart; - while (!done) { - if (twoattributes.find(cxml)) { - xml.StartElement("NamedMeasurement"); - xml.Attribute(twoattributes.match(1).c_str(), twoattributes.match(2)); - xml.Attribute(twoattributes.match(3).c_str(), twoattributes.match(4)); - xml.Element("Value", twoattributes.match(5)); - xml.EndElement(); - cxml.erase(twoattributes.start(), - twoattributes.end() - twoattributes.start()); - } else if (threeattributes.find(cxml)) { - xml.StartElement("NamedMeasurement"); - xml.Attribute(threeattributes.match(1).c_str(), - threeattributes.match(2)); - xml.Attribute(threeattributes.match(3).c_str(), - threeattributes.match(4)); - xml.Attribute(threeattributes.match(5).c_str(), - threeattributes.match(6)); - xml.Element("Value", twoattributes.match(7)); - xml.EndElement(); - cxml.erase(threeattributes.start(), - threeattributes.end() - threeattributes.start()); - } else if (fourattributes.find(cxml)) { +void cmCTestTestHandler::RecordCustomTestMeasurements(cmXMLWriter& xml, + std::string content) +{ + while (this->SingleTestMeasurementRegex.find(content)) { + // Extract regex match from content and parse it as an XML element. + auto measurement_str = this->SingleTestMeasurementRegex.match(1); + auto parser = cmCTestTestMeasurementXMLParser(); + parser.Parse(measurement_str.c_str()); + + if (parser.ElementName == "CTestMeasurement" || + parser.ElementName == "DartMeasurement") { xml.StartElement("NamedMeasurement"); - xml.Attribute(fourattributes.match(1).c_str(), fourattributes.match(2)); - xml.Attribute(fourattributes.match(3).c_str(), fourattributes.match(4)); - xml.Attribute(fourattributes.match(5).c_str(), fourattributes.match(6)); - xml.Attribute(fourattributes.match(7).c_str(), fourattributes.match(8)); - xml.Element("Value", twoattributes.match(9)); + xml.Attribute("type", parser.MeasurementType); + xml.Attribute("name", parser.MeasurementName); + xml.Element("Value", parser.CharacterData); xml.EndElement(); - cxml.erase(fourattributes.start(), - fourattributes.end() - fourattributes.start()); - } else if (cdatastart.find(cxml) && cdataend.find(cxml)) { - xml.StartElement("NamedMeasurement"); - xml.Attribute(cdatastart.match(1).c_str(), cdatastart.match(2)); - xml.Attribute(cdatastart.match(3).c_str(), cdatastart.match(4)); - xml.StartElement("Value"); - xml.CData( - cxml.substr(cdatastart.end(), cdataend.start() - cdatastart.end())); - xml.EndElement(); // Value - xml.EndElement(); // NamedMeasurement - cxml.erase(cdatastart.start(), cdataend.end() - cdatastart.start()); - } else if (measurementfile.find(cxml)) { - const std::string& filename = - cmCTest::CleanString(measurementfile.match(5)); - if (cmSystemTools::FileExists(filename)) { + } else if (parser.ElementName == "CTestMeasurementFile" || + parser.ElementName == "DartMeasurementFile") { + const std::string& filename = cmCTest::CleanString(parser.CharacterData); + if (!cmSystemTools::FileExists(filename)) { + xml.StartElement("NamedMeasurement"); + xml.Attribute("name", parser.MeasurementName); + xml.Attribute("text", "text/string"); + xml.Element("Value", "File " + filename + " not found"); + xml.EndElement(); + cmCTestOptionalLog( + this->CTest, HANDLER_OUTPUT, + "File \"" << filename << "\" not found." << std::endl, this->Quiet); + } else { long len = cmSystemTools::FileLength(filename); if (len == 0) { - std::string k1 = measurementfile.match(1); - std::string v1 = measurementfile.match(2); - std::string k2 = measurementfile.match(3); - std::string v2 = measurementfile.match(4); - if (cmSystemTools::LowerCase(k1) == "type") { - v1 = "text/string"; - } - if (cmSystemTools::LowerCase(k2) == "type") { - v2 = "text/string"; - } - xml.StartElement("NamedMeasurement"); - xml.Attribute(k1.c_str(), v1); - xml.Attribute(k2.c_str(), v2); + xml.Attribute("name", parser.MeasurementName); + xml.Attribute("type", "text/string"); xml.Attribute("encoding", "none"); xml.Element("Value", "Image " + filename + " is empty"); xml.EndElement(); } else { - cmsys::ifstream ifs(filename.c_str(), - std::ios::in + if (parser.MeasurementType == "file") { + // Treat this measurement like an "ATTACHED_FILE" when the type + // is explicitly "file" (not an image). + this->AttachFile(xml, filename, parser.MeasurementName); + } else { + cmsys::ifstream ifs(filename.c_str(), + std::ios::in #ifdef _WIN32 - | std::ios::binary + | std::ios::binary #endif - ); - auto file_buffer = cm::make_unique<unsigned char[]>(len + 1); - ifs.read(reinterpret_cast<char*>(file_buffer.get()), len); - auto encoded_buffer = cm::make_unique<unsigned char[]>( - static_cast<int>(static_cast<double>(len) * 1.5 + 5.0)); - - size_t rlen = cmsysBase64_Encode(file_buffer.get(), len, - encoded_buffer.get(), 1); - - xml.StartElement("NamedMeasurement"); - xml.Attribute(measurementfile.match(1).c_str(), - measurementfile.match(2)); - xml.Attribute(measurementfile.match(3).c_str(), - measurementfile.match(4)); - xml.Attribute("encoding", "base64"); - std::ostringstream ostr; - for (size_t cc = 0; cc < rlen; cc++) { - ostr << encoded_buffer[cc]; - if (cc % 60 == 0 && cc) { - ostr << std::endl; + ); + auto file_buffer = cm::make_unique<unsigned char[]>(len + 1); + ifs.read(reinterpret_cast<char*>(file_buffer.get()), len); + auto encoded_buffer = cm::make_unique<unsigned char[]>( + static_cast<int>(static_cast<double>(len) * 1.5 + 5.0)); + + size_t rlen = cmsysBase64_Encode(file_buffer.get(), len, + encoded_buffer.get(), 1); + + xml.StartElement("NamedMeasurement"); + xml.Attribute("name", parser.MeasurementName); + xml.Attribute("type", parser.MeasurementType); + xml.Attribute("encoding", "base64"); + std::ostringstream ostr; + for (size_t cc = 0; cc < rlen; cc++) { + ostr << encoded_buffer[cc]; + if (cc % 60 == 0 && cc) { + ostr << std::endl; + } } + xml.Element("Value", ostr.str()); + xml.EndElement(); // NamedMeasurement } - xml.Element("Value", ostr.str()); - xml.EndElement(); // NamedMeasurement - } - } else { - int idx = 4; - if (measurementfile.match(1) == "name") { - idx = 2; } - xml.StartElement("NamedMeasurement"); - xml.Attribute("name", measurementfile.match(idx)); - xml.Attribute("text", "text/string"); - xml.Element("Value", "File " + filename + " not found"); - xml.EndElement(); - cmCTestOptionalLog( - this->CTest, HANDLER_OUTPUT, - "File \"" << filename << "\" not found." << std::endl, this->Quiet); } - cxml.erase(measurementfile.start(), - measurementfile.end() - measurementfile.start()); - } else { - done = true; } + + // Remove this element from content. + cmSystemTools::ReplaceString(content, measurement_str.c_str(), ""); } } -void cmCTestTestHandler::SetIncludeRegExp(const char* arg) +void cmCTestTestHandler::SetIncludeRegExp(const std::string& arg) { this->IncludeRegExp = arg; } -void cmCTestTestHandler::SetExcludeRegExp(const char* arg) +void cmCTestTestHandler::SetExcludeRegExp(const std::string& arg) { this->ExcludeRegExp = arg; } -void cmCTestTestHandler::SetTestsToRunInformation(const char* in) +void cmCTestTestHandler::SetTestsToRunInformation(cmProp in) { if (!in) { return; } - this->TestsToRunString = in; + this->TestsToRunString = *in; // if the argument is a file, then read it and use the contents as the // string if (cmSystemTools::FileExists(in)) { - cmsys::ifstream fin(in); + cmsys::ifstream fin(in->c_str()); unsigned long filelen = cmSystemTools::FileLength(in); auto buff = cm::make_unique<char[]>(filelen + 1); fin.getline(buff.get(), filelen); @@ -2190,7 +2180,7 @@ bool cmCTestTestHandler::SetTestsProperties( // Ensure we have complete triples otherwise the data is corrupt. if (triples.size() % 3 == 0) { - cmState state; + cmState state(cmState::Unknown); rt.Backtrace = cmListFileBacktrace(state.CreateBaseSnapshot()); // the first entry represents the top of the trace so we need to @@ -2270,6 +2260,8 @@ bool cmCTestTestHandler::SetTestsProperties( cmExpandList(val, rt.Depends); } else if (key == "ENVIRONMENT"_s) { cmExpandList(val, rt.Environment); + } else if (key == "ENVIRONMENT_MODIFICATION"_s) { + cmExpandList(val, rt.EnvironmentModification); } else if (key == "LABELS"_s) { std::vector<std::string> Labels = cmExpandedList(val); rt.Labels.insert(rt.Labels.end(), Labels.begin(), Labels.end()); @@ -2444,3 +2436,125 @@ bool cmCTestTestHandler::cmCTestTestResourceRequirement::operator!=( { return !(*this == other); } + +void cmCTestTestHandler::SetJUnitXMLFileName(const std::string& filename) +{ + this->JUnitXMLFileName = filename; +} + +bool cmCTestTestHandler::WriteJUnitXML() +{ + if (this->JUnitXMLFileName.empty()) { + return true; + } + + // Open new XML file for writing. + cmGeneratedFileStream xmlfile; + xmlfile.SetTempExt("tmp"); + xmlfile.Open(this->JUnitXMLFileName); + if (!xmlfile) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Problem opening file: " << this->JUnitXMLFileName + << std::endl); + return false; + } + cmXMLWriter xml(xmlfile); + + // Iterate over the test results to get the number of tests that + // passed, failed, etc. + auto num_tests = 0; + auto num_passed = 0; + auto num_failed = 0; + auto num_notrun = 0; + auto num_disabled = 0; + SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end()); + for (cmCTestTestResult const& result : resultsSet) { + num_tests++; + if (result.Status == cmCTestTestHandler::COMPLETED) { + num_passed++; + } else if (result.Status == cmCTestTestHandler::NOT_RUN) { + if (result.CompletionStatus == "Disabled") { + num_disabled++; + } else { + num_notrun++; + } + } else { + num_failed++; + } + } + + // Write <testsuite> element. + xml.StartDocument(); + xml.StartElement("testsuite"); + + xml.Attribute("name", + cmCTest::SafeBuildIdField( + this->CTest->GetCTestConfiguration("BuildName"))); + xml.BreakAttributes(); + + xml.Attribute("tests", num_tests); + xml.Attribute("failures", num_failed); + + // CTest disabled => JUnit disabled + xml.Attribute("disabled", num_disabled); + + // Otherwise, CTest notrun => JUnit skipped. + // The distinction between JUnit disabled vs. skipped is that + // skipped tests can have a message associated with them + // (why the test was skipped). + xml.Attribute("skipped", num_notrun); + + xml.Attribute("hostname", this->CTest->GetCTestConfiguration("Site")); + xml.Attribute( + "time", + std::chrono::duration_cast<std::chrono::seconds>(this->ElapsedTestingTime) + .count()); + const std::time_t start_test_time_t = + std::chrono::system_clock::to_time_t(this->StartTestTime); + cmTimestamp cmts; + xml.Attribute("timestamp", + cmts.CreateTimestampFromTimeT(start_test_time_t, + "%Y-%m-%dT%H:%M:%S", false)); + + // Write <testcase> elements. + for (cmCTestTestResult const& result : resultsSet) { + xml.StartElement("testcase"); + xml.Attribute("name", result.Name); + xml.Attribute("classname", result.Name); + xml.Attribute("time", result.ExecutionTime.count()); + + std::string status; + if (result.Status == cmCTestTestHandler::COMPLETED) { + status = "run"; + } else if (result.Status == cmCTestTestHandler::NOT_RUN) { + if (result.CompletionStatus == "Disabled") { + status = "disabled"; + } else { + status = "notrun"; + } + } else { + status = "fail"; + } + xml.Attribute("status", status); + + if (status == "notrun") { + xml.StartElement("skipped"); + xml.Attribute("message", result.CompletionStatus); + xml.EndElement(); // </skipped> + } else if (status == "fail") { + xml.StartElement("failure"); + xml.Attribute("message", result.Reason); + xml.EndElement(); // </failure> + } + + // Note: compressed test output is unconditionally disabled when + // --output-junit is specified. + xml.Element("system-out", result.Output); + xml.EndElement(); // </testcase> + } + + xml.EndElement(); // </testsuite> + xml.EndDocument(); + + return true; +} diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index aa29eeb..585a336 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -22,6 +22,7 @@ #include "cmCTestResourceSpec.h" #include "cmDuration.h" #include "cmListFileCache.h" +#include "cmProperty.h" class cmMakefile; class cmXMLWriter; @@ -65,8 +66,8 @@ public: /// them on void UseIncludeRegExp(); void UseExcludeRegExp(); - void SetIncludeRegExp(const char*); - void SetExcludeRegExp(const char*); + void SetIncludeRegExp(const std::string&); + void SetExcludeRegExp(const std::string&); void SetMaxIndex(int n) { this->MaxIndex = n; } int GetMaxIndex() { return this->MaxIndex; } @@ -81,7 +82,7 @@ public: } //! pass the -I argument down - void SetTestsToRunInformation(const char*); + void SetTestsToRunInformation(cmProp); cmCTestTestHandler(); @@ -151,6 +152,7 @@ public: // return code of test which will mark test as "not run" int SkipReturnCode; std::vector<std::string> Environment; + std::vector<std::string> EnvironmentModification; std::vector<std::string> Labels; std::set<std::string> LockedResources; std::set<std::string> FixturesSetup; @@ -175,8 +177,9 @@ public: std::string ExceptionStatus; bool CompressOutput; std::string CompletionStatus; + std::string CustomCompletionStatus; std::string Output; - std::string DartString; + std::string TestMeasurementsOutput; int TestCount; cmCTestTestProperties* Properties; }; @@ -209,6 +212,9 @@ public: using ListOfTests = std::vector<cmCTestTestProperties>; + // Support for writing test results in JUnit XML format. + void SetJUnitXMLFileName(const std::string& id); + protected: using SetOfTests = std::set<cmCTestTestHandler::cmCTestTestResult, cmCTestTestResultLess>; @@ -234,6 +240,8 @@ protected: cmCTestTestResult const& result); // Write attached test files into the xml void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result); + void AttachFile(cmXMLWriter& xml, std::string const& file, + std::string const& name); //! Clean test output to specified length void CleanTestOutput(std::string& output, size_t length); @@ -270,9 +278,14 @@ public: private: /** - * Generate the Dart compatible output + * Write test results in CTest's Test.xml format */ - virtual void GenerateDartOutput(cmXMLWriter& xml); + virtual void GenerateCTestXML(cmXMLWriter& xml); + + /** + * Write test results in JUnit XML format + */ + bool WriteJUnitXML(); void PrintLabelOrSubprojectSummary(bool isSubProject); @@ -320,20 +333,16 @@ private: std::vector<int> TestsToRun; - bool UseIncludeLabelRegExpFlag; - bool UseExcludeLabelRegExpFlag; bool UseIncludeRegExpFlag; bool UseExcludeRegExpFlag; bool UseExcludeRegExpFirst; - std::string IncludeLabelRegExp; - std::string ExcludeLabelRegExp; std::string IncludeRegExp; std::string ExcludeRegExp; std::string ExcludeFixtureRegExp; std::string ExcludeFixtureSetupRegExp; std::string ExcludeFixtureCleanupRegExp; - cmsys::RegularExpression IncludeLabelRegularExpression; - cmsys::RegularExpression ExcludeLabelRegularExpression; + std::vector<cmsys::RegularExpression> IncludeLabelRegularExpressions; + std::vector<cmsys::RegularExpression> ExcludeLabelRegularExpressions; cmsys::RegularExpression IncludeTestsRegularExpression; cmsys::RegularExpression ExcludeTestsRegularExpression; @@ -341,8 +350,7 @@ private: cmCTestResourceSpec ResourceSpec; std::string ResourceSpecFile; - void GenerateRegressionImages(cmXMLWriter& xml, const std::string& dart); - cmsys::RegularExpression DartStuff1; + void RecordCustomTestMeasurements(cmXMLWriter& xml, std::string content); void CheckLabelFilter(cmCTestTestProperties& it); void CheckLabelFilterExclude(cmCTestTestProperties& it); void CheckLabelFilterInclude(cmCTestTestProperties& it); @@ -351,11 +359,16 @@ private: bool UseUnion; ListOfTests TestList; size_t TotalNumberOfTests; - cmsys::RegularExpression DartStuff; + cmsys::RegularExpression AllTestMeasurementsRegex; + cmsys::RegularExpression SingleTestMeasurementRegex; + cmsys::RegularExpression CustomCompletionStatusRegex; + cmsys::RegularExpression CustomLabelRegex; std::ostream* LogFile; cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never; int RepeatCount = 1; bool RerunFailed; + + std::string JUnitXMLFileName; }; diff --git a/Source/CTest/cmCTestTestMeasurementXMLParser.cxx b/Source/CTest/cmCTestTestMeasurementXMLParser.cxx new file mode 100644 index 0000000..636be24 --- /dev/null +++ b/Source/CTest/cmCTestTestMeasurementXMLParser.cxx @@ -0,0 +1,26 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ + +#include "cmCTestTestMeasurementXMLParser.h" + +#include <cstring> + +void cmCTestTestMeasurementXMLParser::StartElement(const std::string& name, + const char** attributes) +{ + this->CharacterData.clear(); + this->ElementName = name; + for (const char** attr = attributes; *attr; attr += 2) { + if (strcmp(attr[0], "name") == 0) { + this->MeasurementName = attr[1]; + } else if (strcmp(attr[0], "type") == 0) { + this->MeasurementType = attr[1]; + } + } +} + +void cmCTestTestMeasurementXMLParser::CharacterDataHandler(const char* data, + int length) +{ + this->CharacterData.append(data, length); +} diff --git a/Source/CTest/cmCTestTestMeasurementXMLParser.h b/Source/CTest/cmCTestTestMeasurementXMLParser.h new file mode 100644 index 0000000..b2c3eb3 --- /dev/null +++ b/Source/CTest/cmCTestTestMeasurementXMLParser.h @@ -0,0 +1,21 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ + +#include <string> + +#include "cmXMLParser.h" + +class cmCTestTestMeasurementXMLParser : public cmXMLParser +{ +public: + cmCTestTestMeasurementXMLParser() {} + std::string CharacterData; + std::string ElementName; + std::string MeasurementName; + std::string MeasurementType; + +protected: + void StartElement(const std::string& name, const char** atts) override; + void EndElement(const std::string& /*name*/) override {} + void CharacterDataHandler(const char* data, int length) override; +}; diff --git a/Source/CTest/cmCTestUpdateCommand.cxx b/Source/CTest/cmCTestUpdateCommand.cxx index 0ba2c41..6655bf7 100644 --- a/Source/CTest/cmCTestUpdateCommand.cxx +++ b/Source/CTest/cmCTestUpdateCommand.cxx @@ -75,12 +75,11 @@ cmCTestGenericHandler* cmCTestUpdateCommand::InitializeHandler() cmCTestUpdateHandler* handler = this->CTest->GetUpdateHandler(); handler->Initialize(); - handler->SetCommand(this); if (source_dir.empty()) { this->SetError("source directory not specified. Please use SOURCE tag"); return nullptr; } - handler->SetOption("SourceDirectory", source_dir.c_str()); + handler->SetOption("SourceDirectory", source_dir); handler->SetQuiet(this->Quiet); return handler; } diff --git a/Source/CTest/cmCTestUpdateHandler.cxx b/Source/CTest/cmCTestUpdateHandler.cxx index 57cc024..7c7c380 100644 --- a/Source/CTest/cmCTestUpdateHandler.cxx +++ b/Source/CTest/cmCTestUpdateHandler.cxx @@ -17,6 +17,7 @@ #include "cmCTestSVN.h" #include "cmCTestVC.h" #include "cmGeneratedFileStream.h" +#include "cmProperty.h" #include "cmStringAlgorithms.h" #include "cmSystemTools.h" #include "cmVersion.h" @@ -108,7 +109,7 @@ int cmCTestUpdateHandler::ProcessHandler() static_cast<void>(fixLocale); // Get source dir - const char* sourceDirectory = this->GetOption("SourceDirectory"); + cmProp sourceDirectory = this->GetOption("SourceDirectory"); if (!sourceDirectory) { cmCTestLog(this->CTest, ERROR_MESSAGE, "Cannot find SourceDirectory key in the DartConfiguration.tcl" @@ -257,7 +258,7 @@ int cmCTestUpdateHandler::ProcessHandler() return updated && loadedMods ? numUpdated : -1; } -int cmCTestUpdateHandler::DetectVCS(const char* dir) +int cmCTestUpdateHandler::DetectVCS(const std::string& dir) { std::string sourceDirectory = dir; cmCTestOptionalLog(this->CTest, DEBUG, diff --git a/Source/CTest/cmCTestUpdateHandler.h b/Source/CTest/cmCTestUpdateHandler.h index 25bbb2f..70269ea 100644 --- a/Source/CTest/cmCTestUpdateHandler.h +++ b/Source/CTest/cmCTestUpdateHandler.h @@ -59,6 +59,6 @@ private: std::string UpdateCommand; int UpdateType; - int DetectVCS(const char* dir); + int DetectVCS(const std::string& dir); bool SelectVCS(); }; |