diff options
Diffstat (limited to 'Source/CTest/cmCTestTestHandler.cxx')
| -rw-r--r-- | Source/CTest/cmCTestTestHandler.cxx | 241 |
1 files changed, 192 insertions, 49 deletions
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 1cb5d00..1596d4a 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -42,6 +42,7 @@ #include "cmStateSnapshot.h" #include "cmStringAlgorithms.h" #include "cmSystemTools.h" +#include "cmTimestamp.h" #include "cmWorkingDirectory.h" #include "cmXMLWriter.h" #include "cmake.h" @@ -287,8 +288,6 @@ cmCTestTestHandler::cmCTestTestHandler() { this->UseUnion = false; - this->UseIncludeLabelRegExpFlag = false; - this->UseExcludeLabelRegExpFlag = false; this->UseIncludeRegExpFlag = false; this->UseExcludeRegExpFlag = false; this->UseExcludeRegExpFirst = false; @@ -301,6 +300,9 @@ cmCTestTestHandler::cmCTestTestHandler() this->LogFile = nullptr; + // Support for JUnit XML output. + this->JUnitXMLFileName = ""; + // regex to detect <DartMeasurement>...</DartMeasurement> this->DartStuff.compile("(<DartMeasurement.*/DartMeasurement[a-zA-Z]*>)"); // regex to detect each individual <DartMeasurement>...</DartMeasurement> @@ -327,13 +329,11 @@ void cmCTestTestHandler::Initialize() this->TestsToRun.clear(); - this->UseIncludeLabelRegExpFlag = false; - this->UseExcludeLabelRegExpFlag = false; this->UseIncludeRegExpFlag = false; this->UseExcludeRegExpFlag = false; this->UseExcludeRegExpFirst = false; - this->IncludeLabelRegularExpression = ""; - this->ExcludeLabelRegularExpression = ""; + this->IncludeLabelRegularExpressions.clear(); + this->ExcludeLabelRegularExpressions.clear(); this->IncludeRegExp.clear(); this->ExcludeRegExp.clear(); this->ExcludeFixtureRegExp.clear(); @@ -460,6 +460,10 @@ int cmCTestTestHandler::ProcessHandler() return 1; } + if (!this->WriteJUnitXML()) { + return 1; + } + if (!this->PostProcessHandler()) { this->LogFile = nullptr; return -1; @@ -479,6 +483,22 @@ int cmCTestTestHandler::ProcessHandler() return 0; } +/* Given a multi-option value `parts`, compile those parts into + * regular expressions in `expressions`. Skip empty values. + * Returns true if there were any expressions. + */ +static bool BuildLabelRE(const std::vector<std::string>& parts, + std::vector<cmsys::RegularExpression>& expressions) +{ + expressions.clear(); + for (const auto& p : parts) { + if (!p.empty()) { + expressions.emplace_back(p); + } + } + return !expressions.empty(); +} + bool cmCTestTestHandler::ProcessOptions() { // Update internal data structure from generic one @@ -519,18 +539,11 @@ bool cmCTestTestHandler::ProcessOptions() this->CTest->SetStopOnFailure(true); } - const char* val; - val = this->GetOption("LabelRegularExpression"); - if (val) { - this->UseIncludeLabelRegExpFlag = true; - this->IncludeLabelRegExp = val; - } - val = this->GetOption("ExcludeLabelRegularExpression"); - if (val) { - this->UseExcludeLabelRegExpFlag = true; - this->ExcludeLabelRegExp = val; - } - val = this->GetOption("IncludeRegularExpression"); + BuildLabelRE(this->GetMultiOption("LabelRegularExpression"), + this->IncludeLabelRegularExpressions); + BuildLabelRE(this->GetMultiOption("ExcludeLabelRegularExpression"), + this->ExcludeLabelRegularExpressions); + const char* val = this->GetOption("IncludeRegularExpression"); if (val) { this->UseIncludeRegExp(); this->SetIncludeRegExp(val); @@ -763,10 +776,40 @@ void cmCTestTestHandler::PrintLabelOrSubprojectSummary(bool doSubProject) cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, "\n", this->Quiet); } +/** + * Check if the labels (from a test) match all the expressions. + * + * Each of the RE's must match at least one label + * (e.g. all of the REs must match **some** label, + * in order for the filter to apply to the test). + */ +static bool MatchLabelsAgainstFilterRE( + const std::vector<std::string>& labels, + const std::vector<cmsys::RegularExpression>& expressions) +{ + for (const auto& re : expressions) { + // check to see if the label regular expression matches + bool found = false; // assume it does not match + cmsys::RegularExpressionMatch match; + // loop over all labels and look for match + for (std::string const& l : labels) { + if (re.find(l.c_str(), match)) { + found = true; + break; + } + } + // if no match was found, exclude the test + if (!found) { + return false; + } + } + return true; +} + void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it) { // if not using Labels to filter then return - if (!this->UseIncludeLabelRegExpFlag) { + if (this->IncludeLabelRegularExpressions.empty()) { return; } // if there are no labels and we are filtering by labels @@ -775,16 +818,9 @@ void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it) it.IsInBasedOnREOptions = false; return; } - // check to see if the label regular expression matches - bool found = false; // assume it does not match - // loop over all labels and look for match - for (std::string const& l : it.Labels) { - if (this->IncludeLabelRegularExpression.find(l)) { - found = true; - } - } // if no match was found, exclude the test - if (!found) { + if (!MatchLabelsAgainstFilterRE(it.Labels, + this->IncludeLabelRegularExpressions)) { it.IsInBasedOnREOptions = false; } } @@ -792,7 +828,7 @@ void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it) void cmCTestTestHandler::CheckLabelFilterExclude(cmCTestTestProperties& it) { // if not using Labels to filter then return - if (!this->UseExcludeLabelRegExpFlag) { + if (this->ExcludeLabelRegularExpressions.empty()) { return; } // if there are no labels and we are excluding by labels @@ -800,16 +836,9 @@ void cmCTestTestHandler::CheckLabelFilterExclude(cmCTestTestProperties& it) if (it.Labels.empty()) { return; } - // check to see if the label regular expression matches - bool found = false; // assume it does not match - // loop over all labels and look for match - for (std::string const& l : it.Labels) { - if (this->ExcludeLabelRegularExpression.find(l)) { - found = true; - } - } // if match was found, exclude the test - if (found) { + if (MatchLabelsAgainstFilterRE(it.Labels, + this->ExcludeLabelRegularExpressions)) { it.IsInBasedOnREOptions = false; } } @@ -1704,19 +1733,11 @@ bool cmCTestTestHandler::ParseResourceGroupsProperty( bool cmCTestTestHandler::GetListOfTests() { - if (!this->IncludeLabelRegExp.empty()) { - this->IncludeLabelRegularExpression.compile( - this->IncludeLabelRegExp.c_str()); - } - if (!this->ExcludeLabelRegExp.empty()) { - this->ExcludeLabelRegularExpression.compile( - this->ExcludeLabelRegExp.c_str()); - } if (!this->IncludeRegExp.empty()) { - this->IncludeTestsRegularExpression.compile(this->IncludeRegExp.c_str()); + this->IncludeTestsRegularExpression.compile(this->IncludeRegExp); } if (!this->ExcludeRegExp.empty()) { - this->ExcludeTestsRegularExpression.compile(this->ExcludeRegExp.c_str()); + this->ExcludeTestsRegularExpression.compile(this->ExcludeRegExp); } cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Constructing a list of tests" << std::endl, this->Quiet); @@ -1868,7 +1889,7 @@ void cmCTestTestHandler::ExpandTestsToRunInformationForRerunFailed() std::string dirName = this->CTest->GetBinaryDir() + "/Testing/Temporary"; cmsys::Directory directory; - if (directory.Load(dirName) == 0) { + if (!directory.Load(dirName)) { cmCTestLog(this->CTest, ERROR_MESSAGE, "Unable to read the contents of " << dirName << std::endl); return; @@ -2444,3 +2465,125 @@ bool cmCTestTestHandler::cmCTestTestResourceRequirement::operator!=( { return !(*this == other); } + +void cmCTestTestHandler::SetJUnitXMLFileName(const std::string& filename) +{ + this->JUnitXMLFileName = filename; +} + +bool cmCTestTestHandler::WriteJUnitXML() +{ + if (this->JUnitXMLFileName.empty()) { + return true; + } + + // Open new XML file for writing. + cmGeneratedFileStream xmlfile; + xmlfile.SetTempExt("tmp"); + xmlfile.Open(this->JUnitXMLFileName); + if (!xmlfile) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "Problem opening file: " << this->JUnitXMLFileName + << std::endl); + return false; + } + cmXMLWriter xml(xmlfile); + + // Iterate over the test results to get the number of tests that + // passed, failed, etc. + auto num_tests = 0; + auto num_passed = 0; + auto num_failed = 0; + auto num_notrun = 0; + auto num_disabled = 0; + SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end()); + for (cmCTestTestResult const& result : resultsSet) { + num_tests++; + if (result.Status == cmCTestTestHandler::COMPLETED) { + num_passed++; + } else if (result.Status == cmCTestTestHandler::NOT_RUN) { + if (result.CompletionStatus == "Disabled") { + num_disabled++; + } else { + num_notrun++; + } + } else { + num_failed++; + } + } + + // Write <testsuite> element. + xml.StartDocument(); + xml.StartElement("testsuite"); + + xml.Attribute("name", + cmCTest::SafeBuildIdField( + this->CTest->GetCTestConfiguration("BuildName"))); + xml.BreakAttributes(); + + xml.Attribute("tests", num_tests); + xml.Attribute("failures", num_failed); + + // CTest disabled => JUnit disabled + xml.Attribute("disabled", num_disabled); + + // Otherwise, CTest notrun => JUnit skipped. + // The distinction between JUnit disabled vs. skipped is that + // skipped tests can have a message associated with them + // (why the test was skipped). + xml.Attribute("skipped", num_notrun); + + xml.Attribute("hostname", this->CTest->GetCTestConfiguration("Site")); + xml.Attribute( + "time", + std::chrono::duration_cast<std::chrono::seconds>(this->ElapsedTestingTime) + .count()); + const std::time_t start_test_time_t = + std::chrono::system_clock::to_time_t(this->StartTestTime); + cmTimestamp cmts; + xml.Attribute("timestamp", + cmts.CreateTimestampFromTimeT(start_test_time_t, + "%Y-%m-%dT%H:%M:%S", false)); + + // Write <testcase> elements. + for (cmCTestTestResult const& result : resultsSet) { + xml.StartElement("testcase"); + xml.Attribute("name", result.Name); + xml.Attribute("classname", result.Name); + xml.Attribute("time", result.ExecutionTime.count()); + + std::string status; + if (result.Status == cmCTestTestHandler::COMPLETED) { + status = "run"; + } else if (result.Status == cmCTestTestHandler::NOT_RUN) { + if (result.CompletionStatus == "Disabled") { + status = "disabled"; + } else { + status = "notrun"; + } + } else { + status = "fail"; + } + xml.Attribute("status", status); + + if (status == "notrun") { + xml.StartElement("skipped"); + xml.Attribute("message", result.CompletionStatus); + xml.EndElement(); // </skipped> + } else if (status == "fail") { + xml.StartElement("failure"); + xml.Attribute("message", result.Reason); + xml.EndElement(); // </failure> + } + + // Note: compressed test output is unconditionally disabled when + // --output-junit is specified. + xml.Element("system-out", result.Output); + xml.EndElement(); // </testcase> + } + + xml.EndElement(); // </testsuite> + xml.EndDocument(); + + return true; +} |