From fc41a95f0803abb2b5daa4f0eb21cf38c625bd26 Mon Sep 17 00:00:00 2001 From: Justin Goshi Date: Thu, 18 Oct 2018 11:34:37 -0700 Subject: CTest: Add --show-only[=format] option to print test info format can be 'human' to print the current text format or 'json-v1' to print the test object model in json format and is useful for IDEs who want to gather information about the tests. Defaults to 'human' format. --- Help/manual/ctest.1.rst | 71 +++++- Help/release/dev/ctest-show-only-json-v1.rst | 6 + Source/CTest/cmCTestMultiProcessHandler.cxx | 331 +++++++++++++++++++++++++++ Source/CTest/cmCTestMultiProcessHandler.h | 1 + Source/CTest/cmCTestRunTest.h | 4 + Source/cmCTest.cxx | 26 +++ Source/cmCTest.h | 6 + Source/ctest.cxx | 5 +- 8 files changed, 448 insertions(+), 2 deletions(-) create mode 100644 Help/release/dev/ctest-show-only-json-v1.rst diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 1ef20ab..8490e3d 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -109,13 +109,23 @@ Options This option tells CTest to write all its output to a log file. -``-N,--show-only`` +``-N,--show-only[=]`` Disable actual execution of tests. This option tells CTest to list the tests that would be run but not actually run them. Useful in conjunction with the ``-R`` and ``-E`` options. + ```` can be one of the following values. + + ``human`` + Human-friendly output. This is not guaranteed to be stable. + This is the default. + + ``json-v1`` + Dump the test information in JSON format. + See `Show as JSON Object Model`_. + ``-L , --label-regex `` Run tests with labels matching regular expression. @@ -1163,6 +1173,65 @@ Configuration settings include: * :module:`CTest` module variable: ``TRIGGER_SITE`` if set, else ``CTEST_TRIGGER_SITE`` +.. _`Show as JSON Object Model`: + +Show as JSON Object Model +========================= + +When the ``--show-only=json-v1`` command line option is given, the test +information is output in JSON format. Version 1.0 of the JSON object +model is defined as follows: + +``kind`` + The string "ctestInfo". + +``version`` + A JSON object specifying the version components. Its members are + + ``major`` + A non-negative integer specifying the major version component. + ``minor`` + A non-negative integer specifying the minor version component. + +``backtraceGraph`` + JSON object representing backtrace information with the + following members: + + ``commands`` + List of command names. + ``files`` + List of file names. + ``nodes`` + List of node JSON objects with members: + + ``command`` + Index into the ``commands`` member of the ``backtraceGraph``. + ``file`` + Index into the ``files`` member of the ``backtraceGraph``. + ``line`` + Line number in the file where the backtrace was added. + ``parent`` + Index into the ``nodes`` member of the ``backtraceGraph`` + representing the parent in the graph. + +``tests`` + A JSON array listing information about each test. Each entry + is a JSON object with members: + + ``name`` + Test name. + ``config`` + Configuration that the test can run on. + Empty string means any config. + ``command`` + List where the first element is the test command and the + remaining elements are the command arguments. + ``backtrace`` + Index into the ``nodes`` member of the ``backtraceGraph``. + ``properties`` + Test properties. + Can contain keys for each of the supported test properties. + See Also ======== diff --git a/Help/release/dev/ctest-show-only-json-v1.rst b/Help/release/dev/ctest-show-only-json-v1.rst new file mode 100644 index 0000000..f593e7e --- /dev/null +++ b/Help/release/dev/ctest-show-only-json-v1.rst @@ -0,0 +1,6 @@ +ctest-show-only-json-v1 +----------------------- + +* :manual:`ctest(1)` gained a ``--show-only=json-v1`` option to show the + list of tests in a machine-readable JSON format. + See the :ref:`Show as JSON Object Model` section of the manual. diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index f026001..8867323 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -6,9 +6,13 @@ #include "cmCTest.h" #include "cmCTestRunTest.h" #include "cmCTestTestHandler.h" +#include "cmDuration.h" +#include "cmListFileCache.h" #include "cmSystemTools.h" #include "cmWorkingDirectory.h" +#include "cm_jsoncpp_value.h" +#include "cm_jsoncpp_writer.h" #include "cm_uv.h" #include "cmUVSignalHackRAII.h" // IWYU pragma: keep @@ -20,13 +24,19 @@ #include #include #include +#include #include #include #include #include #include +#include #include +namespace cmsys { +class RegularExpression; +} + class TestComparator { public: @@ -725,9 +735,330 @@ void cmCTestMultiProcessHandler::MarkFinished() cmSystemTools::RemoveFile(fname); } +static Json::Value DumpToJsonArray(const std::set& values) +{ + Json::Value jsonArray = Json::arrayValue; + for (auto& it : values) { + jsonArray.append(it); + } + return jsonArray; +} + +static Json::Value DumpToJsonArray(const std::vector& values) +{ + Json::Value jsonArray = Json::arrayValue; + for (auto& it : values) { + jsonArray.append(it); + } + return jsonArray; +} + +static Json::Value DumpRegExToJsonArray( + const std::vector>& values) +{ + Json::Value jsonArray = Json::arrayValue; + for (auto& it : values) { + jsonArray.append(it.second); + } + return jsonArray; +} + +static Json::Value DumpMeasurementToJsonArray( + const std::map& values) +{ + Json::Value jsonArray = Json::arrayValue; + for (auto& it : values) { + Json::Value measurement = Json::objectValue; + measurement["measurement"] = it.first; + measurement["value"] = it.second; + jsonArray.append(measurement); + } + return jsonArray; +} + +static Json::Value DumpTimeoutAfterMatch( + cmCTestTestHandler::cmCTestTestProperties& testProperties) +{ + Json::Value timeoutAfterMatch = Json::objectValue; + timeoutAfterMatch["timeout"] = testProperties.AlternateTimeout.count(); + timeoutAfterMatch["regex"] = + DumpRegExToJsonArray(testProperties.TimeoutRegularExpressions); + return timeoutAfterMatch; +} + +static Json::Value DumpCTestProperty(std::string const& name, + Json::Value value) +{ + Json::Value property = Json::objectValue; + property["name"] = name; + property["value"] = std::move(value); + return property; +} + +static Json::Value DumpCTestProperties( + cmCTestTestHandler::cmCTestTestProperties& testProperties) +{ + Json::Value properties = Json::arrayValue; + if (!testProperties.AttachOnFail.empty()) { + properties.append(DumpCTestProperty( + "ATTACHED_FILES_ON_FAIL", DumpToJsonArray(testProperties.AttachOnFail))); + } + if (!testProperties.AttachedFiles.empty()) { + properties.append(DumpCTestProperty( + "ATTACHED_FILES", DumpToJsonArray(testProperties.AttachedFiles))); + } + if (testProperties.Cost != 0.0f) { + properties.append( + DumpCTestProperty("COST", static_cast(testProperties.Cost))); + } + if (!testProperties.Depends.empty()) { + properties.append( + DumpCTestProperty("DEPENDS", DumpToJsonArray(testProperties.Depends))); + } + if (testProperties.Disabled) { + properties.append(DumpCTestProperty("DISABLED", testProperties.Disabled)); + } + if (!testProperties.Environment.empty()) { + properties.append(DumpCTestProperty( + "ENVIRONMENT", DumpToJsonArray(testProperties.Environment))); + } + if (!testProperties.ErrorRegularExpressions.empty()) { + properties.append(DumpCTestProperty( + "FAIL_REGULAR_EXPRESSION", + DumpRegExToJsonArray(testProperties.ErrorRegularExpressions))); + } + if (!testProperties.FixturesCleanup.empty()) { + properties.append(DumpCTestProperty( + "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup))); + } + if (!testProperties.FixturesRequired.empty()) { + properties.append(DumpCTestProperty( + "FIXTURES_REQUIRED", DumpToJsonArray(testProperties.FixturesRequired))); + } + if (!testProperties.FixturesSetup.empty()) { + properties.append(DumpCTestProperty( + "FIXTURES_SETUP", DumpToJsonArray(testProperties.FixturesSetup))); + } + if (!testProperties.Labels.empty()) { + properties.append( + DumpCTestProperty("LABELS", DumpToJsonArray(testProperties.Labels))); + } + if (!testProperties.Measurements.empty()) { + properties.append(DumpCTestProperty( + "MEASUREMENT", DumpMeasurementToJsonArray(testProperties.Measurements))); + } + if (!testProperties.RequiredRegularExpressions.empty()) { + properties.append(DumpCTestProperty( + "PASS_REGULAR_EXPRESSION", + DumpRegExToJsonArray(testProperties.RequiredRegularExpressions))); + } + if (testProperties.WantAffinity) { + properties.append( + DumpCTestProperty("PROCESSOR_AFFINITY", testProperties.WantAffinity)); + } + if (testProperties.Processors != 1) { + properties.append( + DumpCTestProperty("PROCESSORS", testProperties.Processors)); + } + if (!testProperties.RequiredFiles.empty()) { + properties["REQUIRED_FILES"] = + DumpToJsonArray(testProperties.RequiredFiles); + } + if (!testProperties.LockedResources.empty()) { + properties.append(DumpCTestProperty( + "RESOURCE_LOCK", DumpToJsonArray(testProperties.LockedResources))); + } + if (testProperties.RunSerial) { + properties.append( + DumpCTestProperty("RUN_SERIAL", testProperties.RunSerial)); + } + if (testProperties.SkipReturnCode != -1) { + properties.append( + DumpCTestProperty("SKIP_RETURN_CODE", testProperties.SkipReturnCode)); + } + if (testProperties.ExplicitTimeout) { + properties.append( + DumpCTestProperty("TIMEOUT", testProperties.Timeout.count())); + } + if (!testProperties.TimeoutRegularExpressions.empty()) { + properties.append(DumpCTestProperty( + "TIMEOUT_AFTER_MATCH", DumpTimeoutAfterMatch(testProperties))); + } + if (testProperties.WillFail) { + properties.append(DumpCTestProperty("WILL_FAIL", testProperties.WillFail)); + } + if (!testProperties.Directory.empty()) { + properties.append( + DumpCTestProperty("WORKING_DIRECTORY", testProperties.Directory)); + } + return properties; +} + +class BacktraceData +{ + std::unordered_map CommandMap; + std::unordered_map FileMap; + std::unordered_map NodeMap; + Json::Value Commands = Json::arrayValue; + Json::Value Files = Json::arrayValue; + Json::Value Nodes = Json::arrayValue; + + Json::ArrayIndex AddCommand(std::string const& command) + { + auto i = this->CommandMap.find(command); + if (i == this->CommandMap.end()) { + i = this->CommandMap.emplace(command, this->Commands.size()).first; + this->Commands.append(command); + } + return i->second; + } + + Json::ArrayIndex AddFile(std::string const& file) + { + auto i = this->FileMap.find(file); + if (i == this->FileMap.end()) { + i = this->FileMap.emplace(file, this->Files.size()).first; + this->Files.append(file); + } + return i->second; + } + +public: + bool Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index); + Json::Value Dump(); +}; + +bool BacktraceData::Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index) +{ + if (bt.Empty()) { + return false; + } + cmListFileContext const* top = &bt.Top(); + auto found = this->NodeMap.find(top); + if (found != this->NodeMap.end()) { + index = found->second; + return true; + } + Json::Value entry = Json::objectValue; + entry["file"] = this->AddFile(top->FilePath); + if (top->Line) { + entry["line"] = static_cast(top->Line); + } + if (!top->Name.empty()) { + entry["command"] = this->AddCommand(top->Name); + } + Json::ArrayIndex parent; + if (this->Add(bt.Pop(), parent)) { + entry["parent"] = parent; + } + index = this->NodeMap[top] = this->Nodes.size(); + this->Nodes.append(std::move(entry)); // NOLINT(*) + return true; +} + +Json::Value BacktraceData::Dump() +{ + Json::Value backtraceGraph; + this->CommandMap.clear(); + this->FileMap.clear(); + this->NodeMap.clear(); + backtraceGraph["commands"] = std::move(this->Commands); + backtraceGraph["files"] = std::move(this->Files); + backtraceGraph["nodes"] = std::move(this->Nodes); + return backtraceGraph; +} + +static void AddBacktrace(BacktraceData& backtraceGraph, Json::Value& object, + cmListFileBacktrace const& bt) +{ + Json::ArrayIndex backtrace; + if (backtraceGraph.Add(bt, backtrace)) { + object["backtrace"] = backtrace; + } +} + +static Json::Value DumpCTestInfo( + cmCTestRunTest& testRun, + cmCTestTestHandler::cmCTestTestProperties& testProperties, + BacktraceData& backtraceGraph) +{ + Json::Value testInfo = Json::objectValue; + // test name should always be present + testInfo["name"] = testProperties.Name; + std::string const& config = testRun.GetCTest()->GetConfigType(); + if (!config.empty()) { + testInfo["config"] = config; + } + std::string const& command = testRun.GetActualCommand(); + if (!command.empty()) { + std::vector commandAndArgs; + commandAndArgs.push_back(command); + const std::vector& args = testRun.GetArguments(); + if (!args.empty()) { + commandAndArgs.reserve(args.size() + 1); + commandAndArgs.insert(commandAndArgs.end(), args.begin(), args.end()); + } + testInfo["command"] = DumpToJsonArray(commandAndArgs); + } + Json::Value properties = DumpCTestProperties(testProperties); + if (!properties.empty()) { + testInfo["properties"] = properties; + } + if (!testProperties.Backtrace.Empty()) { + AddBacktrace(backtraceGraph, testInfo, testProperties.Backtrace); + } + return testInfo; +} + +static Json::Value DumpVersion(int major, int minor) +{ + Json::Value version = Json::objectValue; + version["major"] = major; + version["minor"] = minor; + return version; +} + +void cmCTestMultiProcessHandler::PrintOutputAsJson() +{ + this->TestHandler->SetMaxIndex(this->FindMaxIndex()); + + Json::Value result = Json::objectValue; + result["kind"] = "ctestInfo"; + result["version"] = DumpVersion(1, 0); + + BacktraceData backtraceGraph; + Json::Value tests = Json::arrayValue; + for (auto& it : this->Properties) { + cmCTestTestHandler::cmCTestTestProperties& p = *it.second; + + // Don't worry if this fails, we are only showing the test list, not + // running the tests + cmWorkingDirectory workdir(p.Directory); + cmCTestRunTest testRun(*this); + testRun.SetIndex(p.Index); + testRun.SetTestProperties(&p); + testRun.ComputeArguments(); + + Json::Value testInfo = DumpCTestInfo(testRun, p, backtraceGraph); + tests.append(testInfo); + } + result["backtraceGraph"] = backtraceGraph.Dump(); + result["tests"] = std::move(tests); + + Json::StreamWriterBuilder builder; + builder["indentation"] = " "; + std::unique_ptr jout(builder.newStreamWriter()); + jout->write(result, &std::cout); +} + // For ShowOnly mode void cmCTestMultiProcessHandler::PrintTestList() { + if (this->CTest->GetOutputAsJson()) { + PrintOutputAsJson(); + return; + } + this->TestHandler->SetMaxIndex(this->FindMaxIndex()); int count = 0; diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h index 3927a8a..93bb880 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.h +++ b/Source/CTest/cmCTestMultiProcessHandler.h @@ -51,6 +51,7 @@ public: void SetParallelLevel(size_t); void SetTestLoad(unsigned long load); virtual void RunTests(); + void PrintOutputAsJson(); void PrintTestList(); void PrintLabels(); diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index 10dceca..c786413 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -78,6 +78,10 @@ public: cmCTest* GetCTest() const { return this->CTest; } + std::string& GetActualCommand() { return this->ActualCommand; } + + const std::vector& GetArguments() { return this->Arguments; } + void FinalizeTest(); bool TimedOutForStopTime() const { return this->TimeoutIsForStopTime; } diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 7c19864..225c99f 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -278,6 +278,8 @@ cmCTest::cmCTest() this->ExtraVerbose = false; this->ProduceXML = false; this->ShowOnly = false; + this->OutputAsJson = false; + this->OutputAsJsonVersion = 1; this->RunConfigurationScript = false; this->UseHTTP10 = false; this->PrintLabels = false; @@ -1930,6 +1932,20 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, if (this->CheckArgument(arg, "-N", "--show-only")) { this->ShowOnly = true; } + if (cmSystemTools::StringStartsWith(arg.c_str(), "--show-only=")) { + this->ShowOnly = true; + + // Check if a specific format is requested. Defaults to human readable + // text. + std::string argWithFormat = "--show-only="; + std::string format = arg.substr(argWithFormat.length()); + if (format == "json-v1") { + // Force quiet mode so the only output is the json object model. + this->Quiet = true; + this->OutputAsJson = true; + this->OutputAsJsonVersion = 1; + } + } if (this->CheckArgument(arg, "-O", "--output-log") && i < args.size() - 1) { i++; @@ -2630,6 +2646,16 @@ bool cmCTest::GetShowOnly() return this->ShowOnly; } +bool cmCTest::GetOutputAsJson() +{ + return this->OutputAsJson; +} + +int cmCTest::GetOutputAsJsonVersion() +{ + return this->OutputAsJsonVersion; +} + int cmCTest::GetMaxTestNameWidth() const { return this->MaxTestNameWidth; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 480204a..2b40ca3 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -215,6 +215,10 @@ public: /** Should we only show what we would do? */ bool GetShowOnly(); + bool GetOutputAsJson(); + + int GetOutputAsJsonVersion(); + bool ShouldUseHTTP10() { return this->UseHTTP10; } bool ShouldPrintLabels() { return this->PrintLabels; } @@ -507,6 +511,8 @@ private: t_TestingHandlers TestingHandlers; bool ShowOnly; + bool OutputAsJson; + int OutputAsJsonVersion; /** Map of configuration properties */ typedef std::map CTestConfigurationMap; diff --git a/Source/ctest.cxx b/Source/ctest.cxx index ca412ae..8ba126f 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -46,7 +46,10 @@ static const char* cmDocumentationOptions[][2] = { "given number of jobs." }, { "-Q,--quiet", "Make ctest quiet." }, { "-O , --output-log ", "Output to log file" }, - { "-N,--show-only", "Disable actual execution of tests." }, + { "-N,--show-only[=format]", + "Disable actual execution of tests. The optional 'format' defines the " + "format of the test information and can be 'human' for the current text " + "format or 'json-v1' for json format. Defaults to 'human'." }, { "-L , --label-regex ", "Run tests with labels matching " "regular expression." }, -- cgit v0.12