summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Help/manual/ctest.1.rst71
-rw-r--r--Help/release/dev/ctest-show-only-json-v1.rst6
-rw-r--r--Source/CTest/cmCTestMultiProcessHandler.cxx331
-rw-r--r--Source/CTest/cmCTestMultiProcessHandler.h1
-rw-r--r--Source/CTest/cmCTestRunTest.h4
-rw-r--r--Source/cmCTest.cxx26
-rw-r--r--Source/cmCTest.h6
-rw-r--r--Source/ctest.cxx5
8 files changed, 448 insertions, 2 deletions
diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst
index 1ef20ab..8490e3d 100644
--- a/Help/manual/ctest.1.rst
+++ b/Help/manual/ctest.1.rst
@@ -109,13 +109,23 @@ Options
This option tells CTest to write all its output to a log file.
-``-N,--show-only``
+``-N,--show-only[=<format>]``
Disable actual execution of tests.
This option tells CTest to list the tests that would be run but not
actually run them. Useful in conjunction with the ``-R`` and ``-E``
options.
+ ``<format>`` can be one of the following values.
+
+ ``human``
+ Human-friendly output. This is not guaranteed to be stable.
+ This is the default.
+
+ ``json-v1``
+ Dump the test information in JSON format.
+ See `Show as JSON Object Model`_.
+
``-L <regex>, --label-regex <regex>``
Run tests with labels matching regular expression.
@@ -1163,6 +1173,65 @@ Configuration settings include:
* :module:`CTest` module variable: ``TRIGGER_SITE`` if set,
else ``CTEST_TRIGGER_SITE``
+.. _`Show as JSON Object Model`:
+
+Show as JSON Object Model
+=========================
+
+When the ``--show-only=json-v1`` command line option is given, the test
+information is output in JSON format. Version 1.0 of the JSON object
+model is defined as follows:
+
+``kind``
+ The string "ctestInfo".
+
+``version``
+ A JSON object specifying the version components. Its members are
+
+ ``major``
+ A non-negative integer specifying the major version component.
+ ``minor``
+ A non-negative integer specifying the minor version component.
+
+``backtraceGraph``
+ JSON object representing backtrace information with the
+ following members:
+
+ ``commands``
+ List of command names.
+ ``files``
+ List of file names.
+ ``nodes``
+ List of node JSON objects with members:
+
+ ``command``
+ Index into the ``commands`` member of the ``backtraceGraph``.
+ ``file``
+ Index into the ``files`` member of the ``backtraceGraph``.
+ ``line``
+ Line number in the file where the backtrace was added.
+ ``parent``
+ Index into the ``nodes`` member of the ``backtraceGraph``
+ representing the parent in the graph.
+
+``tests``
+ A JSON array listing information about each test. Each entry
+ is a JSON object with members:
+
+ ``name``
+ Test name.
+ ``config``
+ Configuration that the test can run on.
+ Empty string means any config.
+ ``command``
+ List where the first element is the test command and the
+ remaining elements are the command arguments.
+ ``backtrace``
+ Index into the ``nodes`` member of the ``backtraceGraph``.
+ ``properties``
+ Test properties.
+ Can contain keys for each of the supported test properties.
+
See Also
========
diff --git a/Help/release/dev/ctest-show-only-json-v1.rst b/Help/release/dev/ctest-show-only-json-v1.rst
new file mode 100644
index 0000000..f593e7e
--- /dev/null
+++ b/Help/release/dev/ctest-show-only-json-v1.rst
@@ -0,0 +1,6 @@
+ctest-show-only-json-v1
+-----------------------
+
+* :manual:`ctest(1)` gained a ``--show-only=json-v1`` option to show the
+ list of tests in a machine-readable JSON format.
+ See the :ref:`Show as JSON Object Model` section of the manual.
diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index f026001..8867323 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -6,9 +6,13 @@
#include "cmCTest.h"
#include "cmCTestRunTest.h"
#include "cmCTestTestHandler.h"
+#include "cmDuration.h"
+#include "cmListFileCache.h"
#include "cmSystemTools.h"
#include "cmWorkingDirectory.h"
+#include "cm_jsoncpp_value.h"
+#include "cm_jsoncpp_writer.h"
#include "cm_uv.h"
#include "cmUVSignalHackRAII.h" // IWYU pragma: keep
@@ -20,13 +24,19 @@
#include <chrono>
#include <cstring>
#include <iomanip>
+#include <iostream>
#include <list>
#include <math.h>
#include <sstream>
#include <stack>
#include <stdlib.h>
+#include <unordered_map>
#include <utility>
+namespace cmsys {
+class RegularExpression;
+}
+
class TestComparator
{
public:
@@ -725,9 +735,330 @@ void cmCTestMultiProcessHandler::MarkFinished()
cmSystemTools::RemoveFile(fname);
}
+static Json::Value DumpToJsonArray(const std::set<std::string>& values)
+{
+ Json::Value jsonArray = Json::arrayValue;
+ for (auto& it : values) {
+ jsonArray.append(it);
+ }
+ return jsonArray;
+}
+
+static Json::Value DumpToJsonArray(const std::vector<std::string>& values)
+{
+ Json::Value jsonArray = Json::arrayValue;
+ for (auto& it : values) {
+ jsonArray.append(it);
+ }
+ return jsonArray;
+}
+
+static Json::Value DumpRegExToJsonArray(
+ const std::vector<std::pair<cmsys::RegularExpression, std::string>>& values)
+{
+ Json::Value jsonArray = Json::arrayValue;
+ for (auto& it : values) {
+ jsonArray.append(it.second);
+ }
+ return jsonArray;
+}
+
+static Json::Value DumpMeasurementToJsonArray(
+ const std::map<std::string, std::string>& values)
+{
+ Json::Value jsonArray = Json::arrayValue;
+ for (auto& it : values) {
+ Json::Value measurement = Json::objectValue;
+ measurement["measurement"] = it.first;
+ measurement["value"] = it.second;
+ jsonArray.append(measurement);
+ }
+ return jsonArray;
+}
+
+static Json::Value DumpTimeoutAfterMatch(
+ cmCTestTestHandler::cmCTestTestProperties& testProperties)
+{
+ Json::Value timeoutAfterMatch = Json::objectValue;
+ timeoutAfterMatch["timeout"] = testProperties.AlternateTimeout.count();
+ timeoutAfterMatch["regex"] =
+ DumpRegExToJsonArray(testProperties.TimeoutRegularExpressions);
+ return timeoutAfterMatch;
+}
+
+static Json::Value DumpCTestProperty(std::string const& name,
+ Json::Value value)
+{
+ Json::Value property = Json::objectValue;
+ property["name"] = name;
+ property["value"] = std::move(value);
+ return property;
+}
+
+static Json::Value DumpCTestProperties(
+ cmCTestTestHandler::cmCTestTestProperties& testProperties)
+{
+ Json::Value properties = Json::arrayValue;
+ if (!testProperties.AttachOnFail.empty()) {
+ properties.append(DumpCTestProperty(
+ "ATTACHED_FILES_ON_FAIL", DumpToJsonArray(testProperties.AttachOnFail)));
+ }
+ if (!testProperties.AttachedFiles.empty()) {
+ properties.append(DumpCTestProperty(
+ "ATTACHED_FILES", DumpToJsonArray(testProperties.AttachedFiles)));
+ }
+ if (testProperties.Cost != 0.0f) {
+ properties.append(
+ DumpCTestProperty("COST", static_cast<double>(testProperties.Cost)));
+ }
+ if (!testProperties.Depends.empty()) {
+ properties.append(
+ DumpCTestProperty("DEPENDS", DumpToJsonArray(testProperties.Depends)));
+ }
+ if (testProperties.Disabled) {
+ properties.append(DumpCTestProperty("DISABLED", testProperties.Disabled));
+ }
+ if (!testProperties.Environment.empty()) {
+ properties.append(DumpCTestProperty(
+ "ENVIRONMENT", DumpToJsonArray(testProperties.Environment)));
+ }
+ if (!testProperties.ErrorRegularExpressions.empty()) {
+ properties.append(DumpCTestProperty(
+ "FAIL_REGULAR_EXPRESSION",
+ DumpRegExToJsonArray(testProperties.ErrorRegularExpressions)));
+ }
+ if (!testProperties.FixturesCleanup.empty()) {
+ properties.append(DumpCTestProperty(
+ "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup)));
+ }
+ if (!testProperties.FixturesRequired.empty()) {
+ properties.append(DumpCTestProperty(
+ "FIXTURES_REQUIRED", DumpToJsonArray(testProperties.FixturesRequired)));
+ }
+ if (!testProperties.FixturesSetup.empty()) {
+ properties.append(DumpCTestProperty(
+ "FIXTURES_SETUP", DumpToJsonArray(testProperties.FixturesSetup)));
+ }
+ if (!testProperties.Labels.empty()) {
+ properties.append(
+ DumpCTestProperty("LABELS", DumpToJsonArray(testProperties.Labels)));
+ }
+ if (!testProperties.Measurements.empty()) {
+ properties.append(DumpCTestProperty(
+ "MEASUREMENT", DumpMeasurementToJsonArray(testProperties.Measurements)));
+ }
+ if (!testProperties.RequiredRegularExpressions.empty()) {
+ properties.append(DumpCTestProperty(
+ "PASS_REGULAR_EXPRESSION",
+ DumpRegExToJsonArray(testProperties.RequiredRegularExpressions)));
+ }
+ if (testProperties.WantAffinity) {
+ properties.append(
+ DumpCTestProperty("PROCESSOR_AFFINITY", testProperties.WantAffinity));
+ }
+ if (testProperties.Processors != 1) {
+ properties.append(
+ DumpCTestProperty("PROCESSORS", testProperties.Processors));
+ }
+ if (!testProperties.RequiredFiles.empty()) {
+ properties["REQUIRED_FILES"] =
+ DumpToJsonArray(testProperties.RequiredFiles);
+ }
+ if (!testProperties.LockedResources.empty()) {
+ properties.append(DumpCTestProperty(
+ "RESOURCE_LOCK", DumpToJsonArray(testProperties.LockedResources)));
+ }
+ if (testProperties.RunSerial) {
+ properties.append(
+ DumpCTestProperty("RUN_SERIAL", testProperties.RunSerial));
+ }
+ if (testProperties.SkipReturnCode != -1) {
+ properties.append(
+ DumpCTestProperty("SKIP_RETURN_CODE", testProperties.SkipReturnCode));
+ }
+ if (testProperties.ExplicitTimeout) {
+ properties.append(
+ DumpCTestProperty("TIMEOUT", testProperties.Timeout.count()));
+ }
+ if (!testProperties.TimeoutRegularExpressions.empty()) {
+ properties.append(DumpCTestProperty(
+ "TIMEOUT_AFTER_MATCH", DumpTimeoutAfterMatch(testProperties)));
+ }
+ if (testProperties.WillFail) {
+ properties.append(DumpCTestProperty("WILL_FAIL", testProperties.WillFail));
+ }
+ if (!testProperties.Directory.empty()) {
+ properties.append(
+ DumpCTestProperty("WORKING_DIRECTORY", testProperties.Directory));
+ }
+ return properties;
+}
+
+class BacktraceData
+{
+ std::unordered_map<std::string, Json::ArrayIndex> CommandMap;
+ std::unordered_map<std::string, Json::ArrayIndex> FileMap;
+ std::unordered_map<cmListFileContext const*, Json::ArrayIndex> NodeMap;
+ Json::Value Commands = Json::arrayValue;
+ Json::Value Files = Json::arrayValue;
+ Json::Value Nodes = Json::arrayValue;
+
+ Json::ArrayIndex AddCommand(std::string const& command)
+ {
+ auto i = this->CommandMap.find(command);
+ if (i == this->CommandMap.end()) {
+ i = this->CommandMap.emplace(command, this->Commands.size()).first;
+ this->Commands.append(command);
+ }
+ return i->second;
+ }
+
+ Json::ArrayIndex AddFile(std::string const& file)
+ {
+ auto i = this->FileMap.find(file);
+ if (i == this->FileMap.end()) {
+ i = this->FileMap.emplace(file, this->Files.size()).first;
+ this->Files.append(file);
+ }
+ return i->second;
+ }
+
+public:
+ bool Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index);
+ Json::Value Dump();
+};
+
+bool BacktraceData::Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index)
+{
+ if (bt.Empty()) {
+ return false;
+ }
+ cmListFileContext const* top = &bt.Top();
+ auto found = this->NodeMap.find(top);
+ if (found != this->NodeMap.end()) {
+ index = found->second;
+ return true;
+ }
+ Json::Value entry = Json::objectValue;
+ entry["file"] = this->AddFile(top->FilePath);
+ if (top->Line) {
+ entry["line"] = static_cast<int>(top->Line);
+ }
+ if (!top->Name.empty()) {
+ entry["command"] = this->AddCommand(top->Name);
+ }
+ Json::ArrayIndex parent;
+ if (this->Add(bt.Pop(), parent)) {
+ entry["parent"] = parent;
+ }
+ index = this->NodeMap[top] = this->Nodes.size();
+ this->Nodes.append(std::move(entry)); // NOLINT(*)
+ return true;
+}
+
+Json::Value BacktraceData::Dump()
+{
+ Json::Value backtraceGraph;
+ this->CommandMap.clear();
+ this->FileMap.clear();
+ this->NodeMap.clear();
+ backtraceGraph["commands"] = std::move(this->Commands);
+ backtraceGraph["files"] = std::move(this->Files);
+ backtraceGraph["nodes"] = std::move(this->Nodes);
+ return backtraceGraph;
+}
+
+static void AddBacktrace(BacktraceData& backtraceGraph, Json::Value& object,
+ cmListFileBacktrace const& bt)
+{
+ Json::ArrayIndex backtrace;
+ if (backtraceGraph.Add(bt, backtrace)) {
+ object["backtrace"] = backtrace;
+ }
+}
+
+static Json::Value DumpCTestInfo(
+ cmCTestRunTest& testRun,
+ cmCTestTestHandler::cmCTestTestProperties& testProperties,
+ BacktraceData& backtraceGraph)
+{
+ Json::Value testInfo = Json::objectValue;
+ // test name should always be present
+ testInfo["name"] = testProperties.Name;
+ std::string const& config = testRun.GetCTest()->GetConfigType();
+ if (!config.empty()) {
+ testInfo["config"] = config;
+ }
+ std::string const& command = testRun.GetActualCommand();
+ if (!command.empty()) {
+ std::vector<std::string> commandAndArgs;
+ commandAndArgs.push_back(command);
+ const std::vector<std::string>& args = testRun.GetArguments();
+ if (!args.empty()) {
+ commandAndArgs.reserve(args.size() + 1);
+ commandAndArgs.insert(commandAndArgs.end(), args.begin(), args.end());
+ }
+ testInfo["command"] = DumpToJsonArray(commandAndArgs);
+ }
+ Json::Value properties = DumpCTestProperties(testProperties);
+ if (!properties.empty()) {
+ testInfo["properties"] = properties;
+ }
+ if (!testProperties.Backtrace.Empty()) {
+ AddBacktrace(backtraceGraph, testInfo, testProperties.Backtrace);
+ }
+ return testInfo;
+}
+
+static Json::Value DumpVersion(int major, int minor)
+{
+ Json::Value version = Json::objectValue;
+ version["major"] = major;
+ version["minor"] = minor;
+ return version;
+}
+
+void cmCTestMultiProcessHandler::PrintOutputAsJson()
+{
+ this->TestHandler->SetMaxIndex(this->FindMaxIndex());
+
+ Json::Value result = Json::objectValue;
+ result["kind"] = "ctestInfo";
+ result["version"] = DumpVersion(1, 0);
+
+ BacktraceData backtraceGraph;
+ Json::Value tests = Json::arrayValue;
+ for (auto& it : this->Properties) {
+ cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
+
+ // Don't worry if this fails, we are only showing the test list, not
+ // running the tests
+ cmWorkingDirectory workdir(p.Directory);
+ cmCTestRunTest testRun(*this);
+ testRun.SetIndex(p.Index);
+ testRun.SetTestProperties(&p);
+ testRun.ComputeArguments();
+
+ Json::Value testInfo = DumpCTestInfo(testRun, p, backtraceGraph);
+ tests.append(testInfo);
+ }
+ result["backtraceGraph"] = backtraceGraph.Dump();
+ result["tests"] = std::move(tests);
+
+ Json::StreamWriterBuilder builder;
+ builder["indentation"] = " ";
+ std::unique_ptr<Json::StreamWriter> jout(builder.newStreamWriter());
+ jout->write(result, &std::cout);
+}
+
// For ShowOnly mode
void cmCTestMultiProcessHandler::PrintTestList()
{
+ if (this->CTest->GetOutputAsJson()) {
+ PrintOutputAsJson();
+ return;
+ }
+
this->TestHandler->SetMaxIndex(this->FindMaxIndex());
int count = 0;
diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h
index 3927a8a..93bb880 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.h
+++ b/Source/CTest/cmCTestMultiProcessHandler.h
@@ -51,6 +51,7 @@ public:
void SetParallelLevel(size_t);
void SetTestLoad(unsigned long load);
virtual void RunTests();
+ void PrintOutputAsJson();
void PrintTestList();
void PrintLabels();
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 10dceca..c786413 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -78,6 +78,10 @@ public:
cmCTest* GetCTest() const { return this->CTest; }
+ std::string& GetActualCommand() { return this->ActualCommand; }
+
+ const std::vector<std::string>& GetArguments() { return this->Arguments; }
+
void FinalizeTest();
bool TimedOutForStopTime() const { return this->TimeoutIsForStopTime; }
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 7c19864..225c99f 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -278,6 +278,8 @@ cmCTest::cmCTest()
this->ExtraVerbose = false;
this->ProduceXML = false;
this->ShowOnly = false;
+ this->OutputAsJson = false;
+ this->OutputAsJsonVersion = 1;
this->RunConfigurationScript = false;
this->UseHTTP10 = false;
this->PrintLabels = false;
@@ -1930,6 +1932,20 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
if (this->CheckArgument(arg, "-N", "--show-only")) {
this->ShowOnly = true;
}
+ if (cmSystemTools::StringStartsWith(arg.c_str(), "--show-only=")) {
+ this->ShowOnly = true;
+
+ // Check if a specific format is requested. Defaults to human readable
+ // text.
+ std::string argWithFormat = "--show-only=";
+ std::string format = arg.substr(argWithFormat.length());
+ if (format == "json-v1") {
+ // Force quiet mode so the only output is the json object model.
+ this->Quiet = true;
+ this->OutputAsJson = true;
+ this->OutputAsJsonVersion = 1;
+ }
+ }
if (this->CheckArgument(arg, "-O", "--output-log") && i < args.size() - 1) {
i++;
@@ -2630,6 +2646,16 @@ bool cmCTest::GetShowOnly()
return this->ShowOnly;
}
+bool cmCTest::GetOutputAsJson()
+{
+ return this->OutputAsJson;
+}
+
+int cmCTest::GetOutputAsJsonVersion()
+{
+ return this->OutputAsJsonVersion;
+}
+
int cmCTest::GetMaxTestNameWidth() const
{
return this->MaxTestNameWidth;
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index 480204a..2b40ca3 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -215,6 +215,10 @@ public:
/** Should we only show what we would do? */
bool GetShowOnly();
+ bool GetOutputAsJson();
+
+ int GetOutputAsJsonVersion();
+
bool ShouldUseHTTP10() { return this->UseHTTP10; }
bool ShouldPrintLabels() { return this->PrintLabels; }
@@ -507,6 +511,8 @@ private:
t_TestingHandlers TestingHandlers;
bool ShowOnly;
+ bool OutputAsJson;
+ int OutputAsJsonVersion;
/** Map of configuration properties */
typedef std::map<std::string, std::string> CTestConfigurationMap;
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index ca412ae..8ba126f 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -46,7 +46,10 @@ static const char* cmDocumentationOptions[][2] = {
"given number of jobs." },
{ "-Q,--quiet", "Make ctest quiet." },
{ "-O <file>, --output-log <file>", "Output to log file" },
- { "-N,--show-only", "Disable actual execution of tests." },
+ { "-N,--show-only[=format]",
+ "Disable actual execution of tests. The optional 'format' defines the "
+ "format of the test information and can be 'human' for the current text "
+ "format or 'json-v1' for json format. Defaults to 'human'." },
{ "-L <regex>, --label-regex <regex>",
"Run tests with labels matching "
"regular expression." },
generated by cgit v0.12 at 2024-12-29 03:40:01 (GMT)