From 00530d74d5d07a320c998d6caccc00cf4e59b06d Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Fri, 2 Nov 2018 09:36:31 -0400
Subject: Tests: Pass python interpreter into RunCMake.CTestCommandLine
This will be useful for adding python-based result checks.
---
Tests/RunCMake/CMakeLists.txt | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index 67fd65a..e222376 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -385,8 +385,9 @@ add_RunCMake_test(CPackConfig)
add_RunCMake_test(CPackInstallProperties)
add_RunCMake_test(ExternalProject)
add_RunCMake_test(FetchContent)
+set(CTestCommandLine_ARGS -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE})
if(NOT CMake_TEST_EXTERNAL_CMAKE)
- set(CTestCommandLine_ARGS -DTEST_AFFINITY=$<TARGET_FILE:testAffinity>)
+ list(APPEND CTestCommandLine_ARGS -DTEST_AFFINITY=$<TARGET_FILE:testAffinity>)
endif()
add_executable(print_stdin print_stdin.c)
add_RunCMake_test(CTestCommandLine -DTEST_PRINT_STDIN=$<TARGET_FILE:print_stdin>)
--
cgit v0.12
From 7b81d8c21e0a0d8756f0afdc0530c2d06ee3bcd4 Mon Sep 17 00:00:00 2001
From: Justin Goshi <jgoshi@microsoft.com>
Date: Thu, 18 Oct 2018 11:33:01 -0700
Subject: TestGenerator: Record support file and line where test was added
Add internal test properties that ctest can use to report where
the test was added in CMake code.
---
Source/CTest/cmCTestTestHandler.cxx | 26 ++++++++++++++++
Source/CTest/cmCTestTestHandler.h | 3 ++
Source/cmTestGenerator.cxx | 59 ++++++++++++++++++++++++++-----------
Source/cmTestGenerator.h | 3 ++
4 files changed, 74 insertions(+), 17 deletions(-)
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index 2e1bb0a..9fd2299 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -2147,6 +2147,32 @@ bool cmCTestTestHandler::SetTestsProperties(
for (std::string const& t : tests) {
for (cmCTestTestProperties& rt : this->TestList) {
if (t == rt.Name) {
+ if (key == "_BACKTRACE_TRIPLES") {
+ std::vector<std::string> triples;
+ // allow empty args in the triples
+ cmSystemTools::ExpandListArgument(val, triples, true);
+
+ // Ensure we have complete triples otherwise the data is corrupt.
+ if (triples.size() % 3 == 0) {
+ cmState state;
+ rt.Backtrace = cmListFileBacktrace(state.CreateBaseSnapshot());
+
+ // the first entry represents the top of the trace so we need to
+ // reconstruct the backtrace in reverse
+ for (size_t i = triples.size(); i >= 3; i -= 3) {
+ cmListFileContext fc;
+ fc.FilePath = triples[i - 3];
+ long line = 0;
+ if (!cmSystemTools::StringToLong(triples[i - 2].c_str(),
+ &line)) {
+ line = 0;
+ }
+ fc.Line = line;
+ fc.Name = triples[i - 1];
+ rt.Backtrace = rt.Backtrace.Push(fc);
+ }
+ }
+ }
if (key == "WILL_FAIL") {
rt.WillFail = cmSystemTools::IsOn(val);
}
diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h
index bcacf23..0b557db 100644
--- a/Source/CTest/cmCTestTestHandler.h
+++ b/Source/CTest/cmCTestTestHandler.h
@@ -7,6 +7,7 @@
#include "cmCTestGenericHandler.h"
#include "cmDuration.h"
+#include "cmListFileCache.h"
#include "cmsys/RegularExpression.hxx"
#include <chrono>
@@ -141,6 +142,8 @@ public:
std::set<std::string> FixturesCleanup;
std::set<std::string> FixturesRequired;
std::set<std::string> RequireSuccessDepends;
+ // Private test generator properties used to track backtraces
+ cmListFileBacktrace Backtrace;
};
struct cmCTestTestResult
diff --git a/Source/cmTestGenerator.cxx b/Source/cmTestGenerator.cxx
index 796d2df..e4ced6e 100644
--- a/Source/cmTestGenerator.cxx
+++ b/Source/cmTestGenerator.cxx
@@ -7,14 +7,16 @@
#include "cmGeneratorExpression.h"
#include "cmGeneratorTarget.h"
+#include "cmListFileCache.h"
#include "cmLocalGenerator.h"
#include "cmOutputConverter.h"
#include "cmProperty.h"
-#include "cmPropertyMap.h"
#include "cmStateTypes.h"
#include "cmSystemTools.h"
#include "cmTest.h"
+class cmPropertyMap;
+
cmTestGenerator::cmTestGenerator(
cmTest* test, std::vector<std::string> const& configurations)
: cmScriptGenerator("CTEST_CONFIGURATION_TYPE", configurations)
@@ -121,16 +123,15 @@ void cmTestGenerator::GenerateScriptForConfig(std::ostream& os,
// Output properties for the test.
cmPropertyMap& pm = this->Test->GetProperties();
- if (!pm.empty()) {
- os << indent << "set_tests_properties(" << this->Test->GetName()
- << " PROPERTIES ";
- for (auto const& i : pm) {
- os << " " << i.first << " "
- << cmOutputConverter::EscapeForCMake(
- ge.Parse(i.second.GetValue())->Evaluate(this->LG, config));
- }
- os << ")" << std::endl;
+ os << indent << "set_tests_properties(" << this->Test->GetName()
+ << " PROPERTIES ";
+ for (auto const& i : pm) {
+ os << " " << i.first << " "
+ << cmOutputConverter::EscapeForCMake(
+ ge.Parse(i.second.GetValue())->Evaluate(this->LG, config));
}
+ this->GenerateInternalProperties(os);
+ os << ")" << std::endl;
}
void cmTestGenerator::GenerateScriptNoConfig(std::ostream& os, Indent indent)
@@ -179,13 +180,37 @@ void cmTestGenerator::GenerateOldStyle(std::ostream& fout, Indent indent)
// Output properties for the test.
cmPropertyMap& pm = this->Test->GetProperties();
- if (!pm.empty()) {
- fout << indent << "set_tests_properties(" << this->Test->GetName()
- << " PROPERTIES ";
- for (auto const& i : pm) {
- fout << " " << i.first << " "
- << cmOutputConverter::EscapeForCMake(i.second.GetValue());
+ fout << indent << "set_tests_properties(" << this->Test->GetName()
+ << " PROPERTIES ";
+ for (auto const& i : pm) {
+ fout << " " << i.first << " "
+ << cmOutputConverter::EscapeForCMake(i.second.GetValue());
+ }
+ this->GenerateInternalProperties(fout);
+ fout << ")" << std::endl;
+}
+
+void cmTestGenerator::GenerateInternalProperties(std::ostream& os)
+{
+ cmListFileBacktrace bt = this->Test->GetBacktrace();
+ if (bt.Empty()) {
+ return;
+ }
+
+ os << " "
+ << "_BACKTRACE_TRIPLES"
+ << " \"";
+
+ bool prependTripleSeparator = false;
+ while (!bt.Empty()) {
+ const auto& entry = bt.Top();
+ if (prependTripleSeparator) {
+ os << ";";
}
- fout << ")" << std::endl;
+ os << entry.FilePath << ";" << entry.Line << ";" << entry.Name;
+ bt = bt.Pop();
+ prependTripleSeparator = true;
}
+
+ os << "\"";
}
diff --git a/Source/cmTestGenerator.h b/Source/cmTestGenerator.h
index 73d05a3..f26d2ff 100644
--- a/Source/cmTestGenerator.h
+++ b/Source/cmTestGenerator.h
@@ -35,6 +35,9 @@ public:
cmTest* GetTest() const;
+private:
+ void GenerateInternalProperties(std::ostream& os);
+
protected:
void GenerateScriptConfigs(std::ostream& os, Indent indent) override;
void GenerateScriptActions(std::ostream& os, Indent indent) override;
--
cgit v0.12
From fc41a95f0803abb2b5daa4f0eb21cf38c625bd26 Mon Sep 17 00:00:00 2001
From: Justin Goshi <jgoshi@microsoft.com>
Date: Thu, 18 Oct 2018 11:34:37 -0700
Subject: CTest: Add --show-only[=format] option to print test info
format can be 'human' to print the current text format or 'json-v1' to
print the test object model in json format and is useful for IDEs who
want to gather information about the tests. Defaults to 'human' format.
---
Help/manual/ctest.1.rst | 71 +++++-
Help/release/dev/ctest-show-only-json-v1.rst | 6 +
Source/CTest/cmCTestMultiProcessHandler.cxx | 331 +++++++++++++++++++++++++++
Source/CTest/cmCTestMultiProcessHandler.h | 1 +
Source/CTest/cmCTestRunTest.h | 4 +
Source/cmCTest.cxx | 26 +++
Source/cmCTest.h | 6 +
Source/ctest.cxx | 5 +-
8 files changed, 448 insertions(+), 2 deletions(-)
create mode 100644 Help/release/dev/ctest-show-only-json-v1.rst
diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst
index 1ef20ab..8490e3d 100644
--- a/Help/manual/ctest.1.rst
+++ b/Help/manual/ctest.1.rst
@@ -109,13 +109,23 @@ Options
This option tells CTest to write all its output to a log file.
-``-N,--show-only``
+``-N,--show-only[=<format>]``
Disable actual execution of tests.
This option tells CTest to list the tests that would be run but not
actually run them. Useful in conjunction with the ``-R`` and ``-E``
options.
+ ``<format>`` can be one of the following values.
+
+ ``human``
+ Human-friendly output. This is not guaranteed to be stable.
+ This is the default.
+
+ ``json-v1``
+ Dump the test information in JSON format.
+ See `Show as JSON Object Model`_.
+
``-L <regex>, --label-regex <regex>``
Run tests with labels matching regular expression.
@@ -1163,6 +1173,65 @@ Configuration settings include:
* :module:`CTest` module variable: ``TRIGGER_SITE`` if set,
else ``CTEST_TRIGGER_SITE``
+.. _`Show as JSON Object Model`:
+
+Show as JSON Object Model
+=========================
+
+When the ``--show-only=json-v1`` command line option is given, the test
+information is output in JSON format. Version 1.0 of the JSON object
+model is defined as follows:
+
+``kind``
+ The string "ctestInfo".
+
+``version``
+ A JSON object specifying the version components. Its members are
+
+ ``major``
+ A non-negative integer specifying the major version component.
+ ``minor``
+ A non-negative integer specifying the minor version component.
+
+``backtraceGraph``
+ JSON object representing backtrace information with the
+ following members:
+
+ ``commands``
+ List of command names.
+ ``files``
+ List of file names.
+ ``nodes``
+ List of node JSON objects with members:
+
+ ``command``
+ Index into the ``commands`` member of the ``backtraceGraph``.
+ ``file``
+ Index into the ``files`` member of the ``backtraceGraph``.
+ ``line``
+ Line number in the file where the backtrace was added.
+ ``parent``
+ Index into the ``nodes`` member of the ``backtraceGraph``
+ representing the parent in the graph.
+
+``tests``
+ A JSON array listing information about each test. Each entry
+ is a JSON object with members:
+
+ ``name``
+ Test name.
+ ``config``
+ Configuration that the test can run on.
+ Empty string means any config.
+ ``command``
+ List where the first element is the test command and the
+ remaining elements are the command arguments.
+ ``backtrace``
+ Index into the ``nodes`` member of the ``backtraceGraph``.
+ ``properties``
+ Test properties.
+ Can contain keys for each of the supported test properties.
+
See Also
========
diff --git a/Help/release/dev/ctest-show-only-json-v1.rst b/Help/release/dev/ctest-show-only-json-v1.rst
new file mode 100644
index 0000000..f593e7e
--- /dev/null
+++ b/Help/release/dev/ctest-show-only-json-v1.rst
@@ -0,0 +1,6 @@
+ctest-show-only-json-v1
+-----------------------
+
+* :manual:`ctest(1)` gained a ``--show-only=json-v1`` option to show the
+ list of tests in a machine-readable JSON format.
+ See the :ref:`Show as JSON Object Model` section of the manual.
diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index f026001..8867323 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -6,9 +6,13 @@
#include "cmCTest.h"
#include "cmCTestRunTest.h"
#include "cmCTestTestHandler.h"
+#include "cmDuration.h"
+#include "cmListFileCache.h"
#include "cmSystemTools.h"
#include "cmWorkingDirectory.h"
+#include "cm_jsoncpp_value.h"
+#include "cm_jsoncpp_writer.h"
#include "cm_uv.h"
#include "cmUVSignalHackRAII.h" // IWYU pragma: keep
@@ -20,13 +24,19 @@
#include <chrono>
#include <cstring>
#include <iomanip>
+#include <iostream>
#include <list>
#include <math.h>
#include <sstream>
#include <stack>
#include <stdlib.h>
+#include <unordered_map>
#include <utility>
+namespace cmsys {
+class RegularExpression;
+}
+
class TestComparator
{
public:
@@ -725,9 +735,330 @@ void cmCTestMultiProcessHandler::MarkFinished()
cmSystemTools::RemoveFile(fname);
}
+static Json::Value DumpToJsonArray(const std::set<std::string>& values)
+{
+ Json::Value jsonArray = Json::arrayValue;
+ for (auto& it : values) {
+ jsonArray.append(it);
+ }
+ return jsonArray;
+}
+
+static Json::Value DumpToJsonArray(const std::vector<std::string>& values)
+{
+ Json::Value jsonArray = Json::arrayValue;
+ for (auto& it : values) {
+ jsonArray.append(it);
+ }
+ return jsonArray;
+}
+
+static Json::Value DumpRegExToJsonArray(
+ const std::vector<std::pair<cmsys::RegularExpression, std::string>>& values)
+{
+ Json::Value jsonArray = Json::arrayValue;
+ for (auto& it : values) {
+ jsonArray.append(it.second);
+ }
+ return jsonArray;
+}
+
+static Json::Value DumpMeasurementToJsonArray(
+ const std::map<std::string, std::string>& values)
+{
+ Json::Value jsonArray = Json::arrayValue;
+ for (auto& it : values) {
+ Json::Value measurement = Json::objectValue;
+ measurement["measurement"] = it.first;
+ measurement["value"] = it.second;
+ jsonArray.append(measurement);
+ }
+ return jsonArray;
+}
+
+static Json::Value DumpTimeoutAfterMatch(
+ cmCTestTestHandler::cmCTestTestProperties& testProperties)
+{
+ Json::Value timeoutAfterMatch = Json::objectValue;
+ timeoutAfterMatch["timeout"] = testProperties.AlternateTimeout.count();
+ timeoutAfterMatch["regex"] =
+ DumpRegExToJsonArray(testProperties.TimeoutRegularExpressions);
+ return timeoutAfterMatch;
+}
+
+static Json::Value DumpCTestProperty(std::string const& name,
+ Json::Value value)
+{
+ Json::Value property = Json::objectValue;
+ property["name"] = name;
+ property["value"] = std::move(value);
+ return property;
+}
+
+static Json::Value DumpCTestProperties(
+ cmCTestTestHandler::cmCTestTestProperties& testProperties)
+{
+ Json::Value properties = Json::arrayValue;
+ if (!testProperties.AttachOnFail.empty()) {
+ properties.append(DumpCTestProperty(
+ "ATTACHED_FILES_ON_FAIL", DumpToJsonArray(testProperties.AttachOnFail)));
+ }
+ if (!testProperties.AttachedFiles.empty()) {
+ properties.append(DumpCTestProperty(
+ "ATTACHED_FILES", DumpToJsonArray(testProperties.AttachedFiles)));
+ }
+ if (testProperties.Cost != 0.0f) {
+ properties.append(
+ DumpCTestProperty("COST", static_cast<double>(testProperties.Cost)));
+ }
+ if (!testProperties.Depends.empty()) {
+ properties.append(
+ DumpCTestProperty("DEPENDS", DumpToJsonArray(testProperties.Depends)));
+ }
+ if (testProperties.Disabled) {
+ properties.append(DumpCTestProperty("DISABLED", testProperties.Disabled));
+ }
+ if (!testProperties.Environment.empty()) {
+ properties.append(DumpCTestProperty(
+ "ENVIRONMENT", DumpToJsonArray(testProperties.Environment)));
+ }
+ if (!testProperties.ErrorRegularExpressions.empty()) {
+ properties.append(DumpCTestProperty(
+ "FAIL_REGULAR_EXPRESSION",
+ DumpRegExToJsonArray(testProperties.ErrorRegularExpressions)));
+ }
+ if (!testProperties.FixturesCleanup.empty()) {
+ properties.append(DumpCTestProperty(
+ "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup)));
+ }
+ if (!testProperties.FixturesRequired.empty()) {
+ properties.append(DumpCTestProperty(
+ "FIXTURES_REQUIRED", DumpToJsonArray(testProperties.FixturesRequired)));
+ }
+ if (!testProperties.FixturesSetup.empty()) {
+ properties.append(DumpCTestProperty(
+ "FIXTURES_SETUP", DumpToJsonArray(testProperties.FixturesSetup)));
+ }
+ if (!testProperties.Labels.empty()) {
+ properties.append(
+ DumpCTestProperty("LABELS", DumpToJsonArray(testProperties.Labels)));
+ }
+ if (!testProperties.Measurements.empty()) {
+ properties.append(DumpCTestProperty(
+ "MEASUREMENT", DumpMeasurementToJsonArray(testProperties.Measurements)));
+ }
+ if (!testProperties.RequiredRegularExpressions.empty()) {
+ properties.append(DumpCTestProperty(
+ "PASS_REGULAR_EXPRESSION",
+ DumpRegExToJsonArray(testProperties.RequiredRegularExpressions)));
+ }
+ if (testProperties.WantAffinity) {
+ properties.append(
+ DumpCTestProperty("PROCESSOR_AFFINITY", testProperties.WantAffinity));
+ }
+ if (testProperties.Processors != 1) {
+ properties.append(
+ DumpCTestProperty("PROCESSORS", testProperties.Processors));
+ }
+ if (!testProperties.RequiredFiles.empty()) {
+ properties["REQUIRED_FILES"] =
+ DumpToJsonArray(testProperties.RequiredFiles);
+ }
+ if (!testProperties.LockedResources.empty()) {
+ properties.append(DumpCTestProperty(
+ "RESOURCE_LOCK", DumpToJsonArray(testProperties.LockedResources)));
+ }
+ if (testProperties.RunSerial) {
+ properties.append(
+ DumpCTestProperty("RUN_SERIAL", testProperties.RunSerial));
+ }
+ if (testProperties.SkipReturnCode != -1) {
+ properties.append(
+ DumpCTestProperty("SKIP_RETURN_CODE", testProperties.SkipReturnCode));
+ }
+ if (testProperties.ExplicitTimeout) {
+ properties.append(
+ DumpCTestProperty("TIMEOUT", testProperties.Timeout.count()));
+ }
+ if (!testProperties.TimeoutRegularExpressions.empty()) {
+ properties.append(DumpCTestProperty(
+ "TIMEOUT_AFTER_MATCH", DumpTimeoutAfterMatch(testProperties)));
+ }
+ if (testProperties.WillFail) {
+ properties.append(DumpCTestProperty("WILL_FAIL", testProperties.WillFail));
+ }
+ if (!testProperties.Directory.empty()) {
+ properties.append(
+ DumpCTestProperty("WORKING_DIRECTORY", testProperties.Directory));
+ }
+ return properties;
+}
+
+class BacktraceData
+{
+ std::unordered_map<std::string, Json::ArrayIndex> CommandMap;
+ std::unordered_map<std::string, Json::ArrayIndex> FileMap;
+ std::unordered_map<cmListFileContext const*, Json::ArrayIndex> NodeMap;
+ Json::Value Commands = Json::arrayValue;
+ Json::Value Files = Json::arrayValue;
+ Json::Value Nodes = Json::arrayValue;
+
+ Json::ArrayIndex AddCommand(std::string const& command)
+ {
+ auto i = this->CommandMap.find(command);
+ if (i == this->CommandMap.end()) {
+ i = this->CommandMap.emplace(command, this->Commands.size()).first;
+ this->Commands.append(command);
+ }
+ return i->second;
+ }
+
+ Json::ArrayIndex AddFile(std::string const& file)
+ {
+ auto i = this->FileMap.find(file);
+ if (i == this->FileMap.end()) {
+ i = this->FileMap.emplace(file, this->Files.size()).first;
+ this->Files.append(file);
+ }
+ return i->second;
+ }
+
+public:
+ bool Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index);
+ Json::Value Dump();
+};
+
+bool BacktraceData::Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index)
+{
+ if (bt.Empty()) {
+ return false;
+ }
+ cmListFileContext const* top = &bt.Top();
+ auto found = this->NodeMap.find(top);
+ if (found != this->NodeMap.end()) {
+ index = found->second;
+ return true;
+ }
+ Json::Value entry = Json::objectValue;
+ entry["file"] = this->AddFile(top->FilePath);
+ if (top->Line) {
+ entry["line"] = static_cast<int>(top->Line);
+ }
+ if (!top->Name.empty()) {
+ entry["command"] = this->AddCommand(top->Name);
+ }
+ Json::ArrayIndex parent;
+ if (this->Add(bt.Pop(), parent)) {
+ entry["parent"] = parent;
+ }
+ index = this->NodeMap[top] = this->Nodes.size();
+ this->Nodes.append(std::move(entry)); // NOLINT(*)
+ return true;
+}
+
+Json::Value BacktraceData::Dump()
+{
+ Json::Value backtraceGraph;
+ this->CommandMap.clear();
+ this->FileMap.clear();
+ this->NodeMap.clear();
+ backtraceGraph["commands"] = std::move(this->Commands);
+ backtraceGraph["files"] = std::move(this->Files);
+ backtraceGraph["nodes"] = std::move(this->Nodes);
+ return backtraceGraph;
+}
+
+static void AddBacktrace(BacktraceData& backtraceGraph, Json::Value& object,
+ cmListFileBacktrace const& bt)
+{
+ Json::ArrayIndex backtrace;
+ if (backtraceGraph.Add(bt, backtrace)) {
+ object["backtrace"] = backtrace;
+ }
+}
+
+static Json::Value DumpCTestInfo(
+ cmCTestRunTest& testRun,
+ cmCTestTestHandler::cmCTestTestProperties& testProperties,
+ BacktraceData& backtraceGraph)
+{
+ Json::Value testInfo = Json::objectValue;
+ // test name should always be present
+ testInfo["name"] = testProperties.Name;
+ std::string const& config = testRun.GetCTest()->GetConfigType();
+ if (!config.empty()) {
+ testInfo["config"] = config;
+ }
+ std::string const& command = testRun.GetActualCommand();
+ if (!command.empty()) {
+ std::vector<std::string> commandAndArgs;
+ commandAndArgs.push_back(command);
+ const std::vector<std::string>& args = testRun.GetArguments();
+ if (!args.empty()) {
+ commandAndArgs.reserve(args.size() + 1);
+ commandAndArgs.insert(commandAndArgs.end(), args.begin(), args.end());
+ }
+ testInfo["command"] = DumpToJsonArray(commandAndArgs);
+ }
+ Json::Value properties = DumpCTestProperties(testProperties);
+ if (!properties.empty()) {
+ testInfo["properties"] = properties;
+ }
+ if (!testProperties.Backtrace.Empty()) {
+ AddBacktrace(backtraceGraph, testInfo, testProperties.Backtrace);
+ }
+ return testInfo;
+}
+
+static Json::Value DumpVersion(int major, int minor)
+{
+ Json::Value version = Json::objectValue;
+ version["major"] = major;
+ version["minor"] = minor;
+ return version;
+}
+
+void cmCTestMultiProcessHandler::PrintOutputAsJson()
+{
+ this->TestHandler->SetMaxIndex(this->FindMaxIndex());
+
+ Json::Value result = Json::objectValue;
+ result["kind"] = "ctestInfo";
+ result["version"] = DumpVersion(1, 0);
+
+ BacktraceData backtraceGraph;
+ Json::Value tests = Json::arrayValue;
+ for (auto& it : this->Properties) {
+ cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
+
+ // Don't worry if this fails, we are only showing the test list, not
+ // running the tests
+ cmWorkingDirectory workdir(p.Directory);
+ cmCTestRunTest testRun(*this);
+ testRun.SetIndex(p.Index);
+ testRun.SetTestProperties(&p);
+ testRun.ComputeArguments();
+
+ Json::Value testInfo = DumpCTestInfo(testRun, p, backtraceGraph);
+ tests.append(testInfo);
+ }
+ result["backtraceGraph"] = backtraceGraph.Dump();
+ result["tests"] = std::move(tests);
+
+ Json::StreamWriterBuilder builder;
+ builder["indentation"] = " ";
+ std::unique_ptr<Json::StreamWriter> jout(builder.newStreamWriter());
+ jout->write(result, &std::cout);
+}
+
// For ShowOnly mode
void cmCTestMultiProcessHandler::PrintTestList()
{
+ if (this->CTest->GetOutputAsJson()) {
+ PrintOutputAsJson();
+ return;
+ }
+
this->TestHandler->SetMaxIndex(this->FindMaxIndex());
int count = 0;
diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h
index 3927a8a..93bb880 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.h
+++ b/Source/CTest/cmCTestMultiProcessHandler.h
@@ -51,6 +51,7 @@ public:
void SetParallelLevel(size_t);
void SetTestLoad(unsigned long load);
virtual void RunTests();
+ void PrintOutputAsJson();
void PrintTestList();
void PrintLabels();
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 10dceca..c786413 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -78,6 +78,10 @@ public:
cmCTest* GetCTest() const { return this->CTest; }
+ std::string& GetActualCommand() { return this->ActualCommand; }
+
+ const std::vector<std::string>& GetArguments() { return this->Arguments; }
+
void FinalizeTest();
bool TimedOutForStopTime() const { return this->TimeoutIsForStopTime; }
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 7c19864..225c99f 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -278,6 +278,8 @@ cmCTest::cmCTest()
this->ExtraVerbose = false;
this->ProduceXML = false;
this->ShowOnly = false;
+ this->OutputAsJson = false;
+ this->OutputAsJsonVersion = 1;
this->RunConfigurationScript = false;
this->UseHTTP10 = false;
this->PrintLabels = false;
@@ -1930,6 +1932,20 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
if (this->CheckArgument(arg, "-N", "--show-only")) {
this->ShowOnly = true;
}
+ if (cmSystemTools::StringStartsWith(arg.c_str(), "--show-only=")) {
+ this->ShowOnly = true;
+
+ // Check if a specific format is requested. Defaults to human readable
+ // text.
+ std::string argWithFormat = "--show-only=";
+ std::string format = arg.substr(argWithFormat.length());
+ if (format == "json-v1") {
+ // Force quiet mode so the only output is the json object model.
+ this->Quiet = true;
+ this->OutputAsJson = true;
+ this->OutputAsJsonVersion = 1;
+ }
+ }
if (this->CheckArgument(arg, "-O", "--output-log") && i < args.size() - 1) {
i++;
@@ -2630,6 +2646,16 @@ bool cmCTest::GetShowOnly()
return this->ShowOnly;
}
+bool cmCTest::GetOutputAsJson()
+{
+ return this->OutputAsJson;
+}
+
+int cmCTest::GetOutputAsJsonVersion()
+{
+ return this->OutputAsJsonVersion;
+}
+
int cmCTest::GetMaxTestNameWidth() const
{
return this->MaxTestNameWidth;
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index 480204a..2b40ca3 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -215,6 +215,10 @@ public:
/** Should we only show what we would do? */
bool GetShowOnly();
+ bool GetOutputAsJson();
+
+ int GetOutputAsJsonVersion();
+
bool ShouldUseHTTP10() { return this->UseHTTP10; }
bool ShouldPrintLabels() { return this->PrintLabels; }
@@ -507,6 +511,8 @@ private:
t_TestingHandlers TestingHandlers;
bool ShowOnly;
+ bool OutputAsJson;
+ int OutputAsJsonVersion;
/** Map of configuration properties */
typedef std::map<std::string, std::string> CTestConfigurationMap;
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index ca412ae..8ba126f 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -46,7 +46,10 @@ static const char* cmDocumentationOptions[][2] = {
"given number of jobs." },
{ "-Q,--quiet", "Make ctest quiet." },
{ "-O <file>, --output-log <file>", "Output to log file" },
- { "-N,--show-only", "Disable actual execution of tests." },
+ { "-N,--show-only[=format]",
+ "Disable actual execution of tests. The optional 'format' defines the "
+ "format of the test information and can be 'human' for the current text "
+ "format or 'json-v1' for json format. Defaults to 'human'." },
{ "-L <regex>, --label-regex <regex>",
"Run tests with labels matching "
"regular expression." },
--
cgit v0.12
From 67209a9291c672f7c6d6cc36f28e6adbc9c42b8c Mon Sep 17 00:00:00 2001
From: Brad King <brad.king@kitware.com>
Date: Thu, 1 Nov 2018 11:42:23 -0400
Subject: Tests: Add cases for ctest --show-only=json-v1
---
Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 29 ++++++
.../RunCMake/CTestCommandLine/ShowAsJson1-check.py | 106 +++++++++++++++++++++
.../RunCMake/CTestCommandLine/ShowAsJson_check.py | 24 +++++
3 files changed, 159 insertions(+)
create mode 100644 Tests/RunCMake/CTestCommandLine/ShowAsJson1-check.py
create mode 100644 Tests/RunCMake/CTestCommandLine/ShowAsJson_check.py
diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index 750ae50..cae14b1 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -173,3 +173,32 @@ function(run_TestStdin)
run_cmake_command(TestStdin ${CMAKE_CTEST_COMMAND} -V)
endfunction()
run_TestStdin()
+
+function(ShowAsJson_check_python v)
+ set(json_file "${RunCMake_TEST_BINARY_DIR}/ctest.json")
+ file(WRITE "${json_file}" "${actual_stdout}")
+ set(actual_stdout "" PARENT_SCOPE)
+ execute_process(
+ COMMAND ${PYTHON_EXECUTABLE} "${RunCMake_SOURCE_DIR}/ShowAsJson${v}-check.py" "${json_file}"
+ RESULT_VARIABLE result
+ OUTPUT_VARIABLE output
+ ERROR_VARIABLE output
+ )
+ if(NOT result EQUAL 0)
+ string(REPLACE "\n" "\n " output " ${output}")
+ set(RunCMake_TEST_FAILED "Unexpected output:\n${output}" PARENT_SCOPE)
+ endif()
+endfunction()
+
+function(run_ShowAsJson)
+ set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/ShowAsJson)
+ set(RunCMake_TEST_NO_CLEAN 1)
+ file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+ file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+ file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "
+ add_test(ShowAsJson \"${CMAKE_COMMAND}\" -E echo)
+ set_tests_properties(ShowAsJson PROPERTIES WILL_FAIL true _BACKTRACE_TRIPLES \"file1;1;add_test;file0;;\")
+")
+ run_cmake_command(ShowAsJsonVersionOne ${CMAKE_CTEST_COMMAND} --show-only=json-v1)
+endfunction()
+run_ShowAsJson()
diff --git a/Tests/RunCMake/CTestCommandLine/ShowAsJson1-check.py b/Tests/RunCMake/CTestCommandLine/ShowAsJson1-check.py
new file mode 100644
index 0000000..d794e7d
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/ShowAsJson1-check.py
@@ -0,0 +1,106 @@
+from ShowAsJson_check import *
+
+def check_kind(k):
+ assert is_string(k)
+ assert k == "ctestInfo"
+
+def check_version(v):
+ assert is_dict(v)
+ assert sorted(v.keys()) == ["major", "minor"]
+ assert is_int(v["major"])
+ assert is_int(v["minor"])
+ assert v["major"] == 1
+ assert v["minor"] == 0
+
+def check_backtracegraph(b):
+ assert is_dict(b)
+ assert sorted(b.keys()) == ["commands", "files", "nodes"]
+ check_backtracegraph_commands(b["commands"])
+ check_backtracegraph_files(b["files"])
+ check_backtracegraph_nodes(b["nodes"])
+
+def check_backtracegraph_commands(c):
+ assert is_list(c)
+ assert len(c) == 1
+ assert is_string(c[0])
+ assert c[0] == "add_test"
+
+def check_backtracegraph_files(f):
+ assert is_list(f)
+ assert len(f) == 2
+ assert is_string(f[0])
+ assert is_string(f[1])
+ assert f[0] == "file1"
+ assert f[1] == "file0"
+
+def check_backtracegraph_nodes(n):
+ assert is_list(n)
+ assert len(n) == 2
+ node = n[0]
+ assert is_dict(node)
+ assert sorted(node.keys()) == ["file"]
+ assert is_int(node["file"])
+ assert node["file"] == 1
+ node = n[1]
+ assert is_dict(node)
+ assert sorted(node.keys()) == ["command", "file", "line", "parent"]
+ assert is_int(node["command"])
+ assert is_int(node["file"])
+ assert is_int(node["line"])
+ assert is_int(node["parent"])
+ assert node["command"] == 0
+ assert node["file"] == 0
+ assert node["line"] == 1
+ assert node["parent"] == 0
+
+def check_command(c):
+ assert is_list(c)
+ assert len(c) == 3
+ assert is_string(c[0])
+ check_re(c[0], "/cmake(\.exe)?$")
+ assert is_string(c[1])
+ assert c[1] == "-E"
+ assert is_string(c[2])
+ assert c[2] == "echo"
+
+def check_willfail_property(p):
+ assert is_dict(p)
+ assert sorted(p.keys()) == ["name", "value"]
+ assert is_string(p["name"])
+ assert is_bool(p["value"])
+ assert p["name"] == "WILL_FAIL"
+ assert p["value"] == True
+
+def check_workingdir_property(p):
+ assert is_dict(p)
+ assert sorted(p.keys()) == ["name", "value"]
+ assert is_string(p["name"])
+ assert is_string(p["value"])
+ assert p["name"] == "WORKING_DIRECTORY"
+ assert p["value"].endswith("Tests/RunCMake/CTestCommandLine/ShowAsJson")
+
+def check_properties(p):
+ assert is_list(p)
+ assert len(p) == 2
+ check_willfail_property(p[0])
+ check_workingdir_property(p[1])
+
+def check_tests(t):
+ assert is_list(t)
+ assert len(t) == 1
+ test = t[0]
+ assert is_dict(test)
+ assert sorted(test.keys()) == ["backtrace", "command", "name", "properties"]
+ assert is_int(test["backtrace"])
+ assert test["backtrace"] == 1
+ check_command(test["command"])
+ assert is_string(test["name"])
+ assert test["name"] == "ShowAsJson"
+ check_properties(test["properties"])
+
+assert is_dict(ctest_json)
+assert sorted(ctest_json.keys()) == ["backtraceGraph", "kind", "tests", "version"]
+check_backtracegraph(ctest_json["backtraceGraph"])
+check_kind(ctest_json["kind"])
+check_version(ctest_json["version"])
+check_tests(ctest_json["tests"])
diff --git a/Tests/RunCMake/CTestCommandLine/ShowAsJson_check.py b/Tests/RunCMake/CTestCommandLine/ShowAsJson_check.py
new file mode 100644
index 0000000..493c9e5
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/ShowAsJson_check.py
@@ -0,0 +1,24 @@
+import sys
+import json
+import re
+
+def is_bool(x):
+ return isinstance(x, bool)
+
+def is_dict(x):
+ return isinstance(x, dict)
+
+def is_list(x):
+ return isinstance(x, list)
+
+def is_int(x):
+ return isinstance(x, int) or isinstance(x, long)
+
+def is_string(x):
+ return isinstance(x, str) or isinstance(x, unicode)
+
+def check_re(x, regex):
+ assert re.search(regex, x)
+
+with open(sys.argv[1]) as f:
+ ctest_json = json.load(f)
--
cgit v0.12