Merge topic 'ctest-show-as-json'

67209a9291 Tests: Add cases for ctest --show-only=json-v1
fc41a95f08 CTest: Add --show-only[=format] option to print test info
7b81d8c21e TestGenerator: Record support file and line where test was added
00530d74d5 Tests: Pass python interpreter into RunCMake.CTestCommandLine

Acked-by: Kitware Robot <kwrobot@kitware.com>
Acked-by: Brad King <brad.king@kitware.com>
Merge-request: !2499

10 files changed, 446 insertions, 18 deletions

diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index f026001..8867323 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -6,9 +6,13 @@
 #include "cmCTest.h"
 #include "cmCTestRunTest.h"
 #include "cmCTestTestHandler.h"
+#include "cmDuration.h"
+#include "cmListFileCache.h"
 #include "cmSystemTools.h"
 #include "cmWorkingDirectory.h"
 
+#include "cm_jsoncpp_value.h"
+#include "cm_jsoncpp_writer.h"
 #include "cm_uv.h"
 
 #include "cmUVSignalHackRAII.h" // IWYU pragma: keep
@@ -20,13 +24,19 @@
 #include <chrono>
 #include <cstring>
 #include <iomanip>
+#include <iostream>
 #include <list>
 #include <math.h>
 #include <sstream>
 #include <stack>
 #include <stdlib.h>
+#include <unordered_map>
 #include <utility>
 
+namespace cmsys {
+class RegularExpression;
+}
+
 class TestComparator
 {
 public:
@@ -725,9 +735,330 @@ void cmCTestMultiProcessHandler::MarkFinished()
   cmSystemTools::RemoveFile(fname);
 }
 
+static Json::Value DumpToJsonArray(const std::set<std::string>& values)
+{
+  Json::Value jsonArray = Json::arrayValue;
+  for (auto& it : values) {
+    jsonArray.append(it);
+  }
+  return jsonArray;
+}
+
+static Json::Value DumpToJsonArray(const std::vector<std::string>& values)
+{
+  Json::Value jsonArray = Json::arrayValue;
+  for (auto& it : values) {
+    jsonArray.append(it);
+  }
+  return jsonArray;
+}
+
+static Json::Value DumpRegExToJsonArray(
+  const std::vector<std::pair<cmsys::RegularExpression, std::string>>& values)
+{
+  Json::Value jsonArray = Json::arrayValue;
+  for (auto& it : values) {
+    jsonArray.append(it.second);
+  }
+  return jsonArray;
+}
+
+static Json::Value DumpMeasurementToJsonArray(
+  const std::map<std::string, std::string>& values)
+{
+  Json::Value jsonArray = Json::arrayValue;
+  for (auto& it : values) {
+    Json::Value measurement = Json::objectValue;
+    measurement["measurement"] = it.first;
+    measurement["value"] = it.second;
+    jsonArray.append(measurement);
+  }
+  return jsonArray;
+}
+
+static Json::Value DumpTimeoutAfterMatch(
+  cmCTestTestHandler::cmCTestTestProperties& testProperties)
+{
+  Json::Value timeoutAfterMatch = Json::objectValue;
+  timeoutAfterMatch["timeout"] = testProperties.AlternateTimeout.count();
+  timeoutAfterMatch["regex"] =
+    DumpRegExToJsonArray(testProperties.TimeoutRegularExpressions);
+  return timeoutAfterMatch;
+}
+
+static Json::Value DumpCTestProperty(std::string const& name,
+                                     Json::Value value)
+{
+  Json::Value property = Json::objectValue;
+  property["name"] = name;
+  property["value"] = std::move(value);
+  return property;
+}
+
+static Json::Value DumpCTestProperties(
+  cmCTestTestHandler::cmCTestTestProperties& testProperties)
+{
+  Json::Value properties = Json::arrayValue;
+  if (!testProperties.AttachOnFail.empty()) {
+    properties.append(DumpCTestProperty(
+      "ATTACHED_FILES_ON_FAIL", DumpToJsonArray(testProperties.AttachOnFail)));
+  }
+  if (!testProperties.AttachedFiles.empty()) {
+    properties.append(DumpCTestProperty(
+      "ATTACHED_FILES", DumpToJsonArray(testProperties.AttachedFiles)));
+  }
+  if (testProperties.Cost != 0.0f) {
+    properties.append(
+      DumpCTestProperty("COST", static_cast<double>(testProperties.Cost)));
+  }
+  if (!testProperties.Depends.empty()) {
+    properties.append(
+      DumpCTestProperty("DEPENDS", DumpToJsonArray(testProperties.Depends)));
+  }
+  if (testProperties.Disabled) {
+    properties.append(DumpCTestProperty("DISABLED", testProperties.Disabled));
+  }
+  if (!testProperties.Environment.empty()) {
+    properties.append(DumpCTestProperty(
+      "ENVIRONMENT", DumpToJsonArray(testProperties.Environment)));
+  }
+  if (!testProperties.ErrorRegularExpressions.empty()) {
+    properties.append(DumpCTestProperty(
+      "FAIL_REGULAR_EXPRESSION",
+      DumpRegExToJsonArray(testProperties.ErrorRegularExpressions)));
+  }
+  if (!testProperties.FixturesCleanup.empty()) {
+    properties.append(DumpCTestProperty(
+      "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup)));
+  }
+  if (!testProperties.FixturesRequired.empty()) {
+    properties.append(DumpCTestProperty(
+      "FIXTURES_REQUIRED", DumpToJsonArray(testProperties.FixturesRequired)));
+  }
+  if (!testProperties.FixturesSetup.empty()) {
+    properties.append(DumpCTestProperty(
+      "FIXTURES_SETUP", DumpToJsonArray(testProperties.FixturesSetup)));
+  }
+  if (!testProperties.Labels.empty()) {
+    properties.append(
+      DumpCTestProperty("LABELS", DumpToJsonArray(testProperties.Labels)));
+  }
+  if (!testProperties.Measurements.empty()) {
+    properties.append(DumpCTestProperty(
+      "MEASUREMENT", DumpMeasurementToJsonArray(testProperties.Measurements)));
+  }
+  if (!testProperties.RequiredRegularExpressions.empty()) {
+    properties.append(DumpCTestProperty(
+      "PASS_REGULAR_EXPRESSION",
+      DumpRegExToJsonArray(testProperties.RequiredRegularExpressions)));
+  }
+  if (testProperties.WantAffinity) {
+    properties.append(
+      DumpCTestProperty("PROCESSOR_AFFINITY", testProperties.WantAffinity));
+  }
+  if (testProperties.Processors != 1) {
+    properties.append(
+      DumpCTestProperty("PROCESSORS", testProperties.Processors));
+  }
+  if (!testProperties.RequiredFiles.empty()) {
+    properties["REQUIRED_FILES"] =
+      DumpToJsonArray(testProperties.RequiredFiles);
+  }
+  if (!testProperties.LockedResources.empty()) {
+    properties.append(DumpCTestProperty(
+      "RESOURCE_LOCK", DumpToJsonArray(testProperties.LockedResources)));
+  }
+  if (testProperties.RunSerial) {
+    properties.append(
+      DumpCTestProperty("RUN_SERIAL", testProperties.RunSerial));
+  }
+  if (testProperties.SkipReturnCode != -1) {
+    properties.append(
+      DumpCTestProperty("SKIP_RETURN_CODE", testProperties.SkipReturnCode));
+  }
+  if (testProperties.ExplicitTimeout) {
+    properties.append(
+      DumpCTestProperty("TIMEOUT", testProperties.Timeout.count()));
+  }
+  if (!testProperties.TimeoutRegularExpressions.empty()) {
+    properties.append(DumpCTestProperty(
+      "TIMEOUT_AFTER_MATCH", DumpTimeoutAfterMatch(testProperties)));
+  }
+  if (testProperties.WillFail) {
+    properties.append(DumpCTestProperty("WILL_FAIL", testProperties.WillFail));
+  }
+  if (!testProperties.Directory.empty()) {
+    properties.append(
+      DumpCTestProperty("WORKING_DIRECTORY", testProperties.Directory));
+  }
+  return properties;
+}
+
+class BacktraceData
+{
+  std::unordered_map<std::string, Json::ArrayIndex> CommandMap;
+  std::unordered_map<std::string, Json::ArrayIndex> FileMap;
+  std::unordered_map<cmListFileContext const*, Json::ArrayIndex> NodeMap;
+  Json::Value Commands = Json::arrayValue;
+  Json::Value Files = Json::arrayValue;
+  Json::Value Nodes = Json::arrayValue;
+
+  Json::ArrayIndex AddCommand(std::string const& command)
+  {
+    auto i = this->CommandMap.find(command);
+    if (i == this->CommandMap.end()) {
+      i = this->CommandMap.emplace(command, this->Commands.size()).first;
+      this->Commands.append(command);
+    }
+    return i->second;
+  }
+
+  Json::ArrayIndex AddFile(std::string const& file)
+  {
+    auto i = this->FileMap.find(file);
+    if (i == this->FileMap.end()) {
+      i = this->FileMap.emplace(file, this->Files.size()).first;
+      this->Files.append(file);
+    }
+    return i->second;
+  }
+
+public:
+  bool Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index);
+  Json::Value Dump();
+};
+
+bool BacktraceData::Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index)
+{
+  if (bt.Empty()) {
+    return false;
+  }
+  cmListFileContext const* top = &bt.Top();
+  auto found = this->NodeMap.find(top);
+  if (found != this->NodeMap.end()) {
+    index = found->second;
+    return true;
+  }
+  Json::Value entry = Json::objectValue;
+  entry["file"] = this->AddFile(top->FilePath);
+  if (top->Line) {
+    entry["line"] = static_cast<int>(top->Line);
+  }
+  if (!top->Name.empty()) {
+    entry["command"] = this->AddCommand(top->Name);
+  }
+  Json::ArrayIndex parent;
+  if (this->Add(bt.Pop(), parent)) {
+    entry["parent"] = parent;
+  }
+  index = this->NodeMap[top] = this->Nodes.size();
+  this->Nodes.append(std::move(entry)); // NOLINT(*)
+  return true;
+}
+
+Json::Value BacktraceData::Dump()
+{
+  Json::Value backtraceGraph;
+  this->CommandMap.clear();
+  this->FileMap.clear();
+  this->NodeMap.clear();
+  backtraceGraph["commands"] = std::move(this->Commands);
+  backtraceGraph["files"] = std::move(this->Files);
+  backtraceGraph["nodes"] = std::move(this->Nodes);
+  return backtraceGraph;
+}
+
+static void AddBacktrace(BacktraceData& backtraceGraph, Json::Value& object,
+                         cmListFileBacktrace const& bt)
+{
+  Json::ArrayIndex backtrace;
+  if (backtraceGraph.Add(bt, backtrace)) {
+    object["backtrace"] = backtrace;
+  }
+}
+
+static Json::Value DumpCTestInfo(
+  cmCTestRunTest& testRun,
+  cmCTestTestHandler::cmCTestTestProperties& testProperties,
+  BacktraceData& backtraceGraph)
+{
+  Json::Value testInfo = Json::objectValue;
+  // test name should always be present
+  testInfo["name"] = testProperties.Name;
+  std::string const& config = testRun.GetCTest()->GetConfigType();
+  if (!config.empty()) {
+    testInfo["config"] = config;
+  }
+  std::string const& command = testRun.GetActualCommand();
+  if (!command.empty()) {
+    std::vector<std::string> commandAndArgs;
+    commandAndArgs.push_back(command);
+    const std::vector<std::string>& args = testRun.GetArguments();
+    if (!args.empty()) {
+      commandAndArgs.reserve(args.size() + 1);
+      commandAndArgs.insert(commandAndArgs.end(), args.begin(), args.end());
+    }
+    testInfo["command"] = DumpToJsonArray(commandAndArgs);
+  }
+  Json::Value properties = DumpCTestProperties(testProperties);
+  if (!properties.empty()) {
+    testInfo["properties"] = properties;
+  }
+  if (!testProperties.Backtrace.Empty()) {
+    AddBacktrace(backtraceGraph, testInfo, testProperties.Backtrace);
+  }
+  return testInfo;
+}
+
+static Json::Value DumpVersion(int major, int minor)
+{
+  Json::Value version = Json::objectValue;
+  version["major"] = major;
+  version["minor"] = minor;
+  return version;
+}
+
+void cmCTestMultiProcessHandler::PrintOutputAsJson()
+{
+  this->TestHandler->SetMaxIndex(this->FindMaxIndex());
+
+  Json::Value result = Json::objectValue;
+  result["kind"] = "ctestInfo";
+  result["version"] = DumpVersion(1, 0);
+
+  BacktraceData backtraceGraph;
+  Json::Value tests = Json::arrayValue;
+  for (auto& it : this->Properties) {
+    cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
+
+    // Don't worry if this fails, we are only showing the test list, not
+    // running the tests
+    cmWorkingDirectory workdir(p.Directory);
+    cmCTestRunTest testRun(*this);
+    testRun.SetIndex(p.Index);
+    testRun.SetTestProperties(&p);
+    testRun.ComputeArguments();
+
+    Json::Value testInfo = DumpCTestInfo(testRun, p, backtraceGraph);
+    tests.append(testInfo);
+  }
+  result["backtraceGraph"] = backtraceGraph.Dump();
+  result["tests"] = std::move(tests);
+
+  Json::StreamWriterBuilder builder;
+  builder["indentation"] = "  ";
+  std::unique_ptr<Json::StreamWriter> jout(builder.newStreamWriter());
+  jout->write(result, &std::cout);
+}
+
 // For ShowOnly mode
 void cmCTestMultiProcessHandler::PrintTestList()
 {
+  if (this->CTest->GetOutputAsJson()) {
+    PrintOutputAsJson();
+    return;
+  }
+
   this->TestHandler->SetMaxIndex(this->FindMaxIndex());
   int count = 0;
 
diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h
index 3927a8a..93bb880 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.h
+++ b/Source/CTest/cmCTestMultiProcessHandler.h
@@ -51,6 +51,7 @@ public:
   void SetParallelLevel(size_t);
   void SetTestLoad(unsigned long load);
   virtual void RunTests();
+  void PrintOutputAsJson();
   void PrintTestList();
   void PrintLabels();
 
diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h
index 10dceca..c786413 100644
--- a/Source/CTest/cmCTestRunTest.h
+++ b/Source/CTest/cmCTestRunTest.h
@@ -78,6 +78,10 @@ public:
 
   cmCTest* GetCTest() const { return this->CTest; }
 
+  std::string& GetActualCommand() { return this->ActualCommand; }
+
+  const std::vector<std::string>& GetArguments() { return this->Arguments; }
+
   void FinalizeTest();
 
   bool TimedOutForStopTime() const { return this->TimeoutIsForStopTime; }
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index 2e1bb0a..9fd2299 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -2147,6 +2147,32 @@ bool cmCTestTestHandler::SetTestsProperties(
     for (std::string const& t : tests) {
       for (cmCTestTestProperties& rt : this->TestList) {
         if (t == rt.Name) {
+          if (key == "_BACKTRACE_TRIPLES") {
+            std::vector<std::string> triples;
+            // allow empty args in the triples
+            cmSystemTools::ExpandListArgument(val, triples, true);
+
+            // Ensure we have complete triples otherwise the data is corrupt.
+            if (triples.size() % 3 == 0) {
+              cmState state;
+              rt.Backtrace = cmListFileBacktrace(state.CreateBaseSnapshot());
+
+              // the first entry represents the top of the trace so we need to
+              // reconstruct the backtrace in reverse
+              for (size_t i = triples.size(); i >= 3; i -= 3) {
+                cmListFileContext fc;
+                fc.FilePath = triples[i - 3];
+                long line = 0;
+                if (!cmSystemTools::StringToLong(triples[i - 2].c_str(),
+                                                 &line)) {
+                  line = 0;
+                }
+                fc.Line = line;
+                fc.Name = triples[i - 1];
+                rt.Backtrace = rt.Backtrace.Push(fc);
+              }
+            }
+          }
           if (key == "WILL_FAIL") {
             rt.WillFail = cmSystemTools::IsOn(val);
           }
diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h
index bcacf23..0b557db 100644
--- a/Source/CTest/cmCTestTestHandler.h
+++ b/Source/CTest/cmCTestTestHandler.h
@@ -7,6 +7,7 @@
 
 #include "cmCTestGenericHandler.h"
 #include "cmDuration.h"
+#include "cmListFileCache.h"
 
 #include "cmsys/RegularExpression.hxx"
 #include <chrono>
@@ -141,6 +142,8 @@ public:
     std::set<std::string> FixturesCleanup;
     std::set<std::string> FixturesRequired;
     std::set<std::string> RequireSuccessDepends;
+    // Private test generator properties used to track backtraces
+    cmListFileBacktrace Backtrace;
   };
 
   struct cmCTestTestResult
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 7c19864..225c99f 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -278,6 +278,8 @@ cmCTest::cmCTest()
   this->ExtraVerbose = false;
   this->ProduceXML = false;
   this->ShowOnly = false;
+  this->OutputAsJson = false;
+  this->OutputAsJsonVersion = 1;
   this->RunConfigurationScript = false;
   this->UseHTTP10 = false;
   this->PrintLabels = false;
@@ -1930,6 +1932,20 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
   if (this->CheckArgument(arg, "-N", "--show-only")) {
     this->ShowOnly = true;
   }
+  if (cmSystemTools::StringStartsWith(arg.c_str(), "--show-only=")) {
+    this->ShowOnly = true;
+
+    // Check if a specific format is requested. Defaults to human readable
+    // text.
+    std::string argWithFormat = "--show-only=";
+    std::string format = arg.substr(argWithFormat.length());
+    if (format == "json-v1") {
+      // Force quiet mode so the only output is the json object model.
+      this->Quiet = true;
+      this->OutputAsJson = true;
+      this->OutputAsJsonVersion = 1;
+    }
+  }
 
   if (this->CheckArgument(arg, "-O", "--output-log") && i < args.size() - 1) {
     i++;
@@ -2630,6 +2646,16 @@ bool cmCTest::GetShowOnly()
   return this->ShowOnly;
 }
 
+bool cmCTest::GetOutputAsJson()
+{
+  return this->OutputAsJson;
+}
+
+int cmCTest::GetOutputAsJsonVersion()
+{
+  return this->OutputAsJsonVersion;
+}
+
 int cmCTest::GetMaxTestNameWidth() const
 {
   return this->MaxTestNameWidth;
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index 480204a..2b40ca3 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -215,6 +215,10 @@ public:
   /** Should we only show what we would do? */
   bool GetShowOnly();
 
+  bool GetOutputAsJson();
+
+  int GetOutputAsJsonVersion();
+
   bool ShouldUseHTTP10() { return this->UseHTTP10; }
 
   bool ShouldPrintLabels() { return this->PrintLabels; }
@@ -507,6 +511,8 @@ private:
   t_TestingHandlers TestingHandlers;
 
   bool ShowOnly;
+  bool OutputAsJson;
+  int OutputAsJsonVersion;
 
   /** Map of configuration properties */
   typedef std::map<std::string, std::string> CTestConfigurationMap;
diff --git a/Source/cmTestGenerator.cxx b/Source/cmTestGenerator.cxx
index 796d2df..e4ced6e 100644
--- a/Source/cmTestGenerator.cxx
+++ b/Source/cmTestGenerator.cxx
@@ -7,14 +7,16 @@
 
 #include "cmGeneratorExpression.h"
 #include "cmGeneratorTarget.h"
+#include "cmListFileCache.h"
 #include "cmLocalGenerator.h"
 #include "cmOutputConverter.h"
 #include "cmProperty.h"
-#include "cmPropertyMap.h"
 #include "cmStateTypes.h"
 #include "cmSystemTools.h"
 #include "cmTest.h"
 
+class cmPropertyMap;
+
 cmTestGenerator::cmTestGenerator(
   cmTest* test, std::vector<std::string> const& configurations)
   : cmScriptGenerator("CTEST_CONFIGURATION_TYPE", configurations)
@@ -121,16 +123,15 @@ void cmTestGenerator::GenerateScriptForConfig(std::ostream& os,
 
   // Output properties for the test.
   cmPropertyMap& pm = this->Test->GetProperties();
-  if (!pm.empty()) {
-    os << indent << "set_tests_properties(" << this->Test->GetName()
-       << " PROPERTIES ";
-    for (auto const& i : pm) {
-      os << " " << i.first << " "
-         << cmOutputConverter::EscapeForCMake(
-              ge.Parse(i.second.GetValue())->Evaluate(this->LG, config));
-    }
-    os << ")" << std::endl;
+  os << indent << "set_tests_properties(" << this->Test->GetName()
+     << " PROPERTIES ";
+  for (auto const& i : pm) {
+    os << " " << i.first << " "
+       << cmOutputConverter::EscapeForCMake(
+            ge.Parse(i.second.GetValue())->Evaluate(this->LG, config));
   }
+  this->GenerateInternalProperties(os);
+  os << ")" << std::endl;
 }
 
 void cmTestGenerator::GenerateScriptNoConfig(std::ostream& os, Indent indent)
@@ -179,13 +180,37 @@ void cmTestGenerator::GenerateOldStyle(std::ostream& fout, Indent indent)
 
   // Output properties for the test.
   cmPropertyMap& pm = this->Test->GetProperties();
-  if (!pm.empty()) {
-    fout << indent << "set_tests_properties(" << this->Test->GetName()
-         << " PROPERTIES ";
-    for (auto const& i : pm) {
-      fout << " " << i.first << " "
-           << cmOutputConverter::EscapeForCMake(i.second.GetValue());
+  fout << indent << "set_tests_properties(" << this->Test->GetName()
+       << " PROPERTIES ";
+  for (auto const& i : pm) {
+    fout << " " << i.first << " "
+         << cmOutputConverter::EscapeForCMake(i.second.GetValue());
+  }
+  this->GenerateInternalProperties(fout);
+  fout << ")" << std::endl;
+}
+
+void cmTestGenerator::GenerateInternalProperties(std::ostream& os)
+{
+  cmListFileBacktrace bt = this->Test->GetBacktrace();
+  if (bt.Empty()) {
+    return;
+  }
+
+  os << " "
+     << "_BACKTRACE_TRIPLES"
+     << " \"";
+
+  bool prependTripleSeparator = false;
+  while (!bt.Empty()) {
+    const auto& entry = bt.Top();
+    if (prependTripleSeparator) {
+      os << ";";
     }
-    fout << ")" << std::endl;
+    os << entry.FilePath << ";" << entry.Line << ";" << entry.Name;
+    bt = bt.Pop();
+    prependTripleSeparator = true;
   }
+
+  os << "\"";
 }
diff --git a/Source/cmTestGenerator.h b/Source/cmTestGenerator.h
index 73d05a3..f26d2ff 100644
--- a/Source/cmTestGenerator.h
+++ b/Source/cmTestGenerator.h
@@ -35,6 +35,9 @@ public:
 
   cmTest* GetTest() const;
 
+private:
+  void GenerateInternalProperties(std::ostream& os);
+
 protected:
   void GenerateScriptConfigs(std::ostream& os, Indent indent) override;
   void GenerateScriptActions(std::ostream& os, Indent indent) override;
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index ca412ae..8ba126f 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -46,7 +46,10 @@ static const char* cmDocumentationOptions[][2] = {
     "given number of jobs." },
   { "-Q,--quiet", "Make ctest quiet." },
   { "-O <file>, --output-log <file>", "Output to log file" },
-  { "-N,--show-only", "Disable actual execution of tests." },
+  { "-N,--show-only[=format]",
+    "Disable actual execution of tests. The optional 'format' defines the "
+    "format of the test information and can be 'human' for the current text "
+    "format or 'json-v1' for json format. Defaults to 'human'." },
   { "-L <regex>, --label-regex <regex>",
     "Run tests with labels matching "
     "regular expression." },