From 85a63143ed005cb7fdf100fdc9650d58f384f69e Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Fri, 14 Feb 2025 09:05:54 -0500 Subject: instrument: don't report target=TARGET_NAME Remove the erroneous default target name when instrumenting custom commands. --- Help/manual/cmake-instrumentation.7.rst | 2 +- Source/CTest/cmCTestLaunch.cxx | 4 +++- Tests/RunCMake/Instrumentation/verify-snippet.cmake | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Help/manual/cmake-instrumentation.7.rst b/Help/manual/cmake-instrumentation.7.rst index 7b2d1e6..bb31858 100644 --- a/Help/manual/cmake-instrumentation.7.rst +++ b/Help/manual/cmake-instrumentation.7.rst @@ -270,7 +270,7 @@ and contain the following data: ``target`` The CMake target associated with the command. Only included when ``role`` is - one of ``compile``, ``link``, ``custom``. + ``compile`` or ``link``. ``targetType`` The :prop_tgt:`TYPE` of the target. Only included when ``role`` is diff --git a/Source/CTest/cmCTestLaunch.cxx b/Source/CTest/cmCTestLaunch.cxx index 2f25de7..200bd7d 100644 --- a/Source/CTest/cmCTestLaunch.cxx +++ b/Source/CTest/cmCTestLaunch.cxx @@ -268,7 +268,9 @@ int cmCTestLaunch::Run() { auto instrumentation = cmInstrumentation(this->Reporter.OptionBuildDir); std::map options; - options["target"] = this->Reporter.OptionTargetName; + if (this->Reporter.OptionTargetName != "TARGET_NAME") { + options["target"] = this->Reporter.OptionTargetName; + } options["source"] = this->Reporter.OptionSource; options["language"] = this->Reporter.OptionLanguage; options["targetType"] = this->Reporter.OptionTargetType; diff --git a/Tests/RunCMake/Instrumentation/verify-snippet.cmake b/Tests/RunCMake/Instrumentation/verify-snippet.cmake index ec997e3..33f9414 100644 --- a/Tests/RunCMake/Instrumentation/verify-snippet.cmake +++ b/Tests/RunCMake/Instrumentation/verify-snippet.cmake @@ -49,7 +49,6 @@ function(snippet_has_fields snippet contents) has_key("${snippet}" "${contents}" language) has_key("${snippet}" "${contents}" config) elseif (filename MATCHES "^custom-*") - has_key("${snippet}" "${contents}" target) has_key("${snippet}" "${contents}" outputs) has_key("${snippet}" "${contents}" outputSizes) elseif (filename MATCHES "^test-*") -- cgit v0.12 From a6d4a9a2aecb09d5ae238800c1a07afd880da2c0 Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Tue, 11 Feb 2025 10:49:19 -0500 Subject: ctest: Include cmake instrumentation data in XML files --- Help/dev/experimental.rst | 8 +- Help/envvar/CTEST_USE_INSTRUMENTATION.rst | 15 ++ Help/envvar/CTEST_USE_VERBOSE_INSTRUMENTATION.rst | 17 ++ Help/manual/cmake-env-variables.7.rst | 2 + Help/manual/cmake-instrumentation.7.rst | 35 ++++ Source/CTest/cmCTestBuildHandler.cxx | 91 ++++++++++ Source/CTest/cmCTestBuildHandler.h | 1 + Source/CTest/cmCTestConfigureCommand.cxx | 7 + Source/CTest/cmCTestRunTest.cxx | 11 +- Source/CTest/cmCTestRunTest.h | 2 - Source/CTest/cmCTestTestHandler.cxx | 15 +- Source/CTest/cmCTestTestHandler.h | 2 + Source/cmCTest.cxx | 139 ++++++++++++++- Source/cmCTest.h | 9 + Source/cmInstrumentation.cxx | 186 ++++++++++++++++++++- Source/cmInstrumentation.h | 19 ++- Source/cmInstrumentationQuery.cxx | 5 +- Source/cmInstrumentationQuery.h | 1 + Tests/RunCMake/CMakeLists.txt | 3 + .../ctest_instrumentation/CMakeLists.txt.in | 10 ++ .../InstrumentationInCTestXML-check.cmake | 41 +++++ .../NoInstrumentationInCTestXML-check.cmake | 11 ++ .../ctest_instrumentation/RunCMakeTest.cmake | 22 +++ Tests/RunCMake/ctest_instrumentation/main.c | 4 + Tests/RunCMake/ctest_instrumentation/test.cmake.in | 17 ++ 25 files changed, 652 insertions(+), 21 deletions(-) create mode 100644 Help/envvar/CTEST_USE_INSTRUMENTATION.rst create mode 100644 Help/envvar/CTEST_USE_VERBOSE_INSTRUMENTATION.rst create mode 100644 Tests/RunCMake/ctest_instrumentation/CMakeLists.txt.in create mode 100644 Tests/RunCMake/ctest_instrumentation/InstrumentationInCTestXML-check.cmake create mode 100644 Tests/RunCMake/ctest_instrumentation/NoInstrumentationInCTestXML-check.cmake create mode 100644 Tests/RunCMake/ctest_instrumentation/RunCMakeTest.cmake create mode 100644 Tests/RunCMake/ctest_instrumentation/main.c create mode 100644 Tests/RunCMake/ctest_instrumentation/test.cmake.in diff --git a/Help/dev/experimental.rst b/Help/dev/experimental.rst index 5f2cb22..907b31f 100644 --- a/Help/dev/experimental.rst +++ b/Help/dev/experimental.rst @@ -129,7 +129,13 @@ set * variable ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` to * value ``a37d1069-1972-4901-b9c9-f194aaf2b6e0``. -To enable instrumentation at the user-level, files should be blaced under +To enable instrumentation at the user-level, files should be placed under either ``/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0`` or ``/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0``. + +To include instrumentation data in CTest XML files (for submission to CDash), +you need to set the following environment variables: + +* ``CTEST_USE_INSTRUMENTATION=1`` +* ``CTEST_EXPERIMENTAL_INSTRUMENTATION=a37d1069-1972-4901-b9c9-f194aaf2b6e0`` diff --git a/Help/envvar/CTEST_USE_INSTRUMENTATION.rst b/Help/envvar/CTEST_USE_INSTRUMENTATION.rst new file mode 100644 index 0000000..c6c7f70 --- /dev/null +++ b/Help/envvar/CTEST_USE_INSTRUMENTATION.rst @@ -0,0 +1,15 @@ +CTEST_USE_INSTRUMENTATION +------------------------- + +.. versionadded:: 4.0 + +.. include:: ENV_VAR.txt + +.. note:: + + This feature is only available when experimental support for instrumentation + has been enabled by the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` gate. + +Setting this environment variable enables +:manual:`instrumentation ` for CTest in +:ref:`Dashboard Client` mode. diff --git a/Help/envvar/CTEST_USE_VERBOSE_INSTRUMENTATION.rst b/Help/envvar/CTEST_USE_VERBOSE_INSTRUMENTATION.rst new file mode 100644 index 0000000..d7f7477 --- /dev/null +++ b/Help/envvar/CTEST_USE_VERBOSE_INSTRUMENTATION.rst @@ -0,0 +1,17 @@ +CTEST_USE_VERBOSE_INSTRUMENTATION +--------------------------------- + +.. versionadded:: 4.0 + +.. include:: ENV_VAR.txt + +.. note:: + + This feature is only available when experimental support for instrumentation + has been enabled by the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` gate. + +Setting this environment variable causes CTest to report the full +command line (including arguments) to CDash for each instrumented command. +By default, CTest truncates the command line at the first space. + +See also :envvar:`CTEST_USE_INSTRUMENTATION` diff --git a/Help/manual/cmake-env-variables.7.rst b/Help/manual/cmake-env-variables.7.rst index fe3e703..140fc83 100644 --- a/Help/manual/cmake-env-variables.7.rst +++ b/Help/manual/cmake-env-variables.7.rst @@ -116,7 +116,9 @@ Environment Variables for CTest /envvar/CTEST_OUTPUT_ON_FAILURE /envvar/CTEST_PARALLEL_LEVEL /envvar/CTEST_PROGRESS_OUTPUT + /envvar/CTEST_USE_INSTRUMENTATION /envvar/CTEST_USE_LAUNCHERS_DEFAULT + /envvar/CTEST_USE_VERBOSE_INSTRUMENTATION /envvar/DASHBOARD_TEST_FROM_CTEST Environment Variables for the CMake curses interface diff --git a/Help/manual/cmake-instrumentation.7.rst b/Help/manual/cmake-instrumentation.7.rst index bb31858..a33d983 100644 --- a/Help/manual/cmake-instrumentation.7.rst +++ b/Help/manual/cmake-instrumentation.7.rst @@ -94,6 +94,37 @@ Instrumentation can be configured at the user-level by placing query files in the :envvar:`CMAKE_CONFIG_DIR` under ``/instrumentation//query/``. +Enabling Instrumentation for CDash Submissions +---------------------------------------------- + +You can enable instrumentation when using CTest in :ref:`Dashboard Client` +mode by setting the :envvar:`CTEST_USE_INSTRUMENTATION` environment variable +to the current UUID for the ``CMAKE_EXPERIMENTAL_INSTRUMENTATION`` feature. +Doing so automatically enables the ``dynamicSystemInformation`` query. + +The following table shows how each type of instrumented command gets mapped +to a corresponding type of CTest XML file. + +=================================================== ================== +:ref:`Snippet Role ` CTest XML File +=================================================== ================== +``configure`` ``Configure.xml`` +``generate`` ``Configure.xml`` +``compile`` ``Build.xml`` +``link`` ``Build.xml`` +``custom`` ``Build.xml`` +``build`` unused! +``cmakeBuild`` ``Build.xml`` +``cmakeInstall`` ``Build.xml`` +``install`` ``Build.xml`` +``ctest`` ``Build.xml`` +``test`` ``Test.xml`` +=================================================== ================== + +By default the command line reported to CDash is truncated at the first space. +You can instead choose to report the full command line (including arguments) +by setting :envvar:`CTEST_USE_VERBOSE_INSTRUMENTATION` to 1. + .. _`cmake-instrumentation API v1`: API v1 @@ -123,6 +154,10 @@ subdirectories: files, they should never be removed by other processes. Data collected here remains until after `Indexing`_ occurs and all `Callbacks`_ are executed. +``cdash/`` + Holds temporary files used internally to generate XML content to be submitted + to CDash. + .. _`cmake-instrumentation v1 Query Files`: v1 Query Files diff --git a/Source/CTest/cmCTestBuildHandler.cxx b/Source/CTest/cmCTestBuildHandler.cxx index af54e56..9cc1110 100644 --- a/Source/CTest/cmCTestBuildHandler.cxx +++ b/Source/CTest/cmCTestBuildHandler.cxx @@ -11,6 +11,7 @@ #include #include +#include #include #include "cmsys/Directory.hxx" @@ -21,6 +22,9 @@ #include "cmDuration.h" #include "cmFileTimeCache.h" #include "cmGeneratedFileStream.h" +#include "cmInstrumentation.h" +#include "cmInstrumentationQuery.h" +#include "cmJSONState.h" #include "cmList.h" #include "cmMakefile.h" #include "cmProcessOutput.h" @@ -429,6 +433,11 @@ int cmCTestBuildHandler::ProcessHandler() } else { this->GenerateXMLLogScraped(xml); } + + this->CTest->GetInstrumentation().CollectTimingData( + cmInstrumentationQuery::Hook::PrepareForCDash); + this->GenerateInstrumentationXML(xml); + this->GenerateXMLFooter(xml, elapsed_build_time); if (!res || retVal || this->TotalErrors > 0) { @@ -595,6 +604,88 @@ void cmCTestBuildHandler::GenerateXMLLogScraped(cmXMLWriter& xml) } } +void cmCTestBuildHandler::GenerateInstrumentationXML(cmXMLWriter& xml) +{ + // Record instrumentation data on a per-target basis. + cmsys::Directory targets_dir; + std::string targets_snippet_dir = cmStrCat( + this->CTest->GetInstrumentation().GetCDashDir(), "/build/targets"); + if (targets_dir.Load(targets_snippet_dir) && + targets_dir.GetNumberOfFiles() > 0) { + xml.StartElement("Targets"); + for (unsigned int i = 0; i < targets_dir.GetNumberOfFiles(); i++) { + if (!targets_dir.FileIsDirectory(i)) { + continue; + } + std::string target_name = targets_dir.GetFile(i); + if (target_name == "." || target_name == "..") { + continue; + } + std::string target_type = "UNKNOWN"; + + xml.StartElement("Target"); + xml.Attribute("name", target_name); + + // Check if we have a link snippet for this target. + cmsys::Directory target_dir; + if (!target_dir.Load(targets_dir.GetFilePath(i))) { + cmSystemTools::Error( + cmStrCat("Error loading directory ", targets_dir.GetFilePath(i))); + } + Json::Value link_item; + for (unsigned int j = 0; j < target_dir.GetNumberOfFiles(); j++) { + std::string fname = target_dir.GetFile(j); + if (fname.rfind("link-", 0) == 0) { + std::string fpath = target_dir.GetFilePath(j); + cmJSONState parseState = cmJSONState(fpath, &link_item); + if (!parseState.errors.empty()) { + cmSystemTools::Error(parseState.GetErrorMessage(true)); + break; + } + + if (!link_item.isObject()) { + std::string error_msg = + cmStrCat("Expected snippet ", fpath, " to contain an object"); + cmSystemTools::Error(error_msg); + break; + } + break; + } + } + + // If so, parse targetType and targetLabels (optional) from it. + if (link_item.isMember("targetType")) { + target_type = link_item["targetType"].asString(); + } + + xml.Attribute("type", target_type); + + if (link_item.isMember("targetLabels") && + !link_item["targetLabels"].empty()) { + xml.StartElement("Labels"); + for (auto const& json_label_item : link_item["targetLabels"]) { + xml.Element("Label", json_label_item.asString()); + } + xml.EndElement(); // Labels + } + + // Write instrumendation data for this target. + std::string target_subdir = cmStrCat("build/targets/", target_name); + this->CTest->ConvertInstrumentationSnippetsToXML(xml, target_subdir); + std::string target_dir_fullpath = cmStrCat( + this->CTest->GetInstrumentation().GetCDashDir(), '/', target_subdir); + if (cmSystemTools::FileIsDirectory(target_dir_fullpath)) { + cmSystemTools::RemoveADirectory(target_dir_fullpath); + } + xml.EndElement(); // Target + } + xml.EndElement(); // Targets + } + + // Also record instrumentation data for custom commands (no target). + this->CTest->ConvertInstrumentationSnippetsToXML(xml, "build/commands"); +} + void cmCTestBuildHandler::GenerateXMLFooter(cmXMLWriter& xml, cmDuration elapsed_build_time) { diff --git a/Source/CTest/cmCTestBuildHandler.h b/Source/CTest/cmCTestBuildHandler.h index 9d9b847..a455346 100644 --- a/Source/CTest/cmCTestBuildHandler.h +++ b/Source/CTest/cmCTestBuildHandler.h @@ -85,6 +85,7 @@ private: void GenerateXMLHeader(cmXMLWriter& xml); void GenerateXMLLaunched(cmXMLWriter& xml); void GenerateXMLLogScraped(cmXMLWriter& xml); + void GenerateInstrumentationXML(cmXMLWriter& xml); void GenerateXMLFooter(cmXMLWriter& xml, cmDuration elapsed_build_time); bool IsLaunchedErrorFile(char const* fname); bool IsLaunchedWarningFile(char const* fname); diff --git a/Source/CTest/cmCTestConfigureCommand.cxx b/Source/CTest/cmCTestConfigureCommand.cxx index 820eeae..4657bbd 100644 --- a/Source/CTest/cmCTestConfigureCommand.cxx +++ b/Source/CTest/cmCTestConfigureCommand.cxx @@ -17,6 +17,8 @@ #include "cmExecutionStatus.h" #include "cmGeneratedFileStream.h" #include "cmGlobalGenerator.h" +#include "cmInstrumentation.h" +#include "cmInstrumentationQuery.h" #include "cmList.h" #include "cmMakefile.h" #include "cmStringAlgorithms.h" @@ -203,6 +205,11 @@ bool cmCTestConfigureCommand::ExecuteConfigure(ConfigureArguments const& args, xml.Element("EndDateTime", endDateTime); xml.Element("EndConfigureTime", endTime); xml.Element("ElapsedMinutes", elapsedMinutes.count()); + + this->CTest->GetInstrumentation().CollectTimingData( + cmInstrumentationQuery::Hook::PrepareForCDash); + this->CTest->ConvertInstrumentationSnippetsToXML(xml, "configure"); + xml.EndElement(); // Configure this->CTest->EndXML(xml); diff --git a/Source/CTest/cmCTestRunTest.cxx b/Source/CTest/cmCTestRunTest.cxx index 42dc0af..4076347 100644 --- a/Source/CTest/cmCTestRunTest.cxx +++ b/Source/CTest/cmCTestRunTest.cxx @@ -21,6 +21,7 @@ #include "cmCTestMemCheckHandler.h" #include "cmCTestMultiProcessHandler.h" #include "cmDuration.h" +#include "cmInstrumentation.h" #include "cmProcess.h" #include "cmStringAlgorithms.h" #include "cmSystemTools.h" @@ -34,7 +35,6 @@ cmCTestRunTest::cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler, , CTest(MultiTestHandler.CTest) , TestHandler(MultiTestHandler.TestHandler) , TestProperties(MultiTestHandler.Properties[Index]) - , Instrumentation(cmSystemTools::GetLogicalWorkingDirectory()) { } @@ -664,8 +664,8 @@ bool cmCTestRunTest::StartTest(size_t completed, size_t total) return false; } this->StartTime = this->CTest->CurrentTime(); - if (this->Instrumentation.HasQuery()) { - this->Instrumentation.GetPreTestStats(); + if (this->CTest->GetInstrumentation().HasQuery()) { + this->CTest->GetInstrumentation().GetPreTestStats(); } return this->ForkProcess(); @@ -1016,12 +1016,13 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) void cmCTestRunTest::FinalizeTest(bool started) { - if (this->Instrumentation.HasQuery()) { - this->Instrumentation.InstrumentTest( + if (this->CTest->GetInstrumentation().HasQuery()) { + std::string data_file = this->CTest->GetInstrumentation().InstrumentTest( this->TestProperties->Name, this->ActualCommand, this->Arguments, this->TestProcess->GetExitValue(), this->TestProcess->GetStartTime(), this->TestProcess->GetSystemStartTime(), this->GetCTest()->GetConfigType()); + this->TestResult.InstrumentationFile = data_file; } this->MultiTestHandler.FinishTestProcess(this->TestProcess->GetRunner(), started); diff --git a/Source/CTest/cmCTestRunTest.h b/Source/CTest/cmCTestRunTest.h index fd5e037..05a5f76 100644 --- a/Source/CTest/cmCTestRunTest.h +++ b/Source/CTest/cmCTestRunTest.h @@ -14,7 +14,6 @@ #include "cmCTest.h" #include "cmCTestMultiProcessHandler.h" #include "cmCTestTestHandler.h" -#include "cmInstrumentation.h" #include "cmProcess.h" /** \class cmRunTest @@ -141,7 +140,6 @@ private: int NumberOfRunsTotal = 1; // default to 1 run of the test bool RunAgain = false; // default to not having to run again size_t TotalNumberOfTests; - cmInstrumentation Instrumentation; }; inline int getNumWidth(size_t n) diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index afe505d..1f45b9d 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -41,6 +41,8 @@ #include "cmExecutionStatus.h" #include "cmGeneratedFileStream.h" #include "cmGlobalGenerator.h" +#include "cmInstrumentation.h" +#include "cmInstrumentationQuery.h" #include "cmList.h" #include "cmMakefile.h" #include "cmState.h" @@ -1381,6 +1383,9 @@ void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml) return; } + this->CTest->GetInstrumentation().CollectTimingData( + cmInstrumentationQuery::Hook::PrepareForCDash); + this->CTest->StartXML(xml, this->CMake, this->AppendXML); this->CTest->GenerateSubprojectsOutput(xml); xml.StartElement("Testing"); @@ -1395,7 +1400,6 @@ void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml) for (cmCTestTestResult& result : this->TestResults) { this->WriteTestResultHeader(xml, result); xml.StartElement("Results"); - if (result.Status != cmCTestTestHandler::NOT_RUN) { if (result.Status != cmCTestTestHandler::COMPLETED || result.ReturnValue) { @@ -1473,6 +1477,15 @@ void cmCTestTestHandler::GenerateCTestXML(cmXMLWriter& xml) xml.Content(result.Output); xml.EndElement(); // Value xml.EndElement(); // Measurement + + if (!result.InstrumentationFile.empty()) { + std::string instrument_file_path = + cmStrCat(this->CTest->GetInstrumentation().GetCDashDir(), "/test/", + result.InstrumentationFile); + this->CTest->ConvertInstrumentationJSONFileToXML(instrument_file_path, + xml); + } + xml.EndElement(); // Results this->AttachFiles(xml, result); diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index 393b3d3..b97d6b7 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -192,6 +192,7 @@ public: std::string CustomCompletionStatus; std::string Output; std::string TestMeasurementsOutput; + std::string InstrumentationFile; int TestCount = 0; cmCTestTestProperties* Properties = nullptr; }; @@ -250,6 +251,7 @@ protected: cmCTestTestResult const& result); void WriteTestResultFooter(cmXMLWriter& xml, cmCTestTestResult const& result); + // Write attached test files into the xml void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result); void AttachFile(cmXMLWriter& xml, std::string const& file, diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 0bb9260..8927fae 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -115,6 +116,8 @@ struct cmCTest::Private bool UseHTTP10 = false; bool PrintLabels = false; bool Failover = false; + bool UseVerboseInstrumentation = false; + cmJSONState parseState; bool FlushTestProgressLine = false; @@ -195,6 +198,8 @@ struct cmCTest::Private cmCTestTestOptions TestOptions; std::vector CommandLineHttpHeaders; + + std::unique_ptr Instrumentation; }; struct tm* cmCTest::GetNightlyTime(std::string const& str, bool tomorrowtag) @@ -320,6 +325,11 @@ cmCTest::cmCTest() if (cmSystemTools::GetEnv("CTEST_PROGRESS_OUTPUT", envValue)) { this->Impl->TestProgressOutput = !cmIsOff(envValue); } + envValue.clear(); + if (cmSystemTools::GetEnv("CTEST_USE_VERBOSE_INSTRUMENTATION", envValue)) { + this->Impl->UseVerboseInstrumentation = !cmIsOff(envValue); + } + envValue.clear(); this->Impl->Parts[PartStart].SetName("Start"); this->Impl->Parts[PartUpdate].SetName("Update"); @@ -2628,8 +2638,6 @@ int cmCTest::Run(std::vector const& args) } #endif - cmInstrumentation instrumentation( - cmSystemTools::GetCurrentWorkingDirectory()); std::function doTest = [this, &cmakeAndTest, &runScripts, &processSteps]() -> int { // now what should cmake do? if --build-and-test was specified then @@ -2650,6 +2658,8 @@ int cmCTest::Run(std::vector const& args) return this->ExecuteTests(); }; + cmInstrumentation instrumentation( + cmSystemTools::GetCurrentWorkingDirectory()); int ret = instrumentation.InstrumentCommand("ctest", args, [doTest]() { return doTest(); }); instrumentation.CollectTimingData(cmInstrumentationQuery::Hook::PostTest); @@ -3673,3 +3683,128 @@ bool cmCTest::StartLogFile(char const* name, int submitIndex, } return true; } + +cmInstrumentation& cmCTest::GetInstrumentation() +{ + if (!this->Impl->Instrumentation) { + this->Impl->Instrumentation = + cm::make_unique(this->GetBinaryDir()); + } + return *this->Impl->Instrumentation; +} + +bool cmCTest::GetUseVerboseInstrumentation() const +{ + return this->Impl->UseVerboseInstrumentation; +} + +void cmCTest::ConvertInstrumentationSnippetsToXML(cmXMLWriter& xml, + std::string const& subdir) +{ + std::string data_dir = + cmStrCat(this->GetInstrumentation().GetCDashDir(), '/', subdir); + + cmsys::Directory d; + if (!d.Load(data_dir) || d.GetNumberOfFiles() == 0) { + return; + } + + xml.StartElement("Commands"); + + for (unsigned int i = 0; i < d.GetNumberOfFiles(); i++) { + std::string fpath = d.GetFilePath(i); + std::string fname = d.GetFile(i); + if (fname.rfind('.', 0) == 0) { + continue; + } + this->ConvertInstrumentationJSONFileToXML(fpath, xml); + } + + xml.EndElement(); // Commands +} + +bool cmCTest::ConvertInstrumentationJSONFileToXML(std::string const& fpath, + cmXMLWriter& xml) +{ + Json::Value root; + this->Impl->parseState = cmJSONState(fpath, &root); + if (!this->Impl->parseState.errors.empty()) { + cmCTestLog(this, ERROR_MESSAGE, + this->Impl->parseState.GetErrorMessage(true) << std::endl); + return false; + } + + if (root.type() != Json::objectValue) { + cmCTestLog(this, ERROR_MESSAGE, + "Expected object, found " << root.type() << " for " + << root.asString() << std::endl); + return false; + } + + std::vector required_members = { + "command", + "role", + "dynamicSystemInformation", + }; + for (std::string const& required_member : required_members) { + if (!root.isMember(required_member)) { + cmCTestLog(this, ERROR_MESSAGE, + fpath << " is missing the '" << required_member << "' key" + << std::endl); + return false; + } + } + + // Do not record command-level data for Test.xml files because + // it is redundant with information actually captured by CTest. + bool generating_test_xml = root["role"] == "test"; + if (!generating_test_xml) { + std::string element_name = root["role"].asString(); + element_name[0] = static_cast(std::toupper(element_name[0])); + xml.StartElement(element_name); + std::vector keys = root.getMemberNames(); + for (auto const& key : keys) { + auto key_type = root[key].type(); + if (key_type == Json::objectValue || key_type == Json::arrayValue) { + continue; + } + if (key == "role" || key == "target" || key == "targetType" || + key == "targetLabels") { + continue; + } + // Truncate the full command line if verbose instrumentation + // was not requested. + if (key == "command" && !this->GetUseVerboseInstrumentation()) { + std::string command_str = root[key].asString(); + std::string truncated = command_str.substr(0, command_str.find(' ')); + if (command_str != truncated) { + truncated = cmStrCat(truncated, " (truncated)"); + } + xml.Attribute(key.c_str(), truncated); + continue; + } + xml.Attribute(key.c_str(), root[key].asString()); + } + } + + // Record dynamicSystemInformation section as XML. + auto dynamic_information = root["dynamicSystemInformation"]; + std::vector keys = dynamic_information.getMemberNames(); + for (auto const& key : keys) { + std::string measurement_name = key; + measurement_name[0] = static_cast(std::toupper(measurement_name[0])); + + xml.StartElement("NamedMeasurement"); + xml.Attribute("type", "numeric/double"); + xml.Attribute("name", measurement_name); + xml.Element("Value", dynamic_information[key].asString()); + xml.EndElement(); // NamedMeasurement + } + + if (!generating_test_xml) { + xml.EndElement(); // role + } + + cmSystemTools::RemoveFile(fpath); + return true; +} diff --git a/Source/cmCTest.h b/Source/cmCTest.h index c6bfede..bcc4087 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -21,6 +21,7 @@ class cmake; class cmGeneratedFileStream; +class cmInstrumentation; class cmMakefile; class cmValue; class cmXMLWriter; @@ -391,6 +392,11 @@ public: bool StartLogFile(char const* name, int submitIndex, cmGeneratedFileStream& xofs); + void ConvertInstrumentationSnippetsToXML(cmXMLWriter& xml, + std::string const& subdir); + bool ConvertInstrumentationJSONFileToXML(std::string const& fpath, + cmXMLWriter& xml); + void AddSiteProperties(cmXMLWriter& xml, cmake* cm); bool GetInteractiveDebugMode() const; @@ -433,6 +439,9 @@ public: cmCTestTestOptions const& GetTestOptions() const; std::vector GetCommandLineHttpHeaders() const; + cmInstrumentation& GetInstrumentation(); + bool GetUseVerboseInstrumentation() const; + private: int GenerateNotesFile(cmake* cm, std::string const& files); diff --git a/Source/cmInstrumentation.cxx b/Source/cmInstrumentation.cxx index 6dffc4c..2c4414f 100644 --- a/Source/cmInstrumentation.cxx +++ b/Source/cmInstrumentation.cxx @@ -20,10 +20,12 @@ #include "cmCryptoHash.h" #include "cmExperimental.h" #include "cmInstrumentationQuery.h" +#include "cmJSONState.h" #include "cmStringAlgorithms.h" #include "cmSystemTools.h" #include "cmTimestamp.h" #include "cmUVProcessChain.h" +#include "cmValue.h" cmInstrumentation::cmInstrumentation(std::string const& binary_dir) { @@ -53,6 +55,75 @@ void cmInstrumentation::LoadQueries() this->hasQuery = this->hasQuery || this->ReadJSONQueries(cmStrCat(this->userTimingDirv1, "/query")); } + + std::string envVal; + if (cmSystemTools::GetEnv("CTEST_USE_INSTRUMENTATION", envVal) && + !cmIsOff(envVal)) { + if (cmSystemTools::GetEnv("CTEST_EXPERIMENTAL_INSTRUMENTATION", envVal)) { + std::string const uuid = cmExperimental::DataForFeature( + cmExperimental::Feature::Instrumentation) + .Uuid; + if (envVal == uuid) { + this->AddHook(cmInstrumentationQuery::Hook::PrepareForCDash); + this->AddQuery( + cmInstrumentationQuery::Query::DynamicSystemInformation); + this->cdashDir = cmStrCat(this->timingDirv1, "/cdash"); + cmSystemTools::MakeDirectory(this->cdashDir); + cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/configure")); + cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/build")); + cmSystemTools::MakeDirectory( + cmStrCat(this->cdashDir, "/build/commands")); + cmSystemTools::MakeDirectory( + cmStrCat(this->cdashDir, "/build/targets")); + cmSystemTools::MakeDirectory(cmStrCat(this->cdashDir, "/test")); + this->cdashSnippetsMap = { { + "configure", + "configure", + }, + { + "generate", + "configure", + }, + { + "compile", + "build", + }, + { + "link", + "build", + }, + { + "custom", + "build", + }, + { + "build", + "skip", + }, + { + "cmakeBuild", + "build", + }, + { + "cmakeInstall", + "build", + }, + { + "install", + "build", + }, + { + "ctest", + "build", + }, + { + "test", + "test", + } }; + this->hasQuery = true; + } + } + } } bool cmInstrumentation::ReadJSONQueries(std::string const& directory) @@ -211,6 +282,11 @@ int cmInstrumentation::CollectTimingData(cmInstrumentationQuery::Hook hook) cmSystemTools::OUTPUT_PASSTHROUGH); } + // Special case for CDash collation + if (this->HasHook(cmInstrumentationQuery::Hook::PrepareForCDash)) { + this->PrepareDataForCDash(directory, index_path); + } + // Delete files for (auto const& f : index["snippets"]) { cmSystemTools::RemoveFile(cmStrCat(directory, "/", f.asString())); @@ -308,7 +384,7 @@ void cmInstrumentation::WriteInstrumentationJson(Json::Value& root, ftmp.close(); } -int cmInstrumentation::InstrumentTest( +std::string cmInstrumentation::InstrumentTest( std::string const& name, std::string const& command, std::vector const& args, int64_t result, std::chrono::steady_clock::time_point steadyStart, @@ -331,11 +407,11 @@ int cmInstrumentation::InstrumentTest( this->InsertDynamicSystemInformation(root, "after"); } - std::string const& file_name = + std::string file_name = cmStrCat("test-", this->ComputeSuffixHash(command_str), this->ComputeSuffixTime(), ".json"); this->WriteInstrumentationJson(root, "data", file_name); - return 1; + return file_name; } void cmInstrumentation::GetPreTestStats() @@ -547,3 +623,107 @@ int cmInstrumentation::CollectTimingAfterBuild(int ppid) this->CollectTimingData(cmInstrumentationQuery::Hook::PostBuild); return ret; } + +void cmInstrumentation::AddHook(cmInstrumentationQuery::Hook hook) +{ + this->hooks.insert(hook); +} + +void cmInstrumentation::AddQuery(cmInstrumentationQuery::Query query) +{ + this->queries.insert(query); +} + +std::string const& cmInstrumentation::GetCDashDir() +{ + return this->cdashDir; +} + +/** Copy the snippets referred to by an index file to a separate + * directory where they will be parsed for submission to CDash. + **/ +void cmInstrumentation::PrepareDataForCDash(std::string const& data_dir, + std::string const& index_path) +{ + Json::Value root; + std::string error_msg; + cmJSONState parseState = cmJSONState(index_path, &root); + if (!parseState.errors.empty()) { + cmSystemTools::Error(parseState.GetErrorMessage(true)); + return; + } + + if (!root.isObject()) { + error_msg = + cmStrCat("Expected index file ", index_path, " to contain an object"); + cmSystemTools::Error(error_msg); + return; + } + + if (!root.isMember("snippets")) { + error_msg = cmStrCat("Expected index file ", index_path, + " to have a key 'snippets'"); + cmSystemTools::Error(error_msg); + return; + } + + std::string dst_dir; + Json::Value snippets = root["snippets"]; + for (auto const& snippet : snippets) { + // Parse the role of this snippet. + std::string snippet_str = snippet.asString(); + std::string snippet_path = cmStrCat(data_dir, '/', snippet_str); + Json::Value snippet_root; + parseState = cmJSONState(snippet_path, &snippet_root); + if (!parseState.errors.empty()) { + cmSystemTools::Error(parseState.GetErrorMessage(true)); + continue; + } + if (!snippet_root.isObject()) { + error_msg = cmStrCat("Expected snippet file ", snippet_path, + " to contain an object"); + cmSystemTools::Error(error_msg); + continue; + } + if (!snippet_root.isMember("role")) { + error_msg = cmStrCat("Expected snippet file ", snippet_path, + " to have a key 'role'"); + cmSystemTools::Error(error_msg); + continue; + } + + std::string snippet_role = snippet_root["role"].asString(); + auto map_element = this->cdashSnippetsMap.find(snippet_role); + if (map_element == this->cdashSnippetsMap.end()) { + std::string message = + "Unexpected snippet type encountered: " + snippet_role; + cmSystemTools::Message(message, "Warning"); + continue; + } + + if (map_element->second == "skip") { + continue; + } + + if (map_element->second == "build") { + // We organize snippets on a per-target basis (when possible) + // for Build.xml. + if (snippet_root.isMember("target")) { + dst_dir = cmStrCat(this->cdashDir, "/build/targets/", + snippet_root["target"].asString()); + cmSystemTools::MakeDirectory(dst_dir); + } else { + dst_dir = cmStrCat(this->cdashDir, "/build/commands"); + } + } else { + dst_dir = cmStrCat(this->cdashDir, '/', map_element->second); + } + + std::string dst = cmStrCat(dst_dir, '/', snippet_str); + cmsys::Status copied = cmSystemTools::CopyFileAlways(snippet_path, dst); + if (!copied) { + error_msg = cmStrCat("Failed to copy ", snippet_path, " to ", dst); + cmSystemTools::Error(error_msg); + } + } +} diff --git a/Source/cmInstrumentation.h b/Source/cmInstrumentation.h index 3a2f9d9..0382495 100644 --- a/Source/cmInstrumentation.h +++ b/Source/cmInstrumentation.h @@ -29,11 +29,13 @@ public: cm::optional> arrayOptions = cm::nullopt, bool reloadQueriesAfterCommand = false); - int InstrumentTest(std::string const& name, std::string const& command, - std::vector const& args, int64_t result, - std::chrono::steady_clock::time_point steadyStart, - std::chrono::system_clock::time_point systemStart, - std::string config); + std::string InstrumentTest(std::string const& name, + std::string const& command, + std::vector const& args, + int64_t result, + std::chrono::steady_clock::time_point steadyStart, + std::chrono::system_clock::time_point systemStart, + std::string config); void GetPreTestStats(); void LoadQueries(); bool HasQuery() const; @@ -49,7 +51,10 @@ public: int CollectTimingData(cmInstrumentationQuery::Hook hook); int SpawnBuildDaemon(); int CollectTimingAfterBuild(int ppid); + void AddHook(cmInstrumentationQuery::Hook hook); + void AddQuery(cmInstrumentationQuery::Query query); std::string errorMsg; + std::string const& GetCDashDir(); private: void WriteInstrumentationJson(Json::Value& index, @@ -66,13 +71,17 @@ private: static std::string GetCommandStr(std::vector const& args); static std::string ComputeSuffixHash(std::string const& command_str); static std::string ComputeSuffixTime(); + void PrepareDataForCDash(std::string const& data_dir, + std::string const& index_path); std::string binaryDir; std::string timingDirv1; std::string userTimingDirv1; + std::string cdashDir; std::set queries; std::set hooks; std::vector callbacks; std::vector queryFiles; + std::map cdashSnippetsMap; Json::Value preTestStats; bool hasQuery = false; }; diff --git a/Source/cmInstrumentationQuery.cxx b/Source/cmInstrumentationQuery.cxx index bee8b37..88872f6 100644 --- a/Source/cmInstrumentationQuery.cxx +++ b/Source/cmInstrumentationQuery.cxx @@ -19,8 +19,9 @@ std::vector const cmInstrumentationQuery::QueryString{ "staticSystemInformation", "dynamicSystemInformation" }; std::vector const cmInstrumentationQuery::HookString{ - "postGenerate", "preBuild", "postBuild", "preCMakeBuild", - "postCMakeBuild", "postTest", "postInstall", "manual" + "postGenerate", "preBuild", "postBuild", + "preCMakeBuild", "postCMakeBuild", "postTest", + "postInstall", "prepareForCDash", "manual" }; namespace ErrorMessages { diff --git a/Source/cmInstrumentationQuery.h b/Source/cmInstrumentationQuery.h index 93a9caf..b8d8936 100644 --- a/Source/cmInstrumentationQuery.h +++ b/Source/cmInstrumentationQuery.h @@ -28,6 +28,7 @@ public: PostCMakeBuild, PostTest, PostInstall, + PrepareForCDash, Manual }; static std::vector const HookString; diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index e664ca3..dbbe53e 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -605,6 +605,9 @@ add_RunCMake_test(ctest_upload) add_RunCMake_test(ctest_environment) add_RunCMake_test(ctest_empty_binary_directory) add_RunCMake_test(ctest_fixtures) +if(CMAKE_GENERATOR MATCHES "Make|Ninja") + add_RunCMake_test(ctest_instrumentation) +endif() add_RunCMake_test(define_property) add_RunCMake_test(file -DCYGWIN=${CYGWIN} -DMSYS=${MSYS}) add_RunCMake_test(file-CHMOD -DMSYS=${MSYS}) diff --git a/Tests/RunCMake/ctest_instrumentation/CMakeLists.txt.in b/Tests/RunCMake/ctest_instrumentation/CMakeLists.txt.in new file mode 100644 index 0000000..057f708 --- /dev/null +++ b/Tests/RunCMake/ctest_instrumentation/CMakeLists.txt.in @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.10) +@CASE_CMAKELISTS_PREFIX_CODE@ +project(CTestInstrumentation@CASE_NAME@) +if(USE_INSTRUMENTATION) + set(CMAKE_EXPERIMENTAL_INSTRUMENTATION "a37d1069-1972-4901-b9c9-f194aaf2b6e0") +endif() +include(CTest) +add_executable(main main.c) +add_test(NAME main COMMAND main) +@CASE_CMAKELISTS_SUFFIX_CODE@ diff --git a/Tests/RunCMake/ctest_instrumentation/InstrumentationInCTestXML-check.cmake b/Tests/RunCMake/ctest_instrumentation/InstrumentationInCTestXML-check.cmake new file mode 100644 index 0000000..643515b --- /dev/null +++ b/Tests/RunCMake/ctest_instrumentation/InstrumentationInCTestXML-check.cmake @@ -0,0 +1,41 @@ +foreach(xml_type Configure Build Test) + file(GLOB xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/${xml_type}.xml") + if(xml_file) + file(READ "${xml_file}" xml_content) + if(NOT xml_content MATCHES "AfterHostMemoryUsed") + set(RunCMake_TEST_FAILED "'AfterHostMemoryUsed' not found in ${xml_type}.xml") + endif() + if(NOT xml_type STREQUAL "Test") + if(NOT xml_content MATCHES "") + set(RunCMake_TEST_FAILED " element not found in ${xml_type}.xml") + endif() + endif() + if (xml_type STREQUAL "Build") + if(NOT xml_content MATCHES "") + set(RunCMake_TEST_FAILED " element not found in Build.xml") + endif() + if(NOT xml_content MATCHES "") + set(RunCMake_TEST_FAILED " element for 'main' not found in Build.xml") + endif() + if(NOT xml_content MATCHES " element not found in Build.xml") + endif() + if(NOT xml_content MATCHES " element not found in Build.xml") + endif() + if(NOT xml_content MATCHES " element not found in Build.xml") + endif() + endif() + else() + set(RunCMake_TEST_FAILED "${xml_type}.xml not found") + endif() +endforeach() + +foreach(dir_to_check "configure" "test" "build/targets" "build/commands") + file(GLOB leftover_cdash_snippets + "${RunCMake_TEST_BINARY_DIR}/.cmake/instrumentation-a37d1069-1972-4901-b9c9-f194aaf2b6e0/v1/cdash/${dir_to_check}/*") + if(leftover_cdash_snippets) + set(RunCMake_TEST_FAILED "Leftover snippets found in cdash dir: ${leftover_cdash_snippets}") + endif() +endforeach() diff --git a/Tests/RunCMake/ctest_instrumentation/NoInstrumentationInCTestXML-check.cmake b/Tests/RunCMake/ctest_instrumentation/NoInstrumentationInCTestXML-check.cmake new file mode 100644 index 0000000..f026f76 --- /dev/null +++ b/Tests/RunCMake/ctest_instrumentation/NoInstrumentationInCTestXML-check.cmake @@ -0,0 +1,11 @@ +foreach(xml_type Configure Build Test) + file(GLOB xml_file "${RunCMake_TEST_BINARY_DIR}/Testing/*/${xml_type}.xml") + if(xml_file) + file(READ "${xml_file}" xml_content) + if(xml_content MATCHES "AfterHostMemoryUsed") + set(RunCMake_TEST_FAILED "'AfterHostMemoryUsed' found in ${xml_type}.xml") + endif() + else() + set(RunCMake_TEST_FAILED "${xml_type}.xml not found") + endif() +endforeach() diff --git a/Tests/RunCMake/ctest_instrumentation/RunCMakeTest.cmake b/Tests/RunCMake/ctest_instrumentation/RunCMakeTest.cmake new file mode 100644 index 0000000..7424b17 --- /dev/null +++ b/Tests/RunCMake/ctest_instrumentation/RunCMakeTest.cmake @@ -0,0 +1,22 @@ +include(RunCTest) + +function(run_InstrumentationInCTestXML USE_INSTRUMENTATION) + if(USE_INSTRUMENTATION) + set(ENV{CTEST_USE_INSTRUMENTATION} "1") + set(ENV{CTEST_EXPERIMENTAL_INSTRUMENTATION} "a37d1069-1972-4901-b9c9-f194aaf2b6e0") + set(RunCMake_USE_INSTRUMENTATION TRUE) + set(CASE_NAME InstrumentationInCTestXML) + else() + set(ENV{CTEST_USE_INSTRUMENTATION} "0") + set(ENV{CTEST_EXPERIMENTAL_INSTRUMENTATION} "0") + set(RunCMake_USE_INSTRUMENTATION FALSE) + set(CASE_NAME NoInstrumentationInCTestXML) + endif() + configure_file(${RunCMake_SOURCE_DIR}/main.c + ${RunCMake_BINARY_DIR}/${CASE_NAME}/main.c COPYONLY) + run_ctest("${CASE_NAME}") + unset(RunCMake_USE_LAUNCHERS) + unset(RunCMake_USE_INSTRUMENTATION) +endfunction() +run_InstrumentationInCTestXML(ON) +run_InstrumentationInCTestXML(OFF) diff --git a/Tests/RunCMake/ctest_instrumentation/main.c b/Tests/RunCMake/ctest_instrumentation/main.c new file mode 100644 index 0000000..8488f4e --- /dev/null +++ b/Tests/RunCMake/ctest_instrumentation/main.c @@ -0,0 +1,4 @@ +int main(void) +{ + return 0; +} diff --git a/Tests/RunCMake/ctest_instrumentation/test.cmake.in b/Tests/RunCMake/ctest_instrumentation/test.cmake.in new file mode 100644 index 0000000..41b9612 --- /dev/null +++ b/Tests/RunCMake/ctest_instrumentation/test.cmake.in @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 3.10) + +set(CTEST_SITE "test-site") +set(CTEST_BUILD_NAME "test-build-name") +set(CTEST_SOURCE_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@") +set(CTEST_BINARY_DIRECTORY "@RunCMake_BINARY_DIR@/@CASE_NAME@-build") +set(CTEST_CMAKE_GENERATOR "@RunCMake_GENERATOR@") +set(CTEST_CMAKE_GENERATOR_PLATFORM "@RunCMake_GENERATOR_PLATFORM@") +set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@") +set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}") +set(CTEST_USE_LAUNCHERS TRUE) +set(CTEST_USE_INSTRUMENTATION "@RunCMake_USE_INSTRUMENTATION@") + +ctest_start(Experimental) +ctest_configure(OPTIONS "-DUSE_INSTRUMENTATION=${CTEST_USE_INSTRUMENTATION}") +ctest_build() +ctest_test() -- cgit v0.12