From cbcb92d1cb4230426761fb7d08f9135546c98881 Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Fri, 4 Jun 2021 11:22:34 -0400 Subject: ctest: add support for attaching files to tests at run time Allow tests to specify files to upload at runtime. Previously this was only possible to specify at configure time with the ATTACHED_FILES test properties. This commit also fixes a bug in how our test data tarballs were generated by CTest. Previously, if you tried to attach a file outside of the binary directory, CTest would generate a tar file with a relative path, and tar would not allow you to extract it. We resolve this problem by creating tar files with a flat directory structure instead. Fixes: #22284 --- Help/command/ctest_test.rst | 13 ++- Help/release/dev/ctest-measurements-docs.rst | 5 + Help/release/dev/ctest-runtime-files.rst | 8 ++ Source/CTest/cmCTestTestHandler.cxx | 106 +++++++++++++-------- Source/CTest/cmCTestTestHandler.h | 2 + Source/cmCTest.cxx | 40 ++++++-- Source/cmCTest.h | 3 + Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 4 + .../ctest_test/TestMeasurements-check.cmake | 5 + 9 files changed, 137 insertions(+), 49 deletions(-) create mode 100644 Help/release/dev/ctest-measurements-docs.rst create mode 100644 Help/release/dev/ctest-runtime-files.rst diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index c61d01e..65555a6 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -248,5 +248,14 @@ separate from the interactive comparison UI. Attached Files """""""""""""" -To associate other types of files with a test, use the -:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test properties. +The following example demonstrates how to upload non-image files to CDash. + +.. code-block:: c++ + + std::cout << + "" << + "/dir/to/data.csv" << std::endl; + +If the name of the file to upload is known at configure time, you can use the +:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test +properties instead. diff --git a/Help/release/dev/ctest-measurements-docs.rst b/Help/release/dev/ctest-measurements-docs.rst new file mode 100644 index 0000000..e47582e --- /dev/null +++ b/Help/release/dev/ctest-measurements-docs.rst @@ -0,0 +1,5 @@ +ctest-measurements-docs +----------------------- + +* :manual:`ctest(1)` gained documentation for its ability to capture + :ref:`Additional Test Measurements`. diff --git a/Help/release/dev/ctest-runtime-files.rst b/Help/release/dev/ctest-runtime-files.rst new file mode 100644 index 0000000..f13baa4 --- /dev/null +++ b/Help/release/dev/ctest-runtime-files.rst @@ -0,0 +1,8 @@ +ctest-runtime-files +------------------- + +* :manual:`ctest(1)` learned to recognize files attached to a test at run time. + Previously it was only possible to attach files to tests at configure time + by using the :prop_test:`ATTACHED_FILES` or + :prop_test:`ATTACHED_FILES_ON_FAIL` test properties. + See :ref:`Additional Test Measurements` for more information. diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 1596d4a..fd38f39 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -1550,19 +1550,29 @@ void cmCTestTestHandler::AttachFiles(cmXMLWriter& xml, result.Properties->AttachOnFail.end()); } for (std::string const& file : result.Properties->AttachedFiles) { - const std::string& base64 = this->CTest->Base64GzipEncodeFile(file); - std::string const fname = cmSystemTools::GetFilenameName(file); - xml.StartElement("NamedMeasurement"); - xml.Attribute("name", "Attached File"); - xml.Attribute("encoding", "base64"); - xml.Attribute("compression", "tar/gzip"); - xml.Attribute("filename", fname); - xml.Attribute("type", "file"); - xml.Element("Value", base64); - xml.EndElement(); // NamedMeasurement + this->AttachFile(xml, file, ""); } } +void cmCTestTestHandler::AttachFile(cmXMLWriter& xml, std::string const& file, + std::string const& name) +{ + const std::string& base64 = this->CTest->Base64GzipEncodeFile(file); + std::string const fname = cmSystemTools::GetFilenameName(file); + xml.StartElement("NamedMeasurement"); + std::string measurement_name = name; + if (measurement_name.empty()) { + measurement_name = "Attached File"; + } + xml.Attribute("name", measurement_name); + xml.Attribute("encoding", "base64"); + xml.Attribute("compression", "tar/gzip"); + xml.Attribute("filename", fname); + xml.Attribute("type", "file"); + xml.Element("Value", base64); + xml.EndElement(); // NamedMeasurement +} + int cmCTestTestHandler::ExecuteCommands(std::vector& vec) { for (std::string const& it : vec) { @@ -2041,11 +2051,11 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml, cmCTest::CleanString(measurementfile.match(5)); if (cmSystemTools::FileExists(filename)) { long len = cmSystemTools::FileLength(filename); + std::string k1 = measurementfile.match(1); + std::string v1 = measurementfile.match(2); + std::string k2 = measurementfile.match(3); + std::string v2 = measurementfile.match(4); if (len == 0) { - std::string k1 = measurementfile.match(1); - std::string v1 = measurementfile.match(2); - std::string k2 = measurementfile.match(3); - std::string v2 = measurementfile.match(4); if (cmSystemTools::LowerCase(k1) == "type") { v1 = "text/string"; } @@ -2060,35 +2070,53 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml, xml.Element("Value", "Image " + filename + " is empty"); xml.EndElement(); } else { - cmsys::ifstream ifs(filename.c_str(), - std::ios::in + std::string type; + std::string name; + if (cmSystemTools::LowerCase(k1) == "type") { + type = v1; + } else if (cmSystemTools::LowerCase(k2) == "type") { + type = v2; + } + if (cmSystemTools::LowerCase(k1) == "name") { + name = v1; + } else if (cmSystemTools::LowerCase(k2) == "name") { + name = v2; + } + if (type == "file") { + // Treat this measurement like an "ATTACHED_FILE" when the type + // is explicitly "file" (not an image). + this->AttachFile(xml, filename, name); + } else { + cmsys::ifstream ifs(filename.c_str(), + std::ios::in #ifdef _WIN32 - | std::ios::binary + | std::ios::binary #endif - ); - auto file_buffer = cm::make_unique(len + 1); - ifs.read(reinterpret_cast(file_buffer.get()), len); - auto encoded_buffer = cm::make_unique( - static_cast(static_cast(len) * 1.5 + 5.0)); - - size_t rlen = cmsysBase64_Encode(file_buffer.get(), len, - encoded_buffer.get(), 1); - - xml.StartElement("NamedMeasurement"); - xml.Attribute(measurementfile.match(1).c_str(), - measurementfile.match(2)); - xml.Attribute(measurementfile.match(3).c_str(), - measurementfile.match(4)); - xml.Attribute("encoding", "base64"); - std::ostringstream ostr; - for (size_t cc = 0; cc < rlen; cc++) { - ostr << encoded_buffer[cc]; - if (cc % 60 == 0 && cc) { - ostr << std::endl; + ); + auto file_buffer = cm::make_unique(len + 1); + ifs.read(reinterpret_cast(file_buffer.get()), len); + auto encoded_buffer = cm::make_unique( + static_cast(static_cast(len) * 1.5 + 5.0)); + + size_t rlen = cmsysBase64_Encode(file_buffer.get(), len, + encoded_buffer.get(), 1); + + xml.StartElement("NamedMeasurement"); + xml.Attribute(measurementfile.match(1).c_str(), + measurementfile.match(2)); + xml.Attribute(measurementfile.match(3).c_str(), + measurementfile.match(4)); + xml.Attribute("encoding", "base64"); + std::ostringstream ostr; + for (size_t cc = 0; cc < rlen; cc++) { + ostr << encoded_buffer[cc]; + if (cc % 60 == 0 && cc) { + ostr << std::endl; + } } + xml.Element("Value", ostr.str()); + xml.EndElement(); // NamedMeasurement } - xml.Element("Value", ostr.str()); - xml.EndElement(); // NamedMeasurement } } else { int idx = 4; diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h index 6841624..9f5b3da 100644 --- a/Source/CTest/cmCTestTestHandler.h +++ b/Source/CTest/cmCTestTestHandler.h @@ -237,6 +237,8 @@ protected: cmCTestTestResult const& result); // Write attached test files into the xml void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result); + void AttachFile(cmXMLWriter& xml, std::string const& file, + std::string const& name); //! Clean test output to specified length void CleanTestOutput(std::string& output, size_t length); diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 7534e37..7c469c8 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -1612,8 +1612,33 @@ int cmCTest::GenerateDoneFile() return 0; } +bool cmCTest::TryToChangeDirectory(std::string const& dir) +{ + cmCTestLog(this, OUTPUT, + "Internal ctest changing into directory: " << dir << std::endl); + cmsys::Status status = cmSystemTools::ChangeDirectory(dir); + if (!status) { + auto msg = "Failed to change working directory to \"" + dir + + "\" : " + status.GetString() + "\n"; + cmCTestLog(this, ERROR_MESSAGE, msg); + return false; + } + return true; +} + std::string cmCTest::Base64GzipEncodeFile(std::string const& file) { + const std::string currDir = cmSystemTools::GetCurrentWorkingDirectory(); + std::string parentDir = cmSystemTools::GetParentDirectory(file); + + // Temporarily change to the file's directory so the tar gets created + // with a flat directory structure. + if (currDir != parentDir) { + if (!this->TryToChangeDirectory(parentDir)) { + return ""; + } + } + std::string tarFile = file + "_temp.tar.gz"; std::vector files; files.push_back(file); @@ -1628,6 +1653,12 @@ std::string cmCTest::Base64GzipEncodeFile(std::string const& file) } std::string base64 = this->Base64EncodeFile(tarFile); cmSystemTools::RemoveFile(tarFile); + + // Change back to the directory we started in. + if (currDir != parentDir) { + cmSystemTools::ChangeDirectory(currDir); + } + return base64; } @@ -2853,14 +2884,7 @@ int cmCTest::ExecuteTests() } if (currDir != workDir) { - cmCTestLog(this, OUTPUT, - "Internal ctest changing into directory: " << workDir - << std::endl); - cmsys::Status status = cmSystemTools::ChangeDirectory(workDir); - if (!status) { - auto msg = "Failed to change working directory to \"" + workDir + - "\" : " + status.GetString() + "\n"; - cmCTestLog(this, ERROR_MESSAGE, msg); + if (!this->TryToChangeDirectory(workDir)) { return 1; } } diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 392eb1c..551c116 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -536,6 +536,9 @@ private: int RunCMakeAndTest(std::string* output); int ExecuteTests(); + /** return true iff change directory was successful */ + bool TryToChangeDirectory(std::string const& dir); + struct Private; std::unique_ptr Impl; }; diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index b559e89..76a63f0 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -161,6 +161,10 @@ add_test( NAME img_measurement COMMAND ${CMAKE_COMMAND} -E echo ]] ${IMAGE_DIR}/cmake-logo-16.png [[) +add_test( + NAME file_measurement + COMMAND ${CMAKE_COMMAND} -E + echo ]] ${IMAGE_DIR}/cmake-logo-16.png [[) ]]) run_ctest(TestMeasurements) endfunction() diff --git a/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake b/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake index 9ff9447..0095db0 100644 --- a/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake +++ b/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake @@ -15,3 +15,8 @@ if(NOT _test_contents MATCHES [[NamedMeasurement name="TestImage" type="image/pn string(APPEND RunCMake_TEST_FAILED "Could not find expected tag for type='image/png' in Test.xml") endif() +# Check file measurement. +if(NOT _test_contents MATCHES [[NamedMeasurement name="my_test_input_data" encoding="base64" compression="tar/gzip" filename="cmake-logo-16.png" type="file"]]) + string(APPEND RunCMake_TEST_FAILED + "Could not find expected tag for type='file' in Test.xml") +endif() -- cgit v0.12