ctest: add support for attaching files to tests at run time

Allow tests to specify files to upload at runtime. Previously this was
only possible to specify at configure time with the ATTACHED_FILES
test properties.

This commit also fixes a bug in how our test data tarballs were generated
by CTest. Previously, if you tried to attach a file outside of the binary
directory, CTest would generate a tar file with a relative path, and tar
would not allow you to extract it. We resolve this problem by creating
tar files with a flat directory structure instead.

Fixes: #22284

9 files changed, 137 insertions, 49 deletions

diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst
index c61d01e..65555a6 100644
--- a/Help/command/ctest_test.rst
+++ b/Help/command/ctest_test.rst
@@ -248,5 +248,14 @@ separate from the interactive comparison UI.
 Attached Files
 """"""""""""""
 
-To associate other types of files with a test, use the
-:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test properties.
+The following example demonstrates how to upload non-image files to CDash.
+
+.. code-block:: c++
+
+   std::cout <<
+     "<DartMeasurementFile type=\"file\" name=\"MyTestInputData\">" <<
+     "/dir/to/data.csv</DartMeasurementFile>" << std::endl;
+
+If the name of the file to upload is known at configure time, you can use the
+:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test
+properties instead.
diff --git a/Help/release/dev/ctest-measurements-docs.rst b/Help/release/dev/ctest-measurements-docs.rst
new file mode 100644
index 0000000..e47582e
--- /dev/null
+++ b/Help/release/dev/ctest-measurements-docs.rst
@@ -0,0 +1,5 @@
+ctest-measurements-docs
+-----------------------
+
+* :manual:`ctest(1)` gained documentation for its ability to capture
+  :ref:`Additional Test Measurements`.
diff --git a/Help/release/dev/ctest-runtime-files.rst b/Help/release/dev/ctest-runtime-files.rst
new file mode 100644
index 0000000..f13baa4
--- /dev/null
+++ b/Help/release/dev/ctest-runtime-files.rst
@@ -0,0 +1,8 @@
+ctest-runtime-files
+-------------------
+
+* :manual:`ctest(1)` learned to recognize files attached to a test at run time.
+  Previously it was only possible to attach files to tests at configure time
+  by using the :prop_test:`ATTACHED_FILES` or
+  :prop_test:`ATTACHED_FILES_ON_FAIL` test properties.
+  See :ref:`Additional Test Measurements` for more information.
diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx
index 1596d4a..fd38f39 100644
--- a/Source/CTest/cmCTestTestHandler.cxx
+++ b/Source/CTest/cmCTestTestHandler.cxx
@@ -1550,19 +1550,29 @@ void cmCTestTestHandler::AttachFiles(cmXMLWriter& xml,
       result.Properties->AttachOnFail.end());
   }
   for (std::string const& file : result.Properties->AttachedFiles) {
-    const std::string& base64 = this->CTest->Base64GzipEncodeFile(file);
-    std::string const fname = cmSystemTools::GetFilenameName(file);
-    xml.StartElement("NamedMeasurement");
-    xml.Attribute("name", "Attached File");
-    xml.Attribute("encoding", "base64");
-    xml.Attribute("compression", "tar/gzip");
-    xml.Attribute("filename", fname);
-    xml.Attribute("type", "file");
-    xml.Element("Value", base64);
-    xml.EndElement(); // NamedMeasurement
+    this->AttachFile(xml, file, "");
   }
 }
 
+void cmCTestTestHandler::AttachFile(cmXMLWriter& xml, std::string const& file,
+                                    std::string const& name)
+{
+  const std::string& base64 = this->CTest->Base64GzipEncodeFile(file);
+  std::string const fname = cmSystemTools::GetFilenameName(file);
+  xml.StartElement("NamedMeasurement");
+  std::string measurement_name = name;
+  if (measurement_name.empty()) {
+    measurement_name = "Attached File";
+  }
+  xml.Attribute("name", measurement_name);
+  xml.Attribute("encoding", "base64");
+  xml.Attribute("compression", "tar/gzip");
+  xml.Attribute("filename", fname);
+  xml.Attribute("type", "file");
+  xml.Element("Value", base64);
+  xml.EndElement(); // NamedMeasurement
+}
+
 int cmCTestTestHandler::ExecuteCommands(std::vector<std::string>& vec)
 {
   for (std::string const& it : vec) {
@@ -2041,11 +2051,11 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml,
         cmCTest::CleanString(measurementfile.match(5));
       if (cmSystemTools::FileExists(filename)) {
         long len = cmSystemTools::FileLength(filename);
+        std::string k1 = measurementfile.match(1);
+        std::string v1 = measurementfile.match(2);
+        std::string k2 = measurementfile.match(3);
+        std::string v2 = measurementfile.match(4);
         if (len == 0) {
-          std::string k1 = measurementfile.match(1);
-          std::string v1 = measurementfile.match(2);
-          std::string k2 = measurementfile.match(3);
-          std::string v2 = measurementfile.match(4);
           if (cmSystemTools::LowerCase(k1) == "type") {
             v1 = "text/string";
           }
@@ -2060,35 +2070,53 @@ void cmCTestTestHandler::GenerateRegressionImages(cmXMLWriter& xml,
           xml.Element("Value", "Image " + filename + " is empty");
           xml.EndElement();
         } else {
-          cmsys::ifstream ifs(filename.c_str(),
-                              std::ios::in
+          std::string type;
+          std::string name;
+          if (cmSystemTools::LowerCase(k1) == "type") {
+            type = v1;
+          } else if (cmSystemTools::LowerCase(k2) == "type") {
+            type = v2;
+          }
+          if (cmSystemTools::LowerCase(k1) == "name") {
+            name = v1;
+          } else if (cmSystemTools::LowerCase(k2) == "name") {
+            name = v2;
+          }
+          if (type == "file") {
+            // Treat this measurement like an "ATTACHED_FILE" when the type
+            // is explicitly "file" (not an image).
+            this->AttachFile(xml, filename, name);
+          } else {
+            cmsys::ifstream ifs(filename.c_str(),
+                                std::ios::in
 #ifdef _WIN32
-                                | std::ios::binary
+                                  | std::ios::binary
 #endif
-          );
-          auto file_buffer = cm::make_unique<unsigned char[]>(len + 1);
-          ifs.read(reinterpret_cast<char*>(file_buffer.get()), len);
-          auto encoded_buffer = cm::make_unique<unsigned char[]>(
-            static_cast<int>(static_cast<double>(len) * 1.5 + 5.0));
-
-          size_t rlen = cmsysBase64_Encode(file_buffer.get(), len,
-                                           encoded_buffer.get(), 1);
-
-          xml.StartElement("NamedMeasurement");
-          xml.Attribute(measurementfile.match(1).c_str(),
-                        measurementfile.match(2));
-          xml.Attribute(measurementfile.match(3).c_str(),
-                        measurementfile.match(4));
-          xml.Attribute("encoding", "base64");
-          std::ostringstream ostr;
-          for (size_t cc = 0; cc < rlen; cc++) {
-            ostr << encoded_buffer[cc];
-            if (cc % 60 == 0 && cc) {
-              ostr << std::endl;
+            );
+            auto file_buffer = cm::make_unique<unsigned char[]>(len + 1);
+            ifs.read(reinterpret_cast<char*>(file_buffer.get()), len);
+            auto encoded_buffer = cm::make_unique<unsigned char[]>(
+              static_cast<int>(static_cast<double>(len) * 1.5 + 5.0));
+
+            size_t rlen = cmsysBase64_Encode(file_buffer.get(), len,
+                                             encoded_buffer.get(), 1);
+
+            xml.StartElement("NamedMeasurement");
+            xml.Attribute(measurementfile.match(1).c_str(),
+                          measurementfile.match(2));
+            xml.Attribute(measurementfile.match(3).c_str(),
+                          measurementfile.match(4));
+            xml.Attribute("encoding", "base64");
+            std::ostringstream ostr;
+            for (size_t cc = 0; cc < rlen; cc++) {
+              ostr << encoded_buffer[cc];
+              if (cc % 60 == 0 && cc) {
+                ostr << std::endl;
+              }
             }
+            xml.Element("Value", ostr.str());
+            xml.EndElement(); // NamedMeasurement
           }
-          xml.Element("Value", ostr.str());
-          xml.EndElement(); // NamedMeasurement
         }
       } else {
         int idx = 4;
diff --git a/Source/CTest/cmCTestTestHandler.h b/Source/CTest/cmCTestTestHandler.h
index 6841624..9f5b3da 100644
--- a/Source/CTest/cmCTestTestHandler.h
+++ b/Source/CTest/cmCTestTestHandler.h
@@ -237,6 +237,8 @@ protected:
                              cmCTestTestResult const& result);
   // Write attached test files into the xml
   void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result);
+  void AttachFile(cmXMLWriter& xml, std::string const& file,
+                  std::string const& name);
 
   //! Clean test output to specified length
   void CleanTestOutput(std::string& output, size_t length);
diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx
index 7534e37..7c469c8 100644
--- a/Source/cmCTest.cxx
+++ b/Source/cmCTest.cxx
@@ -1612,8 +1612,33 @@ int cmCTest::GenerateDoneFile()
   return 0;
 }
 
+bool cmCTest::TryToChangeDirectory(std::string const& dir)
+{
+  cmCTestLog(this, OUTPUT,
+             "Internal ctest changing into directory: " << dir << std::endl);
+  cmsys::Status status = cmSystemTools::ChangeDirectory(dir);
+  if (!status) {
+    auto msg = "Failed to change working directory to \"" + dir +
+      "\" : " + status.GetString() + "\n";
+    cmCTestLog(this, ERROR_MESSAGE, msg);
+    return false;
+  }
+  return true;
+}
+
 std::string cmCTest::Base64GzipEncodeFile(std::string const& file)
 {
+  const std::string currDir = cmSystemTools::GetCurrentWorkingDirectory();
+  std::string parentDir = cmSystemTools::GetParentDirectory(file);
+
+  // Temporarily change to the file's directory so the tar gets created
+  // with a flat directory structure.
+  if (currDir != parentDir) {
+    if (!this->TryToChangeDirectory(parentDir)) {
+      return "";
+    }
+  }
+
   std::string tarFile = file + "_temp.tar.gz";
   std::vector<std::string> files;
   files.push_back(file);
@@ -1628,6 +1653,12 @@ std::string cmCTest::Base64GzipEncodeFile(std::string const& file)
   }
   std::string base64 = this->Base64EncodeFile(tarFile);
   cmSystemTools::RemoveFile(tarFile);
+
+  // Change back to the directory we started in.
+  if (currDir != parentDir) {
+    cmSystemTools::ChangeDirectory(currDir);
+  }
+
   return base64;
 }
 
@@ -2853,14 +2884,7 @@ int cmCTest::ExecuteTests()
     }
 
     if (currDir != workDir) {
-      cmCTestLog(this, OUTPUT,
-                 "Internal ctest changing into directory: " << workDir
-                                                            << std::endl);
-      cmsys::Status status = cmSystemTools::ChangeDirectory(workDir);
-      if (!status) {
-        auto msg = "Failed to change working directory to \"" + workDir +
-          "\" : " + status.GetString() + "\n";
-        cmCTestLog(this, ERROR_MESSAGE, msg);
+      if (!this->TryToChangeDirectory(workDir)) {
         return 1;
       }
     }
diff --git a/Source/cmCTest.h b/Source/cmCTest.h
index 392eb1c..551c116 100644
--- a/Source/cmCTest.h
+++ b/Source/cmCTest.h
@@ -536,6 +536,9 @@ private:
   int RunCMakeAndTest(std::string* output);
   int ExecuteTests();
 
+  /** return true iff change directory was successful */
+  bool TryToChangeDirectory(std::string const& dir);
+
   struct Private;
   std::unique_ptr<Private> Impl;
 };
diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
index b559e89..76a63f0 100644
--- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
+++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
@@ -161,6 +161,10 @@ add_test(
   NAME img_measurement
   COMMAND ${CMAKE_COMMAND} -E
   echo <DartMeasurementFile name="TestImage" type="image/png">]] ${IMAGE_DIR}/cmake-logo-16.png [[</DartMeasurementFile>)
+add_test(
+  NAME file_measurement
+  COMMAND ${CMAKE_COMMAND} -E
+  echo <DartMeasurementFile name="my_test_input_data" type="file">]] ${IMAGE_DIR}/cmake-logo-16.png [[</DartMeasurementFile>)
   ]])
   run_ctest(TestMeasurements)
 endfunction()
diff --git a/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake b/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake
index 9ff9447..0095db0 100644
--- a/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake
+++ b/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake
@@ -15,3 +15,8 @@ if(NOT _test_contents MATCHES [[NamedMeasurement name="TestImage" type="image/pn
   string(APPEND RunCMake_TEST_FAILED
     "Could not find expected <NamedMeasurement> tag for type='image/png' in Test.xml")
 endif()
+# Check file measurement.
+if(NOT _test_contents MATCHES [[NamedMeasurement name="my_test_input_data" encoding="base64" compression="tar/gzip" filename="cmake-logo-16.png" type="file"]])
+  string(APPEND RunCMake_TEST_FAILED
+    "Could not find expected <NamedMeasurement> tag for type='file' in Test.xml")
+endif()