Tests: Write cthwalloc helper tool

2 files changed, 428 insertions, 0 deletions

diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index bd068fd..45c549b 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -335,6 +335,38 @@ add_RunCMake_test(no_install_prefix)
 add_RunCMake_test(configure_file)
 add_RunCMake_test(CTestTimeoutAfterMatch)
 
+# cthwalloc links against CMakeLib and CTestLib, which means it can't be built
+# if CMake_TEST_EXTERNAL_CMAKE is activated (the compiler might be different.)
+# So, it has to be provided in the original build tree.
+if(CMake_TEST_EXTERNAL_CMAKE)
+  set(no_package_root_path)
+  if(NOT CMAKE_VERSION VERSION_LESS 3.12)
+    set(no_package_root_path NO_PACKAGE_ROOT_PATH)
+  endif()
+  find_program(cthwalloc cthwalloc PATHS ${CMake_TEST_EXTERNAL_CMAKE}
+    NO_DEFAULT_PATH
+    ${no_package_root_path}
+    NO_CMAKE_PATH
+    NO_CMAKE_ENVIRONMENT_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH
+    NO_CMAKE_SYSTEM_PATH
+    NO_CMAKE_FIND_ROOT_PATH
+    )
+  if(cthwalloc)
+    add_executable(cthwalloc IMPORTED)
+    set_property(TARGET cthwalloc PROPERTY IMPORTED_LOCATION ${cthwalloc})
+  endif()
+else()
+  add_executable(cthwalloc CTestHardwareAllocation/cthwalloc.cxx)
+  target_link_libraries(cthwalloc CTestLib)
+  target_include_directories(cthwalloc PRIVATE
+    ${CMake_BINARY_DIR}/Source
+    ${CMake_SOURCE_DIR}/Source
+    ${CMake_SOURCE_DIR}/Source/CTest
+    )
+  set_property(TARGET cthwalloc PROPERTY RUNTIME_OUTPUT_DIRECTORY ${CMake_BIN_DIR})
+endif()
+
 find_package(Qt4 QUIET)
 find_package(Qt5Core QUIET)
 if (QT4_FOUND AND Qt5Core_FOUND AND NOT Qt5Core_VERSION VERSION_LESS 5.1.0)
diff --git a/Tests/RunCMake/CTestHardwareAllocation/cthwalloc.cxx b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc.cxx
new file mode 100644
index 0000000..eee2c7f
--- /dev/null
+++ b/Tests/RunCMake/CTestHardwareAllocation/cthwalloc.cxx
@@ -0,0 +1,396 @@
+#include <cassert>
+#include <chrono>
+#include <cstddef>
+#include <cstdlib>
+#include <iostream>
+#include <map>
+#include <set>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "cmsys/Encoding.hxx"
+#include "cmsys/FStream.hxx"
+
+#include "cmCTestHardwareAllocator.h"
+#include "cmCTestHardwareSpec.h"
+#include "cmCTestMultiProcessHandler.h"
+#include "cmCTestTestHandler.h"
+#include "cmFileLock.h"
+#include "cmFileLockResult.h"
+#include "cmStringAlgorithms.h"
+#include "cmSystemTools.h"
+
+/*
+ * This helper program is used to verify that the CTest hardware allocation
+ * feature is working correctly. It consists of two stages:
+ *
+ * 1) write - This stage receives the PROCESSES property of the test and
+ *    compares it with the values passed in the CTEST_PROCESS_* environment
+ *    variables. If it received all of the resources it expected, then it
+ *    writes this information to a log file, which will be read in the verify
+ *    stage.
+ * 2) verify - This stage compares the log file with the hardware spec file to
+ *    make sure that no resources were over-subscribed, deallocated without
+ *    being allocated, or allocated without being deallocated.
+ */
+
+static int usage(const char* argv0)
+{
+  std::cout << "Usage: " << argv0 << " (write|verify) <args...>" << std::endl;
+  return 1;
+}
+
+static int usageWrite(const char* argv0)
+{
+  std::cout << "Usage: " << argv0
+            << " write <log-file> <test-name> <sleep-time-secs>"
+               " [<processes-property>]"
+            << std::endl;
+  return 1;
+}
+
+static int usageVerify(const char* argv0)
+{
+  std::cout << "Usage: " << argv0
+            << " verify <log-file> <hardware-spec-file> [<test-names>]"
+            << std::endl;
+  return 1;
+}
+
+static int doWrite(int argc, char const* const* argv)
+{
+  if (argc < 5 || argc > 6) {
+    return usageWrite(argv[0]);
+  }
+  std::string logFile = argv[2];
+  std::string testName = argv[3];
+  unsigned int sleepTime = std::atoi(argv[4]);
+  std::vector<std::map<
+    std::string, std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>>
+    hardware;
+  if (argc == 6) {
+    // Parse processes property
+    std::string processesProperty = argv[5];
+    std::vector<
+      std::vector<cmCTestTestHandler::cmCTestTestResourceRequirement>>
+      processes;
+    bool result =
+      cmCTestTestHandler::ParseProcessesProperty(processesProperty, processes);
+    (void)result;
+    assert(result);
+
+    // Verify process count
+    const char* processCountEnv = cmSystemTools::GetEnv("CTEST_PROCESS_COUNT");
+    if (!processCountEnv) {
+      std::cout << "CTEST_PROCESS_COUNT should be defined" << std::endl;
+      return 1;
+    }
+    int processCount = std::atoi(processCountEnv);
+    if (processes.size() != std::size_t(processCount)) {
+      std::cout << "CTEST_PROCESS_COUNT does not match expected processes"
+                << std::endl
+                << "Expected: " << processes.size() << std::endl
+                << "Actual: " << processCount << std::endl;
+      return 1;
+    }
+
+    if (!cmSystemTools::Touch(logFile + ".lock", true)) {
+      std::cout << "Could not create lock file" << std::endl;
+      return 1;
+    }
+    cmFileLock lock;
+    auto lockResult =
+      lock.Lock(logFile + ".lock", static_cast<unsigned long>(-1));
+    if (!lockResult.IsOk()) {
+      std::cout << "Could not lock file" << std::endl;
+      return 1;
+    }
+    std::size_t i = 0;
+    cmsys::ofstream fout(logFile.c_str(), std::ios::app);
+    fout << "begin " << testName << std::endl;
+    for (auto& process : processes) {
+      try {
+        // Build and verify set of expected resources
+        std::set<std::string> expectedResources;
+        for (auto const& it : process) {
+          expectedResources.insert(it.ResourceType);
+        }
+
+        std::string prefix = "CTEST_PROCESS_";
+        prefix += std::to_string(i);
+        const char* actualResourcesCStr = cmSystemTools::GetEnv(prefix);
+        if (!actualResourcesCStr) {
+          std::cout << prefix << " should be defined" << std::endl;
+          return 1;
+        }
+
+        auto actualResourcesVec =
+          cmSystemTools::SplitString(actualResourcesCStr, ',');
+        std::set<std::string> actualResources;
+        for (auto const& r : actualResourcesVec) {
+          if (!r.empty()) {
+            actualResources.insert(r);
+          }
+        }
+
+        if (actualResources != expectedResources) {
+          std::cout << prefix << " did not list expected resources"
+                    << std::endl;
+          return 1;
+        }
+
+        // Verify that we got what we asked for and write it to the log
+        prefix += '_';
+        std::map<std::string,
+                 std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>
+          hwEntry;
+        for (auto const& type : actualResources) {
+          auto it = process.begin();
+
+          std::string varName = prefix;
+          varName += cmSystemTools::UpperCase(type);
+          const char* varVal = cmSystemTools::GetEnv(varName);
+          if (!varVal) {
+            std::cout << varName << " should be defined" << std::endl;
+            return 1;
+          }
+
+          auto received = cmSystemTools::SplitString(varVal, ';');
+          for (auto const& r : received) {
+            while (it->ResourceType != type || it->UnitsNeeded == 0) {
+              ++it;
+              if (it == process.end()) {
+                std::cout << varName << " did not list expected resources"
+                          << std::endl;
+                return 1;
+              }
+            }
+            auto split = cmSystemTools::SplitString(r, ',');
+            if (split.size() != 2) {
+              std::cout << varName << " was ill-formed" << std::endl;
+              return 1;
+            }
+            if (!cmHasLiteralPrefix(split[0], "id:")) {
+              std::cout << varName << " was ill-formed" << std::endl;
+              return 1;
+            }
+            auto id = split[0].substr(3);
+            if (!cmHasLiteralPrefix(split[1], "slots:")) {
+              std::cout << varName << " was ill-formed" << std::endl;
+              return 1;
+            }
+            auto slots = split[1].substr(6);
+            unsigned int amount = std::atoi(slots.c_str());
+            if (amount != static_cast<unsigned int>(it->SlotsNeeded)) {
+              std::cout << varName << " did not list expected resources"
+                        << std::endl;
+              return 1;
+            }
+            --it->UnitsNeeded;
+
+            fout << "alloc " << type << " " << id << " " << amount
+                 << std::endl;
+            hwEntry[type].push_back({ id, amount });
+          }
+
+          bool ended = false;
+          while (it->ResourceType != type || it->UnitsNeeded == 0) {
+            ++it;
+            if (it == process.end()) {
+              ended = true;
+              break;
+            }
+          }
+
+          if (!ended) {
+            std::cout << varName << " did not list expected resources"
+                      << std::endl;
+            return 1;
+          }
+        }
+        hardware.push_back(hwEntry);
+
+        ++i;
+      } catch (...) {
+        std::cout << "Unknown error while processing resources" << std::endl;
+        return 1;
+      }
+    }
+
+    auto unlockResult = lock.Release();
+    if (!unlockResult.IsOk()) {
+      std::cout << "Could not unlock file" << std::endl;
+      return 1;
+    }
+  } else {
+    if (cmSystemTools::GetEnv("CTEST_PROCESS_COUNT")) {
+      std::cout << "CTEST_PROCESS_COUNT should not be defined" << std::endl;
+      return 1;
+    }
+  }
+
+  std::this_thread::sleep_for(std::chrono::seconds(sleepTime));
+
+  if (argc == 6) {
+    if (!cmSystemTools::Touch(logFile + ".lock", true)) {
+      std::cout << "Could not create lock file" << std::endl;
+      return 1;
+    }
+    cmFileLock lock;
+    auto lockResult =
+      lock.Lock(logFile + ".lock", static_cast<unsigned long>(-1));
+    if (!lockResult.IsOk()) {
+      std::cout << "Could not lock file" << std::endl;
+      return 1;
+    }
+    cmsys::ofstream fout(logFile.c_str(), std::ios::app);
+    for (auto const& process : hardware) {
+      for (auto const& it : process) {
+        for (auto const& it2 : it.second) {
+          fout << "dealloc " << it.first << " " << it2.Id << " " << it2.Slots
+               << std::endl;
+        }
+      }
+    }
+
+    fout << "end " << testName << std::endl;
+
+    auto unlockResult = lock.Release();
+    if (!unlockResult.IsOk()) {
+      std::cout << "Could not unlock file" << std::endl;
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+static int doVerify(int argc, char const* const* argv)
+{
+  if (argc < 4 || argc > 5) {
+    return usageVerify(argv[0]);
+  }
+  std::string logFile = argv[2];
+  std::string hwFile = argv[3];
+  std::string testNames;
+  if (argc == 5) {
+    testNames = argv[4];
+  }
+  auto testNameList = cmExpandedList(testNames, false);
+  std::set<std::string> testNameSet(testNameList.begin(), testNameList.end());
+
+  cmCTestHardwareSpec spec;
+  if (!spec.ReadFromJSONFile(hwFile)) {
+    std::cout << "Could not read hardware spec " << hwFile << std::endl;
+    return 1;
+  }
+
+  cmCTestHardwareAllocator allocator;
+  allocator.InitializeFromHardwareSpec(spec);
+
+  cmsys::ifstream fin(logFile.c_str(), std::ios::in);
+  if (!fin) {
+    std::cout << "Could not open log file " << logFile << std::endl;
+    return 1;
+  }
+
+  std::string command;
+  std::string resourceName;
+  std::string resourceId;
+  std::string testName;
+  unsigned int amount;
+  std::set<std::string> inProgressTests;
+  std::set<std::string> completedTests;
+  try {
+    while (fin >> command) {
+      if (command == "begin") {
+        if (!(fin >> testName)) {
+          std::cout << "Could not read begin line" << std::endl;
+          return 1;
+        }
+        if (!testNameSet.count(testName) || inProgressTests.count(testName) ||
+            completedTests.count(testName)) {
+          std::cout << "Could not begin test" << std::endl;
+          return 1;
+        }
+        inProgressTests.insert(testName);
+      } else if (command == "alloc") {
+        if (!(fin >> resourceName) || !(fin >> resourceId) ||
+            !(fin >> amount)) {
+          std::cout << "Could not read alloc line" << std::endl;
+          return 1;
+        }
+        if (!allocator.AllocateResource(resourceName, resourceId, amount)) {
+          std::cout << "Could not allocate resources" << std::endl;
+          return 1;
+        }
+      } else if (command == "dealloc") {
+        if (!(fin >> resourceName) || !(fin >> resourceId) ||
+            !(fin >> amount)) {
+          std::cout << "Could not read dealloc line" << std::endl;
+          return 1;
+        }
+        if (!allocator.DeallocateResource(resourceName, resourceId, amount)) {
+          std::cout << "Could not deallocate resources" << std::endl;
+          return 1;
+        }
+      } else if (command == "end") {
+        if (!(fin >> testName)) {
+          std::cout << "Could not read end line" << std::endl;
+          return 1;
+        }
+        if (!inProgressTests.erase(testName)) {
+          std::cout << "Could not end test" << std::endl;
+          return 1;
+        }
+        if (!completedTests.insert(testName).second) {
+          std::cout << "Could not end test" << std::endl;
+          return 1;
+        }
+      }
+    }
+  } catch (...) {
+    std::cout << "Unknown error while reading log file" << std::endl;
+    return 1;
+  }
+
+  auto const& avail = allocator.GetResources();
+  for (auto const& it : avail) {
+    for (auto const& it2 : it.second) {
+      if (it2.second.Locked != 0) {
+        std::cout << "Resource was not unlocked" << std::endl;
+        return 1;
+      }
+    }
+  }
+
+  if (completedTests != testNameSet) {
+    std::cout << "Tests were not ended" << std::endl;
+    return 1;
+  }
+
+  return 0;
+}
+
+int main(int argc, char const* const* argv)
+{
+  cmsys::Encoding::CommandLineArguments args =
+    cmsys::Encoding::CommandLineArguments::Main(argc, argv);
+  argc = args.argc();
+  argv = args.argv();
+
+  if (argc < 2) {
+    return usage(argv[0]);
+  }
+
+  std::string argv1 = argv[1];
+  if (argv1 == "write") {
+    return doWrite(argc, argv);
+  }
+  if (argv1 == "verify") {
+    return doVerify(argc, argv);
+  }
+  return usage(argv[0]);
+}