CTest: Allocate hardware to tests

1 files changed, 136 insertions, 4 deletions

diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index 1902500..aee6d67 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -3,8 +3,10 @@
 #include "cmCTestMultiProcessHandler.h"
 
 #include <algorithm>
+#include <cassert>
 #include <chrono>
 #include <cmath>
+#include <cstddef>
 #include <cstdlib>
 #include <cstring>
 #include <iomanip>
@@ -27,6 +29,7 @@
 #include "cmAffinity.h"
 #include "cmAlgorithms.h"
 #include "cmCTest.h"
+#include "cmCTestBinPacker.h"
 #include "cmCTestRunTest.h"
 #include "cmCTestTestHandler.h"
 #include "cmDuration.h"
@@ -133,6 +136,12 @@ void cmCTestMultiProcessHandler::RunTests()
   uv_run(&this->Loop, UV_RUN_DEFAULT);
   uv_loop_close(&this->Loop);
 
+  if (!this->StopTimePassed) {
+    assert(this->Completed == this->Total);
+    assert(this->Tests.empty());
+  }
+  assert(this->AllHardwareAvailable());
+
   this->MarkFinished();
   this->UpdateCostData();
 }
@@ -168,6 +177,10 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
   }
   testRun->SetIndex(test);
   testRun->SetTestProperties(this->Properties[test]);
+  if (this->TestHandler->UseHardwareSpec) {
+    testRun->SetUseAllocatedHardware(true);
+    testRun->SetAllocatedHardware(this->AllocatedHardware[test]);
+  }
 
   // Find any failed dependencies for this test. We assume the more common
   // scenario has no failed tests, so make it the outer loop.
@@ -179,7 +192,13 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
 
   // Always lock the resources we'll be using, even if we fail to set the
   // working directory because FinishTestProcess() will try to unlock them
-  this->LockResources(test);
+  this->AllocateResources(test);
+
+  if (!this->TestsHaveSufficientHardware[test]) {
+    testRun->StartFailure("Insufficient hardware");
+    this->FinishTestProcess(testRun, false);
+    return false;
+  }
 
   cmWorkingDirectory workdir(this->Properties[test]->Directory);
   if (workdir.Failed()) {
@@ -199,6 +218,110 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
   return false;
 }
 
+bool cmCTestMultiProcessHandler::AllocateHardware(int index)
+{
+  if (!this->TestHandler->UseHardwareSpec) {
+    return true;
+  }
+
+  std::map<std::string, std::vector<cmCTestBinPackerAllocation>> allocations;
+  if (!this->TryAllocateHardware(index, allocations)) {
+    return false;
+  }
+
+  auto& allocatedHardware = this->AllocatedHardware[index];
+  allocatedHardware.resize(this->Properties[index]->Processes.size());
+  for (auto const& it : allocations) {
+    for (auto const& alloc : it.second) {
+      bool result = this->HardwareAllocator.AllocateResource(
+        it.first, alloc.Id, alloc.SlotsNeeded);
+      (void)result;
+      assert(result);
+      allocatedHardware[alloc.ProcessIndex][it.first].push_back(
+        { alloc.Id, static_cast<unsigned int>(alloc.SlotsNeeded) });
+    }
+  }
+
+  return true;
+}
+
+bool cmCTestMultiProcessHandler::TryAllocateHardware(
+  int index,
+  std::map<std::string, std::vector<cmCTestBinPackerAllocation>>& allocations)
+{
+  allocations.clear();
+
+  std::size_t processIndex = 0;
+  for (auto const& process : this->Properties[index]->Processes) {
+    for (auto const& requirement : process) {
+      for (int i = 0; i < requirement.UnitsNeeded; ++i) {
+        allocations[requirement.ResourceType].push_back(
+          { processIndex, requirement.SlotsNeeded, "" });
+      }
+    }
+    ++processIndex;
+  }
+
+  auto const& availableHardware = this->HardwareAllocator.GetResources();
+  for (auto& it : allocations) {
+    if (!availableHardware.count(it.first)) {
+      return false;
+    }
+    if (!cmAllocateCTestHardwareRoundRobin(availableHardware.at(it.first),
+                                           it.second)) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+void cmCTestMultiProcessHandler::DeallocateHardware(int index)
+{
+  if (!this->TestHandler->UseHardwareSpec) {
+    return;
+  }
+
+  {
+    auto& allocatedHardware = this->AllocatedHardware[index];
+    for (auto const& processAlloc : allocatedHardware) {
+      for (auto const& it : processAlloc) {
+        auto resourceType = it.first;
+        for (auto const& it2 : it.second) {
+          bool success = this->HardwareAllocator.DeallocateResource(
+            resourceType, it2.Id, it2.Slots);
+          (void)success;
+          assert(success);
+        }
+      }
+    }
+  }
+  this->AllocatedHardware.erase(index);
+}
+
+bool cmCTestMultiProcessHandler::AllHardwareAvailable()
+{
+  for (auto const& it : this->HardwareAllocator.GetResources()) {
+    for (auto const& it2 : it.second) {
+      if (it2.second.Locked != 0) {
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+void cmCTestMultiProcessHandler::CheckHardwareAvailable()
+{
+  for (auto test : this->SortedTests) {
+    std::map<std::string, std::vector<cmCTestBinPackerAllocation>> allocations;
+    this->TestsHaveSufficientHardware[test] =
+      !this->TestHandler->UseHardwareSpec ||
+      this->TryAllocateHardware(test, allocations);
+  }
+}
+
 bool cmCTestMultiProcessHandler::CheckStopTimePassed()
 {
   if (!this->StopTimePassed) {
@@ -223,7 +346,7 @@ void cmCTestMultiProcessHandler::SetStopTimePassed()
   }
 }
 
-void cmCTestMultiProcessHandler::LockResources(int index)
+void cmCTestMultiProcessHandler::AllocateResources(int index)
 {
   this->LockedResources.insert(
     this->Properties[index]->LockedResources.begin(),
@@ -234,7 +357,7 @@ void cmCTestMultiProcessHandler::LockResources(int index)
   }
 }
 
-void cmCTestMultiProcessHandler::UnlockResources(int index)
+void cmCTestMultiProcessHandler::DeallocateResources(int index)
 {
   for (std::string const& i : this->Properties[index]->LockedResources) {
     this->LockedResources.erase(i);
@@ -281,12 +404,20 @@ bool cmCTestMultiProcessHandler::StartTest(int test)
     }
   }
 
+  // Allocate hardware
+  if (this->TestsHaveSufficientHardware[test] &&
+      !this->AllocateHardware(test)) {
+    this->DeallocateHardware(test);
+    return false;
+  }
+
   // if there are no depends left then run this test
   if (this->Tests[test].empty()) {
     return this->StartTestProcess(test);
   }
   // This test was not able to start because it is waiting
   // on depends to run
+  this->DeallocateHardware(test);
   return false;
 }
 
@@ -471,7 +602,8 @@ void cmCTestMultiProcessHandler::FinishTestProcess(cmCTestRunTest* runner,
   this->TestFinishMap[test] = true;
   this->TestRunningMap[test] = false;
   this->WriteCheckpoint(test);
-  this->UnlockResources(test);
+  this->DeallocateHardware(test);
+  this->DeallocateResources(test);
   this->RunningCount -= GetProcessorsUsed(test);
 
   for (auto p : properties->Affinity) {