ctest: Optionally avoid starting tests that may exceed a given CPU load

Add a TestLoad setting to CTest that can be set via a new --test-load
command-line option, CTEST_TEST_LOAD variable, or TEST_LOAD option to
the ctest_test command.  Teach cmCTestMultiProcessHandler to measure
the CPU load and avoid starting tests that may take more than the
spare load currently available.  The expression

 <current_load> + <test_processors> <= <max-load>

must be true to start a new test.

Co-Author: Zack Galbreath <zack.galbreath@kitware.com>

1 files changed, 117 insertions, 7 deletions

diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx
index 28aec3d..4832186 100644
--- a/Source/CTest/cmCTestMultiProcessHandler.cxx
+++ b/Source/CTest/cmCTestMultiProcessHandler.cxx
@@ -13,12 +13,15 @@
 #include "cmProcess.h"
 #include "cmStandardIncludes.h"
 #include "cmCTest.h"
+#include "cmCTestScriptHandler.h"
 #include "cmSystemTools.h"
 #include <stdlib.h>
 #include <stack>
 #include <list>
 #include <float.h>
+#include <math.h>
 #include <cmsys/FStream.hxx>
+#include <cmsys/SystemInformation.hxx>
 
 class TestComparator
 {
@@ -40,6 +43,7 @@ private:
 cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
 {
   this->ParallelLevel = 1;
+  this->TestLoad = 0;
   this->Completed = 0;
   this->RunningCount = 0;
   this->StopTimePassed = false;
@@ -84,6 +88,11 @@ void cmCTestMultiProcessHandler::SetParallelLevel(size_t level)
   this->ParallelLevel = level < 1 ? 1 : level;
 }
 
+void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load)
+{
+  this->TestLoad = load;
+}
+
 //---------------------------------------------------------
 void cmCTestMultiProcessHandler::RunTests()
 {
@@ -213,6 +222,11 @@ inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test)
   return processors;
 }
 
+std::string cmCTestMultiProcessHandler::GetName(int test)
+{
+  return this->Properties[test]->Name;
+}
+
 //---------------------------------------------------------
 bool cmCTestMultiProcessHandler::StartTest(int test)
 {
@@ -259,6 +273,46 @@ void cmCTestMultiProcessHandler::StartNextTests()
     return;
     }
 
+  bool allTestsFailedTestLoadCheck = false;
+  bool usedFakeLoadForTesting = false;
+  size_t minProcessorsRequired = this->ParallelLevel;
+  std::string testWithMinProcessors = "";
+
+  cmsys::SystemInformation info;
+
+  unsigned long systemLoad = 0;
+  size_t spareLoad = 0;
+  if (this->TestLoad > 0)
+    {
+    // Activate possible wait.
+    allTestsFailedTestLoadCheck = true;
+
+    // Check for a fake load average value used in testing.
+    if (const char* fake_load_value =
+        cmSystemTools::GetEnv("__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING"))
+      {
+      usedFakeLoadForTesting = true;
+      if (!cmSystemTools::StringToULong(fake_load_value, &systemLoad))
+        {
+        cmSystemTools::Error("Failed to parse fake load value: ",
+                             fake_load_value);
+        }
+      }
+    // If it's not set, look up the true load average.
+    else
+      {
+      systemLoad = static_cast<unsigned long>(ceil(info.GetLoadAverage()));
+      }
+    spareLoad = (this->TestLoad > systemLoad ?
+                 this->TestLoad - systemLoad : 0);
+
+    // Don't start more tests than the spare load can support.
+    if (numToStart > spareLoad)
+      {
+      numToStart = spareLoad;
+      }
+    }
+
   TestList copy = this->SortedTests;
   for(TestList::iterator test = copy.begin(); test != copy.end(); ++test)
     {
@@ -274,18 +328,74 @@ void cmCTestMultiProcessHandler::StartNextTests()
       }
 
     size_t processors = GetProcessorsUsed(*test);
+    bool testLoadOk = true;
+    if (this->TestLoad > 0)
+      {
+      if (processors <= spareLoad)
+        {
+        cmCTestLog(this->CTest, DEBUG,
+                    "OK to run " << GetName(*test) <<
+                    ", it requires " << processors <<
+                    " procs & system load is: " <<
+                    systemLoad << std::endl);
+        allTestsFailedTestLoadCheck = false;
+        }
+      else
+        {
+        testLoadOk = false;
+        }
+      }
 
-    if(processors <= numToStart && this->StartTest(*test))
+    if (processors <= minProcessorsRequired)
       {
-        if(this->StopTimePassed)
-          {
-          return;
-          }
-        numToStart -= processors;
+      minProcessorsRequired = processors;
+      testWithMinProcessors = GetName(*test);
+      }
+
+    if(testLoadOk && processors <= numToStart && this->StartTest(*test))
+      {
+      if(this->StopTimePassed)
+        {
+        return;
+        }
+
+      numToStart -= processors;
       }
     else if(numToStart == 0)
       {
-      return;
+      break;
+      }
+    }
+
+  if (allTestsFailedTestLoadCheck)
+    {
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***** WAITING, ");
+    if (this->SerialTestRunning)
+      {
+      cmCTestLog(this->CTest, HANDLER_OUTPUT,
+                 "Waiting for RUN_SERIAL test to finish.");
+      }
+    else
+      {
+      cmCTestLog(this->CTest, HANDLER_OUTPUT,
+                 "System Load: " << systemLoad << ", "
+                 "Max Allowed Load: " << this->TestLoad << ", "
+                 "Smallest test " << testWithMinProcessors <<
+                 " requires " << minProcessorsRequired);
+      }
+    cmCTestLog(this->CTest, HANDLER_OUTPUT, "*****" << std::endl);
+
+    if (usedFakeLoadForTesting)
+      {
+      // Break out of the infinite loop of waiting for our fake load
+      // to come down.
+      this->StopTimePassed = true;
+      }
+    else
+      {
+      // Wait between 1 and 5 seconds before trying again.
+      cmCTestScriptHandler::SleepInSeconds(
+        cmSystemTools::RandomSeed() % 5 + 1);
       }
     }
 }