ctest: Optionally avoid starting tests that may exceed a given CPU load

Add a TestLoad setting to CTest that can be set via a new --test-load
command-line option, CTEST_TEST_LOAD variable, or TEST_LOAD option to
the ctest_test command.  Teach cmCTestMultiProcessHandler to measure
the CPU load and avoid starting tests that may take more than the
spare load currently available.  The expression

 <current_load> + <test_processors> <= <max-load>

must be true to start a new test.

Co-Author: Zack Galbreath <zack.galbreath@kitware.com>

16 files changed, 88 insertions, 0 deletions

diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt
new file mode 100644
index 0000000..eafba1c
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt
@@ -0,0 +1 @@
+No tests were found!!!
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt
new file mode 100644
index 0000000..e203c10
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt
@@ -0,0 +1,2 @@
+Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadFail-build
+\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 4, Smallest test RunCMakeVersion requires 1\*\*\*\*\*$
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt
new file mode 100644
index 0000000..7f2d7f6
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt
@@ -0,0 +1 @@
+^Invalid value for 'CTEST_TEST_LOAD' : ERR2
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt
new file mode 100644
index 0000000..b54220c
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt
new file mode 100644
index 0000000..c221eed
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadPass-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
index d906290..21d0447 100644
--- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
+++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
@@ -1,6 +1,8 @@
 include(RunCTest)
+set(RunCMake_TEST_TIMEOUT 60)
 
 set(CASE_CTEST_TEST_ARGS "")
+set(CASE_CTEST_TEST_LOAD "")
 
 function(run_ctest_test CASE_NAME)
   set(CASE_CTEST_TEST_ARGS "${ARGN}")
@@ -8,3 +10,42 @@ function(run_ctest_test CASE_NAME)
 endfunction()
 
 run_ctest_test(TestQuiet QUIET)
+
+# Tests for the 'Test Load' feature of ctest
+#
+# Spoof a load average value to make these tests more reliable.
+set(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING} 5)
+
+# Verify that new tests are started when the load average falls below
+# our threshold.
+run_ctest_test(TestLoadPass TEST_LOAD 6)
+
+# Verify that new tests are not started when the load average exceeds
+# our threshold.
+run_ctest_test(TestLoadFail TEST_LOAD 2)
+
+# Verify that when an invalid "TEST_LOAD" value is given, a warning
+# message is displayed and the value is ignored.
+run_ctest_test(TestLoadInvalid TEST_LOAD "ERR1")
+
+# Verify that new tests are started when the load average falls below
+# our threshold.
+set(CASE_CTEST_TEST_LOAD 7)
+run_ctest_test(CTestTestLoadPass)
+
+# Verify that new tests are not started when the load average exceeds
+# our threshold.
+set(CASE_CTEST_TEST_LOAD 4)
+run_ctest_test(CTestTestLoadFail)
+
+# Verify that when an invalid "CTEST_TEST_LOAD" value is given,
+# a warning message is displayed and the value is ignored.
+set(CASE_CTEST_TEST_LOAD "ERR2")
+run_ctest_test(CTestTestLoadInvalid)
+
+# Verify that the "TEST_LOAD" value has higher precedence than
+# the "CTEST_TEST_LOAD" value
+set(CASE_CTEST_TEST_LOAD "ERR3")
+run_ctest_test(TestLoadOrder TEST_LOAD "ERR4")
+
+unset(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING})
diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-result.txt b/Tests/RunCMake/ctest_test/TestLoadFail-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadFail-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt
new file mode 100644
index 0000000..eafba1c
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt
@@ -0,0 +1 @@
+No tests were found!!!
diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt
new file mode 100644
index 0000000..4d7ce48
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt
@@ -0,0 +1,2 @@
+Test project .*/Tests/RunCMake/ctest_test/TestLoadFail-build
+\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 2, Smallest test RunCMakeVersion requires 1\*\*\*\*\*$
diff --git a/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt
new file mode 100644
index 0000000..40ddb3a
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt
@@ -0,0 +1 @@
+^Invalid value for 'TEST_LOAD' : ERR1
diff --git a/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt
new file mode 100644
index 0000000..c4fd35b
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/TestLoadInvalid-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt
new file mode 100644
index 0000000..1de730e
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt
@@ -0,0 +1 @@
+^Invalid value for 'TEST_LOAD' : ERR4
diff --git a/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt
new file mode 100644
index 0000000..22da092
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/TestLoadOrder-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt
new file mode 100644
index 0000000..e5048f4
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/TestLoadPass-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/test.cmake.in b/Tests/RunCMake/ctest_test/test.cmake.in
index 1350abe..a8de6a3 100644
--- a/Tests/RunCMake/ctest_test/test.cmake.in
+++ b/Tests/RunCMake/ctest_test/test.cmake.in
@@ -8,6 +8,7 @@ set(CTEST_CMAKE_GENERATOR               "@RunCMake_GENERATOR@")
 set(CTEST_CMAKE_GENERATOR_PLATFORM      "@RunCMake_GENERATOR_PLATFORM@")
 set(CTEST_CMAKE_GENERATOR_TOOLSET       "@RunCMake_GENERATOR_TOOLSET@")
 set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+set(CTEST_TEST_LOAD                     "@CASE_CTEST_TEST_LOAD@")
 
 set(ctest_test_args "@CASE_CTEST_TEST_ARGS@")
 ctest_start(Experimental)