Merge topic 'ctest-test-load'

f62d301b ctest: Optionally avoid starting tests that may exceed a given CPU load
07c550ca cmCTestMultiProcessHandler: Refactor RUN_SERIAL implementation
8bf5a80b cmSystemTools: Add StringToULong helper
dffc307c Tests: Teach RunCMake infrastructure to optionally timeout

24 files changed, 147 insertions, 0 deletions

diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
index a3ce139..aba1daf 100644
--- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake
@@ -1,4 +1,5 @@
 include(RunCMake)
+set(RunCMake_TEST_TIMEOUT 60)
 
 unset(ENV{CTEST_PARALLEL_LEVEL})
 unset(ENV{CTEST_OUTPUT_ON_FAILURE})
@@ -52,3 +53,35 @@ add_test(MergeOutput \"${CMAKE_COMMAND}\" -P \"${RunCMake_SOURCE_DIR}/MergeOutpu
   run_cmake_command(MergeOutput ${CMAKE_CTEST_COMMAND} -V)
 endfunction()
 run_MergeOutput()
+
+
+function(run_TestLoad name load)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestLoad)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
+  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "
+  add_test(TestLoad1 \"${CMAKE_COMMAND}\" -E echo \"test of --test-load\")
+  add_test(TestLoad2 \"${CMAKE_COMMAND}\" -E echo \"test of --test-load\")
+")
+  run_cmake_command(${name} ${CMAKE_CTEST_COMMAND} -j2 --test-load ${load} --test-timeout 5)
+endfunction()
+
+# Tests for the --test-load feature of ctest
+#
+# Spoof a load average value to make these tests more reliable.
+set(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING} 5)
+
+# Verify that new tests are not started when the load average exceeds
+# our threshold.
+run_TestLoad(test-load-fail 2)
+
+# Verify that warning message is displayed but tests still start when
+# an invalid argument is given.
+run_TestLoad(test-load-invalid 'two')
+
+# Verify that new tests are started when the load average falls below
+# our threshold.
+run_TestLoad(test-load-pass 10)
+
+unset(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING})
diff --git a/Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt b/Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt
new file mode 100644
index 0000000..eafba1c
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt
@@ -0,0 +1 @@
+No tests were found!!!
diff --git a/Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt b/Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt
new file mode 100644
index 0000000..153da09
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt
@@ -0,0 +1,2 @@
+^Test project .*/Tests/RunCMake/CTestCommandLine/TestLoad
+\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 2, Smallest test TestLoad[1-2] requires 1\*\*\*\*\*
diff --git a/Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt
new file mode 100644
index 0000000..caab3b9
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt
@@ -0,0 +1 @@
+Invalid value for 'Test Load' : 'two'
diff --git a/Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt
new file mode 100644
index 0000000..7ee3dae
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt
@@ -0,0 +1,7 @@
+^Test project .*/Tests/RunCMake/CTestCommandLine/TestLoad
+    Start 1: TestLoad1
+    Start 2: TestLoad2
+1/2 Test #[1-2]: TestLoad[1-2] ........................   Passed +[0-9.]+ sec
+2/2 Test #[1-2]: TestLoad[1-2] ........................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
diff --git a/Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt b/Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt
new file mode 100644
index 0000000..10f3293
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt
@@ -0,0 +1 @@
+^$
diff --git a/Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt b/Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt
new file mode 100644
index 0000000..7ee3dae
--- /dev/null
+++ b/Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt
@@ -0,0 +1,7 @@
+^Test project .*/Tests/RunCMake/CTestCommandLine/TestLoad
+    Start 1: TestLoad1
+    Start 2: TestLoad2
+1/2 Test #[1-2]: TestLoad[1-2] ........................   Passed +[0-9.]+ sec
+2/2 Test #[1-2]: TestLoad[1-2] ........................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 2
diff --git a/Tests/RunCMake/RunCMake.cmake b/Tests/RunCMake/RunCMake.cmake
index 70c0d6c..46bc494 100644
--- a/Tests/RunCMake/RunCMake.cmake
+++ b/Tests/RunCMake/RunCMake.cmake
@@ -66,6 +66,11 @@ function(run_cmake test)
   else()
     set(actual_stderr_var actual_stderr)
   endif()
+  if(DEFINED RunCMake_TEST_TIMEOUT)
+    set(maybe_timeout TIMEOUT ${RunCMake_TEST_TIMEOUT})
+  else()
+    set(maybe_timeout "")
+  endif()
   if(RunCMake_TEST_COMMAND)
     execute_process(
       COMMAND ${RunCMake_TEST_COMMAND}
@@ -73,6 +78,7 @@ function(run_cmake test)
       OUTPUT_VARIABLE actual_stdout
       ERROR_VARIABLE ${actual_stderr_var}
       RESULT_VARIABLE actual_result
+      ${maybe_timeout}
       )
   else()
     execute_process(
@@ -87,6 +93,7 @@ function(run_cmake test)
       OUTPUT_VARIABLE actual_stdout
       ERROR_VARIABLE ${actual_stderr_var}
       RESULT_VARIABLE actual_result
+      ${maybe_timeout}
       )
   endif()
   set(msg "")
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt
new file mode 100644
index 0000000..eafba1c
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt
@@ -0,0 +1 @@
+No tests were found!!!
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt
new file mode 100644
index 0000000..e203c10
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt
@@ -0,0 +1,2 @@
+Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadFail-build
+\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 4, Smallest test RunCMakeVersion requires 1\*\*\*\*\*$
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt
new file mode 100644
index 0000000..7f2d7f6
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt
@@ -0,0 +1 @@
+^Invalid value for 'CTEST_TEST_LOAD' : ERR2
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt
new file mode 100644
index 0000000..b54220c
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt
new file mode 100644
index 0000000..c221eed
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadPass-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
index d906290..21d0447 100644
--- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
+++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
@@ -1,6 +1,8 @@
 include(RunCTest)
+set(RunCMake_TEST_TIMEOUT 60)
 
 set(CASE_CTEST_TEST_ARGS "")
+set(CASE_CTEST_TEST_LOAD "")
 
 function(run_ctest_test CASE_NAME)
   set(CASE_CTEST_TEST_ARGS "${ARGN}")
@@ -8,3 +10,42 @@ function(run_ctest_test CASE_NAME)
 endfunction()
 
 run_ctest_test(TestQuiet QUIET)
+
+# Tests for the 'Test Load' feature of ctest
+#
+# Spoof a load average value to make these tests more reliable.
+set(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING} 5)
+
+# Verify that new tests are started when the load average falls below
+# our threshold.
+run_ctest_test(TestLoadPass TEST_LOAD 6)
+
+# Verify that new tests are not started when the load average exceeds
+# our threshold.
+run_ctest_test(TestLoadFail TEST_LOAD 2)
+
+# Verify that when an invalid "TEST_LOAD" value is given, a warning
+# message is displayed and the value is ignored.
+run_ctest_test(TestLoadInvalid TEST_LOAD "ERR1")
+
+# Verify that new tests are started when the load average falls below
+# our threshold.
+set(CASE_CTEST_TEST_LOAD 7)
+run_ctest_test(CTestTestLoadPass)
+
+# Verify that new tests are not started when the load average exceeds
+# our threshold.
+set(CASE_CTEST_TEST_LOAD 4)
+run_ctest_test(CTestTestLoadFail)
+
+# Verify that when an invalid "CTEST_TEST_LOAD" value is given,
+# a warning message is displayed and the value is ignored.
+set(CASE_CTEST_TEST_LOAD "ERR2")
+run_ctest_test(CTestTestLoadInvalid)
+
+# Verify that the "TEST_LOAD" value has higher precedence than
+# the "CTEST_TEST_LOAD" value
+set(CASE_CTEST_TEST_LOAD "ERR3")
+run_ctest_test(TestLoadOrder TEST_LOAD "ERR4")
+
+unset(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING})
diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-result.txt b/Tests/RunCMake/ctest_test/TestLoadFail-result.txt
new file mode 100644
index 0000000..b57e2de
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadFail-result.txt
@@ -0,0 +1 @@
+(-1|255)
diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt
new file mode 100644
index 0000000..eafba1c
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt
@@ -0,0 +1 @@
+No tests were found!!!
diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt
new file mode 100644
index 0000000..4d7ce48
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt
@@ -0,0 +1,2 @@
+Test project .*/Tests/RunCMake/ctest_test/TestLoadFail-build
+\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 2, Smallest test RunCMakeVersion requires 1\*\*\*\*\*$
diff --git a/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt
new file mode 100644
index 0000000..40ddb3a
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt
@@ -0,0 +1 @@
+^Invalid value for 'TEST_LOAD' : ERR1
diff --git a/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt
new file mode 100644
index 0000000..c4fd35b
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/TestLoadInvalid-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt
new file mode 100644
index 0000000..1de730e
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt
@@ -0,0 +1 @@
+^Invalid value for 'TEST_LOAD' : ERR4
diff --git a/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt
new file mode 100644
index 0000000..22da092
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/TestLoadOrder-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt
new file mode 100644
index 0000000..e5048f4
--- /dev/null
+++ b/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt
@@ -0,0 +1,7 @@
+Test project .*/Tests/RunCMake/ctest_test/TestLoadPass-build
+    Start 1: RunCMakeVersion
+1/1 Test #1: RunCMakeVersion ..................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$
diff --git a/Tests/RunCMake/ctest_test/test.cmake.in b/Tests/RunCMake/ctest_test/test.cmake.in
index 1350abe..a8de6a3 100644
--- a/Tests/RunCMake/ctest_test/test.cmake.in
+++ b/Tests/RunCMake/ctest_test/test.cmake.in
@@ -8,6 +8,7 @@ set(CTEST_CMAKE_GENERATOR               "@RunCMake_GENERATOR@")
 set(CTEST_CMAKE_GENERATOR_PLATFORM      "@RunCMake_GENERATOR_PLATFORM@")
 set(CTEST_CMAKE_GENERATOR_TOOLSET       "@RunCMake_GENERATOR_TOOLSET@")
 set(CTEST_BUILD_CONFIGURATION           "$ENV{CMAKE_CONFIG_TYPE}")
+set(CTEST_TEST_LOAD                     "@CASE_CTEST_TEST_LOAD@")
 
 set(ctest_test_args "@CASE_CTEST_TEST_ARGS@")
 ctest_start(Experimental)