diff options
40 files changed, 392 insertions, 7 deletions
diff --git a/Help/command/ctest_memcheck.rst b/Help/command/ctest_memcheck.rst index 2800511..29bdf7d 100644 --- a/Help/command/ctest_memcheck.rst +++ b/Help/command/ctest_memcheck.rst @@ -14,6 +14,7 @@ Perform the :ref:`CTest MemCheck Step` as a :ref:`Dashboard Client`. [EXCLUDE_LABEL <label-exclude-regex>] [INCLUDE_LABEL <label-include-regex>] [PARALLEL_LEVEL <level>] + [TEST_LOAD <threshold>] [SCHEDULE_RANDOM <ON|OFF>] [STOP_TIME <time-of-day>] [RETURN_VALUE <result-var>] diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index af422b6..162db69 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -14,6 +14,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`. [EXCLUDE_LABEL <label-exclude-regex>] [INCLUDE_LABEL <label-include-regex>] [PARALLEL_LEVEL <level>] + [TEST_LOAD <threshold>] [SCHEDULE_RANDOM <ON|OFF>] [STOP_TIME <time-of-day>] [RETURN_VALUE <result-var>] @@ -61,6 +62,13 @@ The options are: Specify a positive number representing the number of tests to be run in parallel. +``TEST_LOAD <threshold>`` + While running tests in parallel, try not to start tests when they + may cause the CPU load to pass above a given threshold. If not + specified the :variable:`CTEST_TEST_LOAD` variable will be checked, + and then the ``--test-load`` command-line argument to :manual:`ctest(1)`. + See also the ``TestLoad`` setting in the :ref:`CTest Test Step`. + ``SCHEDULE_RANDOM <ON|OFF>`` Launch tests in a random order. This may be useful for detecting implicit test dependencies. diff --git a/Help/manual/cmake-variables.7.rst b/Help/manual/cmake-variables.7.rst index 0e6222f..adbc40b 100644 --- a/Help/manual/cmake-variables.7.rst +++ b/Help/manual/cmake-variables.7.rst @@ -386,6 +386,7 @@ Variables for CTest /variable/CTEST_SVN_COMMAND /variable/CTEST_SVN_OPTIONS /variable/CTEST_SVN_UPDATE_OPTIONS + /variable/CTEST_TEST_LOAD /variable/CTEST_TEST_TIMEOUT /variable/CTEST_TRIGGER_SITE /variable/CTEST_UPDATE_COMMAND diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index c91321b..50c856a 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -66,6 +66,13 @@ Options number of jobs. This option can also be set by setting the environment variable ``CTEST_PARALLEL_LEVEL``. +``--test-load <level>`` + While running tests in parallel (e.g. with ``-j``), try not to start + tests when they may cause the CPU load to pass above a given threshold. + + When ``ctest`` is run as a `Dashboard Client`_ this sets the + ``TestLoad`` option of the `CTest Test Step`_. + ``-Q,--quiet`` Make ctest quiet. @@ -776,6 +783,13 @@ Arguments to the command may specify some of the step settings. Configuration settings include: +``TestLoad`` + While running tests in parallel (e.g. with ``-j``), try not to start + tests when they may cause the CPU load to pass above a given threshold. + + * `CTest Script`_ variable: :variable:`CTEST_TEST_LOAD` + * :module:`CTest` module variable: ``CTEST_TEST_LOAD`` + ``TimeOut`` The default timeout for each test if not specified by the :prop_test:`TIMEOUT` test property. diff --git a/Help/release/dev/ctest-test-load-option.rst b/Help/release/dev/ctest-test-load-option.rst new file mode 100644 index 0000000..069f49a --- /dev/null +++ b/Help/release/dev/ctest-test-load-option.rst @@ -0,0 +1,9 @@ +ctest-test-load-option +---------------------- + +* CTest learned to optionally measure the CPU load during parallel + testing and avoid starting tests that may cause the load to exceed + a given threshold. See the :manual:`ctest(1)` command ``--test-load`` + option, the ``TestLoad`` setting of the :ref:`CTest Test Step`, + the :variable:`CTEST_TEST_LOAD` variable, and the ``TEST_LOAD`` + option of the :command:`ctest_test` command. diff --git a/Help/variable/CTEST_TEST_LOAD.rst b/Help/variable/CTEST_TEST_LOAD.rst new file mode 100644 index 0000000..80823fe --- /dev/null +++ b/Help/variable/CTEST_TEST_LOAD.rst @@ -0,0 +1,7 @@ +CTEST_TEST_LOAD +--------------- + +Specify the ``TestLoad`` setting in the :ref:`CTest Test Step` +of a :manual:`ctest(1)` dashboard client script. This sets the +default value for the ``TEST_LOAD`` option of the :command:`ctest_test` +command. diff --git a/Modules/DartConfiguration.tcl.in b/Modules/DartConfiguration.tcl.in index 37a0a40..918b407 100644 --- a/Modules/DartConfiguration.tcl.in +++ b/Modules/DartConfiguration.tcl.in @@ -95,6 +95,10 @@ SlurmRunCommand: @SLURM_SRUN_COMMAND@ # Currently set to 25 minutes TimeOut: @DART_TESTING_TIMEOUT@ +# During parallel testing CTest will not start a new test if doing +# so would cause the system load to exceed this value. +TestLoad: @CTEST_TEST_LOAD@ + UseLaunchers: @CTEST_USE_LAUNCHERS@ CurlOptions: @CTEST_CURL_OPTIONS@ # warning, if you add new options here that have to do with submit, diff --git a/Source/CTest/cmCTestGenericHandler.cxx b/Source/CTest/cmCTestGenericHandler.cxx index 81eb0a8..ad79ba2 100644 --- a/Source/CTest/cmCTestGenericHandler.cxx +++ b/Source/CTest/cmCTestGenericHandler.cxx @@ -23,6 +23,7 @@ cmCTestGenericHandler::cmCTestGenericHandler() this->SubmitIndex = 0; this->AppendXML = false; this->Quiet = false; + this->TestLoad = 0; } //---------------------------------------------------------------------- @@ -70,6 +71,7 @@ void cmCTestGenericHandler::SetPersistentOption(const std::string& op, void cmCTestGenericHandler::Initialize() { this->AppendXML = false; + this->TestLoad = 0; this->Options.clear(); t_StringToString::iterator it; for ( it = this->PersistentOptions.begin(); diff --git a/Source/CTest/cmCTestGenericHandler.h b/Source/CTest/cmCTestGenericHandler.h index 8567dd7..4b7ae79 100644 --- a/Source/CTest/cmCTestGenericHandler.h +++ b/Source/CTest/cmCTestGenericHandler.h @@ -89,6 +89,8 @@ public: void SetAppendXML(bool b) { this->AppendXML = b; } void SetQuiet(bool b) { this->Quiet = b; } bool GetQuiet() { return this->Quiet; } + void SetTestLoad(unsigned long load) { this->TestLoad = load; } + unsigned long GetTestLoad() const { return this->TestLoad; } protected: bool StartResultingXML(cmCTest::Part part, @@ -97,6 +99,7 @@ protected: bool AppendXML; bool Quiet; + unsigned long TestLoad; cmSystemTools::OutputOption HandlerVerbose; cmCTest *CTest; t_StringToString Options; diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 28aec3d..4832186 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -13,12 +13,15 @@ #include "cmProcess.h" #include "cmStandardIncludes.h" #include "cmCTest.h" +#include "cmCTestScriptHandler.h" #include "cmSystemTools.h" #include <stdlib.h> #include <stack> #include <list> #include <float.h> +#include <math.h> #include <cmsys/FStream.hxx> +#include <cmsys/SystemInformation.hxx> class TestComparator { @@ -40,6 +43,7 @@ private: cmCTestMultiProcessHandler::cmCTestMultiProcessHandler() { this->ParallelLevel = 1; + this->TestLoad = 0; this->Completed = 0; this->RunningCount = 0; this->StopTimePassed = false; @@ -84,6 +88,11 @@ void cmCTestMultiProcessHandler::SetParallelLevel(size_t level) this->ParallelLevel = level < 1 ? 1 : level; } +void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load) +{ + this->TestLoad = load; +} + //--------------------------------------------------------- void cmCTestMultiProcessHandler::RunTests() { @@ -213,6 +222,11 @@ inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test) return processors; } +std::string cmCTestMultiProcessHandler::GetName(int test) +{ + return this->Properties[test]->Name; +} + //--------------------------------------------------------- bool cmCTestMultiProcessHandler::StartTest(int test) { @@ -259,6 +273,46 @@ void cmCTestMultiProcessHandler::StartNextTests() return; } + bool allTestsFailedTestLoadCheck = false; + bool usedFakeLoadForTesting = false; + size_t minProcessorsRequired = this->ParallelLevel; + std::string testWithMinProcessors = ""; + + cmsys::SystemInformation info; + + unsigned long systemLoad = 0; + size_t spareLoad = 0; + if (this->TestLoad > 0) + { + // Activate possible wait. + allTestsFailedTestLoadCheck = true; + + // Check for a fake load average value used in testing. + if (const char* fake_load_value = + cmSystemTools::GetEnv("__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING")) + { + usedFakeLoadForTesting = true; + if (!cmSystemTools::StringToULong(fake_load_value, &systemLoad)) + { + cmSystemTools::Error("Failed to parse fake load value: ", + fake_load_value); + } + } + // If it's not set, look up the true load average. + else + { + systemLoad = static_cast<unsigned long>(ceil(info.GetLoadAverage())); + } + spareLoad = (this->TestLoad > systemLoad ? + this->TestLoad - systemLoad : 0); + + // Don't start more tests than the spare load can support. + if (numToStart > spareLoad) + { + numToStart = spareLoad; + } + } + TestList copy = this->SortedTests; for(TestList::iterator test = copy.begin(); test != copy.end(); ++test) { @@ -274,18 +328,74 @@ void cmCTestMultiProcessHandler::StartNextTests() } size_t processors = GetProcessorsUsed(*test); + bool testLoadOk = true; + if (this->TestLoad > 0) + { + if (processors <= spareLoad) + { + cmCTestLog(this->CTest, DEBUG, + "OK to run " << GetName(*test) << + ", it requires " << processors << + " procs & system load is: " << + systemLoad << std::endl); + allTestsFailedTestLoadCheck = false; + } + else + { + testLoadOk = false; + } + } - if(processors <= numToStart && this->StartTest(*test)) + if (processors <= minProcessorsRequired) { - if(this->StopTimePassed) - { - return; - } - numToStart -= processors; + minProcessorsRequired = processors; + testWithMinProcessors = GetName(*test); + } + + if(testLoadOk && processors <= numToStart && this->StartTest(*test)) + { + if(this->StopTimePassed) + { + return; + } + + numToStart -= processors; } else if(numToStart == 0) { - return; + break; + } + } + + if (allTestsFailedTestLoadCheck) + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, "***** WAITING, "); + if (this->SerialTestRunning) + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, + "Waiting for RUN_SERIAL test to finish."); + } + else + { + cmCTestLog(this->CTest, HANDLER_OUTPUT, + "System Load: " << systemLoad << ", " + "Max Allowed Load: " << this->TestLoad << ", " + "Smallest test " << testWithMinProcessors << + " requires " << minProcessorsRequired); + } + cmCTestLog(this->CTest, HANDLER_OUTPUT, "*****" << std::endl); + + if (usedFakeLoadForTesting) + { + // Break out of the infinite loop of waiting for our fake load + // to come down. + this->StopTimePassed = true; + } + else + { + // Wait between 1 and 5 seconds before trying again. + cmCTestScriptHandler::SleepInSeconds( + cmSystemTools::RandomSeed() % 5 + 1); } } } diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h index 243318f..ed3e155 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.h +++ b/Source/CTest/cmCTestMultiProcessHandler.h @@ -37,6 +37,7 @@ public: void SetTests(TestMap& tests, PropertiesMap& properties); // Set the max number of tests that can be run at the same time. void SetParallelLevel(size_t); + void SetTestLoad(unsigned long load); virtual void RunTests(); void PrintTestList(); void PrintLabels(); @@ -93,6 +94,7 @@ protected: bool CheckCycles(); int FindMaxIndex(); inline size_t GetProcessorsUsed(int index); + std::string GetName(int index); void LockResources(int index); void UnlockResources(int index); @@ -116,6 +118,7 @@ protected: std::set<std::string> LockedResources; std::vector<cmCTestTestHandler::cmCTestTestResult>* TestResults; size_t ParallelLevel; // max number of process that can be run at once + unsigned long TestLoad; std::set<cmCTestRunTest*> RunningTests; // current running tests cmCTestTestHandler * TestHandler; cmCTest* CTest; diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index 8b357ac..b7d8318 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -26,6 +26,7 @@ cmCTestTestCommand::cmCTestTestCommand() this->Arguments[ctt_PARALLEL_LEVEL] = "PARALLEL_LEVEL"; this->Arguments[ctt_SCHEDULE_RANDOM] = "SCHEDULE_RANDOM"; this->Arguments[ctt_STOP_TIME] = "STOP_TIME"; + this->Arguments[ctt_TEST_LOAD] = "TEST_LOAD"; this->Arguments[ctt_LAST] = 0; this->Last = ctt_LAST; } @@ -103,6 +104,38 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() { this->CTest->SetStopTime(this->Values[ctt_STOP_TIME]); } + + // Test load is determined by: TEST_LOAD argument, + // or CTEST_TEST_LOAD script variable, or ctest --test-load + // command line argument... in that order. + unsigned long testLoad; + const char* ctestTestLoad + = this->Makefile->GetDefinition("CTEST_TEST_LOAD"); + if(this->Values[ctt_TEST_LOAD] && *this->Values[ctt_TEST_LOAD]) + { + if (!cmSystemTools::StringToULong(this->Values[ctt_TEST_LOAD], &testLoad)) + { + testLoad = 0; + cmCTestLog(this->CTest, WARNING, "Invalid value for 'TEST_LOAD' : " + << this->Values[ctt_TEST_LOAD] << std::endl); + } + } + else if(ctestTestLoad && *ctestTestLoad) + { + if (!cmSystemTools::StringToULong(ctestTestLoad, &testLoad)) + { + testLoad = 0; + cmCTestLog(this->CTest, WARNING, + "Invalid value for 'CTEST_TEST_LOAD' : " << + ctestTestLoad << std::endl); + } + } + else + { + testLoad = this->CTest->GetTestLoad(); + } + handler->SetTestLoad(testLoad); + handler->SetQuiet(this->Quiet); return handler; } diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index a1e5f36..0dfca97 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -60,6 +60,7 @@ protected: ctt_PARALLEL_LEVEL, ctt_SCHEDULE_RANDOM, ctt_STOP_TIME, + ctt_TEST_LOAD, ctt_LAST }; }; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 70b7f5c..59576f4 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -1062,6 +1062,14 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<std::string> &passed, parallel->SetParallelLevel(this->CTest->GetParallelLevel()); parallel->SetTestHandler(this); parallel->SetQuiet(this->Quiet); + if(this->TestLoad > 0) + { + parallel->SetTestLoad(this->TestLoad); + } + else + { + parallel->SetTestLoad(this->CTest->GetTestLoad()); + } *this->LogFile << "Start testing: " << this->CTest->CurrentTime() << std::endl diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index e3b7a2b..5887ba8 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -294,6 +294,7 @@ cmCTest::cmCTest() this->LabelSummary = true; this->ParallelLevel = 1; this->ParallelLevelSetInCli = false; + this->TestLoad = 0; this->SubmitIndex = 0; this->Failover = false; this->BatchJobs = false; @@ -393,6 +394,11 @@ void cmCTest::SetParallelLevel(int level) this->ParallelLevel = level < 1 ? 1 : level; } +void cmCTest::SetTestLoad(unsigned long load) +{ + this->TestLoad = load; +} + //---------------------------------------------------------------------------- bool cmCTest::ShouldCompressTestOutput() { @@ -820,6 +826,20 @@ bool cmCTest::UpdateCTestConfiguration() cmSystemTools::ChangeDirectory(this->BinaryDir); } this->TimeOut = atoi(this->GetCTestConfiguration("TimeOut").c_str()); + std::string const& testLoad = this->GetCTestConfiguration("TestLoad"); + if (!testLoad.empty()) + { + unsigned long load; + if (cmSystemTools::StringToULong(testLoad.c_str(), &load)) + { + this->SetTestLoad(load); + } + else + { + cmCTestLog(this, WARNING, "Invalid value for 'Test Load' : " + << testLoad << std::endl); + } + } if ( this->ProduceXML ) { this->CompressXMLFiles = cmSystemTools::IsOn( @@ -2051,6 +2071,21 @@ bool cmCTest::HandleCommandLineArguments(size_t &i, } } + if(this->CheckArgument(arg, "--test-load") && i < args.size() - 1) + { + i++; + unsigned long load; + if (cmSystemTools::StringToULong(args[i].c_str(), &load)) + { + this->SetTestLoad(load); + } + else + { + cmCTestLog(this, WARNING, + "Invalid value for 'Test Load' : " << args[i] << std::endl); + } + } + if(this->CheckArgument(arg, "--no-compress-output")) { this->CompressTestOutput = false; diff --git a/Source/cmCTest.h b/Source/cmCTest.h index db3ea10..47245ae 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -161,6 +161,9 @@ public: int GetParallelLevel() { return this->ParallelLevel; } void SetParallelLevel(int); + unsigned long GetTestLoad() { return this->TestLoad; } + void SetTestLoad(unsigned long); + /** * Check if CTest file exists */ @@ -499,6 +502,8 @@ private: int ParallelLevel; bool ParallelLevelSetInCli; + unsigned long TestLoad; + int CompatibilityMode; // information for the --build-and-test options diff --git a/Source/ctest.cxx b/Source/ctest.cxx index e784759..afcbd61 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -98,6 +98,7 @@ static const char * cmDocumentationOptions[][2] = {"--test-command", "The test to run with the --build-and-test option."}, {"--test-timeout", "The time limit in seconds, internal use only."}, + {"--test-load", "CPU load threshold for starting new parallel tests."}, {"--tomorrow-tag", "Nightly or experimental starts with next day tag."}, {"--ctest-config", "The configuration file used to initialize CTest state " "when submitting dashboards."}, diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index a3ce139..aba1daf 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -1,4 +1,5 @@ include(RunCMake) +set(RunCMake_TEST_TIMEOUT 60) unset(ENV{CTEST_PARALLEL_LEVEL}) unset(ENV{CTEST_OUTPUT_ON_FAILURE}) @@ -52,3 +53,35 @@ add_test(MergeOutput \"${CMAKE_COMMAND}\" -P \"${RunCMake_SOURCE_DIR}/MergeOutpu run_cmake_command(MergeOutput ${CMAKE_CTEST_COMMAND} -V) endfunction() run_MergeOutput() + + +function(run_TestLoad name load) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestLoad) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" " + add_test(TestLoad1 \"${CMAKE_COMMAND}\" -E echo \"test of --test-load\") + add_test(TestLoad2 \"${CMAKE_COMMAND}\" -E echo \"test of --test-load\") +") + run_cmake_command(${name} ${CMAKE_CTEST_COMMAND} -j2 --test-load ${load} --test-timeout 5) +endfunction() + +# Tests for the --test-load feature of ctest +# +# Spoof a load average value to make these tests more reliable. +set(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING} 5) + +# Verify that new tests are not started when the load average exceeds +# our threshold. +run_TestLoad(test-load-fail 2) + +# Verify that warning message is displayed but tests still start when +# an invalid argument is given. +run_TestLoad(test-load-invalid 'two') + +# Verify that new tests are started when the load average falls below +# our threshold. +run_TestLoad(test-load-pass 10) + +unset(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING}) diff --git a/Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt b/Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt new file mode 100644 index 0000000..eafba1c --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-fail-stderr.txt @@ -0,0 +1 @@ +No tests were found!!! diff --git a/Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt b/Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt new file mode 100644 index 0000000..153da09 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-fail-stdout.txt @@ -0,0 +1,2 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/TestLoad +\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 2, Smallest test TestLoad[1-2] requires 1\*\*\*\*\* diff --git a/Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt new file mode 100644 index 0000000..caab3b9 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stderr.txt @@ -0,0 +1 @@ +Invalid value for 'Test Load' : 'two' diff --git a/Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt new file mode 100644 index 0000000..7ee3dae --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-invalid-stdout.txt @@ -0,0 +1,7 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/TestLoad + Start 1: TestLoad1 + Start 2: TestLoad2 +1/2 Test #[1-2]: TestLoad[1-2] ........................ Passed +[0-9.]+ sec +2/2 Test #[1-2]: TestLoad[1-2] ........................ Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 2 diff --git a/Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt b/Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt new file mode 100644 index 0000000..10f3293 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-pass-stderr.txt @@ -0,0 +1 @@ +^$ diff --git a/Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt b/Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt new file mode 100644 index 0000000..7ee3dae --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/test-load-pass-stdout.txt @@ -0,0 +1,7 @@ +^Test project .*/Tests/RunCMake/CTestCommandLine/TestLoad + Start 1: TestLoad1 + Start 2: TestLoad2 +1/2 Test #[1-2]: TestLoad[1-2] ........................ Passed +[0-9.]+ sec +2/2 Test #[1-2]: TestLoad[1-2] ........................ Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 2 diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt new file mode 100644 index 0000000..eafba1c --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stderr.txt @@ -0,0 +1 @@ +No tests were found!!! diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt new file mode 100644 index 0000000..e203c10 --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadFail-stdout.txt @@ -0,0 +1,2 @@ +Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadFail-build +\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 4, Smallest test RunCMakeVersion requires 1\*\*\*\*\*$ diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt new file mode 100644 index 0000000..7f2d7f6 --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stderr.txt @@ -0,0 +1 @@ +^Invalid value for 'CTEST_TEST_LOAD' : ERR2 diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt new file mode 100644 index 0000000..b54220c --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadInvalid-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt b/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt new file mode 100644 index 0000000..c221eed --- /dev/null +++ b/Tests/RunCMake/ctest_test/CTestTestLoadPass-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/CTestTestLoadPass-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index d906290..21d0447 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -1,6 +1,8 @@ include(RunCTest) +set(RunCMake_TEST_TIMEOUT 60) set(CASE_CTEST_TEST_ARGS "") +set(CASE_CTEST_TEST_LOAD "") function(run_ctest_test CASE_NAME) set(CASE_CTEST_TEST_ARGS "${ARGN}") @@ -8,3 +10,42 @@ function(run_ctest_test CASE_NAME) endfunction() run_ctest_test(TestQuiet QUIET) + +# Tests for the 'Test Load' feature of ctest +# +# Spoof a load average value to make these tests more reliable. +set(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING} 5) + +# Verify that new tests are started when the load average falls below +# our threshold. +run_ctest_test(TestLoadPass TEST_LOAD 6) + +# Verify that new tests are not started when the load average exceeds +# our threshold. +run_ctest_test(TestLoadFail TEST_LOAD 2) + +# Verify that when an invalid "TEST_LOAD" value is given, a warning +# message is displayed and the value is ignored. +run_ctest_test(TestLoadInvalid TEST_LOAD "ERR1") + +# Verify that new tests are started when the load average falls below +# our threshold. +set(CASE_CTEST_TEST_LOAD 7) +run_ctest_test(CTestTestLoadPass) + +# Verify that new tests are not started when the load average exceeds +# our threshold. +set(CASE_CTEST_TEST_LOAD 4) +run_ctest_test(CTestTestLoadFail) + +# Verify that when an invalid "CTEST_TEST_LOAD" value is given, +# a warning message is displayed and the value is ignored. +set(CASE_CTEST_TEST_LOAD "ERR2") +run_ctest_test(CTestTestLoadInvalid) + +# Verify that the "TEST_LOAD" value has higher precedence than +# the "CTEST_TEST_LOAD" value +set(CASE_CTEST_TEST_LOAD "ERR3") +run_ctest_test(TestLoadOrder TEST_LOAD "ERR4") + +unset(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING}) diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-result.txt b/Tests/RunCMake/ctest_test/TestLoadFail-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadFail-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt new file mode 100644 index 0000000..eafba1c --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadFail-stderr.txt @@ -0,0 +1 @@ +No tests were found!!! diff --git a/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt new file mode 100644 index 0000000..4d7ce48 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadFail-stdout.txt @@ -0,0 +1,2 @@ +Test project .*/Tests/RunCMake/ctest_test/TestLoadFail-build +\*\*\*\*\* WAITING, System Load: 5, Max Allowed Load: 2, Smallest test RunCMakeVersion requires 1\*\*\*\*\*$ diff --git a/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt new file mode 100644 index 0000000..40ddb3a --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadInvalid-stderr.txt @@ -0,0 +1 @@ +^Invalid value for 'TEST_LOAD' : ERR1 diff --git a/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt new file mode 100644 index 0000000..c4fd35b --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadInvalid-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/TestLoadInvalid-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt b/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt new file mode 100644 index 0000000..1de730e --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadOrder-stderr.txt @@ -0,0 +1 @@ +^Invalid value for 'TEST_LOAD' : ERR4 diff --git a/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt new file mode 100644 index 0000000..22da092 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadOrder-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/TestLoadOrder-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt b/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt new file mode 100644 index 0000000..e5048f4 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestLoadPass-stdout.txt @@ -0,0 +1,7 @@ +Test project .*/Tests/RunCMake/ctest_test/TestLoadPass-build + Start 1: RunCMakeVersion +1/1 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec$ diff --git a/Tests/RunCMake/ctest_test/test.cmake.in b/Tests/RunCMake/ctest_test/test.cmake.in index 1350abe..a8de6a3 100644 --- a/Tests/RunCMake/ctest_test/test.cmake.in +++ b/Tests/RunCMake/ctest_test/test.cmake.in @@ -8,6 +8,7 @@ set(CTEST_CMAKE_GENERATOR "@RunCMake_GENERATOR@") set(CTEST_CMAKE_GENERATOR_PLATFORM "@RunCMake_GENERATOR_PLATFORM@") set(CTEST_CMAKE_GENERATOR_TOOLSET "@RunCMake_GENERATOR_TOOLSET@") set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}") +set(CTEST_TEST_LOAD "@CASE_CTEST_TEST_LOAD@") set(ctest_test_args "@CASE_CTEST_TEST_ARGS@") ctest_start(Experimental) |