From 5de1e21659090ca83e39e223d351e353347eb88e Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 6 Mar 2024 11:16:33 -0500 Subject: ctest: Allow passing -j without value to choose a contextual default Under job server integration, added by commit 80fe56c481 (ctest: Add support for running under a make job server on POSIX systems, 2023-11-15, v3.29.0-rc1~324^2), use a very high default so that parallelism is effectively limited only by available job server tokens. Otherwise, choose a default limit based on the number of processors. Also allow passing `0` to specify unbounded parallelism. Fixes: #25739 --- Help/command/ctest_test.rst | 13 +++-- Help/envvar/CTEST_PARALLEL_LEVEL.rst | 11 ++++ Help/manual/ctest.1.rst | 24 +++++--- Help/release/3.29.rst | 6 ++ Source/CTest/cmCTestMultiProcessHandler.cxx | 66 +++++++++++++++++++--- Source/CTest/cmCTestMultiProcessHandler.h | 12 +++- Source/CTest/cmCTestTestCommand.cxx | 4 +- Source/CTest/cmCTestTestCommand.h | 4 +- Source/CTest/cmCTestTestHandler.cxx | 18 +++++- Source/cmCTest.cxx | 55 +++++++++++++----- Source/cmCTest.h | 5 +- Source/ctest.cxx | 6 +- .../CTestCommandLine/Parallel-0-stdout.txt | 9 +++ .../CTestCommandLine/Parallel-4-stdout.txt | 7 +++ .../CTestCommandLine/Parallel-N-stdout.txt | 10 ++++ .../CTestCommandLine/Parallel-bad-result.txt | 1 + .../CTestCommandLine/Parallel-bad-stderr.txt | 1 + .../CTestCommandLine/Parallel-empty-stdout.txt | 5 ++ .../CTestCommandLine/Parallel-env-0-stdout.txt | 9 +++ .../CTestCommandLine/Parallel-env-3-stdout.txt | 6 ++ .../CTestCommandLine/Parallel-env-bad-stdout.txt | 4 ++ .../CTestCommandLine/Parallel-env-empty-stdout.txt | 5 ++ .../CTestCommandLine/Parallel-j-bad-result.txt | 1 + .../CTestCommandLine/Parallel-j-bad-stderr.txt | 1 + .../CTestCommandLine/Parallel-j-stdout.txt | 5 ++ Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 38 +++++++++++++ Tests/RunCMake/Make/CTestJobServer.make | 6 +- Tests/RunCMake/Make/RunCMakeTest.cmake | 3 + Tests/RunCMake/ctest_test/Parallel0-stdout.txt | 9 +++ Tests/RunCMake/ctest_test/Parallel4-stdout.txt | 7 +++ Tests/RunCMake/ctest_test/ParallelBad-result.txt | 1 + Tests/RunCMake/ctest_test/ParallelBad-stderr.txt | 1 + Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt | 5 ++ Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt | 9 +++ Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt | 6 ++ .../RunCMake/ctest_test/ParallelEnvBad-stdout.txt | 4 ++ .../ctest_test/ParallelEnvEmpty-stdout.txt | 5 ++ Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt | 5 ++ Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 34 +++++++++++ Tests/RunCMake/ctest_test/test.cmake.in | 2 +- 40 files changed, 372 insertions(+), 51 deletions(-) create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-0-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-4-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-N-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-bad-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-bad-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-empty-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-env-0-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-env-3-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-env-bad-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-env-empty-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-j-bad-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-j-bad-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-j-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/Parallel0-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/Parallel4-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelBad-result.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelBad-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEnvBad-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEnvEmpty-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index 4d56577..c1862df 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -18,7 +18,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`. [EXCLUDE_FIXTURE ] [EXCLUDE_FIXTURE_SETUP ] [EXCLUDE_FIXTURE_CLEANUP ] - [PARALLEL_LEVEL ] + [PARALLEL_LEVEL []] [RESOURCE_SPEC_FILE ] [TEST_LOAD ] [SCHEDULE_RANDOM ] @@ -104,9 +104,14 @@ The options are: Same as ``EXCLUDE_FIXTURE`` except only matching cleanup tests are excluded. -``PARALLEL_LEVEL `` - Specify a positive number representing the number of tests to - be run in parallel. +``PARALLEL_LEVEL []`` + Run tests in parallel, limited to a given level of parallelism. + + .. versionadded:: 3.29 + + The ```` may be omitted, or ``0``, to let ctest use a default + level of parallelism, or unbounded parallelism, respectively, as + documented by the :option:`ctest --parallel` option. ``RESOURCE_SPEC_FILE `` .. versionadded:: 3.16 diff --git a/Help/envvar/CTEST_PARALLEL_LEVEL.rst b/Help/envvar/CTEST_PARALLEL_LEVEL.rst index 126bdb3..0ef01d5 100644 --- a/Help/envvar/CTEST_PARALLEL_LEVEL.rst +++ b/Help/envvar/CTEST_PARALLEL_LEVEL.rst @@ -8,4 +8,15 @@ For example, if ``CTEST_PARALLEL_LEVEL`` is set to 8, CTest will run up to 8 tests concurrently as if ``ctest`` were invoked with the :option:`--parallel 8 ` option. +.. versionchanged:: 3.29 + + The value may be empty, or ``0``, to let ctest use a default level of + parallelism, or unbounded parallelism, respectively, as documented by + the :option:`ctest --parallel` option. + + On Windows, environment variables cannot be set to an empty string. + CTest will interpret a whitespace-only string as empty. + + In CMake 3.28 and earlier, an empty or ``0`` value was equivalent to ``1``. + See :manual:`ctest(1)` for more information on parallel test execution. diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 602a82d..edbd5a4 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -118,17 +118,27 @@ Run Tests previously interrupted. If no interruption occurred, the ``-F`` option will have no effect. -.. option:: -j , --parallel +.. option:: -j [], --parallel [] - Run the tests in parallel using the given number of jobs. + Run tests in parallel, optionally limited to a given level of parallelism. - This option tells CTest to run the tests in parallel using given - number of jobs. This option can also be set by setting the - :envvar:`CTEST_PARALLEL_LEVEL` environment variable. + .. versionadded:: 3.29 - This option can be used with the :prop_test:`PROCESSORS` test property. + The ```` may be omitted, or ``0``, in which case: - See `Label and Subproject Summary`_. + * Under `Job Server Integration`_, parallelism is limited by + available job tokens. + + * Otherwise, if the value is omitted, parallelism is limited + by the number of processors, or 2, whichever is larger. + + * Otherwise, if the value is ``0``, parallelism is unbounded. + + This option may instead be specified by the :envvar:`CTEST_PARALLEL_LEVEL` + environment variable. + + This option can be used with the :prop_test:`PROCESSORS` test property. + See the `Label and Subproject Summary`_. .. option:: --resource-spec-file diff --git a/Help/release/3.29.rst b/Help/release/3.29.rst index 659237e..42f6491 100644 --- a/Help/release/3.29.rst +++ b/Help/release/3.29.rst @@ -150,6 +150,12 @@ CTest * :manual:`ctest(1)` now supports :ref:`job server integration ` on POSIX systems. +* The :option:`ctest -j` option may now be given without a value to let + ctest choose a default level of parallelism, or with ``0`` to let ctest + use unbounded parallelism. The corresponding :envvar:`CTEST_PARALLEL_LEVEL` + environment variable, if set to the empty string, is now equivalent to + passing ``-j`` with no value. + * The :command:`ctest_test` command gained options ``INCLUDE_FROM_FILE`` and ``EXCLUDE_FROM_FILE`` to run or exclude tests named in a file. diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 6efe008..53c5f76 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -43,6 +43,17 @@ #include "cmUVJobServerClient.h" #include "cmWorkingDirectory.h" +namespace { +// For unspecified parallelism, limit to the number of processors, +// but with a minimum greater than 1 so there is some parallelism. +constexpr unsigned long kParallelLevelMinimum = 2u; + +// For "unbounded" parallelism, limit to a very high value. +// Under a job server, parallelism is effectively limited +// only by available job server tokens. +constexpr unsigned long kParallelLevelUnbounded = 0x10000u; +} + namespace cmsys { class RegularExpression; } @@ -72,6 +83,7 @@ cmCTestMultiProcessHandler::cmCTestMultiProcessHandler( , TestHandler(handler) , ProcessorsAvailable(cmAffinity::GetProcessorsAvailable()) , HaveAffinity(this->ProcessorsAvailable.size()) + , ParallelLevelDefault(kParallelLevelMinimum) { } @@ -97,9 +109,43 @@ void cmCTestMultiProcessHandler::SetTests(TestMap tests, } // Set the max number of tests that can be run at the same time. -void cmCTestMultiProcessHandler::SetParallelLevel(size_t level) +void cmCTestMultiProcessHandler::SetParallelLevel(cm::optional level) +{ + this->ParallelLevel = level; + + if (!this->ParallelLevel) { + // '-j' was given with no value. Limit by number of processors. + cmsys::SystemInformation info; + info.RunCPUCheck(); + unsigned long processorCount = info.GetNumberOfLogicalCPU(); + + if (cm::optional fakeProcessorCount = + cmSystemTools::GetEnvVar( + "__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING")) { + unsigned long pc = 0; + if (cmStrToULong(*fakeProcessorCount, &pc)) { + processorCount = pc; + } else { + cmSystemTools::Error("Failed to parse fake processor count: " + + *fakeProcessorCount); + } + } + + this->ParallelLevelDefault = + std::max(kParallelLevelMinimum, processorCount); + } +} + +size_t cmCTestMultiProcessHandler::GetParallelLevel() const { - this->ParallelLevel = level < 1 ? 1 : level; + if ((this->ParallelLevel && *this->ParallelLevel == 0) || + (!this->ParallelLevel && this->JobServerClient)) { + return kParallelLevelUnbounded; + } + if (this->ParallelLevel) { + return *this->ParallelLevel; + } + return this->ParallelLevelDefault; } void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load) @@ -446,10 +492,11 @@ void cmCTestMultiProcessHandler::UnlockResources(int index) inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test) { size_t processors = static_cast(this->Properties[test]->Processors); + size_t const parallelLevel = this->GetParallelLevel(); // If processors setting is set higher than the -j // setting, we default to using all of the process slots. - if (processors > this->ParallelLevel) { - processors = this->ParallelLevel; + if (processors > parallelLevel) { + processors = parallelLevel; } // Cap tests that want affinity to the maximum affinity available. if (this->HaveAffinity && processors > this->HaveAffinity && @@ -503,8 +550,9 @@ void cmCTestMultiProcessHandler::StartNextTests() size_t numToStart = 0; - if (this->RunningCount < this->ParallelLevel) { - numToStart = this->ParallelLevel - this->RunningCount; + size_t const parallelLevel = this->GetParallelLevel(); + if (this->RunningCount < parallelLevel) { + numToStart = parallelLevel - this->RunningCount; } if (numToStart == 0) { @@ -518,7 +566,7 @@ void cmCTestMultiProcessHandler::StartNextTests() } bool allTestsFailedTestLoadCheck = false; - size_t minProcessorsRequired = this->ParallelLevel; + size_t minProcessorsRequired = this->GetParallelLevel(); std::string testWithMinProcessors; cmsys::SystemInformation info; @@ -813,7 +861,7 @@ void cmCTestMultiProcessHandler::ReadCostData() this->Properties[index]->PreviousRuns = prev; // When not running in parallel mode, don't use cost data - if (this->ParallelLevel > 1 && this->Properties[index] && + if (this->GetParallelLevel() > 1 && this->Properties[index] && this->Properties[index]->Cost == 0) { this->Properties[index]->Cost = cost; } @@ -842,7 +890,7 @@ int cmCTestMultiProcessHandler::SearchByName(std::string const& name) void cmCTestMultiProcessHandler::CreateTestCostList() { - if (this->ParallelLevel > 1) { + if (this->GetParallelLevel() > 1) { this->CreateParallelTestCostList(); } else { this->CreateSerialTestCostList(); diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h index bbbfc7b..d4759f1 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.h +++ b/Source/CTest/cmCTestMultiProcessHandler.h @@ -63,7 +63,7 @@ public: // Set the tests void SetTests(TestMap tests, PropertiesMap properties); // Set the max number of tests that can be run at the same time. - void SetParallelLevel(size_t); + void SetParallelLevel(cm::optional level); void SetTestLoad(unsigned long load); virtual void RunTests(); void PrintOutputAsJson(); @@ -201,7 +201,15 @@ private: ResourceAvailabilityErrors; cmCTestResourceAllocator ResourceAllocator; std::vector* TestResults; - size_t ParallelLevel = 1; // max number of process that can be run at once + + // Get the maximum number of processors that may be used at once. + size_t GetParallelLevel() const; + + // With no '-j' option, default to serial testing. + cm::optional ParallelLevel = 1; + + // Fallback parallelism limit when '-j' is given with no value. + size_t ParallelLevelDefault; // 'make' jobserver client. If connected, we acquire a token // for each test before running its process. diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index ed16354..98ce862 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -105,8 +105,8 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() if (this->StopOnFailure) { handler->SetOption("StopOnFailure", "ON"); } - if (!this->ParallelLevel.empty()) { - handler->SetOption("ParallelLevel", this->ParallelLevel); + if (this->ParallelLevel) { + handler->SetOption("ParallelLevel", *this->ParallelLevel); } if (!this->Repeat.empty()) { handler->SetOption("Repeat", this->Repeat); diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index ce054ed..23661c5 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -8,7 +8,9 @@ #include #include +#include +#include "cmArgumentParserTypes.h" #include "cmCTestHandlerCommand.h" #include "cmCommand.h" @@ -56,7 +58,7 @@ protected: std::string ExcludeFixture; std::string ExcludeFixtureSetup; std::string ExcludeFixtureCleanup; - std::string ParallelLevel; + cm::optional> ParallelLevel; std::string Repeat; std::string ScheduleRandom; std::string StopTime; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 921ab1e..61a18a2 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -548,9 +548,21 @@ bool cmCTestTestHandler::ProcessOptions() return false; } } - if (this->GetOption("ParallelLevel")) { - this->CTest->SetParallelLevel( - std::stoi(*this->GetOption("ParallelLevel"))); + if (cmValue parallelLevel = this->GetOption("ParallelLevel")) { + if (parallelLevel.IsEmpty()) { + // An empty value tells ctest to choose a default. + this->CTest->SetParallelLevel(cm::nullopt); + } else { + // A non-empty value must be a non-negative integer. + unsigned long plevel = 0; + if (!cmStrToULong(*parallelLevel, &plevel)) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "ParallelLevel invalid value: " << *parallelLevel + << std::endl); + return false; + } + this->CTest->SetParallelLevel(plevel); + } } if (this->GetOption("StopOnFailure")) { diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 7acee4b..1ae3cb5 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -179,7 +179,7 @@ struct cmCTest::Private int MaxTestNameWidth = 30; - int ParallelLevel = 1; + cm::optional ParallelLevel = 1; bool ParallelLevelSetInCli = false; unsigned long TestLoad = 0; @@ -380,14 +380,14 @@ cmCTest::cmCTest() cmCTest::~cmCTest() = default; -int cmCTest::GetParallelLevel() const +cm::optional cmCTest::GetParallelLevel() const { return this->Impl->ParallelLevel; } -void cmCTest::SetParallelLevel(int level) +void cmCTest::SetParallelLevel(cm::optional level) { - this->Impl->ParallelLevel = level < 1 ? 1 : level; + this->Impl->ParallelLevel = level; } unsigned long cmCTest::GetTestLoad() const @@ -1892,14 +1892,31 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, std::string arg = args[i]; if (this->CheckArgument(arg, "-F"_s)) { this->Impl->Failover = true; - } else if (this->CheckArgument(arg, "-j"_s, "--parallel") && - i < args.size() - 1) { - i++; - int plevel = atoi(args[i].c_str()); - this->SetParallelLevel(plevel); + } else if (this->CheckArgument(arg, "-j"_s, "--parallel")) { + cm::optional parallelLevel; + // No value or an empty value tells ctest to choose a default. + if (i + 1 < args.size() && !cmHasLiteralPrefix(args[i + 1], "-")) { + ++i; + if (!args[i].empty()) { + // A non-empty value must be a non-negative integer. + unsigned long plevel = 0; + if (!cmStrToULong(args[i], &plevel)) { + errormsg = + cmStrCat("'", arg, "' given invalid value '", args[i], "'"); + return false; + } + parallelLevel = plevel; + } + } + this->SetParallelLevel(parallelLevel); this->Impl->ParallelLevelSetInCli = true; } else if (cmHasPrefix(arg, "-j")) { - int plevel = atoi(arg.substr(2).c_str()); + // The value must be a non-negative integer. + unsigned long plevel = 0; + if (!cmStrToULong(arg.substr(2), &plevel)) { + errormsg = cmStrCat("'", arg, "' given invalid value '", args[i], "'"); + return false; + } this->SetParallelLevel(plevel); this->Impl->ParallelLevelSetInCli = true; } @@ -2799,10 +2816,20 @@ int cmCTest::Run(std::vector& args, std::string* output) // handle CTEST_PARALLEL_LEVEL environment variable if (!this->Impl->ParallelLevelSetInCli) { - std::string parallel; - if (cmSystemTools::GetEnv("CTEST_PARALLEL_LEVEL", parallel)) { - int plevel = atoi(parallel.c_str()); - this->SetParallelLevel(plevel); + if (cm::optional parallelEnv = + cmSystemTools::GetEnvVar("CTEST_PARALLEL_LEVEL")) { + if (parallelEnv->empty() || + parallelEnv->find_first_not_of(" \t") == std::string::npos) { + // An empty value tells ctest to choose a default. + this->SetParallelLevel(cm::nullopt); + } else { + // A non-empty value must be a non-negative integer. + // Otherwise, ignore it. + unsigned long plevel = 0; + if (cmStrToULong(*parallelEnv, &plevel)) { + this->SetParallelLevel(plevel); + } + } } } diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 1644d84..2ab810c 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -12,6 +12,7 @@ #include #include +#include #include #include "cmDuration.h" @@ -116,8 +117,8 @@ public: cmDuration GetGlobalTimeout() const; /** how many test to run at the same time */ - int GetParallelLevel() const; - void SetParallelLevel(int); + cm::optional GetParallelLevel() const; + void SetParallelLevel(cm::optional level); unsigned long GetTestLoad() const; void SetTestLoad(unsigned long); diff --git a/Source/ctest.cxx b/Source/ctest.cxx index c1c2071..f6a11b4 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -48,9 +48,9 @@ const cmDocumentationEntry cmDocumentationOptions[] = { "Truncate 'tail' (default), 'middle' or 'head' of test output once " "maximum output size is reached" }, { "-F", "Enable failover." }, - { "-j , --parallel ", - "Run the tests in parallel using the " - "given number of jobs." }, + { "-j [], --parallel []", + "Run tests in parallel, " + "optionally limited to a given level of parallelism." }, { "-Q,--quiet", "Make ctest quiet." }, { "-O , --output-log ", "Output to log file" }, { "--output-junit ", "Output test results to JUnit XML file." }, diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-0-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-0-stdout.txt new file mode 100644 index 0000000..37f728e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-0-stdout.txt @@ -0,0 +1,9 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-0 + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-4-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-4-stdout.txt new file mode 100644 index 0000000..d90cbd8 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-4-stdout.txt @@ -0,0 +1,7 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-4 + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-N-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-N-stdout.txt new file mode 100644 index 0000000..2c16db7 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-N-stdout.txt @@ -0,0 +1,10 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-N + Test #1: test1 + Test #2: test2 + Test #3: test3 + Test #4: test4 + Test #5: test5 + Test #6: test6 + +Total Tests: 6 diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-bad-result.txt b/Tests/RunCMake/CTestCommandLine/Parallel-bad-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-bad-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-bad-stderr.txt b/Tests/RunCMake/CTestCommandLine/Parallel-bad-stderr.txt new file mode 100644 index 0000000..121248b --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-bad-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--parallel' given invalid value 'bad'$ diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-empty-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-empty-stdout.txt new file mode 100644 index 0000000..f380c17 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-empty-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-empty + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-env-0-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-env-0-stdout.txt new file mode 100644 index 0000000..1eb05ac --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-env-0-stdout.txt @@ -0,0 +1,9 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-env-0 + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-env-3-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-env-3-stdout.txt new file mode 100644 index 0000000..d6fc03b --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-env-3-stdout.txt @@ -0,0 +1,6 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-env-3 + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-env-bad-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-env-bad-stdout.txt new file mode 100644 index 0000000..def3313 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-env-bad-stdout.txt @@ -0,0 +1,4 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-env-bad + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-env-empty-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-env-empty-stdout.txt new file mode 100644 index 0000000..85b880d --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-env-empty-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-env-empty + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-result.txt b/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-stderr.txt b/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-stderr.txt new file mode 100644 index 0000000..228f7cf --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '-j' given invalid value 'bad'$ diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-j-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-j-stdout.txt new file mode 100644 index 0000000..39dd34a --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-j-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-j + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index c09fff8..724c5e4 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -240,6 +240,44 @@ add_test(Echo \"${CMAKE_COMMAND}\" -E echo \"EchoTest\") endfunction() run_SerialFailed() +function(run_Parallel case) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Parallel-${case}) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" " +foreach(i RANGE 1 6) + add_test(test\${i} \"${CMAKE_COMMAND}\" -E true) +endforeach() +") + run_cmake_command(Parallel-${case} ${CMAKE_CTEST_COMMAND} ${ARGN}) +endfunction() +# Spoof a number of processors to make these tests predictable. +set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 1) +run_Parallel(bad --parallel bad) +run_Parallel(j-bad -j bad) +set(RunCMake_TEST_RAW_ARGS [[--parallel ""]]) +run_Parallel(empty) # With 1 processor, defaults to 2. +unset(RunCMake_TEST_RAW_ARGS) +run_Parallel(j -j) # With 1 processor, defaults to 2. +run_Parallel(0 -j0) +run_Parallel(4 --parallel 4) +run_Parallel(N --parallel -N) +set(ENV{CTEST_PARALLEL_LEVEL} bad) +run_Parallel(env-bad) +if(CMAKE_HOST_WIN32) + set(ENV{CTEST_PARALLEL_LEVEL} " ") +else() + set(ENV{CTEST_PARALLEL_LEVEL} "") +endif() +run_Parallel(env-empty) # With 1 processor, defaults to 2. +set(ENV{CTEST_PARALLEL_LEVEL} 0) +run_Parallel(env-0) +set(ENV{CTEST_PARALLEL_LEVEL} 3) +run_Parallel(env-3) +unset(ENV{CTEST_PARALLEL_LEVEL}) +unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) + function(run_TestLoad name load) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestLoad) set(RunCMake_TEST_NO_CLEAN 1) diff --git a/Tests/RunCMake/Make/CTestJobServer.make b/Tests/RunCMake/Make/CTestJobServer.make index 7fc5e28..24fe774 100644 --- a/Tests/RunCMake/Make/CTestJobServer.make +++ b/Tests/RunCMake/Make/CTestJobServer.make @@ -1,11 +1,11 @@ NoPipe: - env MAKEFLAGS= $(CMAKE_CTEST_COMMAND) -j6 + env MAKEFLAGS= $(CMAKE_CTEST_COMMAND) -j0 .PHONY: NoPipe NoTests: - +$(CMAKE_CTEST_COMMAND) -j6 -R NoTests + +$(CMAKE_CTEST_COMMAND) -j -R NoTests .PHONY: NoTests Tests: - +$(CMAKE_CTEST_COMMAND) -j6 + +$(CMAKE_CTEST_COMMAND) -j .PHONY: Tests diff --git a/Tests/RunCMake/Make/RunCMakeTest.cmake b/Tests/RunCMake/Make/RunCMakeTest.cmake index cfaf759..9673329 100644 --- a/Tests/RunCMake/Make/RunCMakeTest.cmake +++ b/Tests/RunCMake/Make/RunCMakeTest.cmake @@ -90,10 +90,13 @@ function(run_CTestJobServer) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/CTestJobServer-build) run_cmake(CTestJobServer) set(RunCMake_TEST_NO_CLEAN 1) + # Spoof a number of processors to make sure jobserver integration is unbounded. + set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 1) run_make_rule(CTestJobServer NoPipe 2) run_make_rule(CTestJobServer NoTests 2) run_make_rule(CTestJobServer Tests 2) run_make_rule(CTestJobServer Tests 3) + unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) endfunction() # Jobservers are currently only supported by GNU makes, except MSYS2 make diff --git a/Tests/RunCMake/ctest_test/Parallel0-stdout.txt b/Tests/RunCMake/ctest_test/Parallel0-stdout.txt new file mode 100644 index 0000000..98230cb --- /dev/null +++ b/Tests/RunCMake/ctest_test/Parallel0-stdout.txt @@ -0,0 +1,9 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/Parallel0-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/Parallel4-stdout.txt b/Tests/RunCMake/ctest_test/Parallel4-stdout.txt new file mode 100644 index 0000000..36b0b85 --- /dev/null +++ b/Tests/RunCMake/ctest_test/Parallel4-stdout.txt @@ -0,0 +1,7 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/Parallel4-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelBad-result.txt b/Tests/RunCMake/ctest_test/ParallelBad-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelBad-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/ParallelBad-stderr.txt b/Tests/RunCMake/ctest_test/ParallelBad-stderr.txt new file mode 100644 index 0000000..2e21a1b --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelBad-stderr.txt @@ -0,0 +1 @@ +^ParallelLevel invalid value: bad$ diff --git a/Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt new file mode 100644 index 0000000..fec0789 --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEmpty-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt new file mode 100644 index 0000000..2e4bc6f --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt @@ -0,0 +1,9 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEnv0-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt new file mode 100644 index 0000000..0193b6c --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt @@ -0,0 +1,6 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEnv3-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelEnvBad-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEnvBad-stdout.txt new file mode 100644 index 0000000..cd7970b --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEnvBad-stdout.txt @@ -0,0 +1,4 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEnvBad-build + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelEnvEmpty-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEnvEmpty-stdout.txt new file mode 100644 index 0000000..e0f92d1 --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEnvEmpty-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEnvEmpty-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt b/Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt new file mode 100644 index 0000000..c388937 --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelOmit-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 012c1a0..ee7114b 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -1,8 +1,10 @@ include(RunCTest) + set(RunCMake_TEST_TIMEOUT 60) set(CASE_CTEST_TEST_ARGS "") set(CASE_CTEST_TEST_LOAD "") +set(CASE_CTEST_TEST_RAW_ARGS "") function(run_ctest_test CASE_NAME) set(CASE_CTEST_TEST_ARGS "${ARGN}") @@ -11,6 +13,7 @@ endfunction() run_ctest_test(TestQuiet QUIET) +set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 4) set(CASE_CMAKELISTS_SUFFIX_CODE [[ foreach(i RANGE 1 6) add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true) @@ -25,6 +28,7 @@ set_property(TEST test6 PROPERTY DEPENDS test1) ]]) run_ctest_test(SerialOrder INCLUDE test) unset(CASE_CMAKELISTS_SUFFIX_CODE) +unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) set(CASE_CMAKELISTS_SUFFIX_CODE [[ add_test(NAME skip COMMAND ${CMAKE_COMMAND} -E true) @@ -33,6 +37,36 @@ set_property(TEST skip PROPERTY SKIP_RETURN_CODE 0) run_ctest_test(SkipReturnCode) unset(CASE_CMAKELISTS_SUFFIX_CODE) +# Spoof a number of processors to make these tests predictable. +set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 1) +set(CASE_CMAKELISTS_SUFFIX_CODE [[ +foreach(i RANGE 1 6) + add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true) +endforeach() +]]) +run_ctest_test(ParallelBad INCLUDE test PARALLEL_LEVEL bad) +set(CASE_CTEST_TEST_RAW_ARGS "PARALLEL_LEVEL \"\"") +run_ctest_test(ParallelEmpty INCLUDE test) # With 1 processor, defaults to 2. +unset(CASE_CTEST_TEST_RAW_ARGS) +run_ctest_test(ParallelOmit INCLUDE test PARALLEL_LEVEL) # With 1 processor, defaults to 2. +run_ctest_test(Parallel0 INCLUDE test PARALLEL_LEVEL 0) +run_ctest_test(Parallel4 INCLUDE test PARALLEL_LEVEL 4) +set(ENV{CTEST_PARALLEL_LEVEL} bad) +run_ctest_test(ParallelEnvBad INCLUDE test) +if(CMAKE_HOST_WIN32) + set(ENV{CTEST_PARALLEL_LEVEL} " ") +else() + set(ENV{CTEST_PARALLEL_LEVEL} "") +endif() +run_ctest_test(ParallelEnvEmpty INCLUDE test) # With 1 processor, defaults to 2. +set(ENV{CTEST_PARALLEL_LEVEL} 0) +run_ctest_test(ParallelEnv0 INCLUDE test) +set(ENV{CTEST_PARALLEL_LEVEL} 3) +run_ctest_test(ParallelEnv3 INCLUDE test) +unset(ENV{CTEST_PARALLEL_LEVEL}) +unset(CASE_CMAKELISTS_SUFFIX_CODE) +unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) + # Tests for the 'Test Load' feature of ctest # # Spoof a load average value to make these tests more reliable. diff --git a/Tests/RunCMake/ctest_test/test.cmake.in b/Tests/RunCMake/ctest_test/test.cmake.in index 16dde1c..d28b1e2 100644 --- a/Tests/RunCMake/ctest_test/test.cmake.in +++ b/Tests/RunCMake/ctest_test/test.cmake.in @@ -19,5 +19,5 @@ if("@CASE_NAME@" STREQUAL "TestChangingLabels") ctest_test(${ctest_test_args} INCLUDE_LABEL "^a$") ctest_test(${ctest_test_args} INCLUDE_LABEL "^b$") else() - ctest_test(${ctest_test_args}) + ctest_test(${ctest_test_args} @CASE_CTEST_TEST_RAW_ARGS@) endif() -- cgit v0.12