From 30dda4941698f6b9dd292b4237b4eff187c80241 Mon Sep 17 00:00:00 2001 From: Brad King Date: Sun, 10 Mar 2024 10:01:44 -0400 Subject: Tests: Convert CTestTestSerialOrder to RunCMake.ctest_test case --- Tests/CMakeLists.txt | 4 --- Tests/CTestTestSerialOrder/CMakeLists.txt | 40 ------------------------ Tests/CTestTestSerialOrder/test.cmake | 31 ------------------ Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 15 +++++++++ Tests/RunCMake/ctest_test/SerialOrder-stdout.txt | 16 ++++++++++ 5 files changed, 31 insertions(+), 75 deletions(-) delete mode 100644 Tests/CTestTestSerialOrder/CMakeLists.txt delete mode 100644 Tests/CTestTestSerialOrder/test.cmake create mode 100644 Tests/RunCMake/ctest_test/SerialOrder-stdout.txt diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 1ee45d6..ae065c8 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -3095,10 +3095,6 @@ if(BUILD_TESTING) PASS_REGULAR_EXPRESSION "\\*\\*\\*Not Run" ) - ADD_TEST_MACRO(CTestTestSerialOrder ${CMAKE_CTEST_COMMAND} - --output-on-failure -C "\${CTestTest_CONFIG}") - set_property(TEST CTestTestSerialOrder PROPERTY ENVIRONMENT CTEST_PARALLEL_LEVEL=) - if(NOT BORLAND) set(CTestLimitDashJ_CTEST_OPTIONS --force-new-ctest-process) add_test_macro(CTestLimitDashJ ${CMAKE_CTEST_COMMAND} -j 4 diff --git a/Tests/CTestTestSerialOrder/CMakeLists.txt b/Tests/CTestTestSerialOrder/CMakeLists.txt deleted file mode 100644 index d46d80e..0000000 --- a/Tests/CTestTestSerialOrder/CMakeLists.txt +++ /dev/null @@ -1,40 +0,0 @@ -cmake_minimum_required(VERSION 3.5) - -project(CTestTestSerialOrder) - -set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt") - -enable_testing() - -function(add_serial_order_test TEST_NAME) - add_test(NAME ${TEST_NAME} - COMMAND ${CMAKE_COMMAND} - "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}" - "-DTEST_NAME=${TEST_NAME}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/test.cmake" - ) - - if(ARGC GREATER 1) - set_tests_properties(${TEST_NAME} PROPERTIES ${ARGN}) - endif() -endfunction() - -add_serial_order_test(initialization COST 1000) -add_serial_order_test(test1) -add_serial_order_test(test2) -add_serial_order_test(test3) -add_serial_order_test(test4 DEPENDS test5) - -add_serial_order_test(test5) -set_tests_properties(test5 PROPERTIES DEPENDS "test6;test7b;test7a") - -add_serial_order_test(test6 COST -2) -add_serial_order_test(test7a COST -1) -add_serial_order_test(test7b COST -1) -add_serial_order_test(test8 COST 10) -add_serial_order_test(test9 COST 20) -add_serial_order_test(test10 COST 0) -add_serial_order_test(test11) -add_serial_order_test(test12 COST 0) - -add_serial_order_test(verification COST -1000) diff --git a/Tests/CTestTestSerialOrder/test.cmake b/Tests/CTestTestSerialOrder/test.cmake deleted file mode 100644 index 8479cae..0000000 --- a/Tests/CTestTestSerialOrder/test.cmake +++ /dev/null @@ -1,31 +0,0 @@ -list(APPEND EXPECTED_OUTPUT - initialization - test9 - test8 - test1 - test2 - test3 - test6 - test7a - test7b - test5 - test4 - test10 - test11 - test12 -) - - -if("${TEST_NAME}" STREQUAL "initialization") - file(WRITE ${TEST_OUTPUT_FILE} "${TEST_NAME}") - -elseif("${TEST_NAME}" STREQUAL "verification") - file(READ ${TEST_OUTPUT_FILE} ACTUAL_OUTPUT) - if(NOT "${ACTUAL_OUTPUT}" STREQUAL "${EXPECTED_OUTPUT}") - message(FATAL_ERROR "Actual test order [${ACTUAL_OUTPUT}] differs from expected test order [${EXPECTED_OUTPUT}]") - endif() - -else() - file(APPEND ${TEST_OUTPUT_FILE} ";${TEST_NAME}") - -endif() diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 4413a4d..54880b5 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -11,6 +11,21 @@ endfunction() run_ctest_test(TestQuiet QUIET) +set(CASE_CMAKELISTS_SUFFIX_CODE [[ +foreach(i RANGE 1 6) + add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true) +endforeach() +set_property(TEST test1 PROPERTY COST -2) +set_property(TEST test2 PROPERTY COST -1) +set_property(TEST test3 PROPERTY COST 0) +set_property(TEST test4 PROPERTY COST 1) +set_property(TEST test5 PROPERTY COST 2) +set_property(TEST test6 PROPERTY COST 3) +set_property(TEST test6 PROPERTY DEPENDS test1) +]]) +run_ctest_test(SerialOrder INCLUDE test) +unset(CASE_CMAKELISTS_SUFFIX_CODE) + # Tests for the 'Test Load' feature of ctest # # Spoof a load average value to make these tests more reliable. diff --git a/Tests/RunCMake/ctest_test/SerialOrder-stdout.txt b/Tests/RunCMake/ctest_test/SerialOrder-stdout.txt new file mode 100644 index 0000000..5f5a0bc --- /dev/null +++ b/Tests/RunCMake/ctest_test/SerialOrder-stdout.txt @@ -0,0 +1,16 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/SerialOrder-build + Start 2: test1 +1/6 Test #2: test1 ............................ Passed +[0-9.]+ sec + Start 7: test6 +2/6 Test #7: test6 ............................ Passed +[0-9.]+ sec + Start 6: test5 +3/6 Test #6: test5 ............................ Passed +[0-9.]+ sec + Start 5: test4 +4/6 Test #5: test4 ............................ Passed +[0-9.]+ sec + Start 4: test3 +5/6 Test #4: test3 ............................ Passed +[0-9.]+ sec + Start 3: test2 +6/6 Test #3: test2 ............................ Passed +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 6 -- cgit v0.12 From ae69801d9620eb6b43146d42f04d1a4b6b40e416 Mon Sep 17 00:00:00 2001 From: Brad King Date: Sun, 10 Mar 2024 10:10:29 -0400 Subject: Tests: Convert CTestTestSkipReturnCode to RunCMake.ctest_test case --- Tests/CMakeLists.txt | 13 ------------ Tests/CTestTestSkipReturnCode/CMakeLists.txt | 8 -------- Tests/CTestTestSkipReturnCode/CTestConfig.cmake | 4 ---- Tests/CTestTestSkipReturnCode/test.cmake.in | 23 ---------------------- Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 7 +++++++ .../RunCMake/ctest_test/SkipReturnCode-stdout.txt | 8 ++++++++ 6 files changed, 15 insertions(+), 48 deletions(-) delete mode 100644 Tests/CTestTestSkipReturnCode/CMakeLists.txt delete mode 100644 Tests/CTestTestSkipReturnCode/CTestConfig.cmake delete mode 100644 Tests/CTestTestSkipReturnCode/test.cmake.in create mode 100644 Tests/RunCMake/ctest_test/SkipReturnCode-stdout.txt diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index ae065c8..0919933 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -3073,19 +3073,6 @@ if(BUILD_TESTING) "Test command:.*Working Directory:.*Environment variables:.*foo=bar.*this=that" ) - configure_file( - "${CMake_SOURCE_DIR}/Tests/CTestTestSkipReturnCode/test.cmake.in" - "${CMake_BINARY_DIR}/Tests/CTestTestSkipReturnCode/test.cmake" - @ONLY ESCAPE_QUOTES) - add_test(CTestTestSkipReturnCode ${CMAKE_CTEST_COMMAND} - -S "${CMake_BINARY_DIR}/Tests/CTestTestSkipReturnCode/test.cmake" -V - --output-log "${CMake_BINARY_DIR}/Tests/CTestTestSkipReturnCode/testOutput.log" - -C \${CTEST_CONFIGURATION_TYPE} - ) - set_tests_properties(CTestTestSkipReturnCode PROPERTIES - PASS_REGULAR_EXPRESSION "CMakeV1 \\.* +Passed.*CMakeV2 \\.+\\*+Skipped") - set_property(TEST CTestTestSkipReturnCode PROPERTY ENVIRONMENT CTEST_PARALLEL_LEVEL=) - ADD_TEST_MACRO(CTestTestSerialInDepends ${CMAKE_CTEST_COMMAND} -j 4 --output-on-failure -C "\${CTestTest_CONFIG}") diff --git a/Tests/CTestTestSkipReturnCode/CMakeLists.txt b/Tests/CTestTestSkipReturnCode/CMakeLists.txt deleted file mode 100644 index 1eeeec6..0000000 --- a/Tests/CTestTestSkipReturnCode/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -cmake_minimum_required(VERSION 3.5) -project(CTestTestSkipReturnCode) -include(CTest) - -add_test (NAME CMakeV1 COMMAND ${CMAKE_COMMAND} "--version") -add_test (NAME CMakeV2 COMMAND ${CMAKE_COMMAND} "--version") - -set_tests_properties(CMakeV2 PROPERTIES SKIP_RETURN_CODE 0) diff --git a/Tests/CTestTestSkipReturnCode/CTestConfig.cmake b/Tests/CTestTestSkipReturnCode/CTestConfig.cmake deleted file mode 100644 index 5bc1e9e..0000000 --- a/Tests/CTestTestSkipReturnCode/CTestConfig.cmake +++ /dev/null @@ -1,4 +0,0 @@ -set (CTEST_NIGHTLY_START_TIME "21:00:00 EDT") -set(CTEST_DROP_METHOD "http") -set(CTEST_DROP_SITE "open.cdash.org") -set(CTEST_DROP_LOCATION "/submit.php?project=PublicDashboard") diff --git a/Tests/CTestTestSkipReturnCode/test.cmake.in b/Tests/CTestTestSkipReturnCode/test.cmake.in deleted file mode 100644 index b45e4a6..0000000 --- a/Tests/CTestTestSkipReturnCode/test.cmake.in +++ /dev/null @@ -1,23 +0,0 @@ -cmake_minimum_required(VERSION 3.5) - -# Settings: -set(CTEST_DASHBOARD_ROOT "@CMake_BINARY_DIR@/Tests/CTestTest") -set(CTEST_SITE "@SITE@") -set(CTEST_BUILD_NAME "CTestTest-@BUILDNAME@-SkipReturnCode") - -set(CTEST_SOURCE_DIRECTORY "@CMake_SOURCE_DIR@/Tests/CTestTestSkipReturnCode") -set(CTEST_BINARY_DIRECTORY "@CMake_BINARY_DIR@/Tests/CTestTestSkipReturnCode") -set(CTEST_CVS_COMMAND "@CVSCOMMAND@") -set(CTEST_CMAKE_GENERATOR "@CMAKE_GENERATOR@") -set(CTEST_CMAKE_GENERATOR_PLATFORM "@CMAKE_GENERATOR_PLATFORM@") -set(CTEST_CMAKE_GENERATOR_TOOLSET "@CMAKE_GENERATOR_TOOLSET@") -set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_CONFIG_TYPE}") -set(CTEST_COVERAGE_COMMAND "@COVERAGE_COMMAND@") -set(CTEST_NOTES_FILES "${CTEST_SCRIPT_DIRECTORY}/${CTEST_SCRIPT_NAME}") - -#CTEST_EMPTY_BINARY_DIRECTORY(${CTEST_BINARY_DIRECTORY}) - -CTEST_START(Experimental) -CTEST_CONFIGURE(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) -CTEST_BUILD(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) -CTEST_TEST(BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 54880b5..012c1a0 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -26,6 +26,13 @@ set_property(TEST test6 PROPERTY DEPENDS test1) run_ctest_test(SerialOrder INCLUDE test) unset(CASE_CMAKELISTS_SUFFIX_CODE) +set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test(NAME skip COMMAND ${CMAKE_COMMAND} -E true) +set_property(TEST skip PROPERTY SKIP_RETURN_CODE 0) +]]) +run_ctest_test(SkipReturnCode) +unset(CASE_CMAKELISTS_SUFFIX_CODE) + # Tests for the 'Test Load' feature of ctest # # Spoof a load average value to make these tests more reliable. diff --git a/Tests/RunCMake/ctest_test/SkipReturnCode-stdout.txt b/Tests/RunCMake/ctest_test/SkipReturnCode-stdout.txt new file mode 100644 index 0000000..98e603a --- /dev/null +++ b/Tests/RunCMake/ctest_test/SkipReturnCode-stdout.txt @@ -0,0 +1,8 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/SkipReturnCode-build + Start 1: RunCMakeVersion +1/2 Test #1: RunCMakeVersion .................. Passed +[0-9.]+ sec + Start 2: skip +2/2 Test #2: skip .............................\*\*\*Skipped +[0-9.]+ sec ++ +100% tests passed, 0 tests failed out of 2 -- cgit v0.12 From 7457b474a1d6eb2165d657852e25f3a8e5a2f8ee Mon Sep 17 00:00:00 2001 From: Brad King Date: Sun, 10 Mar 2024 10:19:27 -0400 Subject: Tests: Remove unnecessary parallel suppression from CTestCoverageCollectGCOV This hasn't been necessary since commit ab7eda2591 (Tests: Remove unnecessary pass regex on CTestCoverageCollectGCOV, 2020-04-29, v3.18.0-rc1~239^2~2). --- Tests/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 0919933..ab31e70 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -2806,7 +2806,6 @@ if(BUILD_TESTING) -S "${CMake_BINARY_DIR}/Tests/CTestCoverageCollectGCOV/test.cmake" -VV --output-log "${CMake_BINARY_DIR}/Tests/CTestCoverageCollectGCOV/testOut.log" ) - set_property(TEST CTestCoverageCollectGCOV PROPERTY ENVIRONMENT CTEST_PARALLEL_LEVEL=) configure_file( "${CMake_SOURCE_DIR}/Tests/CTestTestEmptyBinaryDirectory/test.cmake.in" -- cgit v0.12 From bbcbcff7d94bb3dd4b47ed5484207dc960911f4c Mon Sep 17 00:00:00 2001 From: Brad King Date: Thu, 7 Mar 2024 08:53:19 -0500 Subject: cmCTestMultiProcessHandler: Modernize member initialization --- Source/CTest/cmCTestMultiProcessHandler.cxx | 17 +++++--------- Source/CTest/cmCTestMultiProcessHandler.h | 35 +++++++++++++---------------- Source/CTest/cmCTestTestHandler.cxx | 5 ++--- 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 7b72f30..6efe008 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -66,18 +66,13 @@ private: cmCTestMultiProcessHandler* Handler; }; -cmCTestMultiProcessHandler::cmCTestMultiProcessHandler() +cmCTestMultiProcessHandler::cmCTestMultiProcessHandler( + cmCTest* ctest, cmCTestTestHandler* handler) + : CTest(ctest) + , TestHandler(handler) + , ProcessorsAvailable(cmAffinity::GetProcessorsAvailable()) + , HaveAffinity(this->ProcessorsAvailable.size()) { - this->ParallelLevel = 1; - this->TestLoad = 0; - this->FakeLoadForTesting = 0; - this->Completed = 0; - this->RunningCount = 0; - this->ProcessorsAvailable = cmAffinity::GetProcessorsAvailable(); - this->HaveAffinity = this->ProcessorsAvailable.size(); - this->HasCycles = false; - this->HasInvalidGeneratedResourceSpec = false; - this->SerialTestRunning = false; } cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler() = default; diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h index 02589ca..bbbfc7b 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.h +++ b/Source/CTest/cmCTestMultiProcessHandler.h @@ -58,7 +58,7 @@ public: unsigned int Slots; }; - cmCTestMultiProcessHandler(); + cmCTestMultiProcessHandler(cmCTest* ctest, cmCTestTestHandler* handler); virtual ~cmCTestMultiProcessHandler(); // Set the tests void SetTests(TestMap tests, PropertiesMap properties); @@ -81,13 +81,6 @@ public: this->TestResults = r; } - void SetCTest(cmCTest* ctest) { this->CTest = ctest; } - - void SetTestHandler(cmCTestTestHandler* handler) - { - this->TestHandler = handler; - } - cmCTestTestHandler* GetTestHandler() { return this->TestHandler; } void SetRepeatMode(cmCTest::Repeat mode, int count) @@ -171,22 +164,26 @@ protected: bool InitResourceAllocator(std::string& error); bool CheckGeneratedResourceSpec(); +private: + cmCTest* CTest; + cmCTestTestHandler* TestHandler; + bool UseResourceSpec = false; cmCTestResourceSpec ResourceSpec; std::string ResourceSpecFile; std::string ResourceSpecSetupFixture; cm::optional ResourceSpecSetupTest; - bool HasInvalidGeneratedResourceSpec; + bool HasInvalidGeneratedResourceSpec = false; // Tests pending selection to start. They may have dependencies. TestMap PendingTests; // List of pending test indexes, ordered by cost. std::list OrderedTests; // Total number of tests we'll be running - size_t Total; + size_t Total = 0; // Number of tests that are complete - size_t Completed; - size_t RunningCount; + size_t Completed = 0; + size_t RunningCount = 0; std::set ProcessorsAvailable; size_t HaveAffinity; bool StopTimePassed = false; @@ -204,7 +201,7 @@ protected: ResourceAvailabilityErrors; cmCTestResourceAllocator ResourceAllocator; std::vector* TestResults; - size_t ParallelLevel; // max number of process that can be run at once + size_t ParallelLevel = 1; // max number of process that can be run at once // 'make' jobserver client. If connected, we acquire a token // for each test before running its process. @@ -214,16 +211,14 @@ protected: // Callback invoked when a token is received. void JobServerReceivedToken(); - unsigned long TestLoad; - unsigned long FakeLoadForTesting; + unsigned long TestLoad = 0; + unsigned long FakeLoadForTesting = 0; cm::uv_loop_ptr Loop; cm::uv_idle_ptr StartNextTestsOnIdle_; cm::uv_timer_ptr StartNextTestsOnTimer_; - cmCTestTestHandler* TestHandler; - cmCTest* CTest; - bool HasCycles; + bool HasCycles = false; cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never; int RepeatCount = 1; - bool Quiet; - bool SerialTestRunning; + bool Quiet = false; + bool SerialTestRunning = false; }; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index dc16f66..921ab1e 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -1359,10 +1359,9 @@ bool cmCTestTestHandler::ProcessDirectory(std::vector& passed, this->StartTestTime = std::chrono::system_clock::now(); auto elapsed_time_start = std::chrono::steady_clock::now(); - auto parallel = cm::make_unique(); - parallel->SetCTest(this->CTest); + auto parallel = + cm::make_unique(this->CTest, this); parallel->SetParallelLevel(this->CTest->GetParallelLevel()); - parallel->SetTestHandler(this); if (this->RepeatMode != cmCTest::Repeat::Never) { parallel->SetRepeatMode(this->RepeatMode, this->RepeatCount); } else { -- cgit v0.12 From 5de1e21659090ca83e39e223d351e353347eb88e Mon Sep 17 00:00:00 2001 From: Brad King Date: Wed, 6 Mar 2024 11:16:33 -0500 Subject: ctest: Allow passing -j without value to choose a contextual default Under job server integration, added by commit 80fe56c481 (ctest: Add support for running under a make job server on POSIX systems, 2023-11-15, v3.29.0-rc1~324^2), use a very high default so that parallelism is effectively limited only by available job server tokens. Otherwise, choose a default limit based on the number of processors. Also allow passing `0` to specify unbounded parallelism. Fixes: #25739 --- Help/command/ctest_test.rst | 13 +++-- Help/envvar/CTEST_PARALLEL_LEVEL.rst | 11 ++++ Help/manual/ctest.1.rst | 24 +++++--- Help/release/3.29.rst | 6 ++ Source/CTest/cmCTestMultiProcessHandler.cxx | 66 +++++++++++++++++++--- Source/CTest/cmCTestMultiProcessHandler.h | 12 +++- Source/CTest/cmCTestTestCommand.cxx | 4 +- Source/CTest/cmCTestTestCommand.h | 4 +- Source/CTest/cmCTestTestHandler.cxx | 18 +++++- Source/cmCTest.cxx | 55 +++++++++++++----- Source/cmCTest.h | 5 +- Source/ctest.cxx | 6 +- .../CTestCommandLine/Parallel-0-stdout.txt | 9 +++ .../CTestCommandLine/Parallel-4-stdout.txt | 7 +++ .../CTestCommandLine/Parallel-N-stdout.txt | 10 ++++ .../CTestCommandLine/Parallel-bad-result.txt | 1 + .../CTestCommandLine/Parallel-bad-stderr.txt | 1 + .../CTestCommandLine/Parallel-empty-stdout.txt | 5 ++ .../CTestCommandLine/Parallel-env-0-stdout.txt | 9 +++ .../CTestCommandLine/Parallel-env-3-stdout.txt | 6 ++ .../CTestCommandLine/Parallel-env-bad-stdout.txt | 4 ++ .../CTestCommandLine/Parallel-env-empty-stdout.txt | 5 ++ .../CTestCommandLine/Parallel-j-bad-result.txt | 1 + .../CTestCommandLine/Parallel-j-bad-stderr.txt | 1 + .../CTestCommandLine/Parallel-j-stdout.txt | 5 ++ Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake | 38 +++++++++++++ Tests/RunCMake/Make/CTestJobServer.make | 6 +- Tests/RunCMake/Make/RunCMakeTest.cmake | 3 + Tests/RunCMake/ctest_test/Parallel0-stdout.txt | 9 +++ Tests/RunCMake/ctest_test/Parallel4-stdout.txt | 7 +++ Tests/RunCMake/ctest_test/ParallelBad-result.txt | 1 + Tests/RunCMake/ctest_test/ParallelBad-stderr.txt | 1 + Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt | 5 ++ Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt | 9 +++ Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt | 6 ++ .../RunCMake/ctest_test/ParallelEnvBad-stdout.txt | 4 ++ .../ctest_test/ParallelEnvEmpty-stdout.txt | 5 ++ Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt | 5 ++ Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 34 +++++++++++ Tests/RunCMake/ctest_test/test.cmake.in | 2 +- 40 files changed, 372 insertions(+), 51 deletions(-) create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-0-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-4-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-N-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-bad-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-bad-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-empty-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-env-0-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-env-3-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-env-bad-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-env-empty-stdout.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-j-bad-result.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-j-bad-stderr.txt create mode 100644 Tests/RunCMake/CTestCommandLine/Parallel-j-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/Parallel0-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/Parallel4-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelBad-result.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelBad-stderr.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEnvBad-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelEnvEmpty-stdout.txt create mode 100644 Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index 4d56577..c1862df 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -18,7 +18,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`. [EXCLUDE_FIXTURE ] [EXCLUDE_FIXTURE_SETUP ] [EXCLUDE_FIXTURE_CLEANUP ] - [PARALLEL_LEVEL ] + [PARALLEL_LEVEL []] [RESOURCE_SPEC_FILE ] [TEST_LOAD ] [SCHEDULE_RANDOM ] @@ -104,9 +104,14 @@ The options are: Same as ``EXCLUDE_FIXTURE`` except only matching cleanup tests are excluded. -``PARALLEL_LEVEL `` - Specify a positive number representing the number of tests to - be run in parallel. +``PARALLEL_LEVEL []`` + Run tests in parallel, limited to a given level of parallelism. + + .. versionadded:: 3.29 + + The ```` may be omitted, or ``0``, to let ctest use a default + level of parallelism, or unbounded parallelism, respectively, as + documented by the :option:`ctest --parallel` option. ``RESOURCE_SPEC_FILE `` .. versionadded:: 3.16 diff --git a/Help/envvar/CTEST_PARALLEL_LEVEL.rst b/Help/envvar/CTEST_PARALLEL_LEVEL.rst index 126bdb3..0ef01d5 100644 --- a/Help/envvar/CTEST_PARALLEL_LEVEL.rst +++ b/Help/envvar/CTEST_PARALLEL_LEVEL.rst @@ -8,4 +8,15 @@ For example, if ``CTEST_PARALLEL_LEVEL`` is set to 8, CTest will run up to 8 tests concurrently as if ``ctest`` were invoked with the :option:`--parallel 8 ` option. +.. versionchanged:: 3.29 + + The value may be empty, or ``0``, to let ctest use a default level of + parallelism, or unbounded parallelism, respectively, as documented by + the :option:`ctest --parallel` option. + + On Windows, environment variables cannot be set to an empty string. + CTest will interpret a whitespace-only string as empty. + + In CMake 3.28 and earlier, an empty or ``0`` value was equivalent to ``1``. + See :manual:`ctest(1)` for more information on parallel test execution. diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 602a82d..edbd5a4 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -118,17 +118,27 @@ Run Tests previously interrupted. If no interruption occurred, the ``-F`` option will have no effect. -.. option:: -j , --parallel +.. option:: -j [], --parallel [] - Run the tests in parallel using the given number of jobs. + Run tests in parallel, optionally limited to a given level of parallelism. - This option tells CTest to run the tests in parallel using given - number of jobs. This option can also be set by setting the - :envvar:`CTEST_PARALLEL_LEVEL` environment variable. + .. versionadded:: 3.29 - This option can be used with the :prop_test:`PROCESSORS` test property. + The ```` may be omitted, or ``0``, in which case: - See `Label and Subproject Summary`_. + * Under `Job Server Integration`_, parallelism is limited by + available job tokens. + + * Otherwise, if the value is omitted, parallelism is limited + by the number of processors, or 2, whichever is larger. + + * Otherwise, if the value is ``0``, parallelism is unbounded. + + This option may instead be specified by the :envvar:`CTEST_PARALLEL_LEVEL` + environment variable. + + This option can be used with the :prop_test:`PROCESSORS` test property. + See the `Label and Subproject Summary`_. .. option:: --resource-spec-file diff --git a/Help/release/3.29.rst b/Help/release/3.29.rst index 659237e..42f6491 100644 --- a/Help/release/3.29.rst +++ b/Help/release/3.29.rst @@ -150,6 +150,12 @@ CTest * :manual:`ctest(1)` now supports :ref:`job server integration ` on POSIX systems. +* The :option:`ctest -j` option may now be given without a value to let + ctest choose a default level of parallelism, or with ``0`` to let ctest + use unbounded parallelism. The corresponding :envvar:`CTEST_PARALLEL_LEVEL` + environment variable, if set to the empty string, is now equivalent to + passing ``-j`` with no value. + * The :command:`ctest_test` command gained options ``INCLUDE_FROM_FILE`` and ``EXCLUDE_FROM_FILE`` to run or exclude tests named in a file. diff --git a/Source/CTest/cmCTestMultiProcessHandler.cxx b/Source/CTest/cmCTestMultiProcessHandler.cxx index 6efe008..53c5f76 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.cxx +++ b/Source/CTest/cmCTestMultiProcessHandler.cxx @@ -43,6 +43,17 @@ #include "cmUVJobServerClient.h" #include "cmWorkingDirectory.h" +namespace { +// For unspecified parallelism, limit to the number of processors, +// but with a minimum greater than 1 so there is some parallelism. +constexpr unsigned long kParallelLevelMinimum = 2u; + +// For "unbounded" parallelism, limit to a very high value. +// Under a job server, parallelism is effectively limited +// only by available job server tokens. +constexpr unsigned long kParallelLevelUnbounded = 0x10000u; +} + namespace cmsys { class RegularExpression; } @@ -72,6 +83,7 @@ cmCTestMultiProcessHandler::cmCTestMultiProcessHandler( , TestHandler(handler) , ProcessorsAvailable(cmAffinity::GetProcessorsAvailable()) , HaveAffinity(this->ProcessorsAvailable.size()) + , ParallelLevelDefault(kParallelLevelMinimum) { } @@ -97,9 +109,43 @@ void cmCTestMultiProcessHandler::SetTests(TestMap tests, } // Set the max number of tests that can be run at the same time. -void cmCTestMultiProcessHandler::SetParallelLevel(size_t level) +void cmCTestMultiProcessHandler::SetParallelLevel(cm::optional level) +{ + this->ParallelLevel = level; + + if (!this->ParallelLevel) { + // '-j' was given with no value. Limit by number of processors. + cmsys::SystemInformation info; + info.RunCPUCheck(); + unsigned long processorCount = info.GetNumberOfLogicalCPU(); + + if (cm::optional fakeProcessorCount = + cmSystemTools::GetEnvVar( + "__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING")) { + unsigned long pc = 0; + if (cmStrToULong(*fakeProcessorCount, &pc)) { + processorCount = pc; + } else { + cmSystemTools::Error("Failed to parse fake processor count: " + + *fakeProcessorCount); + } + } + + this->ParallelLevelDefault = + std::max(kParallelLevelMinimum, processorCount); + } +} + +size_t cmCTestMultiProcessHandler::GetParallelLevel() const { - this->ParallelLevel = level < 1 ? 1 : level; + if ((this->ParallelLevel && *this->ParallelLevel == 0) || + (!this->ParallelLevel && this->JobServerClient)) { + return kParallelLevelUnbounded; + } + if (this->ParallelLevel) { + return *this->ParallelLevel; + } + return this->ParallelLevelDefault; } void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load) @@ -446,10 +492,11 @@ void cmCTestMultiProcessHandler::UnlockResources(int index) inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test) { size_t processors = static_cast(this->Properties[test]->Processors); + size_t const parallelLevel = this->GetParallelLevel(); // If processors setting is set higher than the -j // setting, we default to using all of the process slots. - if (processors > this->ParallelLevel) { - processors = this->ParallelLevel; + if (processors > parallelLevel) { + processors = parallelLevel; } // Cap tests that want affinity to the maximum affinity available. if (this->HaveAffinity && processors > this->HaveAffinity && @@ -503,8 +550,9 @@ void cmCTestMultiProcessHandler::StartNextTests() size_t numToStart = 0; - if (this->RunningCount < this->ParallelLevel) { - numToStart = this->ParallelLevel - this->RunningCount; + size_t const parallelLevel = this->GetParallelLevel(); + if (this->RunningCount < parallelLevel) { + numToStart = parallelLevel - this->RunningCount; } if (numToStart == 0) { @@ -518,7 +566,7 @@ void cmCTestMultiProcessHandler::StartNextTests() } bool allTestsFailedTestLoadCheck = false; - size_t minProcessorsRequired = this->ParallelLevel; + size_t minProcessorsRequired = this->GetParallelLevel(); std::string testWithMinProcessors; cmsys::SystemInformation info; @@ -813,7 +861,7 @@ void cmCTestMultiProcessHandler::ReadCostData() this->Properties[index]->PreviousRuns = prev; // When not running in parallel mode, don't use cost data - if (this->ParallelLevel > 1 && this->Properties[index] && + if (this->GetParallelLevel() > 1 && this->Properties[index] && this->Properties[index]->Cost == 0) { this->Properties[index]->Cost = cost; } @@ -842,7 +890,7 @@ int cmCTestMultiProcessHandler::SearchByName(std::string const& name) void cmCTestMultiProcessHandler::CreateTestCostList() { - if (this->ParallelLevel > 1) { + if (this->GetParallelLevel() > 1) { this->CreateParallelTestCostList(); } else { this->CreateSerialTestCostList(); diff --git a/Source/CTest/cmCTestMultiProcessHandler.h b/Source/CTest/cmCTestMultiProcessHandler.h index bbbfc7b..d4759f1 100644 --- a/Source/CTest/cmCTestMultiProcessHandler.h +++ b/Source/CTest/cmCTestMultiProcessHandler.h @@ -63,7 +63,7 @@ public: // Set the tests void SetTests(TestMap tests, PropertiesMap properties); // Set the max number of tests that can be run at the same time. - void SetParallelLevel(size_t); + void SetParallelLevel(cm::optional level); void SetTestLoad(unsigned long load); virtual void RunTests(); void PrintOutputAsJson(); @@ -201,7 +201,15 @@ private: ResourceAvailabilityErrors; cmCTestResourceAllocator ResourceAllocator; std::vector* TestResults; - size_t ParallelLevel = 1; // max number of process that can be run at once + + // Get the maximum number of processors that may be used at once. + size_t GetParallelLevel() const; + + // With no '-j' option, default to serial testing. + cm::optional ParallelLevel = 1; + + // Fallback parallelism limit when '-j' is given with no value. + size_t ParallelLevelDefault; // 'make' jobserver client. If connected, we acquire a token // for each test before running its process. diff --git a/Source/CTest/cmCTestTestCommand.cxx b/Source/CTest/cmCTestTestCommand.cxx index ed16354..98ce862 100644 --- a/Source/CTest/cmCTestTestCommand.cxx +++ b/Source/CTest/cmCTestTestCommand.cxx @@ -105,8 +105,8 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler() if (this->StopOnFailure) { handler->SetOption("StopOnFailure", "ON"); } - if (!this->ParallelLevel.empty()) { - handler->SetOption("ParallelLevel", this->ParallelLevel); + if (this->ParallelLevel) { + handler->SetOption("ParallelLevel", *this->ParallelLevel); } if (!this->Repeat.empty()) { handler->SetOption("Repeat", this->Repeat); diff --git a/Source/CTest/cmCTestTestCommand.h b/Source/CTest/cmCTestTestCommand.h index ce054ed..23661c5 100644 --- a/Source/CTest/cmCTestTestCommand.h +++ b/Source/CTest/cmCTestTestCommand.h @@ -8,7 +8,9 @@ #include #include +#include +#include "cmArgumentParserTypes.h" #include "cmCTestHandlerCommand.h" #include "cmCommand.h" @@ -56,7 +58,7 @@ protected: std::string ExcludeFixture; std::string ExcludeFixtureSetup; std::string ExcludeFixtureCleanup; - std::string ParallelLevel; + cm::optional> ParallelLevel; std::string Repeat; std::string ScheduleRandom; std::string StopTime; diff --git a/Source/CTest/cmCTestTestHandler.cxx b/Source/CTest/cmCTestTestHandler.cxx index 921ab1e..61a18a2 100644 --- a/Source/CTest/cmCTestTestHandler.cxx +++ b/Source/CTest/cmCTestTestHandler.cxx @@ -548,9 +548,21 @@ bool cmCTestTestHandler::ProcessOptions() return false; } } - if (this->GetOption("ParallelLevel")) { - this->CTest->SetParallelLevel( - std::stoi(*this->GetOption("ParallelLevel"))); + if (cmValue parallelLevel = this->GetOption("ParallelLevel")) { + if (parallelLevel.IsEmpty()) { + // An empty value tells ctest to choose a default. + this->CTest->SetParallelLevel(cm::nullopt); + } else { + // A non-empty value must be a non-negative integer. + unsigned long plevel = 0; + if (!cmStrToULong(*parallelLevel, &plevel)) { + cmCTestLog(this->CTest, ERROR_MESSAGE, + "ParallelLevel invalid value: " << *parallelLevel + << std::endl); + return false; + } + this->CTest->SetParallelLevel(plevel); + } } if (this->GetOption("StopOnFailure")) { diff --git a/Source/cmCTest.cxx b/Source/cmCTest.cxx index 7acee4b..1ae3cb5 100644 --- a/Source/cmCTest.cxx +++ b/Source/cmCTest.cxx @@ -179,7 +179,7 @@ struct cmCTest::Private int MaxTestNameWidth = 30; - int ParallelLevel = 1; + cm::optional ParallelLevel = 1; bool ParallelLevelSetInCli = false; unsigned long TestLoad = 0; @@ -380,14 +380,14 @@ cmCTest::cmCTest() cmCTest::~cmCTest() = default; -int cmCTest::GetParallelLevel() const +cm::optional cmCTest::GetParallelLevel() const { return this->Impl->ParallelLevel; } -void cmCTest::SetParallelLevel(int level) +void cmCTest::SetParallelLevel(cm::optional level) { - this->Impl->ParallelLevel = level < 1 ? 1 : level; + this->Impl->ParallelLevel = level; } unsigned long cmCTest::GetTestLoad() const @@ -1892,14 +1892,31 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, std::string arg = args[i]; if (this->CheckArgument(arg, "-F"_s)) { this->Impl->Failover = true; - } else if (this->CheckArgument(arg, "-j"_s, "--parallel") && - i < args.size() - 1) { - i++; - int plevel = atoi(args[i].c_str()); - this->SetParallelLevel(plevel); + } else if (this->CheckArgument(arg, "-j"_s, "--parallel")) { + cm::optional parallelLevel; + // No value or an empty value tells ctest to choose a default. + if (i + 1 < args.size() && !cmHasLiteralPrefix(args[i + 1], "-")) { + ++i; + if (!args[i].empty()) { + // A non-empty value must be a non-negative integer. + unsigned long plevel = 0; + if (!cmStrToULong(args[i], &plevel)) { + errormsg = + cmStrCat("'", arg, "' given invalid value '", args[i], "'"); + return false; + } + parallelLevel = plevel; + } + } + this->SetParallelLevel(parallelLevel); this->Impl->ParallelLevelSetInCli = true; } else if (cmHasPrefix(arg, "-j")) { - int plevel = atoi(arg.substr(2).c_str()); + // The value must be a non-negative integer. + unsigned long plevel = 0; + if (!cmStrToULong(arg.substr(2), &plevel)) { + errormsg = cmStrCat("'", arg, "' given invalid value '", args[i], "'"); + return false; + } this->SetParallelLevel(plevel); this->Impl->ParallelLevelSetInCli = true; } @@ -2799,10 +2816,20 @@ int cmCTest::Run(std::vector& args, std::string* output) // handle CTEST_PARALLEL_LEVEL environment variable if (!this->Impl->ParallelLevelSetInCli) { - std::string parallel; - if (cmSystemTools::GetEnv("CTEST_PARALLEL_LEVEL", parallel)) { - int plevel = atoi(parallel.c_str()); - this->SetParallelLevel(plevel); + if (cm::optional parallelEnv = + cmSystemTools::GetEnvVar("CTEST_PARALLEL_LEVEL")) { + if (parallelEnv->empty() || + parallelEnv->find_first_not_of(" \t") == std::string::npos) { + // An empty value tells ctest to choose a default. + this->SetParallelLevel(cm::nullopt); + } else { + // A non-empty value must be a non-negative integer. + // Otherwise, ignore it. + unsigned long plevel = 0; + if (cmStrToULong(*parallelEnv, &plevel)) { + this->SetParallelLevel(plevel); + } + } } } diff --git a/Source/cmCTest.h b/Source/cmCTest.h index 1644d84..2ab810c 100644 --- a/Source/cmCTest.h +++ b/Source/cmCTest.h @@ -12,6 +12,7 @@ #include #include +#include #include #include "cmDuration.h" @@ -116,8 +117,8 @@ public: cmDuration GetGlobalTimeout() const; /** how many test to run at the same time */ - int GetParallelLevel() const; - void SetParallelLevel(int); + cm::optional GetParallelLevel() const; + void SetParallelLevel(cm::optional level); unsigned long GetTestLoad() const; void SetTestLoad(unsigned long); diff --git a/Source/ctest.cxx b/Source/ctest.cxx index c1c2071..f6a11b4 100644 --- a/Source/ctest.cxx +++ b/Source/ctest.cxx @@ -48,9 +48,9 @@ const cmDocumentationEntry cmDocumentationOptions[] = { "Truncate 'tail' (default), 'middle' or 'head' of test output once " "maximum output size is reached" }, { "-F", "Enable failover." }, - { "-j , --parallel ", - "Run the tests in parallel using the " - "given number of jobs." }, + { "-j [], --parallel []", + "Run tests in parallel, " + "optionally limited to a given level of parallelism." }, { "-Q,--quiet", "Make ctest quiet." }, { "-O , --output-log ", "Output to log file" }, { "--output-junit ", "Output test results to JUnit XML file." }, diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-0-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-0-stdout.txt new file mode 100644 index 0000000..37f728e --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-0-stdout.txt @@ -0,0 +1,9 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-0 + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-4-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-4-stdout.txt new file mode 100644 index 0000000..d90cbd8 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-4-stdout.txt @@ -0,0 +1,7 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-4 + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-N-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-N-stdout.txt new file mode 100644 index 0000000..2c16db7 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-N-stdout.txt @@ -0,0 +1,10 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-N + Test #1: test1 + Test #2: test2 + Test #3: test3 + Test #4: test4 + Test #5: test5 + Test #6: test6 + +Total Tests: 6 diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-bad-result.txt b/Tests/RunCMake/CTestCommandLine/Parallel-bad-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-bad-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-bad-stderr.txt b/Tests/RunCMake/CTestCommandLine/Parallel-bad-stderr.txt new file mode 100644 index 0000000..121248b --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-bad-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '--parallel' given invalid value 'bad'$ diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-empty-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-empty-stdout.txt new file mode 100644 index 0000000..f380c17 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-empty-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-empty + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-env-0-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-env-0-stdout.txt new file mode 100644 index 0000000..1eb05ac --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-env-0-stdout.txt @@ -0,0 +1,9 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-env-0 + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-env-3-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-env-3-stdout.txt new file mode 100644 index 0000000..d6fc03b --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-env-3-stdout.txt @@ -0,0 +1,6 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-env-3 + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-env-bad-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-env-bad-stdout.txt new file mode 100644 index 0000000..def3313 --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-env-bad-stdout.txt @@ -0,0 +1,4 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-env-bad + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-env-empty-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-env-empty-stdout.txt new file mode 100644 index 0000000..85b880d --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-env-empty-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-env-empty + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-result.txt b/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-result.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-result.txt @@ -0,0 +1 @@ +1 diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-stderr.txt b/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-stderr.txt new file mode 100644 index 0000000..228f7cf --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-j-bad-stderr.txt @@ -0,0 +1 @@ +^CMake Error: '-j' given invalid value 'bad'$ diff --git a/Tests/RunCMake/CTestCommandLine/Parallel-j-stdout.txt b/Tests/RunCMake/CTestCommandLine/Parallel-j-stdout.txt new file mode 100644 index 0000000..39dd34a --- /dev/null +++ b/Tests/RunCMake/CTestCommandLine/Parallel-j-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/CTestCommandLine/Parallel-j + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake index c09fff8..724c5e4 100644 --- a/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake +++ b/Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake @@ -240,6 +240,44 @@ add_test(Echo \"${CMAKE_COMMAND}\" -E echo \"EchoTest\") endfunction() run_SerialFailed() +function(run_Parallel case) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/Parallel-${case}) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" " +foreach(i RANGE 1 6) + add_test(test\${i} \"${CMAKE_COMMAND}\" -E true) +endforeach() +") + run_cmake_command(Parallel-${case} ${CMAKE_CTEST_COMMAND} ${ARGN}) +endfunction() +# Spoof a number of processors to make these tests predictable. +set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 1) +run_Parallel(bad --parallel bad) +run_Parallel(j-bad -j bad) +set(RunCMake_TEST_RAW_ARGS [[--parallel ""]]) +run_Parallel(empty) # With 1 processor, defaults to 2. +unset(RunCMake_TEST_RAW_ARGS) +run_Parallel(j -j) # With 1 processor, defaults to 2. +run_Parallel(0 -j0) +run_Parallel(4 --parallel 4) +run_Parallel(N --parallel -N) +set(ENV{CTEST_PARALLEL_LEVEL} bad) +run_Parallel(env-bad) +if(CMAKE_HOST_WIN32) + set(ENV{CTEST_PARALLEL_LEVEL} " ") +else() + set(ENV{CTEST_PARALLEL_LEVEL} "") +endif() +run_Parallel(env-empty) # With 1 processor, defaults to 2. +set(ENV{CTEST_PARALLEL_LEVEL} 0) +run_Parallel(env-0) +set(ENV{CTEST_PARALLEL_LEVEL} 3) +run_Parallel(env-3) +unset(ENV{CTEST_PARALLEL_LEVEL}) +unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) + function(run_TestLoad name load) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/TestLoad) set(RunCMake_TEST_NO_CLEAN 1) diff --git a/Tests/RunCMake/Make/CTestJobServer.make b/Tests/RunCMake/Make/CTestJobServer.make index 7fc5e28..24fe774 100644 --- a/Tests/RunCMake/Make/CTestJobServer.make +++ b/Tests/RunCMake/Make/CTestJobServer.make @@ -1,11 +1,11 @@ NoPipe: - env MAKEFLAGS= $(CMAKE_CTEST_COMMAND) -j6 + env MAKEFLAGS= $(CMAKE_CTEST_COMMAND) -j0 .PHONY: NoPipe NoTests: - +$(CMAKE_CTEST_COMMAND) -j6 -R NoTests + +$(CMAKE_CTEST_COMMAND) -j -R NoTests .PHONY: NoTests Tests: - +$(CMAKE_CTEST_COMMAND) -j6 + +$(CMAKE_CTEST_COMMAND) -j .PHONY: Tests diff --git a/Tests/RunCMake/Make/RunCMakeTest.cmake b/Tests/RunCMake/Make/RunCMakeTest.cmake index cfaf759..9673329 100644 --- a/Tests/RunCMake/Make/RunCMakeTest.cmake +++ b/Tests/RunCMake/Make/RunCMakeTest.cmake @@ -90,10 +90,13 @@ function(run_CTestJobServer) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/CTestJobServer-build) run_cmake(CTestJobServer) set(RunCMake_TEST_NO_CLEAN 1) + # Spoof a number of processors to make sure jobserver integration is unbounded. + set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 1) run_make_rule(CTestJobServer NoPipe 2) run_make_rule(CTestJobServer NoTests 2) run_make_rule(CTestJobServer Tests 2) run_make_rule(CTestJobServer Tests 3) + unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) endfunction() # Jobservers are currently only supported by GNU makes, except MSYS2 make diff --git a/Tests/RunCMake/ctest_test/Parallel0-stdout.txt b/Tests/RunCMake/ctest_test/Parallel0-stdout.txt new file mode 100644 index 0000000..98230cb --- /dev/null +++ b/Tests/RunCMake/ctest_test/Parallel0-stdout.txt @@ -0,0 +1,9 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/Parallel0-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/Parallel4-stdout.txt b/Tests/RunCMake/ctest_test/Parallel4-stdout.txt new file mode 100644 index 0000000..36b0b85 --- /dev/null +++ b/Tests/RunCMake/ctest_test/Parallel4-stdout.txt @@ -0,0 +1,7 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/Parallel4-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelBad-result.txt b/Tests/RunCMake/ctest_test/ParallelBad-result.txt new file mode 100644 index 0000000..b57e2de --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelBad-result.txt @@ -0,0 +1 @@ +(-1|255) diff --git a/Tests/RunCMake/ctest_test/ParallelBad-stderr.txt b/Tests/RunCMake/ctest_test/ParallelBad-stderr.txt new file mode 100644 index 0000000..2e21a1b --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelBad-stderr.txt @@ -0,0 +1 @@ +^ParallelLevel invalid value: bad$ diff --git a/Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt new file mode 100644 index 0000000..fec0789 --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEmpty-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEmpty-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt new file mode 100644 index 0000000..2e4bc6f --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEnv0-stdout.txt @@ -0,0 +1,9 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEnv0-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt new file mode 100644 index 0000000..0193b6c --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEnv3-stdout.txt @@ -0,0 +1,6 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEnv3-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelEnvBad-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEnvBad-stdout.txt new file mode 100644 index 0000000..cd7970b --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEnvBad-stdout.txt @@ -0,0 +1,4 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEnvBad-build + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelEnvEmpty-stdout.txt b/Tests/RunCMake/ctest_test/ParallelEnvEmpty-stdout.txt new file mode 100644 index 0000000..e0f92d1 --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelEnvEmpty-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelEnvEmpty-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt b/Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt new file mode 100644 index 0000000..c388937 --- /dev/null +++ b/Tests/RunCMake/ctest_test/ParallelOmit-stdout.txt @@ -0,0 +1,5 @@ +Test project [^ +]*/Tests/RunCMake/ctest_test/ParallelOmit-build + Start [0-9]+: test[0-9]+ + Start [0-9]+: test[0-9]+ +1/6 Test #[0-9]+: test[0-9]+ ............................ Passed +[0-9.]+ sec diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 012c1a0..ee7114b 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -1,8 +1,10 @@ include(RunCTest) + set(RunCMake_TEST_TIMEOUT 60) set(CASE_CTEST_TEST_ARGS "") set(CASE_CTEST_TEST_LOAD "") +set(CASE_CTEST_TEST_RAW_ARGS "") function(run_ctest_test CASE_NAME) set(CASE_CTEST_TEST_ARGS "${ARGN}") @@ -11,6 +13,7 @@ endfunction() run_ctest_test(TestQuiet QUIET) +set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 4) set(CASE_CMAKELISTS_SUFFIX_CODE [[ foreach(i RANGE 1 6) add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true) @@ -25,6 +28,7 @@ set_property(TEST test6 PROPERTY DEPENDS test1) ]]) run_ctest_test(SerialOrder INCLUDE test) unset(CASE_CMAKELISTS_SUFFIX_CODE) +unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) set(CASE_CMAKELISTS_SUFFIX_CODE [[ add_test(NAME skip COMMAND ${CMAKE_COMMAND} -E true) @@ -33,6 +37,36 @@ set_property(TEST skip PROPERTY SKIP_RETURN_CODE 0) run_ctest_test(SkipReturnCode) unset(CASE_CMAKELISTS_SUFFIX_CODE) +# Spoof a number of processors to make these tests predictable. +set(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING} 1) +set(CASE_CMAKELISTS_SUFFIX_CODE [[ +foreach(i RANGE 1 6) + add_test(NAME test${i} COMMAND ${CMAKE_COMMAND} -E true) +endforeach() +]]) +run_ctest_test(ParallelBad INCLUDE test PARALLEL_LEVEL bad) +set(CASE_CTEST_TEST_RAW_ARGS "PARALLEL_LEVEL \"\"") +run_ctest_test(ParallelEmpty INCLUDE test) # With 1 processor, defaults to 2. +unset(CASE_CTEST_TEST_RAW_ARGS) +run_ctest_test(ParallelOmit INCLUDE test PARALLEL_LEVEL) # With 1 processor, defaults to 2. +run_ctest_test(Parallel0 INCLUDE test PARALLEL_LEVEL 0) +run_ctest_test(Parallel4 INCLUDE test PARALLEL_LEVEL 4) +set(ENV{CTEST_PARALLEL_LEVEL} bad) +run_ctest_test(ParallelEnvBad INCLUDE test) +if(CMAKE_HOST_WIN32) + set(ENV{CTEST_PARALLEL_LEVEL} " ") +else() + set(ENV{CTEST_PARALLEL_LEVEL} "") +endif() +run_ctest_test(ParallelEnvEmpty INCLUDE test) # With 1 processor, defaults to 2. +set(ENV{CTEST_PARALLEL_LEVEL} 0) +run_ctest_test(ParallelEnv0 INCLUDE test) +set(ENV{CTEST_PARALLEL_LEVEL} 3) +run_ctest_test(ParallelEnv3 INCLUDE test) +unset(ENV{CTEST_PARALLEL_LEVEL}) +unset(CASE_CMAKELISTS_SUFFIX_CODE) +unset(ENV{__CTEST_FAKE_PROCESSOR_COUNT_FOR_TESTING) + # Tests for the 'Test Load' feature of ctest # # Spoof a load average value to make these tests more reliable. diff --git a/Tests/RunCMake/ctest_test/test.cmake.in b/Tests/RunCMake/ctest_test/test.cmake.in index 16dde1c..d28b1e2 100644 --- a/Tests/RunCMake/ctest_test/test.cmake.in +++ b/Tests/RunCMake/ctest_test/test.cmake.in @@ -19,5 +19,5 @@ if("@CASE_NAME@" STREQUAL "TestChangingLabels") ctest_test(${ctest_test_args} INCLUDE_LABEL "^a$") ctest_test(${ctest_test_args} INCLUDE_LABEL "^b$") else() - ctest_test(${ctest_test_args}) + ctest_test(${ctest_test_args} @CASE_CTEST_TEST_RAW_ARGS@) endif() -- cgit v0.12