From e494763997c2e28ead2269f21a1c6a66a815ac2e Mon Sep 17 00:00:00 2001 From: James Johnston Date: Sun, 5 Jul 2015 16:32:20 -0400 Subject: ExternalProject: Added new USES_TERMINAL options Added new USES_TERMINAL option to the ExternalProject_Add_Step function. This option passes USES_TERMINAL to the underlying add_custom_command call so that the Ninja console pool is used. Also, corresponding new USES_TERMINAL_ options were added to the ExternalProject_Add function. Justification: if using Ninja with a CMake superbuild, it's often desirable to limit the superbuild to ONE sub-Ninja process at a time to avoid oversubscribing the CPU. Using the console pool also makes it easy to monitor the progress of the sub-Ninja process. Independent USES_TERMINAL_ arguments are passed to ExternalProject_Add instead of one USES_TERMINAL argument that controls everything. Users may wish to run some steps in parallel but not others (e.g. parallelize configure but not build). --- Help/release/dev/ExternalProject-USES_TERMINAL.rst | 7 ++ Modules/ExternalProject.cmake | 83 ++++++++++++++++++ Tests/RunCMake/ExternalProject/RunCMakeTest.cmake | 1 + .../ExternalProject/UsesTerminal-check.cmake | 97 ++++++++++++++++++++++ Tests/RunCMake/ExternalProject/UsesTerminal.cmake | 45 ++++++++++ 5 files changed, 233 insertions(+) create mode 100644 Help/release/dev/ExternalProject-USES_TERMINAL.rst create mode 100644 Tests/RunCMake/ExternalProject/UsesTerminal-check.cmake create mode 100644 Tests/RunCMake/ExternalProject/UsesTerminal.cmake diff --git a/Help/release/dev/ExternalProject-USES_TERMINAL.rst b/Help/release/dev/ExternalProject-USES_TERMINAL.rst new file mode 100644 index 0000000..415540d --- /dev/null +++ b/Help/release/dev/ExternalProject-USES_TERMINAL.rst @@ -0,0 +1,7 @@ +ExternalProject-USES_TERMINAL +----------------------------- + +* The :module:`ExternalProject` module learned new ``USES_TERMINAL`` + arguments for giving steps exclusive terminal access. Especially + useful with the :generator:`Ninja` generator to monitor CMake + superbuild progress and prevent CPU oversubscription. diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake index eee1841..f6844be 100644 --- a/Modules/ExternalProject.cmake +++ b/Modules/ExternalProject.cmake @@ -175,6 +175,23 @@ Create custom targets to build projects in external trees ``LOG_INSTALL 1`` Wrap install in script to log output + Steps can be given direct access to the terminal if possible. With + the :generator:`Ninja` generator, this places the steps in the + ``console`` :prop_gbl:`pool `. Options are: + + ``USES_TERMINAL_DOWNLOAD 1`` + Give download terminal access. + ``USES_TERMINAL_UPDATE 1`` + Give update terminal access. + ``USES_TERMINAL_CONFIGURE 1`` + Give configure terminal access. + ``USES_TERMINAL_BUILD 1`` + Give build terminal access. + ``USES_TERMINAL_TEST 1`` + Give test terminal access. + ``USES_TERMINAL_INSTALL 1`` + Give install terminal access. + Other options are: ``STEP_TARGETS ...`` @@ -256,6 +273,8 @@ Create custom targets to build projects in external trees Working directory for command ``LOG 1`` Wrap step in script to log output + ``USES_TERMINAL 1`` + Give the step direct access to the terminal if possible. The command line, comment, working directory, and byproducts of every standard and custom step are processed to replace tokens ````, @@ -1463,6 +1482,14 @@ function(ExternalProject_Add_Step name step) get_property(comment TARGET ${name} PROPERTY _EP_${step}_COMMENT) endif() + # Uses terminal? + get_property(uses_terminal TARGET ${name} PROPERTY _EP_${step}_USES_TERMINAL) + if(uses_terminal) + set(uses_terminal USES_TERMINAL) + else() + set(uses_terminal "") + endif() + # Run every time? get_property(always TARGET ${name} PROPERTY _EP_${step}_ALWAYS) if(always) @@ -1505,6 +1532,7 @@ function(ExternalProject_Add_Step name step) DEPENDS ${depends} WORKING_DIRECTORY ${work_dir} VERBATIM + ${uses_terminal} ) set_property(TARGET ${name} APPEND PROPERTY _EP_STEPS ${step}) @@ -1890,6 +1918,14 @@ function(_ep_add_download_command name) set(log "") endif() + get_property(uses_terminal TARGET ${name} PROPERTY + _EP_USES_TERMINAL_DOWNLOAD) + if(uses_terminal) + set(uses_terminal USES_TERMINAL 1) + else() + set(uses_terminal "") + endif() + ExternalProject_Add_Step(${name} download COMMENT ${comment} COMMAND ${cmd} @@ -1897,6 +1933,7 @@ function(_ep_add_download_command name) DEPENDS ${depends} DEPENDEES mkdir ${log} + ${uses_terminal} ) endfunction() @@ -2001,6 +2038,14 @@ Update to Mercurial >= 2.1.1. set(log "") endif() + get_property(uses_terminal TARGET ${name} PROPERTY + _EP_USES_TERMINAL_UPDATE) + if(uses_terminal) + set(uses_terminal USES_TERMINAL 1) + else() + set(uses_terminal "") + endif() + ExternalProject_Add_Step(${name} update COMMENT ${comment} COMMAND ${cmd} @@ -2009,6 +2054,7 @@ Update to Mercurial >= 2.1.1. WORKING_DIRECTORY ${work_dir} DEPENDEES download ${log} + ${uses_terminal} ) if(always AND update_disconnected) @@ -2021,6 +2067,7 @@ Update to Mercurial >= 2.1.1. WORKING_DIRECTORY ${work_dir} DEPENDEES download ${log} + ${uses_terminal} ) set_property(SOURCE ${skip-update_stamp_file} PROPERTY SYMBOLIC 1) endif() @@ -2149,6 +2196,14 @@ function(_ep_add_configure_command name) set(log "") endif() + get_property(uses_terminal TARGET ${name} PROPERTY + _EP_USES_TERMINAL_CONFIGURE) + if(uses_terminal) + set(uses_terminal USES_TERMINAL 1) + else() + set(uses_terminal "") + endif() + get_property(update_disconnected_set TARGET ${name} PROPERTY _EP_UPDATE_DISCONNECTED SET) if(update_disconnected_set) get_property(update_disconnected TARGET ${name} PROPERTY _EP_UPDATE_DISCONNECTED) @@ -2167,6 +2222,7 @@ function(_ep_add_configure_command name) DEPENDEES ${update_dep} patch DEPENDS ${file_deps} ${log} + ${uses_terminal} ) endfunction() @@ -2188,6 +2244,14 @@ function(_ep_add_build_command name) set(log "") endif() + get_property(uses_terminal TARGET ${name} PROPERTY + _EP_USES_TERMINAL_BUILD) + if(uses_terminal) + set(uses_terminal USES_TERMINAL 1) + else() + set(uses_terminal "") + endif() + get_property(build_always TARGET ${name} PROPERTY _EP_BUILD_ALWAYS) if(build_always) set(always 1) @@ -2204,6 +2268,7 @@ function(_ep_add_build_command name) DEPENDEES configure ALWAYS ${always} ${log} + ${uses_terminal} ) endfunction() @@ -2225,11 +2290,20 @@ function(_ep_add_install_command name) set(log "") endif() + get_property(uses_terminal TARGET ${name} PROPERTY + _EP_USES_TERMINAL_INSTALL) + if(uses_terminal) + set(uses_terminal USES_TERMINAL 1) + else() + set(uses_terminal "") + endif() + ExternalProject_Add_Step(${name} install COMMAND ${cmd} WORKING_DIRECTORY ${binary_dir} DEPENDEES build ${log} + ${uses_terminal} ) endfunction() @@ -2277,6 +2351,14 @@ function(_ep_add_test_command name) set(log "") endif() + get_property(uses_terminal TARGET ${name} PROPERTY + _EP_USES_TERMINAL_TEST) + if(uses_terminal) + set(uses_terminal USES_TERMINAL 1) + else() + set(uses_terminal "") + endif() + ExternalProject_Add_Step(${name} test COMMAND ${cmd} WORKING_DIRECTORY ${binary_dir} @@ -2284,6 +2366,7 @@ function(_ep_add_test_command name) ${dependers_args} ${exclude_args} ${log} + ${uses_terminal} ) endif() endfunction() diff --git a/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake b/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake index e038409..47d6129 100644 --- a/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake +++ b/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake @@ -11,3 +11,4 @@ run_cmake(Add_StepDependencies) run_cmake(Add_StepDependencies_iface) run_cmake(Add_StepDependencies_iface_step) run_cmake(Add_StepDependencies_no_target) +run_cmake(UsesTerminal) diff --git a/Tests/RunCMake/ExternalProject/UsesTerminal-check.cmake b/Tests/RunCMake/ExternalProject/UsesTerminal-check.cmake new file mode 100644 index 0000000..201d822 --- /dev/null +++ b/Tests/RunCMake/ExternalProject/UsesTerminal-check.cmake @@ -0,0 +1,97 @@ +cmake_minimum_required(VERSION 3.3) + +# If we are using the Ninja generator, we can check and verify that the +# USES_TERMINAL option actually works by examining the Ninja build file. +# This is the only way, since CMake doesn't offer a way to examine the +# options on a custom command after it has been added. Furthermore, +# there isn't an easy way to test for this by actually running Ninja. +# +# Other generators don't currently support USES_TERMINAL at this time. +# This file can be improved to support them if they do. Until then, we +# simply assume success for new generator types. +# +# For Ninja, there is a complication. If the Ninja generator detects a +# version of Ninja < 1.5, it won't actually emit the console pool command, +# because those Ninja versions don't yet support the console pool. In +# that case, we also have to assume success. + +# Check Ninja build output to verify whether or not a target step is in the +# console pool. +macro(CheckNinjaStep _target _step _require) + if("${_build}" MATCHES +" DESC = Performing ${_step} step for '${_target}' + pool = console" + ) + if(NOT ${_require}) + set(RunCMake_TEST_FAILED "${_target} ${_step} step is in console pool") + return() + endif() + else() + if(${_require}) + set(RunCMake_TEST_FAILED "${_target} ${_step} step not in console pool") + return() + endif() + endif() +endmacro() + +# Check Ninja build output to verify whether each target step is in the +# console pool. +macro(CheckNinjaTarget _target + _download _update _configure _build _test _install + ) + CheckNinjaStep(${_target} download ${_download}) + CheckNinjaStep(${_target} update ${_update}) + CheckNinjaStep(${_target} configure ${_configure}) + CheckNinjaStep(${_target} build ${_build}) + CheckNinjaStep(${_target} test ${_test}) + CheckNinjaStep(${_target} install ${_install}) +endmacro() + +# Load build/make file, depending on generator +if(RunCMake_GENERATOR STREQUAL Ninja) + # Check the Ninja version. If < 1.5, console pool isn't supported and + # so the generator would not emit console pool usage. That would cause + # this test to fail. + execute_process(COMMAND ${RunCMake_MAKE_PROGRAM} --version + RESULT_VARIABLE _version_result + OUTPUT_VARIABLE _version + ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(_version_result OR _version VERSION_EQUAL "0") + set(RunCMake_TEST_FAILED "Failed to get Ninja version") + return() + endif() + if(_version VERSION_LESS "1.5") + return() # console pool not supported on Ninja < 1.5 + endif() + + # Read the Ninja build file + set(_build_file "${RunCMake_TEST_BINARY_DIR}/build.ninja") + + if(NOT EXISTS "${_build_file}") + set(RunCMake_TEST_FAILED "Ninja build file not created") + return() + endif() + + file(READ "${_build_file}" _build) + + set(_target_check_macro CheckNinjaTarget) +elseif((RunCMake_GENERATOR STREQUAL "") OR NOT DEFINED RunCMake_GENERATOR) + # protection in case somebody renamed RunCMake_GENERATOR + set(RunCMake_TEST_FAILED "Unknown generator") + return() +else() + # We don't yet know how to test USES_TERMINAL on this generator. + return() +endif() + +# Actual tests: +CheckNinjaTarget(TerminalTest1 + true true true true true true ) +CheckNinjaTarget(TerminalTest2 + true false true false true false) +CheckNinjaTarget(TerminalTest3 + false true false true false true ) +CheckNinjaTarget(TerminalTest4 + false false false false false false) diff --git a/Tests/RunCMake/ExternalProject/UsesTerminal.cmake b/Tests/RunCMake/ExternalProject/UsesTerminal.cmake new file mode 100644 index 0000000..cd87403 --- /dev/null +++ b/Tests/RunCMake/ExternalProject/UsesTerminal.cmake @@ -0,0 +1,45 @@ +if(NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Debug) +endif() +include(ExternalProject) + +# Test various combinations of USES_TERMINAL with ExternalProject_Add. + +macro(DoTerminalTest _target) + ExternalProject_Add(${_target} + DOWNLOAD_COMMAND "${CMAKE_COMMAND}" -E echo "download" + UPDATE_COMMAND "${CMAKE_COMMAND}" -E echo "update" + CONFIGURE_COMMAND "${CMAKE_COMMAND}" -E echo "configure" + BUILD_COMMAND "${CMAKE_COMMAND}" -E echo "build" + TEST_COMMAND "${CMAKE_COMMAND}" -E echo "test" + INSTALL_COMMAND "${CMAKE_COMMAND}" -E echo "install" + ${ARGN} + ) +endmacro() + +# USES_TERMINAL on all steps +DoTerminalTest(TerminalTest1 + USES_TERMINAL_DOWNLOAD 1 + USES_TERMINAL_UPDATE 1 + USES_TERMINAL_CONFIGURE 1 + USES_TERMINAL_BUILD 1 + USES_TERMINAL_TEST 1 + USES_TERMINAL_INSTALL 1 + ) + +# USES_TERMINAL on every other step, starting with download +DoTerminalTest(TerminalTest2 + USES_TERMINAL_DOWNLOAD 1 + USES_TERMINAL_CONFIGURE 1 + USES_TERMINAL_TEST 1 + ) + +# USES_TERMINAL on every other step, starting with update +DoTerminalTest(TerminalTest3 + USES_TERMINAL_UPDATE 1 + USES_TERMINAL_BUILD 1 + USES_TERMINAL_INSTALL 1 + ) + +# USES_TERMINAL on no step +DoTerminalTest(TerminalTest4) -- cgit v0.12