From 99be022428dcbf505cf889ff5d6f8de62821f652 Mon Sep 17 00:00:00 2001 From: Chris Mahoney Date: Wed, 9 Aug 2023 14:28:03 +0000 Subject: Tests/RunCMake/Make: simplify GNUMakeJobSeverAware check function --- ...etectJobServer-absent-parallel-build-stderr.txt | 1 - Tests/RunCMake/Make/DetectJobServer-absent.cmake | 13 ---- Tests/RunCMake/Make/DetectJobServer-present.cmake | 4 +- .../Make/GNUMakeJobServerAware-check.cmake | 22 ++---- Tests/RunCMake/Make/RunCMakeTest.cmake | 27 +------ Tests/RunCMake/detect_jobserver.c | 89 ++++++++-------------- 6 files changed, 44 insertions(+), 112 deletions(-) delete mode 100644 Tests/RunCMake/Make/DetectJobServer-absent-parallel-build-stderr.txt delete mode 100644 Tests/RunCMake/Make/DetectJobServer-absent.cmake diff --git a/Tests/RunCMake/Make/DetectJobServer-absent-parallel-build-stderr.txt b/Tests/RunCMake/Make/DetectJobServer-absent-parallel-build-stderr.txt deleted file mode 100644 index c63bde3..0000000 --- a/Tests/RunCMake/Make/DetectJobServer-absent-parallel-build-stderr.txt +++ /dev/null @@ -1 +0,0 @@ -^(Warning: (Borland's make|NMake|Watcom's WMake) does not support parallel builds\. Ignoring parallel build command line option\.)?$ diff --git a/Tests/RunCMake/Make/DetectJobServer-absent.cmake b/Tests/RunCMake/Make/DetectJobServer-absent.cmake deleted file mode 100644 index e3dddc0..0000000 --- a/Tests/RunCMake/Make/DetectJobServer-absent.cmake +++ /dev/null @@ -1,13 +0,0 @@ -# Verifies that the jobserver connection is absent -add_custom_command(OUTPUT custom_command.txt - JOB_SERVER_AWARE OFF - COMMENT "Should not detect jobserver" - COMMAND ${DETECT_JOBSERVER} --absent "custom_command.txt" -) - -# trigger the custom command to run -add_custom_target(dummy ALL - JOB_SERVER_AWARE OFF - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/custom_command.txt - COMMAND ${DETECT_JOBSERVER} --absent "custom_target.txt" -) diff --git a/Tests/RunCMake/Make/DetectJobServer-present.cmake b/Tests/RunCMake/Make/DetectJobServer-present.cmake index a33658f..cfaf7be 100644 --- a/Tests/RunCMake/Make/DetectJobServer-present.cmake +++ b/Tests/RunCMake/Make/DetectJobServer-present.cmake @@ -2,12 +2,12 @@ add_custom_command(OUTPUT custom_command.txt JOB_SERVER_AWARE ON COMMENT "Should detect jobserver support" - COMMAND ${DETECT_JOBSERVER} --present "custom_command.txt" + COMMAND ${DETECT_JOBSERVER} "custom_command.txt" ) # trigger the custom command to run add_custom_target(dummy ALL JOB_SERVER_AWARE ON DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/custom_command.txt - COMMAND ${DETECT_JOBSERVER} --present "custom_target.txt" + COMMAND ${DETECT_JOBSERVER} "custom_target.txt" ) diff --git a/Tests/RunCMake/Make/GNUMakeJobServerAware-check.cmake b/Tests/RunCMake/Make/GNUMakeJobServerAware-check.cmake index da18f00..dbc1555 100644 --- a/Tests/RunCMake/Make/GNUMakeJobServerAware-check.cmake +++ b/Tests/RunCMake/Make/GNUMakeJobServerAware-check.cmake @@ -1,21 +1,13 @@ set(BUILD_DIR "${RunCMake_BINARY_DIR}/GNUMakeJobServerAware-build") -function(check target line) - # Read the file and split it into a list of lines - file(READ ${BUILD_DIR}/${target} contents) - STRING(REGEX REPLACE ";" "\\\\;" contents "${contents}") - STRING(REGEX REPLACE "\n" ";" contents "${contents}") +function(check target regex) + file(STRINGS ${BUILD_DIR}/${target} lines + REGEX ${regex} + ) - set(found FALSE) - foreach(entry ${contents}) - if("${entry}" MATCHES "${line}") - set(found TRUE) - break() - endif() - endforeach() - - if(NOT found) - message(FATAL_ERROR "Could not find '${line}' in ${BUILD_DIR}/${target}\n${contents}") + list(LENGTH lines len) + if(len EQUAL 0) + message(FATAL_ERROR "Could not find matching lines '${regex}' in ${BUILD_DIR}/${target}") endif() endfunction() diff --git a/Tests/RunCMake/Make/RunCMakeTest.cmake b/Tests/RunCMake/Make/RunCMakeTest.cmake index 12904c8..5d1ba48 100644 --- a/Tests/RunCMake/Make/RunCMakeTest.cmake +++ b/Tests/RunCMake/Make/RunCMakeTest.cmake @@ -71,39 +71,18 @@ if(NOT RunCMake_GENERATOR STREQUAL "Watcom WMake") run_CMP0113(NEW) endif() -function(detect_jobserver_present is_parallel) +function(detect_jobserver_present) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/DetectJobServer-present-build) set(RunCMake_TEST_NO_CLEAN 1) set(RunCMake_TEST_OPTIONS "-DDETECT_JOBSERVER=${DETECT_JOBSERVER}") run_cmake(DetectJobServer-present) - if (is_parallel) - run_cmake_command(DetectJobServer-present-parallel-build ${CMAKE_COMMAND} --build . -j4) - else() - run_cmake_command(DetectJobServer-present-build ${CMAKE_COMMAND} --build .) - endif() -endfunction() - -function(detect_jobserver_absent is_parallel) - set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/DetectJobServer-absent-build) - set(RunCMake_TEST_NO_CLEAN 1) - set(RunCMake_TEST_OPTIONS "-DDETECT_JOBSERVER=${DETECT_JOBSERVER}") - run_cmake(DetectJobServer-absent) - if (is_parallel) - run_cmake_command(DetectJobServer-absent-parallel-build ${CMAKE_COMMAND} --build . -j4) - else() - run_cmake_command(DetectJobServer-absent-build ${CMAKE_COMMAND} --build .) - endif() + run_cmake_command(DetectJobServer-present-parallel-build ${CMAKE_COMMAND} --build . -j4) endfunction() # Jobservers are currently only supported by GNU makes, except MSYS2 make if(MAKE_IS_GNU AND NOT RunCMake_GENERATOR MATCHES "MSYS Makefiles") - detect_jobserver_present(ON) -else() - detect_jobserver_absent(ON) + detect_jobserver_present() endif() -# No matter which generator is used, the jobserver should not be present if a -# parallel build is not requested -detect_jobserver_absent(OFF) if(MAKE_IS_GNU) # In GNU makes, `JOB_SERVER_AWARE` support is implemented by prefixing diff --git a/Tests/RunCMake/detect_jobserver.c b/Tests/RunCMake/detect_jobserver.c index a6c1a7c..8cbfe2e 100644 --- a/Tests/RunCMake/detect_jobserver.c +++ b/Tests/RunCMake/detect_jobserver.c @@ -11,22 +11,24 @@ #include #define MAX_MESSAGE_LENGTH 1023 -#define USAGE "Usage: %s [--present|--absent] \n" +#define USAGE "Usage: %s \n" -// Extracts --jobserver-auth= or --jobserver-fds= from -// MAKEFLAGS. The returned pointer points to the start of Returns NULL -// if MAKEFLAGS is not set or does not contain --jobserver-auth or -// --jobserver-fds +// Extracts the jobserver details from the MAKEFLAGS environment variable. +// +// Returns a pointer to either a string of the form "R,W" where R and W are fds +// or "fifo:PATH". +// +// Returns NULL if MAKEFLAGS is not set or does not contain recognized +// jobserver flags. char* jobserver_auth(char* message) { - const char* jobserver_auth = "--jobserver-auth="; - const char* jobserver_fds = "--jobserver-fds="; - char* auth; - char* fds; - char* start; + const char* jobserver_flags[3] = { "--jobserver-auth=", "--jobserver-fds=", + "-J" }; + char* start = NULL; char* end; char* result; size_t len; + int i; char* makeflags = getenv("MAKEFLAGS"); if (makeflags == NULL) { @@ -34,18 +36,24 @@ char* jobserver_auth(char* message) return NULL; } - // write MAKEFLAGS to stdout for debugging fprintf(stdout, "MAKEFLAGS: %s\n", makeflags); - auth = strstr(makeflags, jobserver_auth); - fds = strstr(makeflags, jobserver_fds); - if (auth == NULL && fds == NULL) { - strncpy(message, "No jobserver found", MAX_MESSAGE_LENGTH); + for (i = 0; i < 3; i++) { + start = strstr(makeflags, jobserver_flags[i]); + if (start != NULL) { + start += strlen(jobserver_flags[i]); + break; + } + } + + if (start == NULL) { + strncpy(message, "No jobserver flags found", MAX_MESSAGE_LENGTH); return NULL; - } else if (auth != NULL) { - start = auth + strlen(jobserver_auth); - } else { - start = fds + strlen(jobserver_fds); + } + + // Skip leading white space + while (*start == ' ' || *start == '\t') { + start++; } end = strchr(start, ' '); @@ -132,38 +140,21 @@ int posix(const char* jobserver, char* message) } #endif -// Takes 2 arguments: -// Either --present or --absent to indicate we expect the jobserver to be -// "present and valid", or "absent or invalid" -// -// if `--present` is passed, the exit code will be 0 if the jobserver is -// present, 1 if it is absent if `--absent` is passed, the exit code will be 0 -// if the jobserver is absent, 1 if it is present in either case, if there is -// some fatal error (e.g the output file cannot be opened), the exit code will -// be 2 +// Takes 1 argument: an outfile to write results to. int main(int argc, char** argv) { char message[MAX_MESSAGE_LENGTH + 1]; char* output_file; FILE* fp; - int expecting_present; - int expecting_absent; char* jobserver; int result; - if (argc != 3) { - fprintf(stderr, USAGE, argv[0]); - return 2; - } - - expecting_present = strcmp(argv[1], "--present") == 0; - expecting_absent = strcmp(argv[1], "--absent") == 0; - if (!expecting_present && !expecting_absent) { + if (argc != 2) { fprintf(stderr, USAGE, argv[0]); return 2; } - output_file = argv[2]; + output_file = argv[1]; fp = fopen(output_file, "w"); if (fp == NULL) { fprintf(stderr, "Error opening output file: %s\n", output_file); @@ -172,11 +163,6 @@ int main(int argc, char** argv) jobserver = jobserver_auth(message); if (jobserver == NULL) { - if (expecting_absent) { - fprintf(stdout, "Success\n"); - return 0; - } - fprintf(stderr, "%s\n", message); return 1; } @@ -187,18 +173,7 @@ int main(int argc, char** argv) result = posix(jobserver, message); #endif free(jobserver); - message[MAX_MESSAGE_LENGTH] = 0; - - if (result == 0 && expecting_present) { - fprintf(stdout, "Success\n"); - return 0; - } - - if (result == 1 && expecting_absent) { - fprintf(stdout, "Success\n"); - return 0; - } + message[MAX_MESSAGE_LENGTH] = '\0'; - fprintf(stderr, "%s\n", message); - return 1; + return result; } -- cgit v0.12 From bc43398e72be08b55d7df1f5195f0c3fc37506da Mon Sep 17 00:00:00 2001 From: Chris Mahoney Date: Wed, 9 Aug 2023 14:28:25 +0000 Subject: ExternalProject: Enable Make Job Server with Explicit Build Command Introduces `BUILD_JOB_SERVER_AWARE` option to `ExternalProject_Add` and `JOB_SERVER_AWARE` to `ExternalProject_Add_Step`. When using an explicit `BUILD_COMMAND` or `COMMAND`, the generated commands won't use `$(MAKE)` thus failing to connect to the outer make's job server. These new options enable explicit job server integration. Co-authored-by: Brad King Fixes: #16273 --- .../dev/ExternalProject-build-jobserver.rst | 10 ++++++ Modules/ExternalProject.cmake | 40 ++++++++++++++++++++++ Tests/RunCMake/CMakeLists.txt | 2 +- .../RunCMake/ExternalProject/DetectJobServer.cmake | 16 +++++++++ Tests/RunCMake/ExternalProject/Foo/CMakeLists.txt | 4 +++ .../GNUMakeJobServerAware-check.cmake | 16 +++++++++ .../ExternalProject/GNUMakeJobServerAware.cmake | 16 +++++++++ Tests/RunCMake/ExternalProject/RunCMakeTest.cmake | 18 ++++++++++ 8 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 Help/release/dev/ExternalProject-build-jobserver.rst create mode 100644 Tests/RunCMake/ExternalProject/DetectJobServer.cmake create mode 100644 Tests/RunCMake/ExternalProject/Foo/CMakeLists.txt create mode 100644 Tests/RunCMake/ExternalProject/GNUMakeJobServerAware-check.cmake create mode 100644 Tests/RunCMake/ExternalProject/GNUMakeJobServerAware.cmake diff --git a/Help/release/dev/ExternalProject-build-jobserver.rst b/Help/release/dev/ExternalProject-build-jobserver.rst new file mode 100644 index 0000000..357da42 --- /dev/null +++ b/Help/release/dev/ExternalProject-build-jobserver.rst @@ -0,0 +1,10 @@ +ExternalProject-build-jobserver +------------------------------- + +* The :module:`ExternalProject` module now includes the + ``BUILD_JOB_SERVER_AWARE`` option for the + :command:`ExternalProject_Add` command. This option enables + the integration of the GNU Make job server when using an + explicit ``BUILD_COMMAND`` with certain :ref:`Makefile Generators`. + Additionally, the :command:`ExternalProject_Add_Step` command + has been updated to support the new ``JOB_SERVER_AWARE`` option. diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake index bec5c8e..dc9d2ce 100644 --- a/Modules/ExternalProject.cmake +++ b/Modules/ExternalProject.cmake @@ -684,6 +684,14 @@ pass ``-v`` to the external project's build step, even if it also uses build step's own underlying call to :command:`add_custom_command`, which has additional documentation. +``BUILD_JOB_SERVER_AWARE `` + .. versionadded:: 3.28 + + Specifies that the build step is aware of the GNU Make job server. + See the :command:`add_custom_command` documentation of its + ``JOB_SERVER_AWARE`` option for details. This option is relevant + only when an explicit ``BUILD_COMMAND`` is specified. + Install Step Options """""""""""""""""""" @@ -1021,6 +1029,13 @@ control needed to implement such step-level capabilities. When enabled, this option specifies that the custom step should always be run (i.e. that it is always considered out of date). + ``JOB_SERVER_AWARE `` + .. versionadded:: 3.28 + + Specifies that the custom step is aware of the GNU Make job server. + See the :command:`add_custom_command` documentation of its + ``JOB_SERVER_AWARE`` option for details. + ``EXCLUDE_FROM_MAIN `` When enabled, this option specifies that the external project's main target does not depend on the custom step. @@ -2366,6 +2381,7 @@ function(ExternalProject_Add_Step name step) INDEPENDENT BYPRODUCTS ALWAYS + JOB_SERVER_AWARE EXCLUDE_FROM_MAIN WORKING_DIRECTORY LOG @@ -2545,6 +2561,16 @@ function(ExternalProject_Add_Step name step) set(maybe_COMMAND_touch "COMMAND \${CMAKE_COMMAND} -E touch \${stamp_file}") endif() + get_property(job_server_aware + TARGET ${name} + PROPERTY _EP_${step}_JOB_SERVER_AWARE + ) + if(job_server_aware) + set(maybe_JOB_SERVER_AWARE "JOB_SERVER_AWARE 1") + else() + set(maybe_JOB_SERVER_AWARE "") + endif() + # Wrap with log script? get_property(log TARGET ${name} PROPERTY _EP_${step}_LOG) if(command AND log) @@ -2571,6 +2597,7 @@ function(ExternalProject_Add_Step name step) COMMENT \${comment} COMMAND ${__cmdQuoted} ${maybe_COMMAND_touch} + ${maybe_JOB_SERVER_AWARE} DEPENDS \${depends} WORKING_DIRECTORY \${work_dir} VERBATIM @@ -3945,6 +3972,17 @@ function(_ep_add_build_command name) PROPERTY _EP_BUILD_BYPRODUCTS ) + get_property(build_job_server_aware + TARGET ${name} + PROPERTY _EP_BUILD_JOB_SERVER_AWARE + ) + if(build_job_server_aware) + set(maybe_JOB_SERVER_AWARE "JOB_SERVER_AWARE 1") + else() + set(maybe_JOB_SERVER_AWARE "") + endif() + + set(__cmdQuoted) foreach(__item IN LISTS cmd) string(APPEND __cmdQuoted " [==[${__item}]==]") @@ -3958,6 +3996,7 @@ function(_ep_add_build_command name) DEPENDEES configure DEPENDS \${file_deps} ALWAYS \${always} + ${maybe_JOB_SERVER_AWARE} ${log} ${uses_terminal} )" @@ -4252,6 +4291,7 @@ function(ExternalProject_Add name) BUILD_IN_SOURCE BUILD_ALWAYS BUILD_BYPRODUCTS + BUILD_JOB_SERVER_AWARE # # Install step options # diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index 8562e2b..a4f6141 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -851,7 +851,7 @@ endif() if(CMake_TEST_RunCMake_ExternalProject_DOWNLOAD_SERVER_TIMEOUT) list(APPEND ExternalProject_ARGS -DDOWNLOAD_SERVER_TIMEOUT=${CMake_TEST_RunCMake_ExternalProject_DOWNLOAD_SERVER_TIMEOUT}) endif() -add_RunCMake_test(ExternalProject) +add_RunCMake_test(ExternalProject -DDETECT_JOBSERVER=$) add_RunCMake_test(FetchContent) add_RunCMake_test(FetchContent_find_package) set(CTestCommandLine_ARGS -DPython_EXECUTABLE=${Python_EXECUTABLE}) diff --git a/Tests/RunCMake/ExternalProject/DetectJobServer.cmake b/Tests/RunCMake/ExternalProject/DetectJobServer.cmake new file mode 100644 index 0000000..c6e1412 --- /dev/null +++ b/Tests/RunCMake/ExternalProject/DetectJobServer.cmake @@ -0,0 +1,16 @@ +include(ExternalProject) +ExternalProject_Add(Foo + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/Foo + BUILD_COMMAND ${DETECT_JOBSERVER} "ep.txt" + BUILD_JOB_SERVER_AWARE 1 + INSTALL_COMMAND "" +) + +# Add a second step to test JOB_SERVER_AWARE +ExternalProject_Add_Step(Foo + second_step + COMMAND ${DETECT_JOBSERVER} "ep_second_step.txt" + DEPENDEES build + ALWAYS 1 + JOB_SERVER_AWARE 1 +) diff --git a/Tests/RunCMake/ExternalProject/Foo/CMakeLists.txt b/Tests/RunCMake/ExternalProject/Foo/CMakeLists.txt new file mode 100644 index 0000000..b38b173 --- /dev/null +++ b/Tests/RunCMake/ExternalProject/Foo/CMakeLists.txt @@ -0,0 +1,4 @@ +cmake_minimum_required(VERSION 3.27) +project(Foo NONE) + +add_custom_target(drive ALL COMMAND ${CMAKE_COMMAND} -E true) diff --git a/Tests/RunCMake/ExternalProject/GNUMakeJobServerAware-check.cmake b/Tests/RunCMake/ExternalProject/GNUMakeJobServerAware-check.cmake new file mode 100644 index 0000000..55a9f0d --- /dev/null +++ b/Tests/RunCMake/ExternalProject/GNUMakeJobServerAware-check.cmake @@ -0,0 +1,16 @@ +set(BUILD_DIR "${RunCMake_BINARY_DIR}/GNUMakeJobServerAware-build") + +function(check target regex) + file(STRINGS ${BUILD_DIR}/${target} lines + REGEX ${regex} + ) + + list(LENGTH lines len) + if(len EQUAL 0) + message(FATAL_ERROR "Could not find matching lines '${regex}' in ${BUILD_DIR}/${target}") + endif() +endfunction() + +check("/CMakeFiles/Foo.dir/build.make" [[\+cd (/d )?"?.*"? && "?.*"? --build "?.*"?]]) +check("/CMakeFiles/Foo.dir/build.make" [[\+cd (/d )?"?.*"? && "?.*"? -E touch "?.*"?]]) +check("/CMakeFiles/Foo.dir/build.make" [[\+"?.*"? -E true]]) diff --git a/Tests/RunCMake/ExternalProject/GNUMakeJobServerAware.cmake b/Tests/RunCMake/ExternalProject/GNUMakeJobServerAware.cmake new file mode 100644 index 0000000..3f688ca --- /dev/null +++ b/Tests/RunCMake/ExternalProject/GNUMakeJobServerAware.cmake @@ -0,0 +1,16 @@ +include(ExternalProject) +ExternalProject_Add(Foo + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/Foo + BUILD_COMMAND ${CMAKE_COMMAND} --build + BUILD_JOB_SERVER_AWARE 1 + INSTALL_COMMAND "" +) + +# Add a second step to test JOB_SERVER_AWARE +ExternalProject_Add_Step(Foo + second_step + COMMAND ${CMAKE_COMMAND} -E true + DEPENDEES build + ALWAYS 1 + JOB_SERVER_AWARE 1 +) diff --git a/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake b/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake index 4afdef8..ffaa46c 100644 --- a/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake +++ b/Tests/RunCMake/ExternalProject/RunCMakeTest.cmake @@ -144,6 +144,24 @@ function(__ep_test_with_build_with_server testName) run_cmake_command(${testName}-build ${CMAKE_COMMAND} --build .) endfunction() +if(RunCMake_GENERATOR MATCHES "(MSYS|MinGW|Unix) Makefiles") + __ep_test_with_build(GNUMakeJobServerAware) +endif() + +function(__ep_test_jobserver) + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/DetectJobServer-build) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + run_cmake_with_options(DetectJobServer -DDETECT_JOBSERVER=${DETECT_JOBSERVER}) + run_cmake_command(DetectJobServer-clean ${CMAKE_COMMAND} --build . --target clean) + run_cmake_command(DetectJobServer-build ${CMAKE_COMMAND} --build . -j4) +endfunction() + +if(RunCMake_GENERATOR MATCHES "(MinGW|Unix) Makefiles") + __ep_test_jobserver() +endif() + __ep_test_with_build(MultiCommand) set(RunCMake_TEST_OUTPUT_MERGE 1) -- cgit v0.12