From 7eb5ab2c63ce80b73e132710786cb5172ad60a0e Mon Sep 17 00:00:00 2001 From: tophoo <30765654+tophoo@users.noreply.github.com> Date: Tue, 14 Nov 2023 20:08:48 +0100 Subject: Autogen: Generalize MaybeWriteMocResponseFile to MaybeWriteResponseFile --- Source/cmQtAutoMocUic.cxx | 103 ++++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/Source/cmQtAutoMocUic.cxx b/Source/cmQtAutoMocUic.cxx index 50110ed..a6d7e19 100644 --- a/Source/cmQtAutoMocUic.cxx +++ b/Source/cmQtAutoMocUic.cxx @@ -333,6 +333,13 @@ public: std::vector const& command, std::string const& output) const; + /* + * Check if command line exceeds maximum length supported by OS + * (if on Windows) and switch to using a response file instead. + */ + void MaybeWriteResponseFile(std::string const& outputFile, + std::vector& cmd) const; + /** @brief Run an external process. Use only during Process() call! */ bool RunProcess(GenT genType, cmWorkerPool::ProcessResultT& result, std::vector const& command, @@ -498,10 +505,6 @@ public: protected: ParseCacheT::FileHandleT CacheEntry; - - private: - void MaybeWriteMocResponseFile(std::string const& outputFile, - std::vector& cmd) const; }; /** uic compiles a file. */ @@ -795,6 +798,51 @@ void cmQtAutoMocUicT::JobT::LogCommandError( this->Gen()->Log().ErrorCommand(genType, message, command, output); } +/* + * Check if command line exceeds maximum length supported by OS + * (if on Windows) and switch to using a response file instead. + */ +void cmQtAutoMocUicT::JobT::MaybeWriteResponseFile( + std::string const& outputFile, std::vector& cmd) const +{ +#ifdef _WIN32 + // Ensure cmd is less than CommandLineLengthMax characters + size_t commandLineLength = cmd.size(); // account for separating spaces + for (std::string const& str : cmd) { + commandLineLength += str.length(); + } + if (commandLineLength >= CommandLineLengthMax) { + // Command line exceeds maximum size allowed by OS + // => create response file + std::string const responseFile = cmStrCat(outputFile, ".rsp"); + + cmsys::ofstream fout(responseFile.c_str()); + if (!fout) { + this->LogError( + GenT::MOC, + cmStrCat("AUTOMOC was unable to create a response file at\n ", + this->MessagePath(responseFile))); + return; + } + + auto it = cmd.begin(); + while (++it != cmd.end()) { + fout << *it << "\n"; + } + fout.close(); + + // Keep all but executable + cmd.resize(1); + + // Specify response file + cmd.emplace_back(cmStrCat('@', responseFile)); + } +#else + static_cast(outputFile); + static_cast(cmd); +#endif +} + bool cmQtAutoMocUicT::JobT::RunProcess(GenT genType, cmWorkerPool::ProcessResultT& result, std::vector const& command, @@ -2034,7 +2082,7 @@ void cmQtAutoMocUicT::JobCompileMocT::Process() // Add source file cmd.push_back(sourceFile); - MaybeWriteMocResponseFile(outputFile, cmd); + MaybeWriteResponseFile(outputFile, cmd); } // Execute moc command @@ -2080,51 +2128,6 @@ void cmQtAutoMocUicT::JobCompileMocT::Process() } } -/* - * Check if command line exceeds maximum length supported by OS - * (if on Windows) and switch to using a response file instead. - */ -void cmQtAutoMocUicT::JobCompileMocT::MaybeWriteMocResponseFile( - std::string const& outputFile, std::vector& cmd) const -{ -#ifdef _WIN32 - // Ensure cmd is less than CommandLineLengthMax characters - size_t commandLineLength = cmd.size(); // account for separating spaces - for (std::string const& str : cmd) { - commandLineLength += str.length(); - } - if (commandLineLength >= CommandLineLengthMax) { - // Command line exceeds maximum size allowed by OS - // => create response file - std::string const responseFile = cmStrCat(outputFile, ".rsp"); - - cmsys::ofstream fout(responseFile.c_str()); - if (!fout) { - this->LogError( - GenT::MOC, - cmStrCat("AUTOMOC was unable to create a response file at\n ", - this->MessagePath(responseFile))); - return; - } - - auto it = cmd.begin(); - while (++it != cmd.end()) { - fout << *it << "\n"; - } - fout.close(); - - // Keep all but executable - cmd.resize(1); - - // Specify response file - cmd.emplace_back(cmStrCat('@', responseFile)); - } -#else - static_cast(outputFile); - static_cast(cmd); -#endif -} - void cmQtAutoMocUicT::JobCompileUicT::Process() { std::string const& sourceFile = this->Mapping->SourceFile->FileName; -- cgit v0.12 From 7a07887055d8acd1d367c640ca069734fdf49fa4 Mon Sep 17 00:00:00 2001 From: tophoo <30765654+tophoo@users.noreply.github.com> Date: Wed, 8 Nov 2023 16:34:19 +0100 Subject: Autogen: Add support for response files for moc predef targets Add support for response files for moc predef targets and make the limit when to use response files for autogen targets configurable. --- Auxiliary/vim/syntax/cmake.vim | 2 ++ Help/manual/cmake-properties.7.rst | 1 + Help/manual/cmake-variables.7.rst | 1 + Help/prop_tgt/AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst | 18 +++++++++++ Help/prop_tgt/AUTOMOC.rst | 4 +++ Help/prop_tgt/AUTOUIC.rst | 4 +++ .../CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst | 10 ++++++ Source/cmQtAutoGenInitializer.cxx | 36 ++++++++++++++++++++++ Source/cmQtAutoGenInitializer.h | 3 ++ Source/cmQtAutoMocUic.cxx | 9 +++++- Source/cmTarget.cxx | 1 + 11 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 Help/prop_tgt/AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst create mode 100644 Help/variable/CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst diff --git a/Auxiliary/vim/syntax/cmake.vim b/Auxiliary/vim/syntax/cmake.vim index a13dfc9..9dad4ac 100644 --- a/Auxiliary/vim/syntax/cmake.vim +++ b/Auxiliary/vim/syntax/cmake.vim @@ -70,6 +70,7 @@ syn keyword cmakeProperty contained \ ATTACHED_FILES \ ATTACHED_FILES_ON_FAIL \ AUTOGEN_BUILD_DIR + \ AUTOGEN_COMMAND_LINE_LENGTH_MAX \ AUTOGEN_ORIGIN_DEPENDS \ AUTOGEN_PARALLEL \ AUTOGEN_SOURCE_GROUP @@ -764,6 +765,7 @@ syn keyword cmakeVariable contained \ CMAKE_ASM_STANDARD_REQUIRED \ CMAKE_ASM_SUPPORTED \ CMAKE_ASM_VISIBILITY_PRESET + \ CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX \ CMAKE_AUTOGEN_ORIGIN_DEPENDS \ CMAKE_AUTOGEN_PARALLEL \ CMAKE_AUTOGEN_USE_SYSTEM_INCLUDE diff --git a/Help/manual/cmake-properties.7.rst b/Help/manual/cmake-properties.7.rst index 276cb13..28d61f7 100644 --- a/Help/manual/cmake-properties.7.rst +++ b/Help/manual/cmake-properties.7.rst @@ -130,6 +130,7 @@ Properties on Targets /prop_tgt/ARCHIVE_OUTPUT_NAME /prop_tgt/ARCHIVE_OUTPUT_NAME_CONFIG /prop_tgt/AUTOGEN_BUILD_DIR + /prop_tgt/AUTOGEN_COMMAND_LINE_LENGTH_MAX /prop_tgt/AUTOGEN_ORIGIN_DEPENDS /prop_tgt/AUTOGEN_PARALLEL /prop_tgt/AUTOGEN_TARGET_DEPENDS diff --git a/Help/manual/cmake-variables.7.rst b/Help/manual/cmake-variables.7.rst index a73494e..5f47c6e 100644 --- a/Help/manual/cmake-variables.7.rst +++ b/Help/manual/cmake-variables.7.rst @@ -400,6 +400,7 @@ Variables that Control the Build /variable/CMAKE_APPLE_SILICON_PROCESSOR /variable/CMAKE_ARCHIVE_OUTPUT_DIRECTORY /variable/CMAKE_ARCHIVE_OUTPUT_DIRECTORY_CONFIG + /variable/CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX /variable/CMAKE_AUTOGEN_ORIGIN_DEPENDS /variable/CMAKE_AUTOGEN_PARALLEL /variable/CMAKE_AUTOGEN_USE_SYSTEM_INCLUDE diff --git a/Help/prop_tgt/AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst b/Help/prop_tgt/AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst new file mode 100644 index 0000000..f1e51a7 --- /dev/null +++ b/Help/prop_tgt/AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst @@ -0,0 +1,18 @@ +AUTOGEN_COMMAND_LINE_LENGTH_MAX +------------------------------- + +.. versionadded:: 3.29 + +Command line length limit for autogen targets, i.e. ``moc`` or ``uic``, +that triggers the use of response files on Windows instead of passing all +arguments to the command line. + +- An empty (or unset) value sets the limit to 32000 +- A positive non zero integer value sets the exact command line length + limit. + +By default ``AUTOGEN_COMMAND_LINE_LENGTH_MAX`` is initialized from +:variable:`CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX`. + +See the :manual:`cmake-qt(7)` manual for more information on using CMake +with Qt. diff --git a/Help/prop_tgt/AUTOMOC.rst b/Help/prop_tgt/AUTOMOC.rst index 0feb2e8..6f58afb 100644 --- a/Help/prop_tgt/AUTOMOC.rst +++ b/Help/prop_tgt/AUTOMOC.rst @@ -247,5 +247,9 @@ will be generated when this variable is ``ON``. This target property controls the number of ``moc`` or ``uic`` processes to start in parallel during builds. +:prop_tgt:`AUTOGEN_COMMAND_LINE_LENGTH_MAX`: +This target property controls the limit when to use response files for +``moc`` or ``uic`` processes on Windows. + See the :manual:`cmake-qt(7)` manual for more information on using CMake with Qt. diff --git a/Help/prop_tgt/AUTOUIC.rst b/Help/prop_tgt/AUTOUIC.rst index dc854b2..95366ee 100644 --- a/Help/prop_tgt/AUTOUIC.rst +++ b/Help/prop_tgt/AUTOUIC.rst @@ -81,5 +81,9 @@ will be generated when this variable is ``ON``. This target property controls the number of ``moc`` or ``uic`` processes to start in parallel during builds. +:prop_tgt:`AUTOGEN_COMMAND_LINE_LENGTH_MAX`: +This target property controls the limit when to use response files for +``moc`` or ``uic`` processes on Windows. + See the :manual:`cmake-qt(7)` manual for more information on using CMake with Qt. diff --git a/Help/variable/CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst b/Help/variable/CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst new file mode 100644 index 0000000..eabda43 --- /dev/null +++ b/Help/variable/CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX.rst @@ -0,0 +1,10 @@ +CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX +------------------------------------- + +.. versionadded:: 3.29 + +Command line length limit for autogen targets, i.e. ``moc`` or ``uic``, +that triggers the use of response files on Windows instead of passing all +arguments to the command line. + +By default ``CMAKE_AUTOGEN_COMMAND_LINE_LENGTH_MAX`` is unset. diff --git a/Source/cmQtAutoGenInitializer.cxx b/Source/cmQtAutoGenInitializer.cxx index 81752a6..39ac1b4 100644 --- a/Source/cmQtAutoGenInitializer.cxx +++ b/Source/cmQtAutoGenInitializer.cxx @@ -485,6 +485,38 @@ bool cmQtAutoGenInitializer::InitCustomTargets() } } +#ifdef _WIN32 + { + const auto& value = + this->GenTarget->GetProperty("AUTOGEN_COMMAND_LINE_LENGTH_MAX"); + if (value.IsSet()) { + using maxCommandLineLengthType = + decltype(this->AutogenTarget.MaxCommandLineLength); + unsigned long propInt = 0; + if (cmStrToULong(value, &propInt) && propInt > 0 && + propInt <= std::numeric_limits::max()) { + this->AutogenTarget.MaxCommandLineLength = + static_cast(propInt); + } else { + // Warn the project author that AUTOGEN_PARALLEL is not valid. + this->Makefile->IssueMessage( + MessageType::AUTHOR_WARNING, + cmStrCat("AUTOGEN_COMMAND_LINE_LENGTH_MAX=\"", *value, + "\" for target \"", this->GenTarget->GetName(), + "\" is not valid. Using no limit for " + "AUTOGEN_COMMAND_LINE_LENGTH_MAX")); + this->AutogenTarget.MaxCommandLineLength = + std::numeric_limits::max(); + } + } else { + // Actually 32767 (see + // https://devblogs.microsoft.com/oldnewthing/20031210-00/?p=41553) but + // we allow for a small margin + this->AutogenTarget.MaxCommandLineLength = 32000; + } + } +#endif + // Autogen target info and settings files { // Info file @@ -1692,6 +1724,10 @@ bool cmQtAutoGenInitializer::SetupWriteAutogenInfo() // General info.SetBool("MULTI_CONFIG", this->MultiConfig); info.SetUInt("PARALLEL", this->AutogenTarget.Parallel); +#ifdef _WIN32 + info.SetUInt("AUTOGEN_COMMAND_LINE_LENGTH_MAX", + this->AutogenTarget.MaxCommandLineLength); +#endif info.SetUInt("VERBOSITY", this->Verbosity); // Directories diff --git a/Source/cmQtAutoGenInitializer.h b/Source/cmQtAutoGenInitializer.h index a44d33f..3f7ab9f 100644 --- a/Source/cmQtAutoGenInitializer.h +++ b/Source/cmQtAutoGenInitializer.h @@ -5,6 +5,7 @@ #include "cmConfigure.h" // IWYU pragma: keep #include +#include #include #include #include @@ -194,6 +195,8 @@ private: bool GlobalTarget = false; // Settings unsigned int Parallel = 1; + unsigned int MaxCommandLineLength = + std::numeric_limits::max(); // Configuration files std::string InfoFile; ConfigString SettingsFile; diff --git a/Source/cmQtAutoMocUic.cxx b/Source/cmQtAutoMocUic.cxx index a6d7e19..83256ed 100644 --- a/Source/cmQtAutoMocUic.cxx +++ b/Source/cmQtAutoMocUic.cxx @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -172,6 +173,8 @@ public: bool MultiConfig = false; IntegerVersion QtVersion = { 4, 0 }; unsigned int ThreadCount = 0; + unsigned int MaxCommandLineLength = + std::numeric_limits::max(); // - Directories std::string AutogenBuildDir; std::string AutogenIncludeDir; @@ -811,7 +814,7 @@ void cmQtAutoMocUicT::JobT::MaybeWriteResponseFile( for (std::string const& str : cmd) { commandLineLength += str.length(); } - if (commandLineLength >= CommandLineLengthMax) { + if (commandLineLength >= this->BaseConst().MaxCommandLineLength) { // Command line exceeds maximum size allowed by OS // => create response file std::string const responseFile = cmStrCat(outputFile, ".rsp"); @@ -2380,6 +2383,10 @@ bool cmQtAutoMocUicT::InitFromInfo(InfoT const& info) !info.GetUInt("QT_VERSION_MINOR", this->BaseConst_.QtVersion.Minor, true) || !info.GetUInt("PARALLEL", this->BaseConst_.ThreadCount, false) || +#ifdef _WIN32 + !info.GetUInt("AUTOGEN_COMMAND_LINE_LENGTH_MAX", + this->BaseConst_.MaxCommandLineLength, false) || +#endif !info.GetString("BUILD_DIR", this->BaseConst_.AutogenBuildDir, true) || !info.GetStringConfig("INCLUDE_DIR", this->BaseConst_.AutogenIncludeDir, true) || diff --git a/Source/cmTarget.cxx b/Source/cmTarget.cxx index e373319..663c9d4 100644 --- a/Source/cmTarget.cxx +++ b/Source/cmTarget.cxx @@ -551,6 +551,7 @@ TargetProperty const StaticTargetProperties[] = { { "ANDROID_PROCESS_MAX"_s, IC::CanCompileSources }, { "ANDROID_SKIP_ANT_STEP"_s, IC::CanCompileSources }, // -- Autogen + { "AUTOGEN_COMMAND_LINE_LENGTH_MAX"_s, IC::CanCompileSources }, { "AUTOGEN_ORIGIN_DEPENDS"_s, IC::CanCompileSources }, { "AUTOGEN_PARALLEL"_s, IC::CanCompileSources }, { "AUTOGEN_USE_SYSTEM_INCLUDE"_s, IC::CanCompileSources }, -- cgit v0.12 From 232610e60ec5546d8c8460a006af9417004def00 Mon Sep 17 00:00:00 2001 From: tophoo <30765654+tophoo@users.noreply.github.com> Date: Wed, 8 Nov 2023 16:35:27 +0100 Subject: Autogen: Use new API for limiting autogen command line lengths --- Source/cmQtAutoGen.cxx | 7 ------- Source/cmQtAutoGen.h | 5 ----- Source/cmQtAutoMocUic.cxx | 2 ++ 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/Source/cmQtAutoGen.cxx b/Source/cmQtAutoGen.cxx index adbdba8..0a394b5 100644 --- a/Source/cmQtAutoGen.cxx +++ b/Source/cmQtAutoGen.cxx @@ -76,13 +76,6 @@ static void MergeOptions(std::vector& baseOpts, unsigned int const cmQtAutoGen::ParallelMax = 64; -#ifdef _WIN32 -// Actually 32767 (see -// https://devblogs.microsoft.com/oldnewthing/20031210-00/?p=41553) but we -// allow for a small margin -size_t const cmQtAutoGen::CommandLineLengthMax = 32000; -#endif - cm::string_view cmQtAutoGen::GeneratorName(GenT genType) { switch (genType) { diff --git a/Source/cmQtAutoGen.h b/Source/cmQtAutoGen.h index d111422..5a23ae9 100644 --- a/Source/cmQtAutoGen.h +++ b/Source/cmQtAutoGen.h @@ -64,11 +64,6 @@ public: /// @brief Maximum number of parallel threads/processes in a generator static unsigned int const ParallelMax; -#ifdef _WIN32 - /// @brief Maximum number of characters on command line - static size_t const CommandLineLengthMax; -#endif - /// @brief Returns the generator name static cm::string_view GeneratorName(GenT genType); /// @brief Returns the generator name in upper case diff --git a/Source/cmQtAutoMocUic.cxx b/Source/cmQtAutoMocUic.cxx index 83256ed..a49125e 100644 --- a/Source/cmQtAutoMocUic.cxx +++ b/Source/cmQtAutoMocUic.cxx @@ -887,6 +887,8 @@ void cmQtAutoMocUicT::JobMocPredefsT::Process() cm::append(cmd, this->MocConst().OptionsDefinitions); // Add includes cm::append(cmd, this->MocConst().OptionsIncludes); + // Check if response file is necessary + MaybeWriteResponseFile(this->MocConst().PredefsFileAbs, cmd); // Execute command if (!this->RunProcess(GenT::MOC, result, cmd, reason.get())) { this->LogCommandError(GenT::MOC, -- cgit v0.12