From fa18069ebd01ecaef6d7500fcb0de29995d6b516 Mon Sep 17 00:00:00 2001 From: Brad King Date: Mon, 9 Nov 2020 16:15:57 -0500 Subject: Ninja: Exclude unused dyndep features during CMake bootstrap --- Source/cmGlobalNinjaGenerator.cxx | 14 +++++++++----- Source/cmcmd.cxx | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index 83d15ab..24b5ccd 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -2024,6 +2024,8 @@ void cmGlobalNinjaGenerator::StripNinjaOutputPathPrefixAsSuffix( cmStripSuffixIfExists(path, this->OutputPathPrefix); } +#if !defined(CMAKE_BOOTSTRAP) + /* We use the following approach to support Fortran. Each target already @@ -2406,11 +2408,6 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile( return true; } -bool cmGlobalNinjaGenerator::EnableCrossConfigBuild() const -{ - return !this->CrossConfigs.empty(); -} - int cmcmd_cmake_ninja_dyndep(std::vector::const_iterator argBeg, std::vector::const_iterator argEnd) { @@ -2492,6 +2489,13 @@ int cmcmd_cmake_ninja_dyndep(std::vector::const_iterator argBeg, return 0; } +#endif + +bool cmGlobalNinjaGenerator::EnableCrossConfigBuild() const +{ + return !this->CrossConfigs.empty(); +} + void cmGlobalNinjaGenerator::AppendDirectoryForConfig( const std::string& prefix, const std::string& config, const std::string& suffix, std::string& dir) diff --git a/Source/cmcmd.cxx b/Source/cmcmd.cxx index a611dd7..81374a1 100644 --- a/Source/cmcmd.cxx +++ b/Source/cmcmd.cxx @@ -1167,7 +1167,7 @@ int cmcmd::ExecuteCMakeCommand(std::vector const& args, return cmcmd::ExecuteLinkScript(args); } -#if !defined(CMAKE_BOOTSTRAP) || defined(CMAKE_BOOTSTRAP_NINJA) +#if !defined(CMAKE_BOOTSTRAP) // Internal CMake ninja dependency scanning support. if (args[1] == "cmake_ninja_depends") { return cmcmd_cmake_ninja_depends(args.begin() + 2, args.end()); -- cgit v0.12 From a02c4ccabc1736712fc90b12c63e4997ccf34284 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Tue, 12 Mar 2019 15:16:46 -0400 Subject: cmScanDepFormat: add reader and writer for the format from P1689R2 This format is currently subject to change, but is not too far from the end goal. Some bits are currently unimplemented (see TODO comments). --- Source/CMakeLists.txt | 1 + Source/cmScanDepFormat.cxx | 289 +++++++++++++++++++++++++++++++++++++++++++++ Source/cmScanDepFormat.h | 30 +++++ 3 files changed, 320 insertions(+) create mode 100644 Source/cmScanDepFormat.cxx create mode 100644 Source/cmScanDepFormat.h diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index cb954e5..d1616ad 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -834,6 +834,7 @@ endif() # Ninja support set(SRCS ${SRCS} + cmScanDepFormat.cxx cmGlobalNinjaGenerator.cxx cmGlobalNinjaGenerator.h cmNinjaTypes.h diff --git a/Source/cmScanDepFormat.cxx b/Source/cmScanDepFormat.cxx new file mode 100644 index 0000000..b157a8b --- /dev/null +++ b/Source/cmScanDepFormat.cxx @@ -0,0 +1,289 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ + +#include "cmScanDepFormat.h" + +#include +#include + +#include +#include +#include + +#include "cmsys/FStream.hxx" + +#include "cm_utf8.h" + +#include "cmGeneratedFileStream.h" +#include "cmStringAlgorithms.h" +#include "cmSystemTools.h" + +static bool ParseFilename(Json::Value const& val, std::string& result) +{ + if (val.isObject()) { + Json::Value const& format = val["format"]; + if (format == "raw8") { + Json::Value const& data = val["data"]; + for (auto const& byte : data) { + result.push_back(static_cast(byte.asUInt())); + } + } else /* TODO: if (format == "raw16") */ { + return false; + } + } else if (val.isString()) { + result = val.asString(); + } else { + return false; + } + + return true; +} + +static Json::Value EncodeFilename(std::string const& path) +{ + if (cm_utf8_is_valid(path.c_str())) { + std::string valid_data; + valid_data.reserve(path.size()); + + for (auto const& byte : path) { + if (std::iscntrl(byte)) { + // Control characters. + valid_data.append("\\u"); + char buf[5]; + std::snprintf(buf, sizeof(buf), "%04x", byte); + valid_data.append(buf); + } else if (byte == '"' || byte == '\\') { + // Special JSON characters. + valid_data.push_back('\\'); + valid_data.push_back(byte); + } else { + // Other data. + valid_data.push_back(byte); + } + } + + return valid_data; + } + + Json::Value data; + data["format"] = "raw8"; + Json::Value& code_units = data["code-units"]; + for (auto const& code_unit : path) { + code_units.append(static_cast(code_unit)); + } + + return data; +} + +#define PARSE_BLOB(val, res) \ + do { \ + if (!ParseFilename(val, res)) { \ + cmSystemTools::Error( \ + cmStrCat("-E cmake_ninja_depends failed to parse ", arg_pp, \ + ": invalid blob")); \ + return false; \ + } \ + } while (0) + +#define PARSE_FILENAME(val, res) \ + do { \ + if (!ParseFilename(val, res)) { \ + cmSystemTools::Error( \ + cmStrCat("-E cmake_ninja_depends failed to parse ", arg_pp, \ + ": invalid filename")); \ + return false; \ + } \ + \ + if (!cmSystemTools::FileIsFullPath(res)) { \ + res = cmStrCat(work_directory, '/', res); \ + } \ + } while (0) + +bool cmScanDepFormat_P1689_Parse(std::string const& arg_pp, cmSourceInfo* info) +{ + Json::Value ppio; + Json::Value const& ppi = ppio; + cmsys::ifstream ppf(arg_pp.c_str(), std::ios::in | std::ios::binary); + { + Json::Reader reader; + if (!reader.parse(ppf, ppio, false)) { + cmSystemTools::Error(cmStrCat("-E cmake_ninja_depends failed to parse ", + arg_pp, + reader.getFormattedErrorMessages())); + return false; + } + } + + Json::Value const& version = ppi["version"]; + if (version.asUInt() != 0) { + cmSystemTools::Error(cmStrCat("-E cmake_ninja_depends failed to parse ", + arg_pp, ": version ", version.asString())); + return false; + } + + Json::Value const& workdir = ppi["work-directory"]; + if (!workdir.isString()) { + cmSystemTools::Error(cmStrCat("-E cmake_ninja_depends failed to parse ", + arg_pp, ": work-directory is not a string")); + return false; + } + std::string work_directory; + PARSE_BLOB(workdir, work_directory); + + Json::Value const& rules = ppi["rules"]; + if (rules.isArray()) { + if (rules.size() != 1) { + cmSystemTools::Error(cmStrCat("-E cmake_ninja_depends failed to parse ", + arg_pp, ": expected 1 source entry")); + return false; + } + + for (auto const& rule : rules) { + Json::Value const& depends = rule["depends"]; + if (depends.isArray()) { + std::string depend_filename; + for (auto const& depend : depends) { + PARSE_FILENAME(depend, depend_filename); + info->Includes.push_back(depend_filename); + } + } + + if (rule.isMember("future-compile")) { + Json::Value const& future_compile = rule["future-compile"]; + + if (future_compile.isMember("outputs")) { + Json::Value const& outputs = future_compile["outputs"]; + if (outputs.isArray()) { + if (outputs.empty()) { + cmSystemTools::Error( + cmStrCat("-E cmake_ninja_depends failed to parse ", arg_pp, + ": expected at least one 1 output")); + return false; + } + + PARSE_FILENAME(outputs[0], info->PrimaryOutput); + } + } + + if (future_compile.isMember("provides")) { + Json::Value const& provides = future_compile["provides"]; + if (provides.isArray()) { + for (auto const& provide : provides) { + cmSourceReqInfo provide_info; + + Json::Value const& logical_name = provide["logical-name"]; + PARSE_BLOB(logical_name, provide_info.LogicalName); + + if (provide.isMember("compiled-module-path")) { + Json::Value const& compiled_module_path = + provide["compiled-module-path"]; + PARSE_FILENAME(compiled_module_path, + provide_info.CompiledModulePath); + } else { + provide_info.CompiledModulePath = + cmStrCat(provide_info.LogicalName, ".mod"); + } + + info->Provides.push_back(provide_info); + } + } + } + + if (future_compile.isMember("requires")) { + Json::Value const& reqs = future_compile["requires"]; + if (reqs.isArray()) { + for (auto const& require : reqs) { + cmSourceReqInfo require_info; + + Json::Value const& logical_name = require["logical-name"]; + PARSE_BLOB(logical_name, require_info.LogicalName); + + if (require.isMember("compiled-module-path")) { + Json::Value const& compiled_module_path = + require["compiled-module-path"]; + PARSE_FILENAME(compiled_module_path, + require_info.CompiledModulePath); + } + + info->Requires.push_back(require_info); + } + } + } + } + } + } + + return true; +} + +bool cmScanDepFormat_P1689_Write(std::string const& path, + std::string const& input, + cmSourceInfo const& info) +{ + Json::Value ddi(Json::objectValue); + ddi["version"] = 0; + ddi["revision"] = 0; + ddi["work-directory"] = + EncodeFilename(cmSystemTools::GetCurrentWorkingDirectory()); + + Json::Value& rules = ddi["rules"] = Json::arrayValue; + + Json::Value rule(Json::objectValue); + Json::Value& inputs = rule["inputs"] = Json::arrayValue; + inputs.append(EncodeFilename(input)); + + Json::Value& rule_outputs = rule["outputs"] = Json::arrayValue; + rule_outputs.append(EncodeFilename(path)); + + Json::Value& depends = rule["depends"] = Json::arrayValue; + for (auto const& include : info.Includes) { + depends.append(EncodeFilename(include)); + } + + Json::Value& future_compile = rule["future-compile"] = Json::objectValue; + + Json::Value& outputs = future_compile["outputs"] = Json::arrayValue; + outputs.append(info.PrimaryOutput); + + Json::Value& provides = future_compile["provides"] = Json::arrayValue; + for (auto const& provide : info.Provides) { + Json::Value provide_obj(Json::objectValue); + auto const encoded = EncodeFilename(provide.LogicalName); + provide_obj["logical-name"] = encoded; + if (provide.CompiledModulePath.empty()) { + provide_obj["compiled-module-path"] = encoded; + } else { + provide_obj["compiled-module-path"] = + EncodeFilename(provide.CompiledModulePath); + } + + // TODO: Source file tracking. See below. + + provides.append(provide_obj); + } + + Json::Value& reqs = future_compile["requires"] = Json::arrayValue; + for (auto const& require : info.Requires) { + Json::Value require_obj(Json::objectValue); + auto const encoded = EncodeFilename(require.LogicalName); + require_obj["logical-name"] = encoded; + if (require.CompiledModulePath.empty()) { + require_obj["compiled-module-path"] = encoded; + } else { + require_obj["compiled-module-path"] = + EncodeFilename(require.CompiledModulePath); + } + + // TODO: Source filename inclusion. Requires collating with the provides + // filenames (as a sanity check if available on both sides). + + reqs.append(require_obj); + } + + rules.append(rule); + + cmGeneratedFileStream ddif(path); + ddif << ddi; + + return !!ddif; +} diff --git a/Source/cmScanDepFormat.h b/Source/cmScanDepFormat.h new file mode 100644 index 0000000..1ad0ecf --- /dev/null +++ b/Source/cmScanDepFormat.h @@ -0,0 +1,30 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ +#pragma once + +#include +#include + +struct cmSourceReqInfo +{ + std::string LogicalName; + std::string CompiledModulePath; +}; + +struct cmSourceInfo +{ + std::string PrimaryOutput; + + // Set of provided and required modules. + std::vector Provides; + std::vector Requires; + + // Set of files included in the translation unit. + std::vector Includes; +}; + +bool cmScanDepFormat_P1689_Parse(std::string const& arg_pp, + cmSourceInfo* info); +bool cmScanDepFormat_P1689_Write(std::string const& path, + std::string const& input, + cmSourceInfo const& info); -- cgit v0.12 From f3eed2c49d38ef95261f687c8c0c549990d37501 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Tue, 12 Mar 2019 15:19:33 -0400 Subject: cmGlobalNinjaGenerator: use P1689 dependency file format for Fortran The module dependency specification format described in the C++ JTC1/SC22/WG21 paper [1] is also suitable for use by Fortran. [1] http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p1689r2.html --- Source/cmGlobalNinjaGenerator.cxx | 121 +++++++++++++++----------------------- Source/cmNinjaTargetGenerator.cxx | 2 +- 2 files changed, 48 insertions(+), 75 deletions(-) diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index 24b5ccd..7ef69f4 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -34,6 +34,7 @@ #include "cmOutputConverter.h" #include "cmProperty.h" #include "cmRange.h" +#include "cmScanDepFormat.h" #include "cmState.h" #include "cmStateDirectory.h" #include "cmStateSnapshot.h" @@ -2105,16 +2106,6 @@ Compilation of source files within a target is split into the following steps: (because the latter consumes the module). */ -struct cmSourceInfo -{ - // Set of provided and required modules. - std::set Provides; - std::set Requires; - - // Set of files included in the translation unit. - std::set Includes; -}; - static std::unique_ptr cmcmd_cmake_ninja_depends_fortran( std::string const& arg_tdi, std::string const& arg_pp); @@ -2122,6 +2113,7 @@ int cmcmd_cmake_ninja_depends(std::vector::const_iterator argBeg, std::vector::const_iterator argEnd) { std::string arg_tdi; + std::string arg_src; std::string arg_pp; std::string arg_dep; std::string arg_obj; @@ -2130,6 +2122,8 @@ int cmcmd_cmake_ninja_depends(std::vector::const_iterator argBeg, for (std::string const& arg : cmMakeRange(argBeg, argEnd)) { if (cmHasLiteralPrefix(arg, "--tdi=")) { arg_tdi = arg.substr(6); + } else if (cmHasLiteralPrefix(arg, "--src=")) { + arg_src = arg.substr(6); } else if (cmHasLiteralPrefix(arg, "--pp=")) { arg_pp = arg.substr(5); } else if (cmHasLiteralPrefix(arg, "--dep=")) { @@ -2170,6 +2164,9 @@ int cmcmd_cmake_ninja_depends(std::vector::const_iterator argBeg, cmSystemTools::Error("-E cmake_ninja_depends requires value for --lang="); return 1; } + if (arg_src.empty()) { + arg_src = cmStrCat("<", arg_obj, " input file>"); + } std::unique_ptr info; if (arg_lang == "Fortran") { @@ -2186,6 +2183,8 @@ int cmcmd_cmake_ninja_depends(std::vector::const_iterator argBeg, return 1; } + info->PrimaryOutput = arg_obj; + { cmGeneratedFileStream depfile(arg_dep); depfile << cmSystemTools::ConvertToUnixOutputPath(arg_pp) << ":"; @@ -2195,24 +2194,7 @@ int cmcmd_cmake_ninja_depends(std::vector::const_iterator argBeg, depfile << "\n"; } - Json::Value ddi(Json::objectValue); - ddi["object"] = arg_obj; - - Json::Value& ddi_provides = ddi["provides"] = Json::arrayValue; - for (std::string const& provide : info->Provides) { - ddi_provides.append(provide); - } - Json::Value& ddi_requires = ddi["requires"] = Json::arrayValue; - for (std::string const& r : info->Requires) { - // Require modules not provided in the same source. - if (!info->Provides.count(r)) { - ddi_requires.append(r); - } - } - - cmGeneratedFileStream ddif(arg_ddi); - ddif << ddi; - if (!ddif) { + if (!cmScanDepFormat_P1689_Write(arg_ddi, arg_src, *info)) { cmSystemTools::Error( cmStrCat("-E cmake_ninja_depends failed to write ", arg_ddi)); return 1; @@ -2270,19 +2252,28 @@ std::unique_ptr cmcmd_cmake_ninja_depends_fortran( } auto info = cm::make_unique(); - info->Provides = finfo.Provides; - info->Requires = finfo.Requires; - info->Includes = finfo.Includes; + for (std::string const& provide : finfo.Provides) { + cmSourceReqInfo src_info; + src_info.LogicalName = provide; + src_info.CompiledModulePath = provide; + info->Provides.emplace_back(src_info); + } + for (std::string const& require : finfo.Requires) { + // Require modules not provided in the same source. + if (finfo.Provides.count(require)) { + continue; + } + cmSourceReqInfo src_info; + src_info.LogicalName = require; + src_info.CompiledModulePath = require; + info->Requires.emplace_back(src_info); + } + for (std::string const& include : finfo.Includes) { + info->Includes.push_back(include); + } return info; } -struct cmDyndepObjectInfo -{ - std::string Object; - std::vector Provides; - std::vector Requires; -}; - bool cmGlobalNinjaGenerator::WriteDyndepFile( std::string const& dir_top_src, std::string const& dir_top_bld, std::string const& dir_cur_src, std::string const& dir_cur_bld, @@ -2304,34 +2295,14 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile( this->LocalGenerators.push_back(std::move(lgd)); } - std::vector objects; + std::vector objects; for (std::string const& arg_ddi : arg_ddis) { - // Load the ddi file and compute the module file paths it provides. - Json::Value ddio; - Json::Value const& ddi = ddio; - cmsys::ifstream ddif(arg_ddi.c_str(), std::ios::in | std::ios::binary); - Json::Reader reader; - if (!reader.parse(ddif, ddio, false)) { - cmSystemTools::Error(cmStrCat("-E cmake_ninja_dyndep failed to parse ", - arg_ddi, - reader.getFormattedErrorMessages())); + cmSourceInfo info; + if (!cmScanDepFormat_P1689_Parse(arg_ddi, &info)) { + cmSystemTools::Error( + cmStrCat("-E cmake_ninja_dyndep failed to parse ddi file ", arg_ddi)); return false; } - - cmDyndepObjectInfo info; - info.Object = ddi["object"].asString(); - Json::Value const& ddi_provides = ddi["provides"]; - if (ddi_provides.isArray()) { - for (auto const& ddi_provide : ddi_provides) { - info.Provides.push_back(ddi_provide.asString()); - } - } - Json::Value const& ddi_requires = ddi["requires"]; - if (ddi_requires.isArray()) { - for (auto const& ddi_require : ddi_requires) { - info.Requires.push_back(ddi_require.asString()); - } - } objects.push_back(std::move(info)); } @@ -2362,11 +2333,12 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile( // We do this after loading the modules provided by linked targets // in case we have one of the same name that must be preferred. Json::Value tm = Json::objectValue; - for (cmDyndepObjectInfo const& object : objects) { - for (std::string const& p : object.Provides) { - std::string const mod = cmStrCat(module_dir, p); - mod_files[p] = mod; - tm[p] = mod; + for (cmSourceInfo const& object : objects) { + for (auto const& p : object.Provides) { + std::string const mod = cmStrCat( + module_dir, cmSystemTools::GetFilenameName(p.CompiledModulePath)); + mod_files[p.LogicalName] = mod; + tm[p.LogicalName] = mod; } } @@ -2376,15 +2348,16 @@ bool cmGlobalNinjaGenerator::WriteDyndepFile( { cmNinjaBuild build("dyndep"); build.Outputs.emplace_back(""); - for (cmDyndepObjectInfo const& object : objects) { - build.Outputs[0] = object.Object; + for (cmSourceInfo const& object : objects) { + build.Outputs[0] = this->ConvertToNinjaPath(object.PrimaryOutput); build.ImplicitOuts.clear(); - for (std::string const& p : object.Provides) { - build.ImplicitOuts.push_back(this->ConvertToNinjaPath(mod_files[p])); + for (auto const& p : object.Provides) { + build.ImplicitOuts.push_back( + this->ConvertToNinjaPath(mod_files[p.LogicalName])); } build.ImplicitDeps.clear(); - for (std::string const& r : object.Requires) { - auto mit = mod_files.find(r); + for (auto const& r : object.Requires) { + auto mit = mod_files.find(r.LogicalName); if (mit != mod_files.end()) { build.ImplicitDeps.push_back(this->ConvertToNinjaPath(mit->second)); } diff --git a/Source/cmNinjaTargetGenerator.cxx b/Source/cmNinjaTargetGenerator.cxx index f2bec8c..17b7efc 100644 --- a/Source/cmNinjaTargetGenerator.cxx +++ b/Source/cmNinjaTargetGenerator.cxx @@ -532,7 +532,7 @@ std::string GetScanCommand(const std::string& cmakeCmd, const std::string& tdi, const std::string& ddiFile) { return cmStrCat(cmakeCmd, " -E cmake_ninja_depends --tdi=", tdi, - " --lang=", lang, " --pp=", ppFile, + " --lang=", lang, " --src=$in", " --pp=", ppFile, " --dep=$DEP_FILE --obj=$OBJ_FILE --ddi=", ddiFile); } -- cgit v0.12