diff options
| -rw-r--r-- | SCons/Tool/ninja/Methods.py | 269 | ||||
| -rw-r--r-- | SCons/Tool/ninja/NinjaState.py | 4 | ||||
| -rw-r--r-- | SCons/Tool/ninja/Overrides.py | 96 | ||||
| -rw-r--r-- | SCons/Tool/ninja/Rules.py | 2 | ||||
| -rw-r--r-- | SCons/Tool/ninja/Utils.py (renamed from SCons/Tool/ninja/Util.py) | 191 | ||||
| -rw-r--r-- | SCons/Tool/ninja/__init__.py | 276 |
6 files changed, 454 insertions, 384 deletions
diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py new file mode 100644 index 0000000..3612236 --- /dev/null +++ b/SCons/Tool/ninja/Methods.py @@ -0,0 +1,269 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import shlex +import textwrap + +import SCons +from SCons.Tool.ninja import NINJA_CUSTOM_HANDLERS, NINJA_RULES, NINJA_POOLS +from SCons.Tool.ninja.Globals import __NINJA_RULE_MAPPING +from SCons.Tool.ninja.Utils import get_targets_sources, get_dependencies, get_order_only, get_outputs, get_inputs, \ + get_rule, get_path, generate_command, get_command_env, get_comstr + + +def register_custom_handler(env, name, handler): + """Register a custom handler for SCons function actions.""" + env[NINJA_CUSTOM_HANDLERS][name] = handler + + +def register_custom_rule_mapping(env, pre_subst_string, rule): + """Register a function to call for a given rule.""" + SCons.Tool.ninja.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule + + +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): + """Allows specification of Ninja rules from inside SCons files.""" + rule_obj = { + "command": command, + "description": description if description else "{} $out".format(rule), + } + + if use_depfile: + rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') + + if deps is not None: + rule_obj["deps"] = deps + + if pool is not None: + rule_obj["pool"] = pool + + if use_response_file: + rule_obj["rspfile"] = "$out.rsp" + rule_obj["rspfile_content"] = response_file_content + + env[NINJA_RULES][rule] = rule_obj + + +def register_custom_pool(env, pool, size): + """Allows the creation of custom Ninja pools""" + env[NINJA_POOLS][pool] = size + + +def set_build_node_callback(env, node, callback): + if not node.is_conftest(): + node.attributes.ninja_build_callback = callback + + +def get_generic_shell_command(env, node, action, targets, sources, executor=None): + return ( + "CMD", + { + # TODO: Why is executor passed in and then ignored below? (bdbaddog) + "cmd": generate_command(env, node, action, targets, sources, executor=None), + "env": get_command_env(env), + }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used solely and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provider + # function for a custom NinjaRuleMapping. + [] + ) + + +def CheckNinjaCompdbExpand(env, context): + """ Configure check testing if ninja's compdb can expand response files""" + + # TODO: When would this be false? + context.Message('Checking if ninja compdb can expand response files... ') + ret, output = context.TryAction( + action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', + extension='.ninja', + text=textwrap.dedent(""" + rule CMD_RSP + command = $cmd @$out.rsp > fake_output.txt + description = Building $out + rspfile = $out.rsp + rspfile_content = $rspc + build fake_output.txt: CMD_RSP fake_input.txt + cmd = echo + pool = console + rspc = "test" + """)) + result = '@fake_output.txt.rsp' not in output + context.Result(result) + return result + + +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + executor = node.get_executor() + tlist, slist = get_targets_sources(node) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + variables = {} + + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be suppled without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) + + ninja_build = { + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": implicit, + "rule": get_rule(node, rule), + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): + """Generate a response file command provider for rule name.""" + + # If win32 using the environment with a response file command will cause + # ninja to fail to create the response file. Additionally since these rules + # generally are not piping through cmd.exe /c any environment variables will + # make CreateProcess fail to start. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + use_command_env = not env["PLATFORM"] == "win32" + if "$" in tool: + tool_is_dynamic = True + + def get_response_file_command(env, node, action, targets, sources, executor=None): + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] + else: + command = generate_command( + env, node, action, targets, sources, executor=executor + ) + cmd_list = shlex.split(command) + + if tool_is_dynamic: + tool_command = env.subst( + tool, target=targets, source=sources, executor=executor + ) + else: + tool_command = tool + + try: + # Add 1 so we always keep the actual tool inside of cmd + tool_idx = cmd_list.index(tool_command) + 1 + except ValueError: + raise Exception( + "Could not find tool {} in {} generated from {}".format( + tool, cmd_list, get_comstr(env, action, targets, sources) + ) + ) + + cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] + rsp_content = " ".join(rsp_content) + + variables = {"rspc": rsp_content, rule: cmd} + if use_command_env: + variables["env"] = get_command_env(env) + + for key, value in custom_env.items(): + variables["env"] += env.subst( + "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor + ) + " " + return rule, variables, [tool_command] + + return get_response_file_command
\ No newline at end of file diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index a7c3584..f906b33 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -35,9 +35,9 @@ import SCons.Tool.ninja.Globals from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function -from .Util import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_command, get_order_only, \ +from .Utils import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_order_only, \ get_outputs, get_inputs, get_dependencies, get_rule, get_command_env - +from . import get_command # pylint: disable=too-many-instance-attributes diff --git a/SCons/Tool/ninja/Overrides.py b/SCons/Tool/ninja/Overrides.py index e69de29..80516a2 100644 --- a/SCons/Tool/ninja/Overrides.py +++ b/SCons/Tool/ninja/Overrides.py @@ -0,0 +1,96 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +This module is to hold logic which overrides default SCons behavoirs to enable +ninja file generation +""" +import SCons + + +def ninja_hack_linkcom(env): + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env.get("LINKCOM", None) == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + + +def ninja_hack_arcom(env): + """ + Force ARCOM so use 's' flag on ar instead of separately running ranlib + """ + if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): + # There is no way to translate the ranlib list action into + # Ninja so add the s flag and disable ranlib. + # + # This is equivalent to Meson. + # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 + old_arflags = str(env["ARFLAGS"]) + if "s" not in old_arflags: + old_arflags += "s" + + env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) + + # Disable running ranlib, since we added 's' above + env["RANLIBCOM"] = "" + + +class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): + """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" + + def __call__(self, target, source, env, for_signature): + return self.cmd + + def _print_cmd_str(*_args, **_kwargs): + """Disable this method""" + pass + + +def ninja_always_serial(self, num, taskmaster): + """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" + # We still set self.num_jobs to num even though it's a lie. The + # only consumer of this attribute is the Parallel Job class AND + # the Main.py function which instantiates a Jobs class. It checks + # if Jobs.num_jobs is equal to options.num_jobs, so if the user + # provides -j12 but we set self.num_jobs = 1 they get an incorrect + # warning about this version of Python not supporting parallel + # builds. So here we lie so the Main.py will not give a false + # warning to users. + self.num_jobs = num + self.job = SCons.Job.Serial(taskmaster) + + +# pylint: disable=too-few-public-methods +class AlwaysExecAction(SCons.Action.FunctionAction): + """Override FunctionAction.__call__ to always execute.""" + + def __call__(self, *args, **kwargs): + kwargs["execute"] = 1 + return super().__call__(*args, **kwargs)
\ No newline at end of file diff --git a/SCons/Tool/ninja/Rules.py b/SCons/Tool/ninja/Rules.py index c1c238e..a2f6bc5 100644 --- a/SCons/Tool/ninja/Rules.py +++ b/SCons/Tool/ninja/Rules.py @@ -21,7 +21,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from .Util import get_outputs, get_rule, get_inputs, get_dependencies +from .Utils import get_outputs, get_rule, get_inputs, get_dependencies def _install_action_function(_env, node): diff --git a/SCons/Tool/ninja/Util.py b/SCons/Tool/ninja/Utils.py index 80d1b16..18d54dc 100644 --- a/SCons/Tool/ninja/Util.py +++ b/SCons/Tool/ninja/Utils.py @@ -21,12 +21,12 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os +import shutil from os.path import join as joinpath import SCons from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption -from SCons.Tool.ninja.Globals import __NINJA_RULE_MAPPING from SCons.Util import is_List, flatten_sequence @@ -249,99 +249,6 @@ def ninja_noop(*_args, **_kwargs): return None -def get_command(env, node, action): # pylint: disable=too-many-branches - """Get the command to execute for node.""" - if node.env: - sub_env = node.env - else: - sub_env = env - executor = node.get_executor() - tlist, slist = get_targets_sources(node) - - # Generate a real CommandAction - if isinstance(action, SCons.Action.CommandGeneratorAction): - # pylint: disable=protected-access - action = action._generate(tlist, slist, sub_env, 1, executor=executor) - - variables = {} - - comstr = get_comstr(sub_env, action, tlist, slist) - if not comstr: - return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) - rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) - - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - - # Now add in the other dependencies related to the command, - # e.g. the compiler binary. The ninja rule can be user provided so - # we must do some validation to resolve the dependency path for ninja. - for provider_dep in provider_deps: - - provider_dep = sub_env.subst(provider_dep) - if not provider_dep: - continue - - # If the tool is a node, then SCons will resolve the path later, if its not - # a node then we assume it generated from build and make sure it is existing. - if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): - implicit.append(provider_dep) - continue - - # in some case the tool could be in the local directory and be suppled without the ext - # such as in windows, so append the executable suffix and check. - prog_suffix = sub_env.get('PROGSUFFIX', '') - provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix - if os.path.exists(provider_dep_ext): - implicit.append(provider_dep_ext) - continue - - # Many commands will assume the binary is in the path, so - # we accept this as a possible input from a given command. - - provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) - if provider_dep_abspath: - implicit.append(provider_dep_abspath) - continue - - # Possibly these could be ignore and the build would still work, however it may not always - # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided - # ninja rule. - raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) - - ninja_build = { - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": implicit, - "rule": get_rule(node, rule), - "variables": variables, - } - - # Don't use sub_env here because we require that NINJA_POOL be set - # on a per-builder call basis to prevent accidental strange - # behavior like env['NINJA_POOL'] = 'console' and sub_env can be - # the global Environment object if node.env is None. - # Example: - # - # Allowed: - # - # env.Command("ls", NINJA_POOL="ls_pool") - # - # Not allowed and ignored: - # - # env["NINJA_POOL"] = "ls_pool" - # env.Command("ls") - # - # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["NINJA_POOL"] - - return ninja_build - - def get_command_env(env): """ Return a string that sets the environment for any environment variables that @@ -413,25 +320,6 @@ def get_comstr(env, action, targets, sources): return action.genstring(targets, sources, env) -def get_generic_shell_command(env, node, action, targets, sources, executor=None): - return ( - "CMD", - { - # TODO: Why is executor passed in and then ignored below? (bdbaddog) - "cmd": generate_command(env, node, action, targets, sources, executor=None), - "env": get_command_env(env), - }, - # Since this function is a rule mapping provider, it must return a list of dependencies, - # and usually this would be the path to a tool, such as a compiler, used for this rule. - # However this function is to generic to be able to reliably extract such deps - # from the command, so we return a placeholder empty list. It should be noted that - # generally this function will not be used solely and is more like a template to generate - # the basics for a custom provider which may have more specific options for a provider - # function for a custom NinjaRuleMapping. - [] - ) - - def generate_command(env, node, action, targets, sources, executor=None): # Actions like CommandAction have a method called process that is # used by SCons to generate the cmd_line they need to run. So @@ -454,4 +342,79 @@ def generate_command(env, node, action, targets, sources, executor=None): cmd = cmd[0:-2].strip() # Escape dollars as necessary - return cmd.replace("$", "$$")
\ No newline at end of file + return cmd.replace("$", "$$") + + +def ninja_csig(original): + """Return a dummy csig""" + + def wrapper(self): + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + return original(self) + return "dummy_ninja_csig" + + return wrapper + + +def ninja_contents(original): + """Return a dummy content without doing IO""" + + def wrapper(self): + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + return original(self) + return bytes("dummy_ninja_contents", encoding="utf-8") + + return wrapper + + +def ninja_stat(_self, path): + """ + Eternally memoized stat call. + + SCons is very aggressive about clearing out cached values. For our + purposes everything should only ever call stat once since we're + running in a no_exec build the file system state should not + change. For these reasons we patch SCons.Node.FS.LocalFS.stat to + use our eternal memoized dictionary. + """ + + try: + return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] + except KeyError: + try: + result = os.stat(path) + except os.error: + result = None + + SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result + return result + + +def ninja_whereis(thing, *_args, **_kwargs): + """Replace env.WhereIs with a much faster version""" + + # Optimize for success, this gets called significantly more often + # when the value is already memoized than when it's not. + try: + return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] + except KeyError: + # TODO: Fix this to respect env['ENV']['PATH']... WPD + # We do not honor any env['ENV'] or env[*] variables in the + # generated ninja file. Ninja passes your raw shell environment + # down to it's subprocess so the only sane option is to do the + # same during generation. At some point, if and when we try to + # upstream this, I'm sure a sticking point will be respecting + # env['ENV'] variables and such but it's actually quite + # complicated. I have a naive version but making it always work + # with shell quoting is nigh impossible. So I've decided to + # cross that bridge when it's absolutely required. + path = shutil.which(thing) + SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path + return path + + +def ninja_print_conf_log(s, target, source, env): + """Command line print only for conftest to generate a correct conf log.""" + if target and target[0].is_conftest(): + action = SCons.Action._ActionAction() + action.print_cmd_line(s, target, source, env)
\ No newline at end of file diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index fd9c13c..ee852a7 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -26,85 +26,25 @@ import importlib import os -import shlex -import shutil import subprocess import sys -import textwrap -from glob import glob import SCons import SCons.Tool.ninja.Globals from SCons.Script import GetOption from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS +from .Methods import register_custom_handler, register_custom_rule_mapping, register_custom_rule, register_custom_pool, \ + set_build_node_callback, get_generic_shell_command, CheckNinjaCompdbExpand, get_command, \ + gen_get_response_file_command from .NinjaState import NinjaState -from .Util import ninja_add_command_line_options, \ - get_path, ninja_noop, get_command, get_command_env, get_comstr, get_generic_shell_command, \ - generate_command +from .Overrides import ninja_hack_linkcom, ninja_hack_arcom, NinjaNoResponseFiles, ninja_always_serial, AlwaysExecAction +from .Utils import ninja_add_command_line_options, \ + get_path, ninja_noop, ninja_print_conf_log, get_command_env, get_comstr, generate_command, ninja_csig, ninja_contents, ninja_stat, ninja_whereis, ninja_csig, ninja_contents NINJA_STATE = None -def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): - """Generate a response file command provider for rule name.""" - - # If win32 using the environment with a response file command will cause - # ninja to fail to create the response file. Additionally since these rules - # generally are not piping through cmd.exe /c any environment variables will - # make CreateProcess fail to start. - # - # On POSIX we can still set environment variables even for compile - # commands so we do so. - use_command_env = not env["PLATFORM"] == "win32" - if "$" in tool: - tool_is_dynamic = True - - def get_response_file_command(env, node, action, targets, sources, executor=None): - if hasattr(action, "process"): - cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] - else: - command = generate_command( - env, node, action, targets, sources, executor=executor - ) - cmd_list = shlex.split(command) - - if tool_is_dynamic: - tool_command = env.subst( - tool, target=targets, source=sources, executor=executor - ) - else: - tool_command = tool - - try: - # Add 1 so we always keep the actual tool inside of cmd - tool_idx = cmd_list.index(tool_command) + 1 - except ValueError: - raise Exception( - "Could not find tool {} in {} generated from {}".format( - tool, cmd_list, get_comstr(env, action, targets, sources) - ) - ) - - cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] - rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] - rsp_content = " ".join(rsp_content) - - variables = {"rspc": rsp_content} - variables[rule] = cmd - if use_command_env: - variables["env"] = get_command_env(env) - - for key, value in custom_env.items(): - variables["env"] += env.subst( - "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor - ) + " " - return rule, variables, [tool_command] - - return get_response_file_command - - def ninja_builder(env, target, source): """Generate a build.ninja for source.""" if not isinstance(source, list): @@ -120,6 +60,7 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": + # TODO: Is this necessary as you set env variable in the ninja build file per target? # this is not great, its doesn't consider specific # node environments, which means on linux the build could # behave differently, because on linux you can set the environment @@ -170,193 +111,6 @@ def ninja_builder(env, target, source): # prone to failure with such a simple check erase_previous = output.startswith('[') -# pylint: disable=too-few-public-methods -class AlwaysExecAction(SCons.Action.FunctionAction): - """Override FunctionAction.__call__ to always execute.""" - - def __call__(self, *args, **kwargs): - kwargs["execute"] = 1 - return super().__call__(*args, **kwargs) - - -def register_custom_handler(env, name, handler): - """Register a custom handler for SCons function actions.""" - env[NINJA_CUSTOM_HANDLERS][name] = handler - - -def register_custom_rule_mapping(env, pre_subst_string, rule): - """Register a function to call for a given rule.""" - SCons.Tool.ninja.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule - - -def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): - """Allows specification of Ninja rules from inside SCons files.""" - rule_obj = { - "command": command, - "description": description if description else "{} $out".format(rule), - } - - if use_depfile: - rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') - - if deps is not None: - rule_obj["deps"] = deps - - if pool is not None: - rule_obj["pool"] = pool - - if use_response_file: - rule_obj["rspfile"] = "$out.rsp" - rule_obj["rspfile_content"] = response_file_content - - env[NINJA_RULES][rule] = rule_obj - - -def register_custom_pool(env, pool, size): - """Allows the creation of custom Ninja pools""" - env[NINJA_POOLS][pool] = size - - -def set_build_node_callback(env, node, callback): - if not node.is_conftest(): - node.attributes.ninja_build_callback = callback - - -def ninja_csig(original): - """Return a dummy csig""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return "dummy_ninja_csig" - - return wrapper - - -def ninja_contents(original): - """Return a dummy content without doing IO""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return bytes("dummy_ninja_contents", encoding="utf-8") - - return wrapper - - -def CheckNinjaCompdbExpand(env, context): - """ Configure check testing if ninja's compdb can expand response files""" - - context.Message('Checking if ninja compdb can expand response files... ') - ret, output = context.TryAction( - action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', - extension='.ninja', - text=textwrap.dedent(""" - rule CMD_RSP - command = $cmd @$out.rsp > fake_output.txt - description = Building $out - rspfile = $out.rsp - rspfile_content = $rspc - build fake_output.txt: CMD_RSP fake_input.txt - cmd = echo - pool = console - rspc = "test" - """)) - result = '@fake_output.txt.rsp' not in output - context.Result(result) - return result - - -def ninja_stat(_self, path): - """ - Eternally memoized stat call. - - SCons is very aggressive about clearing out cached values. For our - purposes everything should only ever call stat once since we're - running in a no_exec build the file system state should not - change. For these reasons we patch SCons.Node.FS.LocalFS.stat to - use our eternal memoized dictionary. - """ - - try: - return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] - except KeyError: - try: - result = os.stat(path) - except os.error: - result = None - - SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result - return result - - -def ninja_whereis(thing, *_args, **_kwargs): - """Replace env.WhereIs with a much faster version""" - - # Optimize for success, this gets called significantly more often - # when the value is already memoized than when it's not. - try: - return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] - except KeyError: - # We do not honor any env['ENV'] or env[*] variables in the - # generated ninja file. Ninja passes your raw shell environment - # down to it's subprocess so the only sane option is to do the - # same during generation. At some point, if and when we try to - # upstream this, I'm sure a sticking point will be respecting - # env['ENV'] variables and such but it's actually quite - # complicated. I have a naive version but making it always work - # with shell quoting is nigh impossible. So I've decided to - # cross that bridge when it's absolutely required. - path = shutil.which(thing) - SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path - return path - - -def ninja_always_serial(self, num, taskmaster): - """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" - # We still set self.num_jobs to num even though it's a lie. The - # only consumer of this attribute is the Parallel Job class AND - # the Main.py function which instantiates a Jobs class. It checks - # if Jobs.num_jobs is equal to options.num_jobs, so if the user - # provides -j12 but we set self.num_jobs = 1 they get an incorrect - # warning about this version of Python not supporting parallel - # builds. So here we lie so the Main.py will not give a false - # warning to users. - self.num_jobs = num - self.job = SCons.Job.Serial(taskmaster) - -def ninja_hack_linkcom(env): - # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks - # without relying on the SCons hacks that SCons uses by default. - if env["PLATFORM"] == "win32": - from SCons.Tool.mslink import compositeLinkAction - - if env.get("LINKCOM", None) == compositeLinkAction: - env[ - "LINKCOM" - ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env[ - "SHLINKCOM" - ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' - - -def ninja_print_conf_log(s, target, source, env): - """Command line print only for conftest to generate a correct conf log.""" - if target and target[0].is_conftest(): - action = SCons.Action._ActionAction() - action.print_cmd_line(s, target, source, env) - - -class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): - """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" - - def __call__(self, target, source, env, for_signature): - return self.cmd - - def _print_cmd_str(*_args, **_kwargs): - """Disable this method""" - pass - def exists(env): """Enable if called.""" @@ -529,20 +283,8 @@ def generate(env): # TODO: switch to using SCons to help determine this (Github Issue #3624) env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] - if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): - # There is no way to translate the ranlib list action into - # Ninja so add the s flag and disable ranlib. - # - # This is equivalent to Meson. - # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 - old_arflags = str(env["ARFLAGS"]) - if "s" not in old_arflags: - old_arflags += "s" - - env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) - - # Disable running ranlib, since we added 's' above - env["RANLIBCOM"] = "" + # Force ARCOM so use 's' flag on ar instead of separately running ranlib + ninja_hack_arcom(env) if GetOption('disable_ninja'): return env |