From d1c74bc17b36e4ce4f163a70ff64100b03e9c6f3 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 28 Jan 2022 06:10:02 -0700 Subject: Man: mention $$ a subst escape [skip appveyor] Some rewordings elsewhere in Variable Substitution section - mainly to a variable that's a function. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 56 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index fc2e24d..eb91d88 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -6414,18 +6414,25 @@ env.Command('marker', 'input_file', action=[MyBuildAction, Touch('$TARGET')]) &scons; performs variable substitution on the string that makes up the action part of the builder. -Variables to be interpolated are indicated in the -string with a leading -$, to distinguish them from plain text +Variables or other text to be substituted are indicated in the +string by a leading $, +to distinguish them from plain text which is not to be substituted. -The name may be surrounded by curly braces -(${}) -to separate the name from surrounding characters if necessary. -Curly braces are required when you use +The substitutable text may be surrounded by curly braces +to separate it from surrounding characters if necessary +(for example ${FOO}BAR). +To avoid substituting a substring that looks like a variable name, +escape it with an additional $, +(for example, $$FOO will be left in the +final string as $FOO). + + +The curly brace notation is required when you use Python list subscripting/slicing notation on a variable to select one or more items from a list, or access a variable's special attributes, -or use Python expression substitution. +or when you use Python expression substitution +(see below for descriptions of these). @@ -6670,9 +6677,10 @@ echo Last build occurred . > $TARGET While &consvars; are normally directly substituted, if a variable refers to a &consvar; -whose value is a &Python; function, -that function is called during substitution. -Such a function must accept four arguments: +whose value is a callable &Python; object (a function +or a class with a __call__ method), +that object is called during substitution. +The callable must accept four arguments: target, source, env and @@ -6681,19 +6689,21 @@ Such a function must accept four arguments: target is a list of target nodes, env is the &consenv; to use for context, and for_signature is -a Boolean value that tells the function +a boolean value that tells the callable if it is being called for the purpose of generating a build signature. Since the build signature is used for rebuild determination, -the function should omit variable elements -that do not affect whether a rebuild should be triggered -(see $( -and $) -above) if for_signature is true. +variable elements that do not affect whether +a rebuild should be triggered +should be omitted from the returned string +if for_signature is true. +See $( +and $) above +for the syntax. &SCons; will insert whatever -the called function returns +the callable returns into the expanded string: @@ -6712,11 +6722,11 @@ will be exactly as it was set: "$FOO baz". You can use this feature to pass arguments to a -Python function by creating a callable class -that stores one or more arguments in an object, -and then uses them when the -__call__() -method is called. +callable variable by creating a callable class +that stores passed arguments in the instance, +and then uses them +(in the __call__ method) +when the instance is called. Note that in this case, the entire variable expansion must be enclosed by curly braces -- cgit v0.12 From 34b62e00d8f0d73c8229d82ce3805f0f72f8380f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 29 Jan 2022 09:20:37 -0700 Subject: man: drop some extra wording [ci skip] An extra stanza was added "or other text" that didn't really make sense in context, remove it again. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index eb91d88..13a06a4 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -6414,7 +6414,7 @@ env.Command('marker', 'input_file', action=[MyBuildAction, Touch('$TARGET')]) &scons; performs variable substitution on the string that makes up the action part of the builder. -Variables or other text to be substituted are indicated in the +Variables to be substituted are indicated in the string by a leading $, to distinguish them from plain text which is not to be substituted. -- cgit v0.12 From d8932c2b87cfef617b2412afc6db024ba6e9560a Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 11 Feb 2022 14:13:05 -0600 Subject: fix up interactive to support ninja scons daemon --- SCons/Node/__init__.py | 1 + SCons/Script/Interactive.py | 149 +++++++++++++++++++++++-------------------- SCons/Script/Main.py | 5 +- SCons/Script/SConsOptions.py | 5 ++ 4 files changed, 88 insertions(+), 72 deletions(-) diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index ec742a6..d0539b4 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -105,6 +105,7 @@ SConscriptNodes = set() # currently used to release parts of a target's info during # clean builds and update runs (see release_target_info). interactive = False +ninja_scons_daemon = False def is_derived_none(node): raise NotImplementedError diff --git a/SCons/Script/Interactive.py b/SCons/Script/Interactive.py index 26a8bcd..ad021fd 100644 --- a/SCons/Script/Interactive.py +++ b/SCons/Script/Interactive.py @@ -188,77 +188,86 @@ version Prints SCons version information. x.extend(n.alter_targets()[0]) nodes.extend(x) - # Clean up so that we can perform the next build correctly. - # - # We do this by walking over all the children of the targets, - # and clearing their state. - # - # We currently have to re-scan each node to find their - # children, because built nodes have already been partially - # cleared and don't remember their children. (In scons - # 0.96.1 and earlier, this wasn't the case, and we didn't - # have to re-scan the nodes.) - # - # Because we have to re-scan each node, we can't clear the - # nodes as we walk over them, because we may end up rescanning - # a cleared node as we scan a later node. Therefore, only - # store the list of nodes that need to be cleared as we walk - # the tree, and clear them in a separate pass. - # - # XXX: Someone more familiar with the inner workings of scons - # may be able to point out a more efficient way to do this. - - SCons.Script.Main.progress_display("scons: Clearing cached node information ...") - - seen_nodes = {} - - def get_unseen_children(node, parent, seen_nodes=seen_nodes): - def is_unseen(node, seen_nodes=seen_nodes): - return node not in seen_nodes - return [child for child in node.children(scan=1) if is_unseen(child)] - - def add_to_seen_nodes(node, parent, seen_nodes=seen_nodes): - seen_nodes[node] = 1 - - # If this file is in a VariantDir and has a - # corresponding source file in the source tree, remember the - # node in the source tree, too. This is needed in - # particular to clear cached implicit dependencies on the - # source file, since the scanner will scan it if the - # VariantDir was created with duplicate=0. - try: - rfile_method = node.rfile - except AttributeError: - return - else: - rfile = rfile_method() - if rfile != node: - seen_nodes[rfile] = 1 - - for node in nodes: - walker = SCons.Node.Walker(node, - kids_func=get_unseen_children, - eval_func=add_to_seen_nodes) - n = walker.get_next() - while n: + if SCons.Node.ninja_scons_daemon: + for n in nodes: + print(f"Node: {n}, State: {SCons.Node.failed}") + if n.get_state() == SCons.Node.failed: + print(n) + n.clear() + n.set_state(SCons.Node.no_state) + n.implicit = None + else: + # Clean up so that we can perform the next build correctly. + # + # We do this by walking over all the children of the targets, + # and clearing their state. + # + # We currently have to re-scan each node to find their + # children, because built nodes have already been partially + # cleared and don't remember their children. (In scons + # 0.96.1 and earlier, this wasn't the case, and we didn't + # have to re-scan the nodes.) + # + # Because we have to re-scan each node, we can't clear the + # nodes as we walk over them, because we may end up rescanning + # a cleared node as we scan a later node. Therefore, only + # store the list of nodes that need to be cleared as we walk + # the tree, and clear them in a separate pass. + # + # XXX: Someone more familiar with the inner workings of scons + # may be able to point out a more efficient way to do this. + + SCons.Script.Main.progress_display("scons: Clearing cached node information ...") + + seen_nodes = {} + + def get_unseen_children(node, parent, seen_nodes=seen_nodes): + def is_unseen(node, seen_nodes=seen_nodes): + return node not in seen_nodes + return [child for child in node.children(scan=1) if is_unseen(child)] + + def add_to_seen_nodes(node, parent, seen_nodes=seen_nodes): + seen_nodes[node] = 1 + + # If this file is in a VariantDir and has a + # corresponding source file in the source tree, remember the + # node in the source tree, too. This is needed in + # particular to clear cached implicit dependencies on the + # source file, since the scanner will scan it if the + # VariantDir was created with duplicate=0. + try: + rfile_method = node.rfile + except AttributeError: + return + else: + rfile = rfile_method() + if rfile != node: + seen_nodes[rfile] = 1 + + for node in nodes: + walker = SCons.Node.Walker(node, + kids_func=get_unseen_children, + eval_func=add_to_seen_nodes) n = walker.get_next() - - for node in seen_nodes.keys(): - # Call node.clear() to clear most of the state - node.clear() - # node.clear() doesn't reset node.state, so call - # node.set_state() to reset it manually - node.set_state(SCons.Node.no_state) - node.implicit = None - - # Debug: Uncomment to verify that all Taskmaster reference - # counts have been reset to zero. - #if node.ref_count != 0: - # from SCons.Debug import Trace - # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) - - SCons.SConsign.Reset() - SCons.Script.Main.progress_display("scons: done clearing node information.") + while n: + n = walker.get_next() + + for node in seen_nodes.keys(): + # Call node.clear() to clear most of the state + node.clear() + # node.clear() doesn't reset node.state, so call + # node.set_state() to reset it manually + node.set_state(SCons.Node.no_state) + node.implicit = None + + # Debug: Uncomment to verify that all Taskmaster reference + # counts have been reset to zero. + #if node.ref_count != 0: + # from SCons.Debug import Trace + # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) + + SCons.SConsign.Reset() + SCons.Script.Main.progress_display("scons: done clearing node information.") def do_clean(self, argv): """\ diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index a340f5b..87d57bc 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -987,7 +987,8 @@ def _main(parser): # This would then cause subtle bugs, as already happened in #2971. if options.interactive: SCons.Node.interactive = True - + if options.ninja_scons_daemon: + SCons.Node.ninja_scons_daemon = True # That should cover (most of) the options. # Next, set up the variables that hold command-line arguments, # so the SConscript files that we read and execute have access to them. @@ -1125,7 +1126,7 @@ def _main(parser): platform = SCons.Platform.platform_module() - if options.interactive: + if options.interactive or options.ninja_scons_daemon: SCons.Script.Interactive.interact(fs, OptionsParser, options, targets, target_top) diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index e2631fb..133d9ab 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -858,6 +858,11 @@ def Parser(version): action="store_true", help="Run in interactive mode") + op.add_option('--ninja-scons-daemon', + dest='ninja_scons_daemon', default=False, + action="store_true", + help="A special interactive mode to support a scons daemon for ninja builds. Intended for use only by the ninja tool.") + op.add_option('-j', '--jobs', nargs=1, type="int", dest="num_jobs", default=1, -- cgit v0.12 From 6c8e521f59a796b1bb06870dce9bce093c15e5fe Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 11 Feb 2022 15:28:19 -0600 Subject: add scons daemon --- SCons/Tool/ninja/NinjaState.py | 198 ++++++++++++++++--------- SCons/Tool/ninja/Utils.py | 4 +- SCons/Tool/ninja/__init__.py | 6 +- SCons/Tool/ninja/ninja_daemon_build.py | 52 +++++++ SCons/Tool/ninja/ninja_run_daemon.py | 71 +++++++++ SCons/Tool/ninja/ninja_scons_daemon.py | 259 +++++++++++++++++++++++++++++++++ test/ninja/force_scons_callback.py | 4 +- 7 files changed, 522 insertions(+), 72 deletions(-) create mode 100644 SCons/Tool/ninja/ninja_daemon_build.py create mode 100644 SCons/Tool/ninja/ninja_run_daemon.py create mode 100644 SCons/Tool/ninja/ninja_scons_daemon.py diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index d7c260e..f2da87a 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -24,11 +24,15 @@ import io import os +import pathlib +import signal +import tempfile import shutil import sys from os.path import splitext from tempfile import NamedTemporaryFile import ninja +import hashlib import SCons from SCons.Script import COMMAND_LINE_TARGETS @@ -59,7 +63,7 @@ class NinjaState: os.pardir, 'data', 'bin', - ninja_bin)) + ninja_bin)) if not os.path.exists(self.ninja_bin_path): # couldn't find it, just give the bin name and hope # its in the path later @@ -84,6 +88,10 @@ class NinjaState: # like CCFLAGS scons_escape = env.get("ESCAPE", lambda x: x) + import random + + PORT = str(random.randint(10000, 60000)) + # if SCons was invoked from python, we expect the first arg to be the scons.py # script, otherwise scons was invoked from the scons script python_bin = '' @@ -184,10 +192,10 @@ class NinjaState: "restat": 1, }, "TEMPLATE": { - "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", - "pool": "scons_pool", - "restat": 1, + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_daemon_build.py'} {PORT} {get_path(env.get('NINJA_DIR'))} $out", + "description": "Defer to SCons to build $out", + "pool": "local_pool", + "restat": 1 }, "SCONS": { "command": "$SCONS_INVOCATION $out", @@ -211,6 +219,29 @@ class NinjaState: # output. "restat": 1, }, + + "SCONS_DAEMON": { + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {PORT} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", + "description": "Starting scons daemon...", + "pool": "local_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, "REGENERATE": { "command": "$SCONS_INVOCATION_W_TARGETS", "description": "Regenerating $self", @@ -245,6 +276,9 @@ class NinjaState: if not node.has_builder(): return False + if isinstance(node, SCons.Node.Python.Value): + return False + if isinstance(node, SCons.Node.Alias.Alias): build = alias_to_ninja_build(node) else: @@ -256,7 +290,24 @@ class NinjaState: node_string = str(node) if node_string in self.builds: - raise InternalError("Node {} added to ninja build state more than once".format(node_string)) + warn_msg = f"Alias {node_string} name the same as File node, ninja does not support this. Renaming Alias {node_string} to {node_string}_alias." + if isinstance(node, SCons.Node.Alias.Alias): + for i, output in enumerate(build["outputs"]): + if output == node_string: + build["outputs"][i] += "_alias" + node_string += "_alias" + print(warn_msg) + elif self.builds[node_string]["rule"] == "phony": + for i, output in enumerate(self.builds[node_string]["outputs"]): + if output == node_string: + self.builds[node_string]["outputs"][i] += "_alias" + tmp_build = self.builds[node_string].copy() + del self.builds[node_string] + node_string += "_alias" + self.builds[node_string] = tmp_build + print(warn_msg) + else: + raise InternalError("Node {} added to ninja build state more than once".format(node_string)) self.builds[node_string] = build self.built.update(build["outputs"]) return True @@ -330,8 +381,12 @@ class NinjaState: ) template_builders = [] + scons_compiledb = False for build in [self.builds[key] for key in sorted(self.builds.keys())]: + if "compile_commands.json" in build["outputs"]: + scons_compiledb = True + if build["rule"] == "TEMPLATE": template_builders.append(build) continue @@ -345,10 +400,10 @@ class NinjaState: # generated sources or else we will create a dependency # cycle. if ( - generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) ): # Make all non-generated source targets depend on # _generated_sources. We use order_only for generated @@ -422,41 +477,10 @@ class NinjaState: ninja.build(**build) - template_builds = dict() + scons_daemon_dirty = str(pathlib.Path(get_path(self.env.get("NINJA_DIR"))) / "scons_daemon_dirty") for template_builder in template_builders: - - # Special handling for outputs and implicit since we need to - # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", "inputs"]: - new_val = template_builds.get(agg_key, []) - - # Use pop so the key is removed and so the update - # below will not overwrite our aggregated values. - cur_val = template_builder.pop(agg_key, []) - if is_List(cur_val): - new_val += cur_val - else: - new_val.append(cur_val) - template_builds[agg_key] = new_val - - # Collect all other keys - template_builds.update(template_builder) - - if template_builds.get("outputs", []): - - # Try to clean up any dependency cycles. If we are passing an - # ouptut node to SCons, it will build any dependencys if ninja - # has not already. - for output in template_builds.get("outputs", []): - inputs = template_builds.get('inputs') - if inputs and output in inputs: - inputs.remove(output) - - implicits = template_builds.get('implicit') - if implicits and output in implicits: - implicits.remove(output) - - ninja.build(**template_builds) + template_builder["implicit"] += [scons_daemon_dirty] + ninja.build(**template_builder) # We have to glob the SCons files here to teach the ninja file # how to regenerate itself. We'll never see ourselves in the @@ -483,30 +507,55 @@ class NinjaState: } ) - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - ninja.build( - "compile_commands.json", - rule="CMD", - pool="console", - implicit=[str(self.ninja_file)], - variables={ - "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( - # NINJA_COMPDB_EXPAND - should only be true for ninja - # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) - # https://github.com/ninja-build/ninja/pull/1223 - # TODO: add check in generate to check version and enable this by default if it's available. - self.ninja_bin_path, str(self.ninja_file), - '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' - ) - }, - ) + if not scons_compiledb: + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + implicit=[str(self.ninja_file)], + variables={ + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + # NINJA_COMPDB_EXPAND - should only be true for ninja + # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) + # https://github.com/ninja-build/ninja/pull/1223 + # TODO: add check in generate to check version and enable this by default if it's available. + self.ninja_bin_path, str(self.ninja_file), + '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) ninja.build( - "compiledb", rule="phony", implicit=["compile_commands.json"], + ["run_scons_daemon", scons_daemon_dirty], + rule="SCONS_DAEMON", ) + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) + pidfile = None + if os.path.exists(scons_daemon_dirty): + pidfile = scons_daemon_dirty + elif os.path.exists(daemon_dir / 'pidfile'): + pidfile = daemon_dir / 'pidfile' + + if pidfile: + with open(pidfile) as f: + pid = int(f.readline()) + try: + os.kill(pid, signal.SIGINT) + except OSError: + pass + + if os.path.exists(scons_daemon_dirty): + os.unlink(scons_daemon_dirty) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that # we generated a ninja build rule for. scons_default_targets = [ @@ -588,7 +637,13 @@ class SConsToNinjaTranslator: elif isinstance(action, COMMAND_TYPES): build = get_command(env, node, action) else: - raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } if build is not None: build["order_only"] = get_order_only(node) @@ -657,7 +712,7 @@ class SConsToNinjaTranslator: return results[0] all_outputs = list({output for build in results for output in build["outputs"]}) - dependencies = list({dep for build in results for dep in build["implicit"]}) + dependencies = list({dep for build in results for dep in build.get("implicit", [])}) if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": cmdline = "" @@ -669,6 +724,9 @@ class SConsToNinjaTranslator: # condition if not cmdstr. So here we strip preceding # and proceeding whitespace to make strings like the # above become empty strings and so will be skipped. + if not cmd.get("variables") or not cmd["variables"].get("cmd"): + continue + cmdstr = cmd["variables"]["cmd"].strip() if not cmdstr: continue @@ -717,4 +775,10 @@ class SConsToNinjaTranslator: "implicit": dependencies, } - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index 3adbb53..888218d 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -123,7 +123,7 @@ def get_dependencies(node, skip_sources=False): get_path(src_file(child)) for child in filter_ninja_nodes(node.children()) if child not in node.sources - ] + ] return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] @@ -370,7 +370,7 @@ def ninja_contents(original): """Return a dummy content without doing IO""" def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + if isinstance(self, SCons.Node.Node) and (self.is_sconscript() or self.is_conftest()): return original(self) return bytes("dummy_ninja_contents", encoding="utf-8") diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 44c2251..3d14faf 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -187,6 +187,10 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_DIR'] = env.Dir(env.get("NINJA_DIR", '#/.ninja')) + env["NINJA_SCONS_DAEMON_KEEP_ALIVE"] = env.get("NINJA_SCONS_DAEMON_KEEP_ALIVE", 180000) + + if GetOption("disable_ninja"): + env.SConsignFile(os.path.join(str(env['NINJA_DIR']),'.ninja.sconsign')) # here we allow multiple environments to construct rules and builds # into the same ninja file @@ -423,7 +427,7 @@ def generate(env): return if target.check_attributes('ninja_file') is None: NINJA_STATE.add_build(target) - else: + else: target.build() SCons.Taskmaster.Task.execute = ninja_execute diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py new file mode 100644 index 0000000..4acd328 --- /dev/null +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -0,0 +1,52 @@ +import http.client +import sys +import time +import os +import logging +import pathlib +import tempfile +import hashlib +import traceback + +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +os.makedirs(daemon_dir, exist_ok=True) + +logging.basicConfig( + filename=daemon_dir / "scons_daemon_request.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + +while True: + try: + logging.debug(f"Sending request: {sys.argv[3]}") + conn = http.client.HTTPConnection("127.0.0.1", port=int(sys.argv[1]), timeout=60) + conn.request("GET", "/?build=" + sys.argv[3]) + response = None + + while not response: + try: + response = conn.getresponse() + except (http.client.RemoteDisconnected, http.client.ResponseNotReady): + time.sleep(0.01) + except http.client.HTTPException as e: + logging.debug(f"Error: {traceback.format_exc()}") + exit(1) + else: + msg = response.read() + status = response.status + if status != 200: + print(msg.decode('utf-8')) + exit(1) + logging.debug(f"Request Done: {sys.argv[3]}") + exit(0) + + except ConnectionRefusedError: + logging.debug(f"Server not ready: {traceback.format_exc()}") + time.sleep(1) + except ConnectionResetError: + logging.debug(f"Server ConnectionResetError") + exit(1) + diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py new file mode 100644 index 0000000..2ab70e6 --- /dev/null +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -0,0 +1,71 @@ +import subprocess +import sys +import os +import pathlib +import tempfile +import hashlib +import logging +import time +import http.client +import traceback + +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +os.makedirs(daemon_dir, exist_ok=True) + +logging.basicConfig( + filename=daemon_dir / "scons_daemon.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + +if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): + cmd = [sys.executable, str(pathlib.Path(__file__).parent / "ninja_scons_daemon.py")] + sys.argv[1:] + logging.debug(f"Starting daemon with {' '.join(cmd)}") + + p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False) + + with open(daemon_dir / "pidfile", "w") as f: + f.write(str(p.pid)) + with open(ninja_builddir / "scons_daemon_dirty", "w") as f: + f.write(str(p.pid)) + + error_msg = f"ERROR: Failed to connect to scons daemon.\n Check {daemon_dir / 'scons_daemon.log'} for more info.\n" + + while True: + try: + logging.debug(f"Attempting to connect scons daemon") + conn = http.client.HTTPConnection("127.0.0.1", port=int(sys.argv[1]), timeout=60) + conn.request("GET", "/?ready=true") + response = None + + try: + response = conn.getresponse() + except (http.client.RemoteDisconnected, http.client.ResponseNotReady): + time.sleep(0.01) + except http.client.HTTPException as e: + logging.debug(f"Error: {traceback.format_exc()}") + sys.stderr.write(error_msg) + exit(1) + else: + msg = response.read() + status = response.status + if status != 200: + print(msg.decode('utf-8')) + exit(1) + logging.debug(f"Request Done: {sys.argv[3]}") + break + + except ConnectionRefusedError: + logging.debug(f"Server not ready: {sys.argv[3]}") + time.sleep(1) + except ConnectionResetError: + logging.debug(f"Server ConnectionResetError") + sys.stderr.write(error_msg) + exit(1) + except: + logging.debug(f"Error: {traceback.format_exc()}") + sys.stderr.write(error_msg) + exit(1) + diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py new file mode 100644 index 0000000..7b839fc --- /dev/null +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -0,0 +1,259 @@ +import http.server +import socketserver +from urllib.parse import urlparse, parse_qs +import time +from threading import Condition +from subprocess import PIPE, Popen +import sys +import os +import threading, queue +import pathlib +import logging +from timeit import default_timer as timer +import traceback +import tempfile +import hashlib + +port = int(sys.argv[1]) +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_keep_alive = int(sys.argv[3]) +args = sys.argv[4:] + +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +os.makedirs(daemon_dir, exist_ok=True) +logging.basicConfig( + filename=daemon_dir / "scons_daemon.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) +def daemon_log(message): + logging.debug(message) + +def custom_readlines(handle, line_separator="\n", chunk_size=1): + buf = "" + while not handle.closed: + data = handle.read(chunk_size) + if not data: + break + buf += data.decode("utf-8") + if line_separator in buf: + chunks = buf.split(line_separator) + buf = chunks.pop() + for chunk in chunks: + yield chunk + line_separator + if buf.endswith("scons>>>"): + yield buf + buf = "" + +def custom_readerr(handle, line_separator="\n", chunk_size=1): + buf = "" + while not handle.closed: + data = handle.read(chunk_size) + if not data: + break + buf += data.decode("utf-8") + if line_separator in buf: + chunks = buf.split(line_separator) + buf = chunks.pop() + for chunk in chunks: + yield chunk + line_separator + +def enqueue_output(out, queue): + for line in iter(custom_readlines(out)): + queue.put(line) + out.close() + +def enqueue_error(err, queue): + for line in iter(custom_readerr(err)): + queue.put(line) + err.close() + +input_q = queue.Queue() +output_q = queue.Queue() +error_q = queue.Queue() + +finished_building = [] +error_nodes = [] + +building_cv = Condition() +error_cv = Condition() + +thread_error = False + +def daemon_thread_func(): + global thread_error + global finished_building + global error_nodes + try: + args_list = args + ["--ninja-scons-daemon"] + daemon_log(f"Starting daemon with args: {' '.join(args_list)}") + daemon_log(f"cwd: {os.getcwd()}") + + p = Popen(args_list, stdout=PIPE, stderr=PIPE, stdin=PIPE) + + t = threading.Thread(target=enqueue_output, args=(p.stdout, output_q)) + t.daemon = True + t.start() + + te = threading.Thread(target=enqueue_error, args=(p.stderr, error_q)) + te.daemon = True + te.start() + + daemon_ready = False + building_node = None + + while p.poll() is None: + + while True: + try: + line = output_q.get(block=False, timeout=0.01) + except queue.Empty: + break + else: + daemon_log("output: " + line.strip()) + + if "scons: building terminated because of errors." in line: + error_output = "" + while True: + try: + error_output += error_q.get(block=False, timeout=0.01) + except queue.Empty: + break + error_nodes += [{'node': building_node, 'error': error_output}] + daemon_ready = True + building_node = None + with building_cv: + building_cv.notify() + + elif line == "scons>>>": + with error_q.mutex: + error_q.queue.clear() + daemon_ready = True + with building_cv: + building_cv.notify() + building_node = None + + while daemon_ready and not input_q.empty(): + + try: + building_node = input_q.get(block=False, timeout=0.01) + except queue.Empty: + break + if "exit" in building_node: + p.stdin.write("exit\n".encode("utf-8")) + p.stdin.flush() + with building_cv: + finished_building += [building_node] + daemon_ready = False + raise + + else: + input_command = "build " + building_node + "\n" + daemon_log("input: " + input_command.strip()) + + p.stdin.write(input_command.encode("utf-8")) + p.stdin.flush() + with building_cv: + finished_building += [building_node] + daemon_ready = False + + time.sleep(0.01) + except: + thread_error = True + daemon_log("SERVER ERROR: " + traceback.format_exc()) + raise + + +daemon_thread = threading.Thread(target=daemon_thread_func) +daemon_thread.daemon = True +daemon_thread.start() + +logging.debug(f"Starting request server on port {port}, keep alive: {daemon_keep_alive}") + +keep_alive_timer = timer() +httpd = None + + +def server_thread_func(): + + class S(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global thread_error + global keep_alive_timer + global error_nodes + + try: + gets = parse_qs(urlparse(self.path).query) + build = gets.get("build") + if build: + keep_alive_timer = timer() + + daemon_log(f"Got request: {build[0]}") + input_q.put(build[0]) + + def pred(): + return build[0] in finished_building + + with building_cv: + building_cv.wait_for(pred) + + for error_node in error_nodes: + if error_node['node'] == build[0]: + self.send_response(500) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(error_node['error'].encode()) + return + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + return + + exitbuild = gets.get("exit") + if exitbuild: + input_q.put('exit') + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + except: + thread_error = True + daemon_log("SERVER ERROR: " + traceback.format_exc()) + raise + + def log_message(self, format, *args): + return + + httpd = socketserver.TCPServer(("127.0.0.1", port), S) + httpd.serve_forever() + +server_thread = threading.Thread(target=server_thread_func) +server_thread.daemon = True +server_thread.start() + +while timer() - keep_alive_timer < daemon_keep_alive and not thread_error: + time.sleep(1) + +if thread_error: + daemon_log(f"Shutting server on port {port} down because thread error.") +else: + daemon_log(f"Shutting server on port {port} down because timed out: {daemon_keep_alive}") + +# if there are errors, don't immediatly shutdown the daemon +# the process which started the server is attempt to connect to +# the daemon before allowing jobs to start being sent. If the daemon +# shutsdown too fast, the launchs script will think it has not +# started yet and sit and wait. If the launch script is able to connect +# and then the connection is dropped, it will immediatly exit with fail. +time.sleep(5) + +if os.path.exists(ninja_builddir / "scons_daemon_dirty"): + os.unlink(ninja_builddir / "scons_daemon_dirty") +if os.path.exists(daemon_dir / "pidfile"): + os.unlink(daemon_dir / "pidfile") + +httpd.shutdown() +server_thread.join() diff --git a/test/ninja/force_scons_callback.py b/test/ninja/force_scons_callback.py index 55c12ca..59b54a0 100644 --- a/test/ninja/force_scons_callback.py +++ b/test/ninja/force_scons_callback.py @@ -53,7 +53,7 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) -if test.stdout().count('scons: Building targets') != 2: +if test.stdout().count('Defer to SCons to build') != 1: test.fail_test() test.must_match('out.txt', 'foo.c' + os.linesep) test.must_match('out2.txt', "test2.cpp" + os.linesep) @@ -74,7 +74,7 @@ test.must_not_exist(test.workpath('out2.txt')) # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program=program, stdout=None) -if test.stdout().count('scons: Building targets') != 1: +if test.stdout().count('Defer to SCons to build') != 1: test.fail_test() test.must_match('out.txt', 'foo.c' + os.linesep) test.must_match('out2.txt', "test2.cpp" + os.linesep) -- cgit v0.12 From 77af301f7f40875837602a1245d23d92266b990c Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Feb 2022 18:08:08 -0800 Subject: Fix issue where only the first interactive 'build' command's results were being written to the sconsign file --- SCons/Script/Interactive.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/SCons/Script/Interactive.py b/SCons/Script/Interactive.py index ad021fd..1015b00 100644 --- a/SCons/Script/Interactive.py +++ b/SCons/Script/Interactive.py @@ -266,7 +266,12 @@ version Prints SCons version information. # from SCons.Debug import Trace # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) - SCons.SConsign.Reset() + # TODO: REMOVE WPD DEBUG 02/14/2022 + # This call was clearing the list of sconsign files to be written, so it would + # only write the results of the first build command. All others wouldn't be written + # to .SConsign. + # Pretty sure commenting this out is the correct fix. + # SCons.SConsign.Reset() SCons.Script.Main.progress_display("scons: done clearing node information.") def do_clean(self, argv): -- cgit v0.12 From 5741258b63df0acd080232fa0bb464b51e7e7282 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 24 Feb 2022 14:19:26 -0600 Subject: Added ninja mingw support and improved CommandGeneratorAction support --- CHANGES.txt | 1 + RELEASE.txt | 1 + SCons/Tool/ninja/Methods.py | 3 ++- SCons/Tool/ninja/__init__.py | 11 ++++++++--- SCons/Util.py | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 149de73..7c3062c 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -29,6 +29,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Daniel Moody: - Add cache-debug messages for push failures. + - Added ninja mingw support and improved ninja CommandGeneratorAction support. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index aeffddf..b1784ea 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -48,6 +48,7 @@ IMPROVEMENTS - Verify that a user specified msvc script (via MSVC_USE_SCRIPT) exists and raise an exception immediately when the user specified msvc script does not exist. - Add cache-debug messages for push failures. +- Added ninja mingw support and improved ninja CommandGeneratorAction support. PACKAGING --------- diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index 073cf71..35cf280 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -134,7 +134,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches variables = {} - comstr = get_comstr(sub_env, action, tlist, slist) + comstr = str(get_comstr(sub_env, action, tlist, slist)) if not comstr: return None @@ -255,6 +255,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom ) cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = [rsp_content_item.replace('\\', '/') for rsp_content_item in rsp_content] rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] rsp_content = " ".join(rsp_content) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 44c2251..893990d 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -214,7 +214,7 @@ def generate(env): # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. - if env["PLATFORM"] == "win32": + if env["PLATFORM"] == "win32" and 'mingw' not in env['TOOLS']: env.Append(CCFLAGS=["/showIncludes"]) else: env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) @@ -267,7 +267,12 @@ def generate(env): def robust_rule_mapping(var, rule, tool): provider = gen_get_response_file_command(env, rule, tool) env.NinjaRuleMapping("${" + var + "}", provider) - env.NinjaRuleMapping(env.get(var, None), provider) + + # some of these construction vars could be generators, e.g. + # CommandGeneratorAction, so if the var is not a string, we + # can't parse the generated string. + if isinstance(env.get(var), str): + env.NinjaRuleMapping(env.get(var, None), provider) robust_rule_mapping("CCCOM", "CC", "$CC") robust_rule_mapping("SHCCCOM", "CC", "$CC") @@ -423,7 +428,7 @@ def generate(env): return if target.check_attributes('ninja_file') is None: NINJA_STATE.add_build(target) - else: + else: target.build() SCons.Taskmaster.Task.execute = ninja_execute diff --git a/SCons/Util.py b/SCons/Util.py index 31202cc..17bb4f2 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -1213,7 +1213,7 @@ class CLVar(UserList): return super().__iadd__(CLVar(other)) def __str__(self): - return ' '.join(self.data) + return ' '.join([str(d) for d in self.data]) class Selector(OrderedDict): -- cgit v0.12 From 2617860ac856dc081d511f75ef1711fb3375c50f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 25 Feb 2022 18:10:47 -0800 Subject: Initial change to use CCDEFFLAGS set by individual compilers rather than hardcoding those flags for all compilers in the ninja tool itself --- SCons/Tool/cc.xml | 13 +++++++++++++ SCons/Tool/clang.py | 8 ++++++-- SCons/Tool/clang.xml | 25 ++++++++++++++++++++++++- SCons/Tool/gcc.py | 3 +++ SCons/Tool/gcc.xml | 1 + SCons/Tool/gxx.py | 3 +++ SCons/Tool/msvc.py | 3 +++ SCons/Tool/msvc.xml | 1 + SCons/Tool/ninja/__init__.py | 7 ++++--- SCons/Tool/ninja/ninja.xml | 1 + 10 files changed, 59 insertions(+), 6 deletions(-) diff --git a/SCons/Tool/cc.xml b/SCons/Tool/cc.xml index a7d6daa..7c8f944 100644 --- a/SCons/Tool/cc.xml +++ b/SCons/Tool/cc.xml @@ -48,6 +48,7 @@ Sets construction variables for generic POSIX C compilers. SHOBJSUFFIX CFILESUFFIX + CCDEPFLAGS PLATFORM @@ -215,4 +216,16 @@ See also &cv-link-CFLAGS; for compiling to static objects. + + + +Options to pass to C or C++ compiler to generate list of dependency files. + + + This is set only by compilers which support this functionality. (&t-link-gcc;, &t-link-clang;, and &t-link-msvc; currently) + + + + + diff --git a/SCons/Tool/clang.py b/SCons/Tool/clang.py index 6c9227c..8c2f728 100644 --- a/SCons/Tool/clang.py +++ b/SCons/Tool/clang.py @@ -43,6 +43,7 @@ from SCons.Tool.MSCommon import msvc_setup_env_once compilers = ['clang'] + def generate(env): """Add Builders and construction variables for clang to an Environment.""" SCons.Tool.cc.generate(env) @@ -58,7 +59,6 @@ def generate(env): # Set-up ms tools paths msvc_setup_env_once(env) - env['CC'] = env.Detect(compilers) or 'clang' if env['PLATFORM'] in ['cygwin', 'win32']: env['SHCCFLAGS'] = SCons.Util.CLVar('$CCFLAGS') @@ -67,7 +67,7 @@ def generate(env): # determine compiler version if env['CC']: - #pipe = SCons.Action._subproc(env, [env['CC'], '-dumpversion'], + # pipe = SCons.Action._subproc(env, [env['CC'], '-dumpversion'], pipe = SCons.Action._subproc(env, [env['CC'], '--version'], stdin='devnull', stderr='devnull', @@ -81,6 +81,10 @@ def generate(env): if match: env['CCVERSION'] = match.group(1) + env['CCDEPFLAGS'] = '.MMD -MF ${TARGET}.d' + + + def exists(env): return env.Detect(compilers) diff --git a/SCons/Tool/clang.xml b/SCons/Tool/clang.xml index 2d989fa..8fdd3c1 100644 --- a/SCons/Tool/clang.xml +++ b/SCons/Tool/clang.xml @@ -1,6 +1,28 @@ diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 893990d..cc69554 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -214,10 +214,11 @@ def generate(env): # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. - if env["PLATFORM"] == "win32" and 'mingw' not in env['TOOLS']: - env.Append(CCFLAGS=["/showIncludes"]) + if 'CCDEPFLAGS' not in env: + # Issue some warning here + pass else: - env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS='$CCDEPFLAGS') env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index c5ee15b..f0ca4f8 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -90,6 +90,7 @@ See its __doc__ string for a discussion of the format. ARFLAGS CC CCCOM + CCDEPFLAGS CCFLAGS CXX CXXCOM -- cgit v0.12 From 5b855a33291742122709d6242f9017d1c43bbb08 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 26 Feb 2022 00:27:24 -0600 Subject: fix a few typos and add tests back --- SCons/Tool/clang.py | 2 +- SCons/Tool/clangxx.py | 3 +- SCons/Tool/gcc.py | 2 +- SCons/Tool/gxx.py | 2 +- test/ninja/mingw_command_generator_action.py | 89 ++++++++++++++++++++++ .../sconstruct_mingw_command_generator_action | 7 ++ 6 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 test/ninja/mingw_command_generator_action.py create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_mingw_command_generator_action diff --git a/SCons/Tool/clang.py b/SCons/Tool/clang.py index 8c2f728..2a12a31 100644 --- a/SCons/Tool/clang.py +++ b/SCons/Tool/clang.py @@ -81,7 +81,7 @@ def generate(env): if match: env['CCVERSION'] = match.group(1) - env['CCDEPFLAGS'] = '.MMD -MF ${TARGET}.d' + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' diff --git a/SCons/Tool/clangxx.py b/SCons/Tool/clangxx.py index 4443b39..88ba6cc 100644 --- a/SCons/Tool/clangxx.py +++ b/SCons/Tool/clangxx.py @@ -51,7 +51,8 @@ def generate(env): SCons.Tool.cxx.generate(env) env['CXX'] = env.Detect(compilers) or 'clang++' - + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + # platform specific settings if env['PLATFORM'] == 'aix': env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS -mminimal-toc') diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 0474d6a..94dfad3 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -57,7 +57,7 @@ def generate(env): if version: env['CCVERSION'] = version - env['CCDEPFLAGS'] = '.MMD -MF ${TARGET}.d' + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' diff --git a/SCons/Tool/gxx.py b/SCons/Tool/gxx.py index 77ed148..cc93f93 100644 --- a/SCons/Tool/gxx.py +++ b/SCons/Tool/gxx.py @@ -64,7 +64,7 @@ def generate(env): if version: env['CXXVERSION'] = version - env['CCDEPFLAGS'] = '.MMD -MF ${TARGET}.d' + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' diff --git a/test/ninja/mingw_command_generator_action.py b/test/ninja/mingw_command_generator_action.py new file mode 100644 index 0000000..58c5106 --- /dev/null +++ b/test/ninja/mingw_command_generator_action.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os +import sys + +import TestSCons +from TestCmd import IS_WINDOWS +import SCons +from SCons.Platform.mingw import MINGW_DEFAULT_PATHS +from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS + +test = TestSCons.TestSCons() + +if sys.platform not in ('cygwin', 'win32',): + test.skip_test("Skipping mingw test on non-Windows platform %s." % sys.platform) + +dp = MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS +gcc = SCons.Tool.find_program_path(test.Environment(), 'gcc', default_paths=dp) +if not gcc: + test.skip_test("Skipping mingw test, no MinGW found.\n") + +# ninja must have the os environment setup to work properly +os.environ["PATH"] += os.pathsep + os.path.dirname(gcc) + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.BIN_DIR, + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_mingw_command_generator_action', 'SConstruct') + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('test' + _exe), stdout="library_function") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.run(program=test.workpath('test' + _exe), stdout="library_function") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_mingw_command_generator_action b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_command_generator_action new file mode 100644 index 0000000..e3fcfe2 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_command_generator_action @@ -0,0 +1,7 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment(tools=['mingw']) +env.Tool('ninja') +dll = env.SharedLibrary(target='test_impl', source='test_impl.c') +env.Program(target='test', source='test1.c', LIBS=[dll]) \ No newline at end of file -- cgit v0.12 From c9bd964430508256a246b3af00358e4fcd42f714 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Sat, 26 Feb 2022 00:53:38 -0600 Subject: Add mingw setup for windows --- .github/workflows/experimental_tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/experimental_tests.yml b/.github/workflows/experimental_tests.yml index aac28d0..3672144 100644 --- a/.github/workflows/experimental_tests.yml +++ b/.github/workflows/experimental_tests.yml @@ -30,6 +30,12 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 + - name: Set up MinGW + uses: egor-tensin/setup-mingw@v2 + if: matrix.os == 'windows-latest' + with: + platform: x64 + - name: Set up Python 3.8 ${{ matrix.os }} uses: actions/setup-python@v2 with: -- cgit v0.12 From bcf17d9bd1d223472a67870a41bb10fe15b28814 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 26 Feb 2022 19:11:40 -0800 Subject: fix typo in gcc/clang CCDEFLAGS --- SCons/Tool/clang.py | 2 +- SCons/Tool/clangxx.py | 3 +++ SCons/Tool/gcc.py | 2 +- SCons/Tool/gxx.py | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/clang.py b/SCons/Tool/clang.py index 8c2f728..2a12a31 100644 --- a/SCons/Tool/clang.py +++ b/SCons/Tool/clang.py @@ -81,7 +81,7 @@ def generate(env): if match: env['CCVERSION'] = match.group(1) - env['CCDEPFLAGS'] = '.MMD -MF ${TARGET}.d' + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' diff --git a/SCons/Tool/clangxx.py b/SCons/Tool/clangxx.py index 4443b39..a78dc6c 100644 --- a/SCons/Tool/clangxx.py +++ b/SCons/Tool/clangxx.py @@ -89,6 +89,9 @@ def generate(env): if match: env['CXXVERSION'] = match.group(1) + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + + def exists(env): return env.Detect(compilers) diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 0474d6a..94dfad3 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -57,7 +57,7 @@ def generate(env): if version: env['CCVERSION'] = version - env['CCDEPFLAGS'] = '.MMD -MF ${TARGET}.d' + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' diff --git a/SCons/Tool/gxx.py b/SCons/Tool/gxx.py index 77ed148..cc93f93 100644 --- a/SCons/Tool/gxx.py +++ b/SCons/Tool/gxx.py @@ -64,7 +64,7 @@ def generate(env): if version: env['CXXVERSION'] = version - env['CCDEPFLAGS'] = '.MMD -MF ${TARGET}.d' + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' -- cgit v0.12 From b27c096277ab49f046a5b934d1a938e9ed5e4325 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 27 Feb 2022 20:48:18 -0800 Subject: fix duplicate CCDEPFLAGS def in clangxx tool --- SCons/Tool/clangxx.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SCons/Tool/clangxx.py b/SCons/Tool/clangxx.py index 775e943..a78dc6c 100644 --- a/SCons/Tool/clangxx.py +++ b/SCons/Tool/clangxx.py @@ -51,8 +51,7 @@ def generate(env): SCons.Tool.cxx.generate(env) env['CXX'] = env.Detect(compilers) or 'clang++' - env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' - + # platform specific settings if env['PLATFORM'] == 'aix': env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS -mminimal-toc') -- cgit v0.12 From 01717fbf21708be8b794aeacebe6664eea8aa138 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 12:00:14 -0700 Subject: [ci skip] Address sider complaints --- SCons/Tool/ninja/NinjaState.py | 2 +- SCons/Tool/ninja/ninja_daemon_build.py | 15 ++++++---- SCons/Tool/ninja/ninja_run_daemon.py | 32 ++++++++++++-------- SCons/Tool/ninja/ninja_scons_daemon.py | 54 +++++++++++++++++++++------------- 4 files changed, 64 insertions(+), 39 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index f2da87a..b6de65c 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -537,7 +537,7 @@ class NinjaState: rule="SCONS_DAEMON", ) - daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) pidfile = None if os.path.exists(scons_daemon_dirty): pidfile = scons_daemon_dirty diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index 4acd328..ff0e2bf 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -9,7 +9,9 @@ import hashlib import traceback ninja_builddir = pathlib.Path(sys.argv[2]) -daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) os.makedirs(daemon_dir, exist_ok=True) logging.basicConfig( @@ -22,7 +24,9 @@ logging.basicConfig( while True: try: logging.debug(f"Sending request: {sys.argv[3]}") - conn = http.client.HTTPConnection("127.0.0.1", port=int(sys.argv[1]), timeout=60) + conn = http.client.HTTPConnection( + "127.0.0.1", port=int(sys.argv[1]), timeout=60 + ) conn.request("GET", "/?build=" + sys.argv[3]) response = None @@ -31,14 +35,14 @@ while True: response = conn.getresponse() except (http.client.RemoteDisconnected, http.client.ResponseNotReady): time.sleep(0.01) - except http.client.HTTPException as e: + except http.client.HTTPException: logging.debug(f"Error: {traceback.format_exc()}") exit(1) else: msg = response.read() status = response.status if status != 200: - print(msg.decode('utf-8')) + print(msg.decode("utf-8")) exit(1) logging.debug(f"Request Done: {sys.argv[3]}") exit(0) @@ -47,6 +51,5 @@ while True: logging.debug(f"Server not ready: {traceback.format_exc()}") time.sleep(1) except ConnectionResetError: - logging.debug(f"Server ConnectionResetError") + logging.debug("Server ConnectionResetError") exit(1) - diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 2ab70e6..442bcad 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -10,7 +10,9 @@ import http.client import traceback ninja_builddir = pathlib.Path(sys.argv[2]) -daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) os.makedirs(daemon_dir, exist_ok=True) logging.basicConfig( @@ -21,10 +23,15 @@ logging.basicConfig( ) if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): - cmd = [sys.executable, str(pathlib.Path(__file__).parent / "ninja_scons_daemon.py")] + sys.argv[1:] + cmd = [ + sys.executable, + str(pathlib.Path(__file__).parent / "ninja_scons_daemon.py"), + ] + sys.argv[1:] logging.debug(f"Starting daemon with {' '.join(cmd)}") - p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False) + p = subprocess.Popen( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False + ) with open(daemon_dir / "pidfile", "w") as f: f.write(str(p.pid)) @@ -34,9 +41,11 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): error_msg = f"ERROR: Failed to connect to scons daemon.\n Check {daemon_dir / 'scons_daemon.log'} for more info.\n" while True: - try: - logging.debug(f"Attempting to connect scons daemon") - conn = http.client.HTTPConnection("127.0.0.1", port=int(sys.argv[1]), timeout=60) + try: + logging.debug("Attempting to connect scons daemon") + conn = http.client.HTTPConnection( + "127.0.0.1", port=int(sys.argv[1]), timeout=60 + ) conn.request("GET", "/?ready=true") response = None @@ -44,7 +53,7 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): response = conn.getresponse() except (http.client.RemoteDisconnected, http.client.ResponseNotReady): time.sleep(0.01) - except http.client.HTTPException as e: + except http.client.HTTPException: logging.debug(f"Error: {traceback.format_exc()}") sys.stderr.write(error_msg) exit(1) @@ -52,20 +61,19 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): msg = response.read() status = response.status if status != 200: - print(msg.decode('utf-8')) + print(msg.decode("utf-8")) exit(1) logging.debug(f"Request Done: {sys.argv[3]}") break - + except ConnectionRefusedError: logging.debug(f"Server not ready: {sys.argv[3]}") time.sleep(1) except ConnectionResetError: - logging.debug(f"Server ConnectionResetError") + logging.debug("Server ConnectionResetError") sys.stderr.write(error_msg) exit(1) - except: + except Exception: logging.debug(f"Error: {traceback.format_exc()}") sys.stderr.write(error_msg) exit(1) - diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 7b839fc..bc14f9e 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -19,7 +19,9 @@ ninja_builddir = pathlib.Path(sys.argv[2]) daemon_keep_alive = int(sys.argv[3]) args = sys.argv[4:] -daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) os.makedirs(daemon_dir, exist_ok=True) logging.basicConfig( filename=daemon_dir / "scons_daemon.log", @@ -27,9 +29,12 @@ logging.basicConfig( format="%(asctime)s %(message)s", level=logging.DEBUG, ) + + def daemon_log(message): logging.debug(message) + def custom_readlines(handle, line_separator="\n", chunk_size=1): buf = "" while not handle.closed: @@ -46,6 +51,7 @@ def custom_readlines(handle, line_separator="\n", chunk_size=1): yield buf buf = "" + def custom_readerr(handle, line_separator="\n", chunk_size=1): buf = "" while not handle.closed: @@ -58,17 +64,20 @@ def custom_readerr(handle, line_separator="\n", chunk_size=1): buf = chunks.pop() for chunk in chunks: yield chunk + line_separator - + + def enqueue_output(out, queue): for line in iter(custom_readlines(out)): queue.put(line) out.close() - + + def enqueue_error(err, queue): for line in iter(custom_readerr(err)): queue.put(line) err.close() + input_q = queue.Queue() output_q = queue.Queue() error_q = queue.Queue() @@ -81,6 +90,7 @@ error_cv = Condition() thread_error = False + def daemon_thread_func(): global thread_error global finished_building @@ -120,12 +130,12 @@ def daemon_thread_func(): error_output += error_q.get(block=False, timeout=0.01) except queue.Empty: break - error_nodes += [{'node': building_node, 'error': error_output}] + error_nodes += [{"node": building_node, "error": error_output}] daemon_ready = True building_node = None with building_cv: building_cv.notify() - + elif line == "scons>>>": with error_q.mutex: error_q.queue.clear() @@ -159,7 +169,7 @@ def daemon_thread_func(): daemon_ready = False time.sleep(0.01) - except: + except Exception: thread_error = True daemon_log("SERVER ERROR: " + traceback.format_exc()) raise @@ -169,14 +179,15 @@ daemon_thread = threading.Thread(target=daemon_thread_func) daemon_thread.daemon = True daemon_thread.start() -logging.debug(f"Starting request server on port {port}, keep alive: {daemon_keep_alive}") +logging.debug( + f"Starting request server on port {port}, keep alive: {daemon_keep_alive}" +) keep_alive_timer = timer() httpd = None def server_thread_func(): - class S(http.server.BaseHTTPRequestHandler): def do_GET(self): global thread_error @@ -197,15 +208,15 @@ def server_thread_func(): with building_cv: building_cv.wait_for(pred) - + for error_node in error_nodes: - if error_node['node'] == build[0]: + if error_node["node"] == build[0]: self.send_response(500) self.send_header("Content-type", "text/html") self.end_headers() - self.wfile.write(error_node['error'].encode()) + self.wfile.write(error_node["error"].encode()) return - + self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() @@ -213,12 +224,12 @@ def server_thread_func(): exitbuild = gets.get("exit") if exitbuild: - input_q.put('exit') - + input_q.put("exit") + self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() - + except: thread_error = True daemon_log("SERVER ERROR: " + traceback.format_exc()) @@ -230,6 +241,7 @@ def server_thread_func(): httpd = socketserver.TCPServer(("127.0.0.1", port), S) httpd.serve_forever() + server_thread = threading.Thread(target=server_thread_func) server_thread.daemon = True server_thread.start() @@ -240,20 +252,22 @@ while timer() - keep_alive_timer < daemon_keep_alive and not thread_error: if thread_error: daemon_log(f"Shutting server on port {port} down because thread error.") else: - daemon_log(f"Shutting server on port {port} down because timed out: {daemon_keep_alive}") + daemon_log( + f"Shutting server on port {port} down because timed out: {daemon_keep_alive}" + ) -# if there are errors, don't immediatly shutdown the daemon +# if there are errors, don't immediately shut down the daemon # the process which started the server is attempt to connect to # the daemon before allowing jobs to start being sent. If the daemon -# shutsdown too fast, the launchs script will think it has not +# shuts down too fast, the launch script will think it has not # started yet and sit and wait. If the launch script is able to connect -# and then the connection is dropped, it will immediatly exit with fail. +# and then the connection is dropped, it will immediately exit with fail. time.sleep(5) if os.path.exists(ninja_builddir / "scons_daemon_dirty"): os.unlink(ninja_builddir / "scons_daemon_dirty") if os.path.exists(daemon_dir / "pidfile"): os.unlink(daemon_dir / "pidfile") - + httpd.shutdown() server_thread.join() -- cgit v0.12 From f3dc33eb69f0f2fc32aa7aa9caaa40d1cb8b658f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 12:07:11 -0700 Subject: [ci skip] Add standard SCons license blurb,and empty docstring --- SCons/Tool/ninja/ninja_daemon_build.py | 34 ++++++++++++++++++++++++++++++++++ SCons/Tool/ninja/ninja_run_daemon.py | 33 +++++++++++++++++++++++++++++++++ SCons/Tool/ninja/ninja_scons_daemon.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index ff0e2bf..b16c039 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -1,3 +1,30 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" + +""" + import http.client import sys import time @@ -53,3 +80,10 @@ while True: except ConnectionResetError: logging.debug("Server ConnectionResetError") exit(1) + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 442bcad..542198a 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -1,3 +1,30 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" + +""" + import subprocess import sys import os @@ -77,3 +104,9 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): logging.debug(f"Error: {traceback.format_exc()}") sys.stderr.write(error_msg) exit(1) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index bc14f9e..00923fb 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -1,3 +1,30 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" + +""" + import http.server import socketserver from urllib.parse import urlparse, parse_qs @@ -271,3 +298,10 @@ if os.path.exists(daemon_dir / "pidfile"): httpd.shutdown() server_thread.join() + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file -- cgit v0.12 From 247d0ba437342088ee6de073a60e3753527a9946 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 12:13:24 -0700 Subject: [ci skip] Address sider complaints --- SCons/Tool/ninja/ninja_daemon_build.py | 3 ++- SCons/Tool/ninja/ninja_run_daemon.py | 2 +- SCons/Tool/ninja/ninja_scons_daemon.py | 8 +++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index b16c039..af0e6b4 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -86,4 +86,5 @@ while True: # tab-width:4 # indent-tabs-mode:nil # End: -# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file +# vim: set expandtab tabstop=4 shiftwidth=4: + diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 542198a..965e50b 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -109,4 +109,4 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): # tab-width:4 # indent-tabs-mode:nil # End: -# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 00923fb..d644fe9 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -33,7 +33,8 @@ from threading import Condition from subprocess import PIPE, Popen import sys import os -import threading, queue +import threading +import queue import pathlib import logging from timeit import default_timer as timer @@ -257,7 +258,7 @@ def server_thread_func(): self.send_header("Content-type", "text/html") self.end_headers() - except: + except Exception: thread_error = True daemon_log("SERVER ERROR: " + traceback.format_exc()) raise @@ -304,4 +305,5 @@ server_thread.join() # tab-width:4 # indent-tabs-mode:nil # End: -# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file +# vim: set expandtab tabstop=4 shiftwidth=4: + -- cgit v0.12 From 8c4394e7e1b28b5c39c7f3e364721a7e25120126 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 14 Mar 2022 14:22:14 -0500 Subject: Added some comments for ninja updates. --- SCons/Tool/ninja/Methods.py | 4 +++- SCons/Util.py | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index 35cf280..c89c42f 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -134,6 +134,8 @@ def get_command(env, node, action): # pylint: disable=too-many-branches variables = {} + # since we will check the ninja rule map for this command str, we must make sure + # its string so its hashable. comstr = str(get_comstr(sub_env, action, tlist, slist)) if not comstr: return None @@ -267,7 +269,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom variables["env"] += env.subst( "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor ) + " " - + if node.get_env().get('NINJA_FORCE_SCONS_BUILD'): ret_rule = 'TEMPLATE' else: diff --git a/SCons/Util.py b/SCons/Util.py index 17bb4f2..093ca41 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -1213,6 +1213,8 @@ class CLVar(UserList): return super().__iadd__(CLVar(other)) def __str__(self): + # Some cases the data can contain Nodes, so make sure they + # processed to string before handing them over to join. return ' '.join([str(d) for d in self.data]) @@ -1684,10 +1686,10 @@ def _attempt_init_of_python_3_9_hash_object(hash_function_object, sys_used=sys): """ if hash_function_object is None: return None - + # https://stackoverflow.com/a/11887885 details how to check versions with the "packaging" library. # however, for our purposes checking the version is greater than or equal to 3.9 is good enough, as - # the API is guaranteed to have support for the 'usedforsecurity' flag in 3.9. See + # the API is guaranteed to have support for the 'usedforsecurity' flag in 3.9. See # https://docs.python.org/3/library/hashlib.html#:~:text=usedforsecurity for the version support notes. if (sys_used.version_info.major > 3) or (sys_used.version_info.major == 3 and sys_used.version_info.minor >= 9): return hash_function_object(usedforsecurity=False) @@ -1716,7 +1718,7 @@ def _set_allowed_viable_default_hashes(hashlib_used, sys_used=sys): # note: if you call this method repeatedly, example using timeout, this is needed. # otherwise it keeps appending valid formats to the string ALLOWED_HASH_FORMATS = [] - + for test_algorithm in DEFAULT_HASH_FORMATS: _test_hash = getattr(hashlib_used, test_algorithm, None) # we know hashlib claims to support it... check to see if we can call it. @@ -1731,7 +1733,7 @@ def _set_allowed_viable_default_hashes(hashlib_used, sys_used=sys): except ValueError as e: _last_error = e continue - + if len(ALLOWED_HASH_FORMATS) == 0: from SCons.Errors import SConsEnvironmentError # pylint: disable=import-outside-toplevel # chain the exception thrown with the most recent error from hashlib. @@ -1817,7 +1819,7 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): 'is reporting; SCons supports: %s. Your local system only ' 'supports: %s' % (hash_format_lower, - ', '.join(DEFAULT_HASH_FORMATS), + ', '.join(DEFAULT_HASH_FORMATS), ', '.join(ALLOWED_HASH_FORMATS)) ) @@ -1825,7 +1827,7 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): # function did not throw, or when it threw, the exception was caught and ignored, or # the global ALLOWED_HASH_FORMATS was changed by an external user. _HASH_FUNCTION = _attempt_get_hash_function(hash_format_lower, hashlib_used, sys_used) - + if _HASH_FUNCTION is None: from SCons.Errors import UserError # pylint: disable=import-outside-toplevel @@ -1842,7 +1844,7 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): # disabled. for choice in ALLOWED_HASH_FORMATS: _HASH_FUNCTION = _attempt_get_hash_function(choice, hashlib_used, sys_used) - + if _HASH_FUNCTION is not None: break else: @@ -1864,7 +1866,7 @@ set_hash_format(None) def get_current_hash_algorithm_used(): """Returns the current hash algorithm name used. - + Where the python version >= 3.9, this is expected to return md5. If python's version is <= 3.8, this returns md5 on non-FIPS-mode platforms, and sha1 or sha256 on FIPS-mode Linux platforms. -- cgit v0.12 From 898d87dc5e026b154ac689d8ddd7400ce07a37bc Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 13:11:39 -0700 Subject: [ci skip] Added NINJA_SCONS_DAEMON_KEEP_ALIVE to docs. --- SCons/Tool/ninja/ninja.xml | 10 ++++++++++ SCons/Tool/ninja/ninja_daemon_build.py | 1 - SCons/Tool/ninja/ninja_scons_daemon.py | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index c5ee15b..b023e86 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -73,6 +73,7 @@ See its __doc__ string for a discussion of the format. _NINJA_REGENERATE_DEPS_FUNC __NINJA_NO IMPLICIT_COMMAND_DEPENDENCIES + NINJA_SCONS_DAEMON_KEEP_ALIVE @@ -335,5 +336,14 @@ See its __doc__ string for a discussion of the format. + + + + The number of seconds for the SCons deamon launched by ninja to stay alive. + (Default: 180000) + + + + diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index af0e6b4..6e1c7be 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -87,4 +87,3 @@ while True: # indent-tabs-mode:nil # End: # vim: set expandtab tabstop=4 shiftwidth=4: - diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index d644fe9..2b13993 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -306,4 +306,3 @@ server_thread.join() # indent-tabs-mode:nil # End: # vim: set expandtab tabstop=4 shiftwidth=4: - -- cgit v0.12 From 3c7a7395d80f97f4e0b2f8d038b2a09bcbeb9e60 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 14 Mar 2022 15:24:16 -0500 Subject: remove scons-ninja-daemon option in favor of interactive --- SCons/Node/__init__.py | 1 - SCons/Script/Interactive.py | 161 ++++++++++++++++----------------- SCons/Script/Main.py | 4 +- SCons/Script/SConsOptions.py | 5 - SCons/Tool/ninja/NinjaState.py | 8 +- SCons/Tool/ninja/ninja_scons_daemon.py | 2 +- 6 files changed, 82 insertions(+), 99 deletions(-) diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index d0539b4..ec742a6 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -105,7 +105,6 @@ SConscriptNodes = set() # currently used to release parts of a target's info during # clean builds and update runs (see release_target_info). interactive = False -ninja_scons_daemon = False def is_derived_none(node): raise NotImplementedError diff --git a/SCons/Script/Interactive.py b/SCons/Script/Interactive.py index 1015b00..d694740 100644 --- a/SCons/Script/Interactive.py +++ b/SCons/Script/Interactive.py @@ -29,7 +29,7 @@ # of its own, which might or might not be a good thing. Nevertheless, # here are some enhancements that will probably be requested some day # and are worth keeping in mind (assuming this takes off): -# +# # - A command to re-read / re-load the SConscript files. This may # involve allowing people to specify command-line options (e.g. -f, # -I, --no-site-dir) that affect how the SConscript files are read. @@ -188,91 +188,82 @@ version Prints SCons version information. x.extend(n.alter_targets()[0]) nodes.extend(x) - if SCons.Node.ninja_scons_daemon: - for n in nodes: - print(f"Node: {n}, State: {SCons.Node.failed}") - if n.get_state() == SCons.Node.failed: - print(n) - n.clear() - n.set_state(SCons.Node.no_state) - n.implicit = None - else: - # Clean up so that we can perform the next build correctly. - # - # We do this by walking over all the children of the targets, - # and clearing their state. - # - # We currently have to re-scan each node to find their - # children, because built nodes have already been partially - # cleared and don't remember their children. (In scons - # 0.96.1 and earlier, this wasn't the case, and we didn't - # have to re-scan the nodes.) - # - # Because we have to re-scan each node, we can't clear the - # nodes as we walk over them, because we may end up rescanning - # a cleared node as we scan a later node. Therefore, only - # store the list of nodes that need to be cleared as we walk - # the tree, and clear them in a separate pass. - # - # XXX: Someone more familiar with the inner workings of scons - # may be able to point out a more efficient way to do this. - - SCons.Script.Main.progress_display("scons: Clearing cached node information ...") - - seen_nodes = {} - - def get_unseen_children(node, parent, seen_nodes=seen_nodes): - def is_unseen(node, seen_nodes=seen_nodes): - return node not in seen_nodes - return [child for child in node.children(scan=1) if is_unseen(child)] - - def add_to_seen_nodes(node, parent, seen_nodes=seen_nodes): - seen_nodes[node] = 1 - - # If this file is in a VariantDir and has a - # corresponding source file in the source tree, remember the - # node in the source tree, too. This is needed in - # particular to clear cached implicit dependencies on the - # source file, since the scanner will scan it if the - # VariantDir was created with duplicate=0. - try: - rfile_method = node.rfile - except AttributeError: - return - else: - rfile = rfile_method() - if rfile != node: - seen_nodes[rfile] = 1 - - for node in nodes: - walker = SCons.Node.Walker(node, - kids_func=get_unseen_children, - eval_func=add_to_seen_nodes) + # Clean up so that we can perform the next build correctly. + # + # We do this by walking over all the children of the targets, + # and clearing their state. + # + # We currently have to re-scan each node to find their + # children, because built nodes have already been partially + # cleared and don't remember their children. (In scons + # 0.96.1 and earlier, this wasn't the case, and we didn't + # have to re-scan the nodes.) + # + # Because we have to re-scan each node, we can't clear the + # nodes as we walk over them, because we may end up rescanning + # a cleared node as we scan a later node. Therefore, only + # store the list of nodes that need to be cleared as we walk + # the tree, and clear them in a separate pass. + # + # XXX: Someone more familiar with the inner workings of scons + # may be able to point out a more efficient way to do this. + + SCons.Script.Main.progress_display("scons: Clearing cached node information ...") + + seen_nodes = {} + + def get_unseen_children(node, parent, seen_nodes=seen_nodes): + def is_unseen(node, seen_nodes=seen_nodes): + return node not in seen_nodes + return [child for child in node.children(scan=1) if is_unseen(child)] + + def add_to_seen_nodes(node, parent, seen_nodes=seen_nodes): + seen_nodes[node] = 1 + + # If this file is in a VariantDir and has a + # corresponding source file in the source tree, remember the + # node in the source tree, too. This is needed in + # particular to clear cached implicit dependencies on the + # source file, since the scanner will scan it if the + # VariantDir was created with duplicate=0. + try: + rfile_method = node.rfile + except AttributeError: + return + else: + rfile = rfile_method() + if rfile != node: + seen_nodes[rfile] = 1 + + for node in nodes: + walker = SCons.Node.Walker(node, + kids_func=get_unseen_children, + eval_func=add_to_seen_nodes) + n = walker.get_next() + while n: n = walker.get_next() - while n: - n = walker.get_next() - - for node in seen_nodes.keys(): - # Call node.clear() to clear most of the state - node.clear() - # node.clear() doesn't reset node.state, so call - # node.set_state() to reset it manually - node.set_state(SCons.Node.no_state) - node.implicit = None - - # Debug: Uncomment to verify that all Taskmaster reference - # counts have been reset to zero. - #if node.ref_count != 0: - # from SCons.Debug import Trace - # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) - - # TODO: REMOVE WPD DEBUG 02/14/2022 - # This call was clearing the list of sconsign files to be written, so it would - # only write the results of the first build command. All others wouldn't be written - # to .SConsign. - # Pretty sure commenting this out is the correct fix. - # SCons.SConsign.Reset() - SCons.Script.Main.progress_display("scons: done clearing node information.") + + for node in seen_nodes.keys(): + # Call node.clear() to clear most of the state + node.clear() + # node.clear() doesn't reset node.state, so call + # node.set_state() to reset it manually + node.set_state(SCons.Node.no_state) + node.implicit = None + + # Debug: Uncomment to verify that all Taskmaster reference + # counts have been reset to zero. + #if node.ref_count != 0: + # from SCons.Debug import Trace + # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) + + # TODO: REMOVE WPD DEBUG 02/14/2022 + # This call was clearing the list of sconsign files to be written, so it would + # only write the results of the first build command. All others wouldn't be written + # to .SConsign. + # Pretty sure commenting this out is the correct fix. + # SCons.SConsign.Reset() + SCons.Script.Main.progress_display("scons: done clearing node information.") def do_clean(self, argv): """\ diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index 87d57bc..ab2dc0e 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -987,8 +987,6 @@ def _main(parser): # This would then cause subtle bugs, as already happened in #2971. if options.interactive: SCons.Node.interactive = True - if options.ninja_scons_daemon: - SCons.Node.ninja_scons_daemon = True # That should cover (most of) the options. # Next, set up the variables that hold command-line arguments, # so the SConscript files that we read and execute have access to them. @@ -1126,7 +1124,7 @@ def _main(parser): platform = SCons.Platform.platform_module() - if options.interactive or options.ninja_scons_daemon: + if options.interactive: SCons.Script.Interactive.interact(fs, OptionsParser, options, targets, target_top) diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 133d9ab..e2631fb 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -858,11 +858,6 @@ def Parser(version): action="store_true", help="Run in interactive mode") - op.add_option('--ninja-scons-daemon', - dest='ninja_scons_daemon', default=False, - action="store_true", - help="A special interactive mode to support a scons daemon for ninja builds. Intended for use only by the ninja tool.") - op.add_option('-j', '--jobs', nargs=1, type="int", dest="num_jobs", default=1, diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index b6de65c..3ef3bb2 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -63,7 +63,7 @@ class NinjaState: os.pardir, 'data', 'bin', - ninja_bin)) + ninja_bin)) if not os.path.exists(self.ninja_bin_path): # couldn't find it, just give the bin name and hope # its in the path later @@ -219,7 +219,7 @@ class NinjaState: # output. "restat": 1, }, - + "SCONS_DAEMON": { "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {PORT} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", "description": "Starting scons daemon...", @@ -551,8 +551,8 @@ class NinjaState: os.kill(pid, signal.SIGINT) except OSError: pass - - if os.path.exists(scons_daemon_dirty): + + if os.path.exists(scons_daemon_dirty): os.unlink(scons_daemon_dirty) diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index d644fe9..69d984d 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -124,7 +124,7 @@ def daemon_thread_func(): global finished_building global error_nodes try: - args_list = args + ["--ninja-scons-daemon"] + args_list = args + ["--interactive"] daemon_log(f"Starting daemon with args: {' '.join(args_list)}") daemon_log(f"cwd: {os.getcwd()}") -- cgit v0.12 From d382175ba00943cbc488a648e973f3e38162397e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 13:27:49 -0700 Subject: [ci skip] Add comment/TODO to code to handle Alias() name conflicts with real files/directories --- SCons/Tool/ninja/NinjaState.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 3ef3bb2..b1d7fdd 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -290,6 +290,10 @@ class NinjaState: node_string = str(node) if node_string in self.builds: + # TODO: If we work out a way to handle Alias() with same name as file this logic can be removed + # This works around adding Alias with the same name as a Node. + # It's not great way to workaround because it force renames the alias, + # but the alternative is broken ninja support. warn_msg = f"Alias {node_string} name the same as File node, ninja does not support this. Renaming Alias {node_string} to {node_string}_alias." if isinstance(node, SCons.Node.Alias.Alias): for i, output in enumerate(build["outputs"]): -- cgit v0.12 From e3cbfd6ae3986513148872308e2ddd0584364da4 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 14:01:46 -0700 Subject: Allow explicitly setting the daemon port with NINJA_SCONS_DAEMON_PORT --- SCons/Tool/ninja/NinjaState.py | 9 +++------ SCons/Tool/ninja/__init__.py | 2 ++ SCons/Tool/ninja/ninja.xml | 11 +++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index b1d7fdd..11e928d 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -87,10 +87,7 @@ class NinjaState: # to make the SCONS_INVOCATION variable properly quoted for things # like CCFLAGS scons_escape = env.get("ESCAPE", lambda x: x) - - import random - - PORT = str(random.randint(10000, 60000)) + scons_daemon_port = int(env.get('NINJA_SCONS_DAEMON_PORT',-1)) # if SCons was invoked from python, we expect the first arg to be the scons.py # script, otherwise scons was invoked from the scons script @@ -192,7 +189,7 @@ class NinjaState: "restat": 1, }, "TEMPLATE": { - "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_daemon_build.py'} {PORT} {get_path(env.get('NINJA_DIR'))} $out", + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_daemon_build.py'} {scons_daemon_port} {get_path(env.get('NINJA_DIR'))} $out", "description": "Defer to SCons to build $out", "pool": "local_pool", "restat": 1 @@ -221,7 +218,7 @@ class NinjaState: }, "SCONS_DAEMON": { - "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {PORT} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {scons_daemon_port} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", "description": "Starting scons daemon...", "pool": "local_pool", # restat diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 3d14faf..9fe9dd4 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -26,6 +26,7 @@ import importlib import os +import random import subprocess import sys @@ -188,6 +189,7 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_DIR'] = env.Dir(env.get("NINJA_DIR", '#/.ninja')) env["NINJA_SCONS_DAEMON_KEEP_ALIVE"] = env.get("NINJA_SCONS_DAEMON_KEEP_ALIVE", 180000) + env["NINJA_SCONS_DAEMON_PORT"] = env.get('NINJA_SCONS_DAEMON_PORT', random.randint(10000,60000)) if GetOption("disable_ninja"): env.SConsignFile(os.path.join(str(env['NINJA_DIR']),'.ninja.sconsign')) diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index b023e86..b497156 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -74,6 +74,7 @@ See its __doc__ string for a discussion of the format. __NINJA_NO IMPLICIT_COMMAND_DEPENDENCIES NINJA_SCONS_DAEMON_KEEP_ALIVE + NINJA_SCONS_DAEMON_PORT @@ -345,5 +346,15 @@ See its __doc__ string for a discussion of the format. + + + + The TCP/IP port for the SCons daemon to listen on. + NOTE: You cannot use a port already being listened to on your build machine. + (Default: random number between 10000,60000) + + + + -- cgit v0.12 From 53a9b7cc192bc589a28ef2f73ee6b8428c1b1549 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 14:27:24 -0700 Subject: update test/ninja/force_scons_callback.py to test using NINJA_SCONS_DAEMON_PORT to explicitly set the daemon port --- SCons/Tool/ninja/__init__.py | 2 +- test/ninja/force_scons_callback.py | 57 ++++++++++++---------- .../sconstruct_force_scons_callback | 11 +++-- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 9fe9dd4..913738a 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -189,7 +189,7 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_DIR'] = env.Dir(env.get("NINJA_DIR", '#/.ninja')) env["NINJA_SCONS_DAEMON_KEEP_ALIVE"] = env.get("NINJA_SCONS_DAEMON_KEEP_ALIVE", 180000) - env["NINJA_SCONS_DAEMON_PORT"] = env.get('NINJA_SCONS_DAEMON_PORT', random.randint(10000,60000)) + env["NINJA_SCONS_DAEMON_PORT"] = env.get('NINJA_SCONS_DAEMON_PORT', random.randint(10000, 60000)) if GetOption("disable_ninja"): env.SConsignFile(os.path.join(str(env['NINJA_DIR']),'.ninja.sconsign')) diff --git a/test/ninja/force_scons_callback.py b/test/ninja/force_scons_callback.py index 59b54a0..c99ed58 100644 --- a/test/ninja/force_scons_callback.py +++ b/test/ninja/force_scons_callback.py @@ -37,47 +37,50 @@ except ImportError: _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +ninja_bin = os.path.abspath( + os.path.join(ninja.__file__, os.pardir, "data", "bin", "ninja" + _exe) +) -test.dir_fixture('ninja-fixture') +test.dir_fixture("ninja-fixture") -test.file_fixture('ninja_test_sconscripts/sconstruct_force_scons_callback', 'SConstruct') +test.file_fixture( + "ninja_test_sconscripts/sconstruct_force_scons_callback", "SConstruct" +) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) -if test.stdout().count('Defer to SCons to build') != 1: +test.must_contain_all_lines(test.stdout(), ["Generating: build.ninja"]) +test.must_contain_all(test.stdout(), "Executing:") +test.must_contain_all(test.stdout(), "ninja%(_exe)s -f" % locals()) +if test.stdout().count("Defer to SCons to build") != 1: test.fail_test() -test.must_match('out.txt', 'foo.c' + os.linesep) -test.must_match('out2.txt', "test2.cpp" + os.linesep) +test.must_match("out.txt", "foo.c" + os.linesep) +test.must_match("out2.txt", "test2.cpp" + os.linesep) # clean build and ninja files -test.run(arguments='-c', stdout=None) -test.must_contain_all_lines(test.stdout(), [ - 'Removed out.txt', - 'Removed out2.txt', - 'Removed build.ninja']) +test.run(arguments="-c", stdout=None) +test.must_contain_all_lines( + test.stdout(), ["Removed out.txt", "Removed out2.txt", "Removed build.ninja"] +) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('out.txt')) -test.must_not_exist(test.workpath('out2.txt')) +test.run(arguments="--disable-execute-ninja", stdout=None) +test.must_contain_all_lines(test.stdout(), ["Generating: build.ninja"]) +test.must_not_exist(test.workpath("out.txt")) +test.must_not_exist(test.workpath("out2.txt")) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +program = test.workpath("run_ninja_env.bat") if IS_WINDOWS else ninja_bin test.run(program=program, stdout=None) -if test.stdout().count('Defer to SCons to build') != 1: +if test.stdout().count("Defer to SCons to build") != 1: test.fail_test() -test.must_match('out.txt', 'foo.c' + os.linesep) -test.must_match('out2.txt', "test2.cpp" + os.linesep) +test.must_match("out.txt", "foo.c" + os.linesep) +test.must_match("out2.txt", "test2.cpp" + os.linesep) + +# only generate the ninja file with specific NINJA_SCONS_DAEMON_PORT +test.run(arguments="PORT=9999 --disable-execute-ninja", stdout=None) +# Verify that port # propagates to call to ninja_run_daemon.py +test.must_contain(test.workpath("build.ninja"), "ninja_run_daemon.py 9999") test.pass_test() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback b/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback index 55729a6..ef3562b 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback +++ b/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback @@ -1,8 +1,11 @@ -SetOption('experimental','ninja') +SetOption("experimental", "ninja") DefaultEnvironment(tools=[]) env = Environment(tools=[]) -env.Tool('ninja') +daemon_port = ARGUMENTS.get("PORT", False) +if daemon_port: + env["NINJA_SCONS_DAEMON_PORT"] = int(daemon_port) +env.Tool("ninja") -env.Command('out.txt', 'foo.c', 'echo $SOURCE> $TARGET', NINJA_FORCE_SCONS_BUILD=True) -env.Command('out2.txt', 'test2.cpp', 'echo $SOURCE> $TARGET') \ No newline at end of file +env.Command("out.txt", "foo.c", "echo $SOURCE> $TARGET", NINJA_FORCE_SCONS_BUILD=True) +env.Command("out2.txt", "test2.cpp", "echo $SOURCE> $TARGET") -- cgit v0.12 From d19f74f8be81e8b1a18033afbb66726674311f6f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 14:38:46 -0700 Subject: [ci skip] Updated CHANGES/RELEASE --- CHANGES.txt | 6 ++++++ RELEASE.txt | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 149de73..e2b26e4 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -29,6 +29,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Daniel Moody: - Add cache-debug messages for push failures. + - Ninja: Changed generated build.ninja file to run SCons only build Actions via + a SCons Deamon. Added logic for starting and connecting to SCons daemon (currently + only used for ninja) + - Ninja: Fix issue where Configure files weren't being properly processed when build run + via ninja. + From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index aeffddf..becceaa 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -34,13 +34,18 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY which are still listed in the manpage. - Help is now sensitive to the size of the terminal window: the width of the help text will scale to wider (or narrower) terminals than 80 characters. - +- Ninja: Changed generated build.ninja file to run SCons only build Actions via + a SCons Deamon. Added logic for starting and connecting to SCons daemon (currently + only used for ninja) FIXES ----- - Fix a number of Python ResourceWarnings which are issued when running SCons and/or it's tests with python 3.9 (or higher) +- Ninja: Fix issue where Configure files weren't being properly processed when build run + via ninja. + IMPROVEMENTS ------------ @@ -70,4 +75,4 @@ Thanks to the following contributors listed below for their contributions to thi ========================================================================================== .. code-block:: text - git shortlog --no-merges -ns 4.0.1..HEAD + git shortlog --no-merges -ns 4.3.0..HEAD -- cgit v0.12 From 2a14bcda67108a86b342a4b4d6f5d4b1d394f3fb Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 14 Mar 2022 12:10:58 -0600 Subject: man: rearrange the variable substitution section [skip appveyor] The idea is to get the syntactial elements presented in a relatively concise order, in the existing text you have to hunt quite a bit for certain parts. Some portions reworded. The mention of $$ as a sbust escape (PR 4091) is also added to the env.subst entry. Signed-off-by: Mats Wichmann --- SCons/Environment.xml | 15 +- doc/man/scons.xml | 398 ++++++++++++++++++++++++++------------------------ 2 files changed, 221 insertions(+), 192 deletions(-) diff --git a/SCons/Environment.xml b/SCons/Environment.xml index d314711..5b41f22 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -2691,7 +2691,7 @@ Prepend values to &consvars; in the current &consenv;, maintaining uniqueness. Works like &f-link-env-Append; (see for details), except that values are added to the front, -rather than the end, of any existing value of the the &consvar;, +rather than the end, of any existing value of the &consvar;, and values already present in the &consvar; will not be added again. If delete_existing @@ -3139,9 +3139,16 @@ files = Split(""" -Performs construction variable interpolation +Performs &consvar; interpolation +(substitution) on input, which can be a string or a sequence. +Substitutable elements take the form +${expression}, +although if there is no ambiguity in recognizing the element, +the braces can be omitted. +A literal $ can be entered by +using $$. @@ -3176,7 +3183,7 @@ pairs -If the input is a sequence +If input is a sequence (list or tuple), the individual elements of the sequence will be expanded, @@ -3388,7 +3395,7 @@ env.Config(target = 'package-config', source = Value(prefix)) def build_value(target, source, env): # A function that "builds" a Python Value by updating - # the the Python value with the contents of the file + # the Python value with the contents of the file # specified as the source of the Builder call ($SOURCE). target[0].write(source[0].get_contents()) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 71ae139..47aba1f 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -145,7 +145,7 @@ to support additional input file types. Information about files involved in the build, including a cryptographic hash of the contents, -is cached for later reuse, +is cached for later reuse, By default content hashes are used to determine if a file has changed since the last build, but this can be controlled by selecting an appropriate @@ -3983,7 +3983,7 @@ actually two bytes. Checks whether the C compiler (as defined by the &cv-link-CC; &consvar;) works, by trying to compile a small source file. -This provides a more rigorous check: +This provides a more rigorous check: by default, &SCons; itself only detects if there is a program with the correct name, not if it is a functioning compiler. Returns a boolean indicating success or failure. @@ -4003,7 +4003,7 @@ be accepted or rejected by the compiler. Checks whether the C++ compiler (as defined by the &cv-link-CXX; &consvar;) works, by trying to compile a small source file. -This provides a more rigorous check: +This provides a more rigorous check: by default, &SCons; itself only detects if there is a program with the correct name, not if it is a functioning compiler. Returns a boolean indicating success or failure. @@ -4023,7 +4023,7 @@ be accepted or rejected by the compiler. Checks whether the shared-object C compiler (as defined by the &cv-link-SHCC; &consvar;) works by trying to compile a small source file. -This provides a more rigorous check: +This provides a more rigorous check: by default, &SCons; itself only detects if there is a program with the correct name, not if it is a functioning compiler. Returns a boolean indicating success or failure. @@ -4045,7 +4045,7 @@ be created. Checks whether the shared-object C++ compiler (as defined by the &cv-link-SHCXX; &consvar;) works by trying to compile a small source file. -This provides a more rigorous check: +This provides a more rigorous check: by default, &SCons; itself only detects if there is a program with the correct name, not if it is a functioning compiler. Returns a boolean indicating success or failure. @@ -4064,7 +4064,7 @@ be created. context.CheckProg(prog_name) -Checks if +Checks if prog_name exists in the path &SCons; will use at build time. (context.env['ENV']['PATH']). @@ -4100,7 +4100,7 @@ it will be be used as the macro replacement value. If value is a string and needs to display with quotes, the quotes need to be included, as in '"string"' -If the optional +If the optional comment is given, it is inserted as a comment above the macro definition (suitable comment marks will be added automatically). @@ -6410,34 +6410,200 @@ env.Command('marker', 'input_file', action=[MyBuildAction, Touch('$TARGET')]) Variable Substitution -Before executing a command, -&scons; -performs variable substitution on the string that makes up -the action part of the builder. -Variables to be substituted are indicated in the -string by a leading $, -to distinguish them from plain text -which is not to be substituted. -The substitutable text may be surrounded by curly braces -to separate it from surrounding characters if necessary -(for example ${FOO}BAR). -To avoid substituting a substring that looks like a variable name, -escape it with an additional $, -(for example, $$FOO will be left in the -final string as $FOO). + +Before executing a command, &scons; +performs parameter expansion (substitution) +on the string that makes up the action part of the builder. +The format of a substitutable parameter is +${expression}. +If expression refers to a variable, +the braces in ${expression} +can be omitted unless the variable name is +immediately followed by a character that could either be interpreted +as part of the name, or is &Python; syntax such as +[ (for indexing/slicing) +or . (for attribute access - +see Special Attributes below). + -The curly brace notation is required when you use -Python list subscripting/slicing notation on a variable -to select one or more items from a list, -or access a variable's special attributes, -or when you use Python expression substitution -(see below for descriptions of these). +If expression refers to a &consvar;, +it is replaced with the value of that variable in the +&consenv; at the time of execution. +If expression looks like +a variable name but is not defined in the &consenv; +it is replaced with an empty string. +If expression refers to one of the +Special Variables +(see below) the corresponding value of the variable is substituted. +expression may also be +a &Python; expression to be evaluated. +See Python Code Substitution +below for a description. +&SCons; uses the following rules when converting &consvars; into +command line strings: + + + + If the value is a string it is interpreted as space delimited + command line arguments. + + + + If the value is a list it is interpreted as a list of command + line arguments. Each element of the list is converted to a string. + + + + Anything that is not a list or string is converted to a string and + interpreted as a single command line argument. + + + + Newline characters (\n) delimit lines. + The newline parsing is done after + all other parsing, so it is not possible for arguments (e.g. file names) to + contain embedded newline characters. + + + + For a literal $ + use $$. + For example, $$FOO will be left in the + final string as $FOO. + + + + +When a build action is executed, a hash of the command line is +saved, together with other information about the target(s) built +by the action, for future use in rebuild determination. +This is called the &buildsig; +(or &build_action; signature). +The escape sequence +$( +subexpression +$) +may be used to indicate parts of a command line +that may change without +causing a rebuild--that is, +which are not to be included when calculating the &buildsig;. +All text from +$( +up to and including the matching +$) +will be removed from the command line +before it is added to the &buildsig; +while only the +$( +and +$) +will be removed before the command is executed. +For example, the command line string: + + +"echo Last build occurred $( $TODAY $). > $TARGET" + + +would execute the command: + + +echo Last build occurred $TODAY. > $TARGET + + +but the build signature added to any target files would be computed from: + + +echo Last build occurred . > $TARGET + + +While &consvars; are normally directly substituted, +if a &consvar; has a value which +is a callable &Python; object +(a function, or a class with a __call__ method), +that object is called during substitution. +The callable must accept four arguments: +target, +source, +env and +for_signature. +source is a list of source nodes, +target is a list of target nodes, +env is the &consenv; to use for context, +and for_signature is +a boolean value that tells the callable +if it is being called for the purpose of generating a build signature. +Since the build signature is used for rebuild determination, +variable elements that do not affect whether +a rebuild should be triggered +should be omitted from the returned string +if for_signature is true. +See $( +and $) above +for the syntax. + + + +&SCons; will insert whatever +the callable returns +into the expanded string: + + + +def foo(target, source, env, for_signature): + return "bar" + +# Will expand $BAR to "bar baz" +env = Environment(FOO=foo, BAR="$FOO baz") + + +As a reminder, substitution happens when +$BAR is actually used in a +builder action. The value of env['BAR'] +will be exactly as it was set: "$FOO baz". +This can make debugging tricky, +as the substituted result is not available at the time +the SConscript files are being interpreted and +thus not available to print(). +However, you can perform the substitution on demand +by calling the &f-link-env-subst; method for this purpose. + + +You can use this feature to pass arguments to a +callable variable by creating a callable class +that stores passed arguments in the instance, +and then uses them +(in the __call__ method) +when the instance is called. +Note that in this case, +the entire variable expansion must +be enclosed by curly braces +so that the arguments will +be associated with the +instantiation of the class: + + +class foo: + def __init__(self, arg): + self.arg = arg + + def __call__(self, target, source, env, for_signature): + return self.arg + " bar" + +# Will expand $BAR to "my argument bar baz" +env=Environment(FOO=foo, BAR="${FOO('my argument')} baz") + + + + + +Substitution: Special Variables + Besides regular &consvars;, scons provides the following -special variables for use in expanding commands: +Special Variables for use in expanding commands: @@ -6509,8 +6675,12 @@ changed since the target was last built. -These names are reserved -and may not be assigned to or used as &consvars;. + +These names are reserved +and may not be assigned to or used as &consvars;. +&SCons; computes them in a context-dependent manner +and they and are not retrieved from a &consenv;. + For example, the following builder call: @@ -6536,6 +6706,11 @@ In the previous example, a string would expand to: bar.c. + + + +Substitution: Special Attributes + A variable name may have the following modifiers appended within the enclosing curly braces @@ -6631,120 +6806,6 @@ Some modifiers can be combined, like ${TARGET.file.suffix}, etc. -The curly brace notation may also be used -to enclose a Python expression to be evaluated. -See below -for a description. - -The special escape sequences -$( -and -$) -may be used to surround parts of a command line -that may change -without -causing a rebuild--that is, -which are not included in the &buildsig; -of target files built with this command. -All text between -$( -and -$) -will be removed from the command line -before it is added to the &buildsig; -and the -$( -and -$) -will be removed before the command is executed. -For example, the command line: - - -echo Last build occurred $( $TODAY $). > $TARGET - - -would execute the command: - - -echo Last build occurred $TODAY. > $TARGET - - -but the command portion of the -the &buildsig; computed for any target files built -by this action would be: - - -echo Last build occurred . > $TARGET - - -While &consvars; are normally directly substituted, -if a variable refers to a &consvar; -whose value is a callable &Python; object (a function -or a class with a __call__ method), -that object is called during substitution. -The callable must accept four arguments: -target, -source, -env and -for_signature. -source is a list of source nodes, -target is a list of target nodes, -env is the &consenv; to use for context, -and for_signature is -a Boolean value that tells the function -if it is being called for the purpose of generating a &buildsig;. -Since the &buildsig; is used for rebuild determination, -the function should omit variable elements -that do not affect whether a rebuild should be triggered -(see $( -and $) -above) if for_signature is true. - - - -&SCons; will insert whatever -the callable returns -into the expanded string: - - - -def foo(target, source, env, for_signature): - return "bar" - -# Will expand $BAR to "bar baz" -env = Environment(FOO=foo, BAR="$FOO baz") - - -As a reminder, substitution happens when -$BAR is actually used in a -builder action. The value of env['BAR'] -will be exactly as it was set: "$FOO baz". - - -You can use this feature to pass arguments to a -callable variable by creating a callable class -that stores passed arguments in the instance, -and then uses them -(in the __call__ method) -when the instance is called. -Note that in this case, -the entire variable expansion must -be enclosed by curly braces -so that the arguments will -be associated with the -instantiation of the class: - - -class foo: - def __init__(self, arg): - self.arg = arg - - def __call__(self, target, source, env, for_signature): - return self.arg + " bar" - -# Will expand $BAR to "my argument bar baz" -env=Environment(FOO=foo, BAR="${FOO('my argument')} baz") - @@ -6752,8 +6813,8 @@ env=Environment(FOO=foo, BAR="${FOO('my argument')} baz") If a substitutable expression using the notation -${something} does not appear to match one of -the other substitution patterns, +${expression} +does not appear to match one of the other substitution patterns, it is evaluated as a Python expression. This uses Python's eval function, with the globals parameter set to @@ -6819,45 +6880,6 @@ which &SCons; passes to eval which returns the value. -&SCons; uses the following rules when converting &consvars; into -command lines: - - - - string - -When the value is a string it is interpreted as a space delimited list of -command line arguments. - - - - - list - -When the value is a list it is interpreted as a list of command line -arguments. Each element of the list is converted to a string. - - - - - other - -Anything that is not a list or string is converted to a string and -interpreted as a single command line argument. - - - - - newline - -Newline characters (\n) delimit lines. -The newline parsing is done after -all other parsing, so it is not possible for arguments (e.g. file names) to -contain embedded newline characters. - - - - Use of the Python eval function is considered to have security implications, since, @@ -6917,7 +6939,7 @@ is optional, the default is no arg. The function can use use str(node) -to fetch the name of the file, +to fetch the name of the file, node.dir to fetch the directory the file is in, node.get_contents() @@ -6938,7 +6960,7 @@ in order to build Nodes with correct paths. Using &f-link-FindPathDirs; with an argument of CPPPATH as the path_function in the &f-Scanner; call means the scanner function will be called with the paths extracted -from &cv-CPPPATH; in the environment env +from &cv-CPPPATH; in the environment env passed as the paths parameter. -- cgit v0.12 From ff81eebb73825091051e5ee846d0d89c60798112 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 16 Aug 2019 08:19:20 -0600 Subject: Use super call instead of direct class call - super used where direct call to superclass existed - convert a few older-style super() (two-argument) uses - in a few places, where there was an intersection with a super change, variables that override a builtin (e.g. "dict") were renamed. Signed-off-by: Mats Wichmann --- CHANGES.txt | 3 + SCons/Action.py | 4 +- SCons/Builder.py | 13 ++-- SCons/Environment.py | 109 +++++++++++++++++--------------- SCons/Errors.py | 6 +- SCons/Job.py | 2 +- SCons/JobTests.py | 6 +- SCons/Memoize.py | 2 +- SCons/Node/Alias.py | 2 +- SCons/Node/FS.py | 12 ++-- SCons/Node/FSTests.py | 4 +- SCons/Node/NodeTests.py | 6 +- SCons/Node/Python.py | 2 +- SCons/SConf.py | 6 +- SCons/SConsign.py | 6 +- SCons/Subst.py | 4 +- SCons/SubstTests.py | 2 +- SCons/Tool/GettextCommon.py | 2 +- SCons/Tool/MSCommon/sdk.py | 4 +- SCons/Tool/msvs.py | 6 +- SCons/UtilTests.py | 2 +- SCons/cppTests.py | 2 +- bin/SConsDoc.py | 2 +- src/test_setup.py | 2 +- testing/framework/TestCommon.py | 2 +- testing/framework/TestRuntest.py | 2 +- testing/framework/TestSCons.py | 4 +- testing/framework/TestSCons_time.py | 2 +- testing/framework/TestSConsign.py | 2 +- testing/framework/TestUnit/taprunner.py | 14 ++-- 30 files changed, 122 insertions(+), 113 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d36e911..d19fb06 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -56,6 +56,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER processors for testing threads. - Fixed crash in C scanner's dictify_CPPDEFINES() function which happens if AppendUnique is called on CPPPATH. (Issue #4108). + - As "code modernization" all of SCons now uses the current super() + zero-argument syntax instead of direct calls to a parent class method + or the super() two-argument syntax. From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/SCons/Action.py b/SCons/Action.py index b7c6bb7..81dc033 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -848,7 +848,7 @@ class CommandAction(_ActionAction): # variables. if SCons.Debug.track_instances: logInstanceCreation(self, 'Action.CommandAction') - _ActionAction.__init__(self, **kw) + super().__init__(**kw) if is_List(cmd): if [c for c in cmd if is_List(c)]: raise TypeError("CommandAction should be given only " @@ -1231,7 +1231,7 @@ class FunctionAction(_ActionAction): # This is weird, just do the best we can. self.funccontents = _object_contents(execfunction) - _ActionAction.__init__(self, **kw) + super().__init__(**kw) def function_name(self): try: diff --git a/SCons/Builder.py b/SCons/Builder.py index 41475ac..ab51c32 100644 --- a/SCons/Builder.py +++ b/SCons/Builder.py @@ -130,8 +130,8 @@ class DictCmdGenerator(SCons.Util.Selector): to return the proper action based on the file suffix of the source file.""" - def __init__(self, dict=None, source_ext_match=1): - SCons.Util.Selector.__init__(self, dict) + def __init__(self, mapping=None, source_ext_match=True): + super().__init__(mapping) self.source_ext_match = source_ext_match def src_suffixes(self): @@ -222,10 +222,11 @@ class OverrideWarner(UserDict): can actually invoke multiple builders. This class only emits the warnings once, no matter how many Builders are invoked. """ - def __init__(self, dict): - UserDict.__init__(self, dict) + def __init__(self, mapping): + super().__init__(mapping) if SCons.Debug.track_instances: logInstanceCreation(self, 'Builder.OverrideWarner') self.already_warned = None + def warn(self): if self.already_warned: return @@ -245,7 +246,7 @@ def Builder(**kw): kw['action'] = SCons.Action.CommandGeneratorAction(kw['generator'], {}) del kw['generator'] elif 'action' in kw: - source_ext_match = kw.get('source_ext_match', 1) + source_ext_match = kw.get('source_ext_match', True) if 'source_ext_match' in kw: del kw['source_ext_match'] if SCons.Util.is_Dict(kw['action']): @@ -876,7 +877,7 @@ class CompositeBuilder(SCons.Util.Proxy): def __init__(self, builder, cmdgen): if SCons.Debug.track_instances: logInstanceCreation(self, 'Builder.CompositeBuilder') - SCons.Util.Proxy.__init__(self, builder) + super().__init__(builder) # cmdgen should always be an instance of DictCmdGenerator. self.cmdgen = cmdgen diff --git a/SCons/Environment.py b/SCons/Environment.py index d0a4baf..34e4968 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -275,12 +275,12 @@ class BuilderDict(UserDict): the Builders. We need to do this because every time someone changes the Builders in the Environment's BUILDERS dictionary, we must update the Environment's attributes.""" - def __init__(self, dict, env): + def __init__(self, mapping, env): # Set self.env before calling the superclass initialization, # because it will end up calling our other methods, which will # need to point the values in this dictionary to self.env. self.env = env - UserDict.__init__(self, dict) + super().__init__(mapping) def __semi_deepcopy__(self): # These cannot be copied since they would both modify the same builder object, and indeed @@ -294,15 +294,15 @@ class BuilderDict(UserDict): pass else: self.env.RemoveMethod(method) - UserDict.__setitem__(self, item, val) + super().__setitem__(item, val) BuilderWrapper(self.env, val, item) def __delitem__(self, item): - UserDict.__delitem__(self, item) + super().__delitem__(item) delattr(self.env, item) - def update(self, dict): - for i, v in dict.items(): + def update(self, mapping): + for i, v in mapping.items(): self.__setitem__(i, v) @@ -637,7 +637,7 @@ class SubstitutionEnvironment: it is assumed to be a command and the rest of the string is executed; the result of that evaluation is then added to the dict. """ - dict = { + mapping = { 'ASFLAGS' : CLVar(''), 'CFLAGS' : CLVar(''), 'CCFLAGS' : CLVar(''), @@ -667,12 +667,12 @@ class SubstitutionEnvironment: arg = self.backtick(arg[1:]) # utility function to deal with -D option - def append_define(name, dict = dict): + def append_define(name, mapping=mapping): t = name.split('=') if len(t) == 1: - dict['CPPDEFINES'].append(name) + mapping['CPPDEFINES'].append(name) else: - dict['CPPDEFINES'].append([t[0], '='.join(t[1:])]) + mapping['CPPDEFINES'].append([t[0], '='.join(t[1:])]) # Loop through the flags and add them to the appropriate option. # This tries to strike a balance between checking for all possible @@ -702,67 +702,67 @@ class SubstitutionEnvironment: append_define(arg) elif append_next_arg_to == '-include': t = ('-include', self.fs.File(arg)) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-imacros': t = ('-imacros', self.fs.File(arg)) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-isysroot': t = ('-isysroot', arg) - dict['CCFLAGS'].append(t) - dict['LINKFLAGS'].append(t) + mapping['CCFLAGS'].append(t) + mapping['LINKFLAGS'].append(t) elif append_next_arg_to == '-isystem': t = ('-isystem', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-iquote': t = ('-iquote', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-idirafter': t = ('-idirafter', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-arch': t = ('-arch', arg) - dict['CCFLAGS'].append(t) - dict['LINKFLAGS'].append(t) + mapping['CCFLAGS'].append(t) + mapping['LINKFLAGS'].append(t) elif append_next_arg_to == '--param': t = ('--param', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) else: - dict[append_next_arg_to].append(arg) + mapping[append_next_arg_to].append(arg) append_next_arg_to = None elif not arg[0] in ['-', '+']: - dict['LIBS'].append(self.fs.File(arg)) + mapping['LIBS'].append(self.fs.File(arg)) elif arg == '-dylib_file': - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) append_next_arg_to = 'LINKFLAGS' elif arg[:2] == '-L': if arg[2:]: - dict['LIBPATH'].append(arg[2:]) + mapping['LIBPATH'].append(arg[2:]) else: append_next_arg_to = 'LIBPATH' elif arg[:2] == '-l': if arg[2:]: - dict['LIBS'].append(arg[2:]) + mapping['LIBS'].append(arg[2:]) else: append_next_arg_to = 'LIBS' elif arg[:2] == '-I': if arg[2:]: - dict['CPPPATH'].append(arg[2:]) + mapping['CPPPATH'].append(arg[2:]) else: append_next_arg_to = 'CPPPATH' elif arg[:4] == '-Wa,': - dict['ASFLAGS'].append(arg[4:]) - dict['CCFLAGS'].append(arg) + mapping['ASFLAGS'].append(arg[4:]) + mapping['CCFLAGS'].append(arg) elif arg[:4] == '-Wl,': if arg[:11] == '-Wl,-rpath=': - dict['RPATH'].append(arg[11:]) + mapping['RPATH'].append(arg[11:]) elif arg[:7] == '-Wl,-R,': - dict['RPATH'].append(arg[7:]) + mapping['RPATH'].append(arg[7:]) elif arg[:6] == '-Wl,-R': - dict['RPATH'].append(arg[6:]) + mapping['RPATH'].append(arg[6:]) else: - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg[:4] == '-Wp,': - dict['CPPFLAGS'].append(arg) + mapping['CPPFLAGS'].append(arg) elif arg[:2] == '-D': if arg[2:]: append_define(arg[2:]) @@ -771,32 +771,32 @@ class SubstitutionEnvironment: elif arg == '-framework': append_next_arg_to = 'FRAMEWORKS' elif arg[:14] == '-frameworkdir=': - dict['FRAMEWORKPATH'].append(arg[14:]) + mapping['FRAMEWORKPATH'].append(arg[14:]) elif arg[:2] == '-F': if arg[2:]: - dict['FRAMEWORKPATH'].append(arg[2:]) + mapping['FRAMEWORKPATH'].append(arg[2:]) else: append_next_arg_to = 'FRAMEWORKPATH' - elif arg in [ + elif arg in ( '-mno-cygwin', '-pthread', '-openmp', '-fmerge-all-constants', '-fopenmp', - ]: - dict['CCFLAGS'].append(arg) - dict['LINKFLAGS'].append(arg) + ): + mapping['CCFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg == '-mwindows': - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg[:5] == '-std=': if '++' in arg[5:]: - key='CXXFLAGS' + key = 'CXXFLAGS' else: - key='CFLAGS' - dict[key].append(arg) + key = 'CFLAGS' + mapping[key].append(arg) elif arg[0] == '+': - dict['CCFLAGS'].append(arg) - dict['LINKFLAGS'].append(arg) + mapping['CCFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg in [ '-include', '-imacros', @@ -809,11 +809,11 @@ class SubstitutionEnvironment: ]: append_next_arg_to = arg else: - dict['CCFLAGS'].append(arg) + mapping['CCFLAGS'].append(arg) for arg in flags: do_parse(arg) - return dict + return mapping def MergeFlags(self, args, unique=True): """Merge flags into construction variables. @@ -2375,7 +2375,7 @@ class OverrideEnvironment(Base): if SCons.Debug.track_instances: logInstanceCreation(self, 'Environment.OverrideEnvironment') self.__dict__['__subject'] = subject if overrides is None: - self.__dict__['overrides'] = dict() + self.__dict__['overrides'] = {} else: self.__dict__['overrides'] = overrides @@ -2472,6 +2472,11 @@ class OverrideEnvironment(Base): self.__dict__['overrides'].update(other) def _update_onlynew(self, other): + """Update a dict with new keys. + + Unlike the .update method, if the key is already present, + it is not replaced. + """ for k, v in other.items(): if k not in self.__dict__['overrides']: self.__dict__['overrides'][k] = v @@ -2526,14 +2531,14 @@ def NoSubstitutionProxy(subject): del kwdict['executor'] else: kwdict['lvars'] = {} - def raw_to_mode(self, dict): + def raw_to_mode(self, mapping): try: - raw = dict['raw'] + raw = mapping['raw'] except KeyError: pass else: - del dict['raw'] - dict['mode'] = raw + del mapping['raw'] + mapping['mode'] = raw def subst(self, string, *args, **kwargs): return string def subst_kw(self, kw, *args, **kwargs): diff --git a/SCons/Errors.py b/SCons/Errors.py index 42db072..04cea38 100644 --- a/SCons/Errors.py +++ b/SCons/Errors.py @@ -99,8 +99,8 @@ class BuildError(Exception): self.action = action self.command = command - Exception.__init__(self, node, errstr, status, exitstatus, filename, - executor, action, command, exc_info) + super().__init__(node, errstr, status, exitstatus, filename, + executor, action, command, exc_info) def __str__(self): if self.filename: @@ -128,7 +128,7 @@ class ExplicitExit(Exception): self.node = node self.status = status self.exitstatus = status - Exception.__init__(self, *args) + super().__init__(*args) def convert_to_BuildError(status, exc_info=None): """Convert a return code to a BuildError Exception. diff --git a/SCons/Job.py b/SCons/Job.py index bcdb88a..b398790 100644 --- a/SCons/Job.py +++ b/SCons/Job.py @@ -238,7 +238,7 @@ else: and a boolean indicating whether the task executed successfully. """ def __init__(self, requestQueue, resultsQueue, interrupted): - threading.Thread.__init__(self) + super().__init__() self.daemon = True self.requestQueue = requestQueue self.resultsQueue = resultsQueue diff --git a/SCons/JobTests.py b/SCons/JobTests.py index 5b5a590..54d7fa4 100644 --- a/SCons/JobTests.py +++ b/SCons/JobTests.py @@ -410,13 +410,13 @@ class DummyNodeInfo: class testnode (SCons.Node.Node): def __init__(self): - SCons.Node.Node.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.executed self.ninfo = DummyNodeInfo() class goodnode (testnode): def __init__(self): - SCons.Node.Node.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.up_to_date self.ninfo = DummyNodeInfo() @@ -430,7 +430,7 @@ class slowgoodnode (goodnode): class badnode (goodnode): def __init__(self): - goodnode.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.failed def build(self, **kw): raise Exception('badnode exception') diff --git a/SCons/Memoize.py b/SCons/Memoize.py index 8c3303f..b02c144 100644 --- a/SCons/Memoize.py +++ b/SCons/Memoize.py @@ -159,7 +159,7 @@ class CountDict(Counter): def __init__(self, cls_name, method_name, keymaker): """ """ - Counter.__init__(self, cls_name, method_name) + super().__init__(cls_name, method_name) self.keymaker = keymaker def count(self, *args, **kw): """ Counts whether the computed key value is already present diff --git a/SCons/Node/Alias.py b/SCons/Node/Alias.py index b5e4eb4..1125c22 100644 --- a/SCons/Node/Alias.py +++ b/SCons/Node/Alias.py @@ -99,7 +99,7 @@ class Alias(SCons.Node.Node): BuildInfo = AliasBuildInfo def __init__(self, name): - SCons.Node.Node.__init__(self) + super().__init__() self.name = name self.changed_since_last_build = 1 self.store_info = 0 diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 5f05a86..c1bfd6a 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -82,7 +82,7 @@ class EntryProxyAttributeError(AttributeError): of the underlying Entry involved in an AttributeError exception. """ def __init__(self, entry_proxy, attribute): - AttributeError.__init__(self) + super().__init__() self.entry_proxy = entry_proxy self.attribute = attribute def __str__(self): @@ -579,7 +579,7 @@ class Base(SCons.Node.Node): signatures.""" if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.Base') - SCons.Node.Node.__init__(self) + super().__init__() # Filenames and paths are probably reused and are intern'ed to save some memory. # Filename with extension as it was specified when the object was @@ -982,7 +982,7 @@ class Entry(Base): 'contentsig'] def __init__(self, name, directory, fs): - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._func_exists = 3 self._func_get_contents = 1 @@ -1574,7 +1574,7 @@ class Dir(Base): def __init__(self, name, directory, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.Dir') - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._morph() def _morph(self): @@ -2563,7 +2563,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): if key != 'dependency_map' and hasattr(self, 'dependency_map'): del self.dependency_map - return super(FileBuildInfo, self).__setattr__(key, value) + return super().__setattr__(key, value) def convert_to_sconsign(self): """ @@ -2674,7 +2674,7 @@ class File(Base): def __init__(self, name, directory, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.File') - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._morph() def Entry(self, name): diff --git a/SCons/Node/FSTests.py b/SCons/Node/FSTests.py index d78940e..b93efc8 100644 --- a/SCons/Node/FSTests.py +++ b/SCons/Node/FSTests.py @@ -2568,7 +2568,7 @@ class FileTestCase(_tempdirTestCase): class ChangedNode(SCons.Node.FS.File): def __init__(self, name, directory=None, fs=None): - SCons.Node.FS.File.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self.name = name self.Tag('found_includes', []) self.stored_info = None @@ -2608,7 +2608,7 @@ class FileTestCase(_tempdirTestCase): class ChangedEnvironment(SCons.Environment.Base): def __init__(self): - SCons.Environment.Base.__init__(self) + super().__init__() self.decide_source = self._changed_timestamp_then_content class FakeNodeInfo: diff --git a/SCons/Node/NodeTests.py b/SCons/Node/NodeTests.py index b36c2e9..ee4d080 100644 --- a/SCons/Node/NodeTests.py +++ b/SCons/Node/NodeTests.py @@ -160,7 +160,7 @@ class NoneBuilder(Builder): class ListBuilder(Builder): def __init__(self, *nodes): - Builder.__init__(self) + super().__init__() self.nodes = nodes def execute(self, target, source, env): if hasattr(self, 'status'): @@ -200,7 +200,7 @@ class MyNode(SCons.Node.Node): simulate a real, functional Node subclass. """ def __init__(self, name): - SCons.Node.Node.__init__(self) + super().__init__() self.name = name self.Tag('found_includes', []) def __str__(self): @@ -1226,7 +1226,7 @@ class NodeTestCase(unittest.TestCase): """Test the get_string() method.""" class TestNode(MyNode): def __init__(self, name, sig): - MyNode.__init__(self, name) + super().__init__(name) self.sig = sig def for_signature(self): diff --git a/SCons/Node/Python.py b/SCons/Node/Python.py index c6850ab..80d2762 100644 --- a/SCons/Node/Python.py +++ b/SCons/Node/Python.py @@ -83,7 +83,7 @@ class Value(SCons.Node.Node): BuildInfo = ValueBuildInfo def __init__(self, value, built_value=None, name=None): - SCons.Node.Node.__init__(self) + super().__init__() self.value = value self.changed_since_last_build = 6 self.store_info = 0 diff --git a/SCons/SConf.py b/SCons/SConf.py index 0ad712d..ad29153 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -142,7 +142,7 @@ SCons.Warnings.enableWarningClass(SConfWarning) # some error definitions class SConfError(SCons.Errors.UserError): def __init__(self,msg): - SCons.Errors.UserError.__init__(self,msg) + super().__init__(msg) class ConfigureDryRunError(SConfError): """Raised when a file or directory needs to be updated during a Configure @@ -152,13 +152,13 @@ class ConfigureDryRunError(SConfError): msg = 'Cannot create configure directory "%s" within a dry-run.' % str(target) else: msg = 'Cannot update configure test "%s" within a dry-run.' % str(target) - SConfError.__init__(self,msg) + super().__init__(msg) class ConfigureCacheError(SConfError): """Raised when a use explicitely requested the cache feature, but the test is run the first time.""" def __init__(self,target): - SConfError.__init__(self, '"%s" is not yet built and cache is forced.' % str(target)) + super().__init__('"%s" is not yet built and cache is forced.' % str(target)) # define actions for building text files diff --git a/SCons/SConsign.py b/SCons/SConsign.py index 5b78855..ecca391 100644 --- a/SCons/SConsign.py +++ b/SCons/SConsign.py @@ -248,7 +248,7 @@ class DB(Base): determined by the database module. """ def __init__(self, dir): - Base.__init__(self) + super().__init__() self.dir = dir @@ -319,7 +319,7 @@ class Dir(Base): """ fp - file pointer to read entries from """ - Base.__init__(self) + super().__init__() if not fp: return @@ -352,7 +352,7 @@ class DirFile(Dir): fp = None try: - Dir.__init__(self, fp, dir) + super().__init__(fp, dir) except KeyboardInterrupt: raise except Exception: diff --git a/SCons/Subst.py b/SCons/Subst.py index 298df38..7535772 100644 --- a/SCons/Subst.py +++ b/SCons/Subst.py @@ -132,7 +132,7 @@ class CmdStringHolder(collections.UserString): proper escape sequences inserted. """ def __init__(self, cmd, literal=None): - collections.UserString.__init__(self, cmd) + super().__init__(cmd) self.literal = literal def is_literal(self): @@ -490,7 +490,7 @@ class ListSubber(collections.UserList): internally. """ def __init__(self, env, mode, conv, gvars): - collections.UserList.__init__(self, []) + super().__init__([]) self.env = env self.mode = mode self.conv = conv diff --git a/SCons/SubstTests.py b/SCons/SubstTests.py index b956a25..1a203a0 100644 --- a/SCons/SubstTests.py +++ b/SCons/SubstTests.py @@ -121,7 +121,7 @@ class SubstTestCase(unittest.TestCase): class MyNode(DummyNode): """Simple node work-alike with some extra stuff for testing.""" def __init__(self, name): - DummyNode.__init__(self, name) + super().__init__(name) class Attribute: pass self.attribute = Attribute() diff --git a/SCons/Tool/GettextCommon.py b/SCons/Tool/GettextCommon.py index 16900e0..058b997 100644 --- a/SCons/Tool/GettextCommon.py +++ b/SCons/Tool/GettextCommon.py @@ -206,7 +206,7 @@ class _POFileBuilder(BuilderBase): del kw['target_alias'] if 'target_factory' not in kw: kw['target_factory'] = _POTargetFactory(env, alias=alias).File - BuilderBase.__init__(self, **kw) + super().__init__(**kw) def _execute(self, env, target, source, *args, **kw): """ Execute builder's actions. diff --git a/SCons/Tool/MSCommon/sdk.py b/SCons/Tool/MSCommon/sdk.py index d95df91..aa94f4d 100644 --- a/SCons/Tool/MSCommon/sdk.py +++ b/SCons/Tool/MSCommon/sdk.py @@ -131,7 +131,7 @@ class WindowsSDK(SDKDefinition): """ HKEY_FMT = r'Software\Microsoft\Microsoft SDKs\Windows\v%s\InstallationFolder' def __init__(self, *args, **kw): - SDKDefinition.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.hkey_data = self.version class PlatformSDK(SDKDefinition): @@ -140,7 +140,7 @@ class PlatformSDK(SDKDefinition): """ HKEY_FMT = r'Software\Microsoft\MicrosoftSDK\InstalledSDKS\%s\Install Dir' def __init__(self, *args, **kw): - SDKDefinition.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.hkey_data = self.uuid # diff --git a/SCons/Tool/msvs.py b/SCons/Tool/msvs.py index 7a827f4..887cb59 100644 --- a/SCons/Tool/msvs.py +++ b/SCons/Tool/msvs.py @@ -320,7 +320,7 @@ class _GenerateV7User(_UserGenerator): self.usrhead = V9UserHeader self.usrconf = V9UserConfiguration self.usrdebg = V9DebugSettings - _UserGenerator.__init__(self, dspfile, source, env) + super().__init__(dspfile, source, env) def UserProject(self): confkeys = sorted(self.configs.keys()) @@ -395,7 +395,7 @@ class _GenerateV10User(_UserGenerator): self.usrhead = V10UserHeader self.usrconf = V10UserConfiguration self.usrdebg = V10DebugSettings - _UserGenerator.__init__(self, dspfile, source, env) + super().__init__(dspfile, source, env) def UserProject(self): confkeys = sorted(self.configs.keys()) @@ -1487,7 +1487,7 @@ class _DSWGenerator: class _GenerateV7DSW(_DSWGenerator): """Generates a Solution file for MSVS .NET""" def __init__(self, dswfile, source, env): - _DSWGenerator.__init__(self, dswfile, source, env) + super().__init__(dswfile, source, env) self.file = None self.version = self.env['MSVS_VERSION'] diff --git a/SCons/UtilTests.py b/SCons/UtilTests.py index 5ed31cb..0cfb70f 100644 --- a/SCons/UtilTests.py +++ b/SCons/UtilTests.py @@ -899,7 +899,7 @@ class HashTestCase(unittest.TestCase): class FIPSHashTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): - super(FIPSHashTestCase, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) ############################### # algorithm mocks, can check if we called with usedforsecurity=False for python >= 3.9 diff --git a/SCons/cppTests.py b/SCons/cppTests.py index 99fa6cf..a9aef9d 100644 --- a/SCons/cppTests.py +++ b/SCons/cppTests.py @@ -853,7 +853,7 @@ class fileTestCase(unittest.TestCase): """) class MyPreProcessor(cpp.DumbPreProcessor): def __init__(self, *args, **kw): - cpp.DumbPreProcessor.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.files = [] def __call__(self, file): self.files.append(file) diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index a465057..82e2c72 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -499,7 +499,7 @@ class Function(Item): class Tool(Item): def __init__(self, name): - Item.__init__(self, name) + super().__init__(name) self.entity = self.name.replace('+', 'X') diff --git a/src/test_setup.py b/src/test_setup.py index 8410be3..5db21a1 100644 --- a/src/test_setup.py +++ b/src/test_setup.py @@ -81,7 +81,7 @@ class MyTestSCons(TestSCons.TestSCons): ] def __init__(self): - TestSCons.TestSCons.__init__(self) + super().__init__() self.root = self.workpath('root') self.prefix = self.root + os.path.splitdrive(sys.prefix)[1] diff --git a/testing/framework/TestCommon.py b/testing/framework/TestCommon.py index 2eeaad0..1999f74 100644 --- a/testing/framework/TestCommon.py +++ b/testing/framework/TestCommon.py @@ -261,7 +261,7 @@ class TestCommon(TestCmd): calling the base class initialization, and then changing directory to the workdir. """ - TestCmd.__init__(self, **kw) + super().__init__(**kw) os.chdir(self.workdir) def options_arguments(self, options, arguments): diff --git a/testing/framework/TestRuntest.py b/testing/framework/TestRuntest.py index 18dcb94..9368953 100644 --- a/testing/framework/TestRuntest.py +++ b/testing/framework/TestRuntest.py @@ -146,7 +146,7 @@ class TestRuntest(TestCommon): del kw['things_to_copy'] orig_cwd = os.getcwd() - TestCommon.__init__(self, **kw) + super().__init__(**kw) dirs = [os.environ.get('SCONS_RUNTEST_DIR', orig_cwd)] diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index 0bf6abd..079e17d 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -321,7 +321,7 @@ class TestSCons(TestCommon): if kw.get('ignore_python_version', -1) != -1: del kw['ignore_python_version'] - TestCommon.__init__(self, **kw) + super().__init__(**kw) if not self.external: import SCons.Node.FS @@ -1755,7 +1755,7 @@ class TimeSCons(TestSCons): if 'verbose' not in kw and not self.calibrate: kw['verbose'] = True - TestSCons.__init__(self, *args, **kw) + super().__init__(*args, **kw) # TODO(sgk): better way to get the script dir than sys.argv[0] self.test_dir = os.path.dirname(sys.argv[0]) diff --git a/testing/framework/TestSCons_time.py b/testing/framework/TestSCons_time.py index a57ca88..e647fe2 100644 --- a/testing/framework/TestSCons_time.py +++ b/testing/framework/TestSCons_time.py @@ -199,7 +199,7 @@ class TestSCons_time(TestCommon): if 'workdir' not in kw: kw['workdir'] = '' - TestCommon.__init__(self, **kw) + super().__init__(**kw) def archive_split(self, path): if path[-7:] == '.tar.gz': diff --git a/testing/framework/TestSConsign.py b/testing/framework/TestSConsign.py index 699e929..b0562bf 100644 --- a/testing/framework/TestSConsign.py +++ b/testing/framework/TestSConsign.py @@ -64,7 +64,7 @@ class TestSConsign(TestSCons): os.chdir(script_dir) self.script_dir = os.getcwd() - TestSCons.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.my_kw = { 'interpreter' : python, # imported from TestSCons diff --git a/testing/framework/TestUnit/taprunner.py b/testing/framework/TestUnit/taprunner.py index 0dde327..001db5c 100644 --- a/testing/framework/TestUnit/taprunner.py +++ b/testing/framework/TestUnit/taprunner.py @@ -43,29 +43,29 @@ class TAPTestResult(TextTestResult): self.stream.flush() def addSuccess(self, test): - super(TextTestResult, self).addSuccess(test) + super().addSuccess(test) self._process(test, "ok") def addFailure(self, test, err): - super(TextTestResult, self).addFailure(test, err) + super().addFailure(test, err) self._process(test, "not ok", "FAIL") # [ ] add structured data about assertion def addError(self, test, err): - super(TextTestResult, self).addError(test, err) + super().addError(test, err) self._process(test, "not ok", "ERROR") # [ ] add structured data about exception def addSkip(self, test, reason): - super(TextTestResult, self).addSkip(test, reason) + super().addSkip(test, reason) self._process(test, "ok", directive=(" # SKIP %s" % reason)) def addExpectedFailure(self, test, err): - super(TextTestResult, self).addExpectedFailure(test, err) + super().addExpectedFailure(test, err) self._process(test, "not ok", directive=" # TODO") def addUnexpectedSuccess(self, test): - super(TextTestResult, self).addUnexpectedSuccess(test) + super().addUnexpectedSuccess(test) self._process(test, "not ok", "FAIL (unexpected success)") """ @@ -90,7 +90,7 @@ class TAPTestRunner(TextTestRunner): for case in test: case.suite = test - return super(TAPTestRunner, self).run(test) + return super().run(test) if __name__ == "__main__": -- cgit v0.12 From 5b0b9b89151f9abb0ed0ae7f47adfe3718088c3c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 15 Mar 2022 08:52:06 -0600 Subject: Fix sider complaint It wants blanks between method definitions, this was one of the compressed proxy classes which was "touched" (not changed). Signed-off-by: Mats Wichmann --- SCons/Environment.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/SCons/Environment.py b/SCons/Environment.py index 34e4968..3507263 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -2521,16 +2521,20 @@ def NoSubstitutionProxy(subject): class _NoSubstitutionProxy(Environment): def __init__(self, subject): self.__dict__['__subject'] = subject + def __getattr__(self, name): return getattr(self.__dict__['__subject'], name) + def __setattr__(self, name, value): return setattr(self.__dict__['__subject'], name, value) + def executor_to_lvars(self, kwdict): if 'executor' in kwdict: kwdict['lvars'] = kwdict['executor'].get_lvars() del kwdict['executor'] else: kwdict['lvars'] = {} + def raw_to_mode(self, mapping): try: raw = mapping['raw'] @@ -2539,10 +2543,13 @@ def NoSubstitutionProxy(subject): else: del mapping['raw'] mapping['mode'] = raw + def subst(self, string, *args, **kwargs): return string + def subst_kw(self, kw, *args, **kwargs): return kw + def subst_list(self, string, *args, **kwargs): nargs = (string, self,) + args nkw = kwargs.copy() @@ -2550,6 +2557,7 @@ def NoSubstitutionProxy(subject): self.executor_to_lvars(nkw) self.raw_to_mode(nkw) return SCons.Subst.scons_subst_list(*nargs, **nkw) + def subst_target_source(self, string, *args, **kwargs): nargs = (string, self,) + args nkw = kwargs.copy() @@ -2557,6 +2565,7 @@ def NoSubstitutionProxy(subject): self.executor_to_lvars(nkw) self.raw_to_mode(nkw) return SCons.Subst.scons_subst(*nargs, **nkw) + return _NoSubstitutionProxy(subject) # Local Variables: -- cgit v0.12 From 17c7e6a59b311b1f3b8c5daedc749f815b2f9d74 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 15 Mar 2022 15:13:50 -0500 Subject: updated startup sequence to check if server process died during start. --- SCons/Tool/ninja/NinjaState.py | 2 +- SCons/Tool/ninja/ninja_run_daemon.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 11e928d..9fb697e 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -218,7 +218,7 @@ class NinjaState: }, "SCONS_DAEMON": { - "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {scons_daemon_port} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {scons_daemon_port} {env.get('NINJA_DIR').abspath} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", "description": "Starting scons daemon...", "pool": "local_pool", # restat diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 965e50b..fbeaf67 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -90,12 +90,15 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): if status != 200: print(msg.decode("utf-8")) exit(1) - logging.debug(f"Request Done: {sys.argv[3]}") + logging.debug(f"Server Responded it was ready!") break except ConnectionRefusedError: - logging.debug(f"Server not ready: {sys.argv[3]}") + logging.debug(f"Server not ready, server PID: {p.pid}") time.sleep(1) + if p.poll is not None: + logging.debug(f"Server process died, aborting: {p.returncode}") + sys.exit(p.returncode) except ConnectionResetError: logging.debug("Server ConnectionResetError") sys.stderr.write(error_msg) -- cgit v0.12 From 0a1a6f14c8c954fa968b6d911fe1a3ad6d4881f8 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 15 Mar 2022 16:12:07 -0500 Subject: adjust iterative speedup to pre-launch scons daemon before taking timing --- SCons/Tool/ninja/NinjaState.py | 3 ++- SCons/Tool/ninja/__init__.py | 4 ++++ test/ninja/iterative_speedup.py | 10 ++++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 9fb697e..fff60be 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -534,10 +534,11 @@ class NinjaState: ) ninja.build( - ["run_scons_daemon", scons_daemon_dirty], + ["run_ninja_scons_daemon_phony", scons_daemon_dirty], rule="SCONS_DAEMON", ) + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) pidfile = None if os.path.exists(scons_daemon_dirty): diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 913738a..9734ffc 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -206,6 +206,7 @@ def generate(env): SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] + def ninja_generate_deps(env): """Return a list of SConscripts TODO: Should we also include files loaded from site_scons/*** @@ -456,3 +457,6 @@ def generate(env): env['TEMPFILEDIR'] = "$NINJA_DIR/.response_files" env["TEMPFILE"] = NinjaNoResponseFiles + + + env.Alias('run-ninja-scons-daemon', 'run_ninja_scons_daemon_phony') diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 05e372c..e5673b0 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -109,7 +109,7 @@ def generate_source(parent_source, current_source): test.write('source_{}.h'.format(current_source), """ #include #include - + int print_function%(current_source)s(); """ % locals()) @@ -125,7 +125,7 @@ def mod_source_return(test_num): int print_function%(test_num)s() - { + { int test = 5 + 5; print_function%(parent_source)s(); return test; @@ -143,7 +143,7 @@ def mod_source_orig(test_num): int print_function%(test_num)s() - { + { return print_function%(parent_source)s(); } """ % locals()) @@ -190,13 +190,15 @@ jobs = '-j' + str(get_num_cpus()) ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] -start = time.perf_counter() test.run(arguments='--disable-execute-ninja', stdout=None) +test.run(program=ninja_program, arguments='run-ninja-scons-daemon', stdout=None) +start = time.perf_counter() test.run(program=ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] test.run(program=test.workpath('print_bin'), stdout="main print") + for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() -- cgit v0.12 From 77ba2d3b905b284350d5773b6fb06d6abcdf5236 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 16 Mar 2022 01:02:07 -0500 Subject: Added doc strings for ninja scripts --- SCons/Tool/ninja/ninja_daemon_build.py | 15 +++++++++------ SCons/Tool/ninja/ninja_run_daemon.py | 13 ++++++++++++- SCons/Tool/ninja/ninja_scons_daemon.py | 11 ++++++++++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index 6e1c7be..f34935b 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# # MIT License # # Copyright The SCons Foundation @@ -22,7 +24,10 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - +This script is intended to execute a single build target. This script should be +called by ninja, passing the port, ninja dir, and build target via arguments. +The script then executes a simple get request to the scons daemon which is listening +on from localhost on the set port. """ import http.client @@ -74,14 +79,12 @@ while True: logging.debug(f"Request Done: {sys.argv[3]}") exit(0) - except ConnectionRefusedError: - logging.debug(f"Server not ready: {traceback.format_exc()}") - time.sleep(1) - except ConnectionResetError: - logging.debug("Server ConnectionResetError") + except Exception: + logging.debug(f"Failed to send command: {traceback.format_exc()}") exit(1) + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index fbeaf67..68d9893 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# # MIT License # # Copyright The SCons Foundation @@ -22,7 +24,16 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +This script is intended to be called by ninja to start up the scons daemon process. It will +launch the server and attempt to connect to it. This process needs to completely detach +from the spawned process so ninja can consider the build edge completed. It should be passed +the args which should be forwarded to the scons daemon process which could be any number of +# arguments. However the first few arguements are required to be port, ninja dir, and keep alive +timeout in seconds. +The scons_daemon_dirty file acts as a pidfile marker letting this script quickly skip over +restarting the server if the server is running. The assumption here is the pidfile should only +exist if the server is running. """ import subprocess @@ -90,7 +101,7 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): if status != 200: print(msg.decode("utf-8")) exit(1) - logging.debug(f"Server Responded it was ready!") + logging.debug("Server Responded it was ready!") break except ConnectionRefusedError: diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 99fcbb9..88f3b22 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# # MIT License # # Copyright The SCons Foundation @@ -22,7 +24,14 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - +This script primarily consists of two threads, the http server thread and the scons interactive +process thread. The http server thread will listen on the passed port for http get request +which should indicate some action for the scons interactive process to take. + +The daemon will keep log files in a tmp directory correlated to the hash of the absolute path +of the ninja build dir passed. The daemon will also use a keep alive time to know when to shut +itself down after the passed timeout of no activity. Any time the server receives a get request, +the keep alive time will be reset. """ import http.server -- cgit v0.12 From 77a1e99d356990b03c2be014658a072a5d2be4c1 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 16 Mar 2022 01:08:36 -0500 Subject: fix spelling mistake --- SCons/Tool/ninja/ninja_run_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 68d9893..057537a 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -28,7 +28,7 @@ This script is intended to be called by ninja to start up the scons daemon proce launch the server and attempt to connect to it. This process needs to completely detach from the spawned process so ninja can consider the build edge completed. It should be passed the args which should be forwarded to the scons daemon process which could be any number of -# arguments. However the first few arguements are required to be port, ninja dir, and keep alive +# arguments. However the first few arguments are required to be port, ninja dir, and keep alive timeout in seconds. The scons_daemon_dirty file acts as a pidfile marker letting this script quickly skip over -- cgit v0.12 From c47a00687a2753c00af8682f00bfe671671074b5 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 17 Mar 2022 12:36:44 -0600 Subject: Add simple unittest for -include Missing test was flagged by code coverage because the line was "touched" due to a variable rename. Signed-off-by: Mats Wichmann --- SCons/EnvironmentTests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index c505422..8359405 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -810,11 +810,13 @@ sys.exit(0) "-fmerge-all-constants " "-fopenmp " "-mno-cygwin -mwindows " - "-arch i386 -isysroot /tmp " + "-arch i386 " + "-isysroot /tmp " "-iquote /usr/include/foo1 " "-isystem /usr/include/foo2 " "-idirafter /usr/include/foo3 " "-imacros /usr/include/foo4 " + "-include /usr/include/foo5 " "--param l1-cache-size=32 --param l2-cache-size=6144 " "+DD64 " "-DFOO -DBAR=value -D BAZ " @@ -832,6 +834,7 @@ sys.exit(0) ('-isystem', '/usr/include/foo2'), ('-idirafter', '/usr/include/foo3'), ('-imacros', env.fs.File('/usr/include/foo4')), + ('-include', env.fs.File('/usr/include/foo5')), ('--param', 'l1-cache-size=32'), ('--param', 'l2-cache-size=6144'), '+DD64'], repr(d['CCFLAGS']) assert d['CXXFLAGS'] == ['-std=c++0x'], repr(d['CXXFLAGS']) -- cgit v0.12 From 915bfb3a55619175e03c3dc3573ad5ba83611e75 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 18 Mar 2022 17:19:04 -0700 Subject: flake8 fixes --- test/ninja/ninja_handle_control_c_rebuild.py | 2 +- test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/ninja/ninja_handle_control_c_rebuild.py b/test/ninja/ninja_handle_control_c_rebuild.py index c2c8c8d..9f6b413 100644 --- a/test/ninja/ninja_handle_control_c_rebuild.py +++ b/test/ninja/ninja_handle_control_c_rebuild.py @@ -22,7 +22,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # """ -This test ensures if ninja get's a control-c (or other interrupting signal) while +This test ensures if ninja gets a control-c (or other interrupting signal) while regenerating the build.ninja, it doesn't remove the build.ninja leaving it in an unworkable state. """ diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja b/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja index 0124576..34d7872 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja +++ b/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja @@ -1,7 +1,7 @@ import os import signal -SetOption('experimental','ninja') +SetOption('experimental', 'ninja') DefaultEnvironment(tools=[]) env = Environment() @@ -9,4 +9,4 @@ env.Tool('ninja') env.Program(target='foo', source='foo.c') if ARGUMENTS.get('NINJA_DISABLE_AUTO_RUN', 0): - os.kill(os.getppid(),signal.SIGINT ) + os.kill(os.getppid(), signal.SIGINT) -- cgit v0.12 From ec47a7085b922d977c13f1e309ee549af43b0782 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 18 Mar 2022 17:23:08 -0700 Subject: Only swap path dirsep if needed --- SCons/Tool/ninja/Methods.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index c89c42f..4c6aa2f 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -257,7 +257,10 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom ) cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] - rsp_content = [rsp_content_item.replace('\\', '/') for rsp_content_item in rsp_content] + + # Canonicalize the path to have forward (posix style) dir sep characters. + if os.altsep: + rsp_content = [rsp_content_item.replace(os.sep, os.altsep) for rsp_content_item in rsp_content] rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] rsp_content = " ".join(rsp_content) -- cgit v0.12 From f7e544e5aac6656775993ab97bebfb2948e7e318 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 18 Mar 2022 08:21:53 -0600 Subject: Guide: update Writing Builders chapter [skip appveyor] Use entities more consistently. Reword a few things. Text implied that site_init.py needs to be manually imported, adjusted wording not to say so. Signed-off-by: Mats Wichmann --- doc/scons.mod | 24 +++---- doc/user/builders-writing.xml | 146 +++++++++++++++++++++--------------------- 2 files changed, 85 insertions(+), 85 deletions(-) diff --git a/doc/scons.mod b/doc/scons.mod index 5579d4d..8a6df17 100644 --- a/doc/scons.mod +++ b/doc/scons.mod @@ -440,15 +440,15 @@ Dictionary"> -Emitter"> -emitter"> +Emitter"> +emitter"> -factory"> +factory"> -Generator"> -generator"> +Generator"> +generator"> -Nodes"> +Nodes"> content signature"> content signatures"> @@ -462,12 +462,12 @@ -action="> -batch_key="> -cmdstr="> -exitstatfunc="> -strfunction="> -varlist="> +action="> +batch_key="> +cmdstr="> +exitstatfunc="> +strfunction="> +varlist="> diff --git a/doc/user/builders-writing.xml b/doc/user/builders-writing.xml index 5a8851d..a53e70e 100644 --- a/doc/user/builders-writing.xml +++ b/doc/user/builders-writing.xml @@ -2,7 +2,7 @@ %scons; - + %builders-mod; @@ -81,14 +81,14 @@ -bld = Builder(action = 'foobuild < $SOURCE > $TARGET') +bld = Builder(action='foobuild < $SOURCE > $TARGET') All the above line does is create a free-standing &Builder; object. - The next section will show us how to actually use it. + The next section will show how to actually use it. @@ -105,7 +105,7 @@ bld = Builder(action = 'foobuild < $SOURCE > $TARGET') for files to be built. This is done through the &cv-link-BUILDERS; &consvar; in an environment. - The &cv-BUILDERS; variable is a Python dictionary + The &cv-link-BUILDERS; variable is a &Python; dictionary that maps the names by which you want to call various &Builder; objects to the objects themselves. For example, if we want to call the @@ -221,7 +221,7 @@ hello.c To be able to use both our own defined &Builder; objects and the default &Builder; objects in the same &consenv;, - you can either add to the &cv-BUILDERS; variable + you can either add to the &cv-link-BUILDERS; variable using the &Append; function: @@ -296,8 +296,8 @@ env.Program('hello.c') suffixes to the target and/or the source file. For example, rather than having to specify explicitly that you want the Foo - &Builder; to build the file.foo - target file from the file.input source file, + &Builder; to build the file.foo + target file from the file.input source file, you can give the .foo and .input suffixes to the &Builder;, making for more compact and readable calls to @@ -361,7 +361,7 @@ env.Foo('file2') In &SCons;, you don't have to call an external command to build a file. - You can, instead, define a Python function + You can, instead, define a &Python; function that a &Builder; object can invoke to build your target file (or files). Such a &buildfunc; definition looks like: @@ -383,7 +383,7 @@ def build_function(target, source, env): - target + target @@ -392,14 +392,14 @@ def build_function(target, source, env): the target or targets to be built by this function. The file names of these target(s) - may be extracted using the Python &str; function. + may be extracted using the &Python; &str; function. - source + source @@ -408,21 +408,21 @@ def build_function(target, source, env): the sources to be used by this function to build the targets. The file names of these source(s) - may be extracted using the Python &str; function. + may be extracted using the &Python; &str; function. - env + env The &consenv; used for building the target(s). The function may use any of the - environment's construction variables + environment's &consvars; in any way to affect how it builds the targets. @@ -446,7 +446,7 @@ def build_function(target, source, env): - Once you've defined the Python function + Once you've defined the &Python; function that will build your target file, defining a &Builder; object for it is as simple as specifying the name of the function, @@ -479,7 +479,7 @@ file.input And notice that the output changes slightly, - reflecting the fact that a Python function, + reflecting the fact that a &Python; function, not an external command, is now called to build the target file: @@ -497,8 +497,8 @@ file.input &SCons; Builder objects can create an action "on the fly" - by using a function called a &generator;. - (Note: this is not the same thing as a Python generator function + by using a function called a &Generator;. + (Note: this is not the same thing as a &Python; generator function described in PEP 255) This provides a great deal of flexibility to construct just the right list of commands @@ -521,7 +521,7 @@ def generate_actions(source, target, env, for_signature): - source + source @@ -531,7 +531,7 @@ def generate_actions(source, target, env, for_signature): by the command or other action generated by this function. The file names of these source(s) - may be extracted using the Python &str; function. + may be extracted using the &Python; &str; function. @@ -539,7 +539,7 @@ def generate_actions(source, target, env, for_signature): - target + target @@ -549,7 +549,7 @@ def generate_actions(source, target, env, for_signature): by the command or other action generated by this function. The file names of these target(s) - may be extracted using the Python &str; function. + may be extracted using the &Python; &str; function. @@ -557,14 +557,14 @@ def generate_actions(source, target, env, for_signature): - env + env The &consenv; used for building the target(s). - The generator may use any of the - environment's construction variables + The &generator; may use any of the + environment's &consvars; in any way to determine what command or other action to return. @@ -574,13 +574,13 @@ def generate_actions(source, target, env, for_signature): - for_signature + for_signature A flag that specifies whether the - generator is being called to contribute to a build signature, + &generator; is being called to contribute to a &buildsig;, as opposed to actually executing the command. @@ -604,8 +604,8 @@ def generate_actions(source, target, env, for_signature): Once you've defined a &generator;, you create a &Builder; to use it - by specifying the generator keyword argument - instead of action. + by specifying the generator keyword argument + instead of action. @@ -652,9 +652,9 @@ env.Foo('file') Note that it's illegal to specify both an - action + action and a - generator + generator for a &Builder;. @@ -672,7 +672,7 @@ env.Foo('file') that takes as its arguments the list of the targets passed to the builder, the list of the sources passed to the builder, - and the construction environment. + and the &consenv;. The emitter function should return the modified lists of targets that should be built and sources from which the targets will be built. @@ -739,7 +739,7 @@ env.Foo('file') - + And would yield the following output: @@ -751,16 +751,15 @@ env.Foo('file') One very flexible thing that you can do is - use a construction variable to specify - different emitter functions for different - construction variable. + use a &consvar; to specify + different emitter functions for different &consenvs;. To do this, specify a string - containing a construction variable + containing a &consvar; expansion as the emitter when you call the &f-link-Builder; function, - and set that construction variable to + and set that &consvar; to the desired emitter function - in different construction environments: + in different &consenvs;: @@ -827,9 +826,9 @@ cat is a powerful concept, but sometimes all you really want is to be able to use an existing builder but change its concept of what targets are created. - In this case, + In this case, trying to recreate the logic of an existing Builder to - supply a special emitter can be a lot of work. + supply a special emitter can be a lot of work. The typical case for this is when you want to use a compiler flag that causes additional files to be generated. For example the GNU linker accepts an option @@ -844,12 +843,12 @@ cat To help with this, &SCons; provides &consvars; which correspond - to a few standard builders: - &cv-link-PROGEMITTER; for &b-link-Program;; - &cv-link-LIBEMITTER; for &b-link-Library;; - &cv-link-SHLIBEMITTER; for &b-link-SharedLibrary; and + to a few standard builders: + &cv-link-PROGEMITTER; for &b-link-Program;; + &cv-link-LIBEMITTER; for &b-link-Library;; + &cv-link-SHLIBEMITTER; for &b-link-SharedLibrary; and &cv-link-LDMODULEEMITTER; for &b-link-LoadableModule;;. - Adding an emitter to one of these will cause it to be + Adding an emitter to one of these will cause it to be invoked in addition to any existing emitter for the corresponding builder. @@ -944,10 +943,10 @@ main() The site_scons directories give you a place to - put Python modules and packages that you can import into your &SConscript; files - (site_scons), + put &Python; modules and packages that you can import into your + &SConscript; files (at the top level) add-on tools that can integrate into &SCons; - (site_scons/site_tools), + (in a site_tools subdirectory), and a site_scons/site_init.py file that gets read before any &SConstruct; or &SConscript; file, allowing you to change &SCons;'s default behavior. @@ -957,8 +956,10 @@ main() Each system type (Windows, Mac, Linux, etc.) searches a canonical - set of directories for site_scons; see the man page for details. - The top-level SConstruct's site_scons dir is always searched last, + set of directories for site_scons; + see the man page for details. + The top-level SConstruct's site_scons dir + (that is, the one in the project) is always searched last, and its dir is placed first in the tool path so it overrides all others. @@ -969,8 +970,8 @@ main() If you get a tool from somewhere (the &SCons; wiki or a third party, for instance) and you'd like to use it in your project, a site_scons dir is the simplest place to put it. - Tools come in two flavors; either a Python function that operates on - an &Environment; or a Python module or package containing two functions, + Tools come in two flavors; either a &Python; function that operates on + an &Environment; or a &Python; module or package containing two functions, exists() and generate(). @@ -1023,7 +1024,7 @@ env.AddHeader('tgt', 'src') -