From d8932c2b87cfef617b2412afc6db024ba6e9560a Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 11 Feb 2022 14:13:05 -0600 Subject: fix up interactive to support ninja scons daemon --- SCons/Node/__init__.py | 1 + SCons/Script/Interactive.py | 149 +++++++++++++++++++++++-------------------- SCons/Script/Main.py | 5 +- SCons/Script/SConsOptions.py | 5 ++ 4 files changed, 88 insertions(+), 72 deletions(-) diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index ec742a6..d0539b4 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -105,6 +105,7 @@ SConscriptNodes = set() # currently used to release parts of a target's info during # clean builds and update runs (see release_target_info). interactive = False +ninja_scons_daemon = False def is_derived_none(node): raise NotImplementedError diff --git a/SCons/Script/Interactive.py b/SCons/Script/Interactive.py index 26a8bcd..ad021fd 100644 --- a/SCons/Script/Interactive.py +++ b/SCons/Script/Interactive.py @@ -188,77 +188,86 @@ version Prints SCons version information. x.extend(n.alter_targets()[0]) nodes.extend(x) - # Clean up so that we can perform the next build correctly. - # - # We do this by walking over all the children of the targets, - # and clearing their state. - # - # We currently have to re-scan each node to find their - # children, because built nodes have already been partially - # cleared and don't remember their children. (In scons - # 0.96.1 and earlier, this wasn't the case, and we didn't - # have to re-scan the nodes.) - # - # Because we have to re-scan each node, we can't clear the - # nodes as we walk over them, because we may end up rescanning - # a cleared node as we scan a later node. Therefore, only - # store the list of nodes that need to be cleared as we walk - # the tree, and clear them in a separate pass. - # - # XXX: Someone more familiar with the inner workings of scons - # may be able to point out a more efficient way to do this. - - SCons.Script.Main.progress_display("scons: Clearing cached node information ...") - - seen_nodes = {} - - def get_unseen_children(node, parent, seen_nodes=seen_nodes): - def is_unseen(node, seen_nodes=seen_nodes): - return node not in seen_nodes - return [child for child in node.children(scan=1) if is_unseen(child)] - - def add_to_seen_nodes(node, parent, seen_nodes=seen_nodes): - seen_nodes[node] = 1 - - # If this file is in a VariantDir and has a - # corresponding source file in the source tree, remember the - # node in the source tree, too. This is needed in - # particular to clear cached implicit dependencies on the - # source file, since the scanner will scan it if the - # VariantDir was created with duplicate=0. - try: - rfile_method = node.rfile - except AttributeError: - return - else: - rfile = rfile_method() - if rfile != node: - seen_nodes[rfile] = 1 - - for node in nodes: - walker = SCons.Node.Walker(node, - kids_func=get_unseen_children, - eval_func=add_to_seen_nodes) - n = walker.get_next() - while n: + if SCons.Node.ninja_scons_daemon: + for n in nodes: + print(f"Node: {n}, State: {SCons.Node.failed}") + if n.get_state() == SCons.Node.failed: + print(n) + n.clear() + n.set_state(SCons.Node.no_state) + n.implicit = None + else: + # Clean up so that we can perform the next build correctly. + # + # We do this by walking over all the children of the targets, + # and clearing their state. + # + # We currently have to re-scan each node to find their + # children, because built nodes have already been partially + # cleared and don't remember their children. (In scons + # 0.96.1 and earlier, this wasn't the case, and we didn't + # have to re-scan the nodes.) + # + # Because we have to re-scan each node, we can't clear the + # nodes as we walk over them, because we may end up rescanning + # a cleared node as we scan a later node. Therefore, only + # store the list of nodes that need to be cleared as we walk + # the tree, and clear them in a separate pass. + # + # XXX: Someone more familiar with the inner workings of scons + # may be able to point out a more efficient way to do this. + + SCons.Script.Main.progress_display("scons: Clearing cached node information ...") + + seen_nodes = {} + + def get_unseen_children(node, parent, seen_nodes=seen_nodes): + def is_unseen(node, seen_nodes=seen_nodes): + return node not in seen_nodes + return [child for child in node.children(scan=1) if is_unseen(child)] + + def add_to_seen_nodes(node, parent, seen_nodes=seen_nodes): + seen_nodes[node] = 1 + + # If this file is in a VariantDir and has a + # corresponding source file in the source tree, remember the + # node in the source tree, too. This is needed in + # particular to clear cached implicit dependencies on the + # source file, since the scanner will scan it if the + # VariantDir was created with duplicate=0. + try: + rfile_method = node.rfile + except AttributeError: + return + else: + rfile = rfile_method() + if rfile != node: + seen_nodes[rfile] = 1 + + for node in nodes: + walker = SCons.Node.Walker(node, + kids_func=get_unseen_children, + eval_func=add_to_seen_nodes) n = walker.get_next() - - for node in seen_nodes.keys(): - # Call node.clear() to clear most of the state - node.clear() - # node.clear() doesn't reset node.state, so call - # node.set_state() to reset it manually - node.set_state(SCons.Node.no_state) - node.implicit = None - - # Debug: Uncomment to verify that all Taskmaster reference - # counts have been reset to zero. - #if node.ref_count != 0: - # from SCons.Debug import Trace - # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) - - SCons.SConsign.Reset() - SCons.Script.Main.progress_display("scons: done clearing node information.") + while n: + n = walker.get_next() + + for node in seen_nodes.keys(): + # Call node.clear() to clear most of the state + node.clear() + # node.clear() doesn't reset node.state, so call + # node.set_state() to reset it manually + node.set_state(SCons.Node.no_state) + node.implicit = None + + # Debug: Uncomment to verify that all Taskmaster reference + # counts have been reset to zero. + #if node.ref_count != 0: + # from SCons.Debug import Trace + # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) + + SCons.SConsign.Reset() + SCons.Script.Main.progress_display("scons: done clearing node information.") def do_clean(self, argv): """\ diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index a340f5b..87d57bc 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -987,7 +987,8 @@ def _main(parser): # This would then cause subtle bugs, as already happened in #2971. if options.interactive: SCons.Node.interactive = True - + if options.ninja_scons_daemon: + SCons.Node.ninja_scons_daemon = True # That should cover (most of) the options. # Next, set up the variables that hold command-line arguments, # so the SConscript files that we read and execute have access to them. @@ -1125,7 +1126,7 @@ def _main(parser): platform = SCons.Platform.platform_module() - if options.interactive: + if options.interactive or options.ninja_scons_daemon: SCons.Script.Interactive.interact(fs, OptionsParser, options, targets, target_top) diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index e2631fb..133d9ab 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -858,6 +858,11 @@ def Parser(version): action="store_true", help="Run in interactive mode") + op.add_option('--ninja-scons-daemon', + dest='ninja_scons_daemon', default=False, + action="store_true", + help="A special interactive mode to support a scons daemon for ninja builds. Intended for use only by the ninja tool.") + op.add_option('-j', '--jobs', nargs=1, type="int", dest="num_jobs", default=1, -- cgit v0.12 From 6c8e521f59a796b1bb06870dce9bce093c15e5fe Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 11 Feb 2022 15:28:19 -0600 Subject: add scons daemon --- SCons/Tool/ninja/NinjaState.py | 198 ++++++++++++++++--------- SCons/Tool/ninja/Utils.py | 4 +- SCons/Tool/ninja/__init__.py | 6 +- SCons/Tool/ninja/ninja_daemon_build.py | 52 +++++++ SCons/Tool/ninja/ninja_run_daemon.py | 71 +++++++++ SCons/Tool/ninja/ninja_scons_daemon.py | 259 +++++++++++++++++++++++++++++++++ test/ninja/force_scons_callback.py | 4 +- 7 files changed, 522 insertions(+), 72 deletions(-) create mode 100644 SCons/Tool/ninja/ninja_daemon_build.py create mode 100644 SCons/Tool/ninja/ninja_run_daemon.py create mode 100644 SCons/Tool/ninja/ninja_scons_daemon.py diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index d7c260e..f2da87a 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -24,11 +24,15 @@ import io import os +import pathlib +import signal +import tempfile import shutil import sys from os.path import splitext from tempfile import NamedTemporaryFile import ninja +import hashlib import SCons from SCons.Script import COMMAND_LINE_TARGETS @@ -59,7 +63,7 @@ class NinjaState: os.pardir, 'data', 'bin', - ninja_bin)) + ninja_bin)) if not os.path.exists(self.ninja_bin_path): # couldn't find it, just give the bin name and hope # its in the path later @@ -84,6 +88,10 @@ class NinjaState: # like CCFLAGS scons_escape = env.get("ESCAPE", lambda x: x) + import random + + PORT = str(random.randint(10000, 60000)) + # if SCons was invoked from python, we expect the first arg to be the scons.py # script, otherwise scons was invoked from the scons script python_bin = '' @@ -184,10 +192,10 @@ class NinjaState: "restat": 1, }, "TEMPLATE": { - "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", - "pool": "scons_pool", - "restat": 1, + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_daemon_build.py'} {PORT} {get_path(env.get('NINJA_DIR'))} $out", + "description": "Defer to SCons to build $out", + "pool": "local_pool", + "restat": 1 }, "SCONS": { "command": "$SCONS_INVOCATION $out", @@ -211,6 +219,29 @@ class NinjaState: # output. "restat": 1, }, + + "SCONS_DAEMON": { + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {PORT} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", + "description": "Starting scons daemon...", + "pool": "local_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, "REGENERATE": { "command": "$SCONS_INVOCATION_W_TARGETS", "description": "Regenerating $self", @@ -245,6 +276,9 @@ class NinjaState: if not node.has_builder(): return False + if isinstance(node, SCons.Node.Python.Value): + return False + if isinstance(node, SCons.Node.Alias.Alias): build = alias_to_ninja_build(node) else: @@ -256,7 +290,24 @@ class NinjaState: node_string = str(node) if node_string in self.builds: - raise InternalError("Node {} added to ninja build state more than once".format(node_string)) + warn_msg = f"Alias {node_string} name the same as File node, ninja does not support this. Renaming Alias {node_string} to {node_string}_alias." + if isinstance(node, SCons.Node.Alias.Alias): + for i, output in enumerate(build["outputs"]): + if output == node_string: + build["outputs"][i] += "_alias" + node_string += "_alias" + print(warn_msg) + elif self.builds[node_string]["rule"] == "phony": + for i, output in enumerate(self.builds[node_string]["outputs"]): + if output == node_string: + self.builds[node_string]["outputs"][i] += "_alias" + tmp_build = self.builds[node_string].copy() + del self.builds[node_string] + node_string += "_alias" + self.builds[node_string] = tmp_build + print(warn_msg) + else: + raise InternalError("Node {} added to ninja build state more than once".format(node_string)) self.builds[node_string] = build self.built.update(build["outputs"]) return True @@ -330,8 +381,12 @@ class NinjaState: ) template_builders = [] + scons_compiledb = False for build in [self.builds[key] for key in sorted(self.builds.keys())]: + if "compile_commands.json" in build["outputs"]: + scons_compiledb = True + if build["rule"] == "TEMPLATE": template_builders.append(build) continue @@ -345,10 +400,10 @@ class NinjaState: # generated sources or else we will create a dependency # cycle. if ( - generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) ): # Make all non-generated source targets depend on # _generated_sources. We use order_only for generated @@ -422,41 +477,10 @@ class NinjaState: ninja.build(**build) - template_builds = dict() + scons_daemon_dirty = str(pathlib.Path(get_path(self.env.get("NINJA_DIR"))) / "scons_daemon_dirty") for template_builder in template_builders: - - # Special handling for outputs and implicit since we need to - # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", "inputs"]: - new_val = template_builds.get(agg_key, []) - - # Use pop so the key is removed and so the update - # below will not overwrite our aggregated values. - cur_val = template_builder.pop(agg_key, []) - if is_List(cur_val): - new_val += cur_val - else: - new_val.append(cur_val) - template_builds[agg_key] = new_val - - # Collect all other keys - template_builds.update(template_builder) - - if template_builds.get("outputs", []): - - # Try to clean up any dependency cycles. If we are passing an - # ouptut node to SCons, it will build any dependencys if ninja - # has not already. - for output in template_builds.get("outputs", []): - inputs = template_builds.get('inputs') - if inputs and output in inputs: - inputs.remove(output) - - implicits = template_builds.get('implicit') - if implicits and output in implicits: - implicits.remove(output) - - ninja.build(**template_builds) + template_builder["implicit"] += [scons_daemon_dirty] + ninja.build(**template_builder) # We have to glob the SCons files here to teach the ninja file # how to regenerate itself. We'll never see ourselves in the @@ -483,30 +507,55 @@ class NinjaState: } ) - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - ninja.build( - "compile_commands.json", - rule="CMD", - pool="console", - implicit=[str(self.ninja_file)], - variables={ - "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( - # NINJA_COMPDB_EXPAND - should only be true for ninja - # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) - # https://github.com/ninja-build/ninja/pull/1223 - # TODO: add check in generate to check version and enable this by default if it's available. - self.ninja_bin_path, str(self.ninja_file), - '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' - ) - }, - ) + if not scons_compiledb: + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + implicit=[str(self.ninja_file)], + variables={ + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + # NINJA_COMPDB_EXPAND - should only be true for ninja + # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) + # https://github.com/ninja-build/ninja/pull/1223 + # TODO: add check in generate to check version and enable this by default if it's available. + self.ninja_bin_path, str(self.ninja_file), + '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) ninja.build( - "compiledb", rule="phony", implicit=["compile_commands.json"], + ["run_scons_daemon", scons_daemon_dirty], + rule="SCONS_DAEMON", ) + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) + pidfile = None + if os.path.exists(scons_daemon_dirty): + pidfile = scons_daemon_dirty + elif os.path.exists(daemon_dir / 'pidfile'): + pidfile = daemon_dir / 'pidfile' + + if pidfile: + with open(pidfile) as f: + pid = int(f.readline()) + try: + os.kill(pid, signal.SIGINT) + except OSError: + pass + + if os.path.exists(scons_daemon_dirty): + os.unlink(scons_daemon_dirty) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that # we generated a ninja build rule for. scons_default_targets = [ @@ -588,7 +637,13 @@ class SConsToNinjaTranslator: elif isinstance(action, COMMAND_TYPES): build = get_command(env, node, action) else: - raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } if build is not None: build["order_only"] = get_order_only(node) @@ -657,7 +712,7 @@ class SConsToNinjaTranslator: return results[0] all_outputs = list({output for build in results for output in build["outputs"]}) - dependencies = list({dep for build in results for dep in build["implicit"]}) + dependencies = list({dep for build in results for dep in build.get("implicit", [])}) if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": cmdline = "" @@ -669,6 +724,9 @@ class SConsToNinjaTranslator: # condition if not cmdstr. So here we strip preceding # and proceeding whitespace to make strings like the # above become empty strings and so will be skipped. + if not cmd.get("variables") or not cmd["variables"].get("cmd"): + continue + cmdstr = cmd["variables"]["cmd"].strip() if not cmdstr: continue @@ -717,4 +775,10 @@ class SConsToNinjaTranslator: "implicit": dependencies, } - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index 3adbb53..888218d 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -123,7 +123,7 @@ def get_dependencies(node, skip_sources=False): get_path(src_file(child)) for child in filter_ninja_nodes(node.children()) if child not in node.sources - ] + ] return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] @@ -370,7 +370,7 @@ def ninja_contents(original): """Return a dummy content without doing IO""" def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + if isinstance(self, SCons.Node.Node) and (self.is_sconscript() or self.is_conftest()): return original(self) return bytes("dummy_ninja_contents", encoding="utf-8") diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 44c2251..3d14faf 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -187,6 +187,10 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_DIR'] = env.Dir(env.get("NINJA_DIR", '#/.ninja')) + env["NINJA_SCONS_DAEMON_KEEP_ALIVE"] = env.get("NINJA_SCONS_DAEMON_KEEP_ALIVE", 180000) + + if GetOption("disable_ninja"): + env.SConsignFile(os.path.join(str(env['NINJA_DIR']),'.ninja.sconsign')) # here we allow multiple environments to construct rules and builds # into the same ninja file @@ -423,7 +427,7 @@ def generate(env): return if target.check_attributes('ninja_file') is None: NINJA_STATE.add_build(target) - else: + else: target.build() SCons.Taskmaster.Task.execute = ninja_execute diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py new file mode 100644 index 0000000..4acd328 --- /dev/null +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -0,0 +1,52 @@ +import http.client +import sys +import time +import os +import logging +import pathlib +import tempfile +import hashlib +import traceback + +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +os.makedirs(daemon_dir, exist_ok=True) + +logging.basicConfig( + filename=daemon_dir / "scons_daemon_request.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + +while True: + try: + logging.debug(f"Sending request: {sys.argv[3]}") + conn = http.client.HTTPConnection("127.0.0.1", port=int(sys.argv[1]), timeout=60) + conn.request("GET", "/?build=" + sys.argv[3]) + response = None + + while not response: + try: + response = conn.getresponse() + except (http.client.RemoteDisconnected, http.client.ResponseNotReady): + time.sleep(0.01) + except http.client.HTTPException as e: + logging.debug(f"Error: {traceback.format_exc()}") + exit(1) + else: + msg = response.read() + status = response.status + if status != 200: + print(msg.decode('utf-8')) + exit(1) + logging.debug(f"Request Done: {sys.argv[3]}") + exit(0) + + except ConnectionRefusedError: + logging.debug(f"Server not ready: {traceback.format_exc()}") + time.sleep(1) + except ConnectionResetError: + logging.debug(f"Server ConnectionResetError") + exit(1) + diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py new file mode 100644 index 0000000..2ab70e6 --- /dev/null +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -0,0 +1,71 @@ +import subprocess +import sys +import os +import pathlib +import tempfile +import hashlib +import logging +import time +import http.client +import traceback + +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +os.makedirs(daemon_dir, exist_ok=True) + +logging.basicConfig( + filename=daemon_dir / "scons_daemon.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + +if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): + cmd = [sys.executable, str(pathlib.Path(__file__).parent / "ninja_scons_daemon.py")] + sys.argv[1:] + logging.debug(f"Starting daemon with {' '.join(cmd)}") + + p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False) + + with open(daemon_dir / "pidfile", "w") as f: + f.write(str(p.pid)) + with open(ninja_builddir / "scons_daemon_dirty", "w") as f: + f.write(str(p.pid)) + + error_msg = f"ERROR: Failed to connect to scons daemon.\n Check {daemon_dir / 'scons_daemon.log'} for more info.\n" + + while True: + try: + logging.debug(f"Attempting to connect scons daemon") + conn = http.client.HTTPConnection("127.0.0.1", port=int(sys.argv[1]), timeout=60) + conn.request("GET", "/?ready=true") + response = None + + try: + response = conn.getresponse() + except (http.client.RemoteDisconnected, http.client.ResponseNotReady): + time.sleep(0.01) + except http.client.HTTPException as e: + logging.debug(f"Error: {traceback.format_exc()}") + sys.stderr.write(error_msg) + exit(1) + else: + msg = response.read() + status = response.status + if status != 200: + print(msg.decode('utf-8')) + exit(1) + logging.debug(f"Request Done: {sys.argv[3]}") + break + + except ConnectionRefusedError: + logging.debug(f"Server not ready: {sys.argv[3]}") + time.sleep(1) + except ConnectionResetError: + logging.debug(f"Server ConnectionResetError") + sys.stderr.write(error_msg) + exit(1) + except: + logging.debug(f"Error: {traceback.format_exc()}") + sys.stderr.write(error_msg) + exit(1) + diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py new file mode 100644 index 0000000..7b839fc --- /dev/null +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -0,0 +1,259 @@ +import http.server +import socketserver +from urllib.parse import urlparse, parse_qs +import time +from threading import Condition +from subprocess import PIPE, Popen +import sys +import os +import threading, queue +import pathlib +import logging +from timeit import default_timer as timer +import traceback +import tempfile +import hashlib + +port = int(sys.argv[1]) +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_keep_alive = int(sys.argv[3]) +args = sys.argv[4:] + +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +os.makedirs(daemon_dir, exist_ok=True) +logging.basicConfig( + filename=daemon_dir / "scons_daemon.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) +def daemon_log(message): + logging.debug(message) + +def custom_readlines(handle, line_separator="\n", chunk_size=1): + buf = "" + while not handle.closed: + data = handle.read(chunk_size) + if not data: + break + buf += data.decode("utf-8") + if line_separator in buf: + chunks = buf.split(line_separator) + buf = chunks.pop() + for chunk in chunks: + yield chunk + line_separator + if buf.endswith("scons>>>"): + yield buf + buf = "" + +def custom_readerr(handle, line_separator="\n", chunk_size=1): + buf = "" + while not handle.closed: + data = handle.read(chunk_size) + if not data: + break + buf += data.decode("utf-8") + if line_separator in buf: + chunks = buf.split(line_separator) + buf = chunks.pop() + for chunk in chunks: + yield chunk + line_separator + +def enqueue_output(out, queue): + for line in iter(custom_readlines(out)): + queue.put(line) + out.close() + +def enqueue_error(err, queue): + for line in iter(custom_readerr(err)): + queue.put(line) + err.close() + +input_q = queue.Queue() +output_q = queue.Queue() +error_q = queue.Queue() + +finished_building = [] +error_nodes = [] + +building_cv = Condition() +error_cv = Condition() + +thread_error = False + +def daemon_thread_func(): + global thread_error + global finished_building + global error_nodes + try: + args_list = args + ["--ninja-scons-daemon"] + daemon_log(f"Starting daemon with args: {' '.join(args_list)}") + daemon_log(f"cwd: {os.getcwd()}") + + p = Popen(args_list, stdout=PIPE, stderr=PIPE, stdin=PIPE) + + t = threading.Thread(target=enqueue_output, args=(p.stdout, output_q)) + t.daemon = True + t.start() + + te = threading.Thread(target=enqueue_error, args=(p.stderr, error_q)) + te.daemon = True + te.start() + + daemon_ready = False + building_node = None + + while p.poll() is None: + + while True: + try: + line = output_q.get(block=False, timeout=0.01) + except queue.Empty: + break + else: + daemon_log("output: " + line.strip()) + + if "scons: building terminated because of errors." in line: + error_output = "" + while True: + try: + error_output += error_q.get(block=False, timeout=0.01) + except queue.Empty: + break + error_nodes += [{'node': building_node, 'error': error_output}] + daemon_ready = True + building_node = None + with building_cv: + building_cv.notify() + + elif line == "scons>>>": + with error_q.mutex: + error_q.queue.clear() + daemon_ready = True + with building_cv: + building_cv.notify() + building_node = None + + while daemon_ready and not input_q.empty(): + + try: + building_node = input_q.get(block=False, timeout=0.01) + except queue.Empty: + break + if "exit" in building_node: + p.stdin.write("exit\n".encode("utf-8")) + p.stdin.flush() + with building_cv: + finished_building += [building_node] + daemon_ready = False + raise + + else: + input_command = "build " + building_node + "\n" + daemon_log("input: " + input_command.strip()) + + p.stdin.write(input_command.encode("utf-8")) + p.stdin.flush() + with building_cv: + finished_building += [building_node] + daemon_ready = False + + time.sleep(0.01) + except: + thread_error = True + daemon_log("SERVER ERROR: " + traceback.format_exc()) + raise + + +daemon_thread = threading.Thread(target=daemon_thread_func) +daemon_thread.daemon = True +daemon_thread.start() + +logging.debug(f"Starting request server on port {port}, keep alive: {daemon_keep_alive}") + +keep_alive_timer = timer() +httpd = None + + +def server_thread_func(): + + class S(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global thread_error + global keep_alive_timer + global error_nodes + + try: + gets = parse_qs(urlparse(self.path).query) + build = gets.get("build") + if build: + keep_alive_timer = timer() + + daemon_log(f"Got request: {build[0]}") + input_q.put(build[0]) + + def pred(): + return build[0] in finished_building + + with building_cv: + building_cv.wait_for(pred) + + for error_node in error_nodes: + if error_node['node'] == build[0]: + self.send_response(500) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(error_node['error'].encode()) + return + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + return + + exitbuild = gets.get("exit") + if exitbuild: + input_q.put('exit') + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + except: + thread_error = True + daemon_log("SERVER ERROR: " + traceback.format_exc()) + raise + + def log_message(self, format, *args): + return + + httpd = socketserver.TCPServer(("127.0.0.1", port), S) + httpd.serve_forever() + +server_thread = threading.Thread(target=server_thread_func) +server_thread.daemon = True +server_thread.start() + +while timer() - keep_alive_timer < daemon_keep_alive and not thread_error: + time.sleep(1) + +if thread_error: + daemon_log(f"Shutting server on port {port} down because thread error.") +else: + daemon_log(f"Shutting server on port {port} down because timed out: {daemon_keep_alive}") + +# if there are errors, don't immediatly shutdown the daemon +# the process which started the server is attempt to connect to +# the daemon before allowing jobs to start being sent. If the daemon +# shutsdown too fast, the launchs script will think it has not +# started yet and sit and wait. If the launch script is able to connect +# and then the connection is dropped, it will immediatly exit with fail. +time.sleep(5) + +if os.path.exists(ninja_builddir / "scons_daemon_dirty"): + os.unlink(ninja_builddir / "scons_daemon_dirty") +if os.path.exists(daemon_dir / "pidfile"): + os.unlink(daemon_dir / "pidfile") + +httpd.shutdown() +server_thread.join() diff --git a/test/ninja/force_scons_callback.py b/test/ninja/force_scons_callback.py index 55c12ca..59b54a0 100644 --- a/test/ninja/force_scons_callback.py +++ b/test/ninja/force_scons_callback.py @@ -53,7 +53,7 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) -if test.stdout().count('scons: Building targets') != 2: +if test.stdout().count('Defer to SCons to build') != 1: test.fail_test() test.must_match('out.txt', 'foo.c' + os.linesep) test.must_match('out2.txt', "test2.cpp" + os.linesep) @@ -74,7 +74,7 @@ test.must_not_exist(test.workpath('out2.txt')) # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program=program, stdout=None) -if test.stdout().count('scons: Building targets') != 1: +if test.stdout().count('Defer to SCons to build') != 1: test.fail_test() test.must_match('out.txt', 'foo.c' + os.linesep) test.must_match('out2.txt', "test2.cpp" + os.linesep) -- cgit v0.12 From 77af301f7f40875837602a1245d23d92266b990c Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Feb 2022 18:08:08 -0800 Subject: Fix issue where only the first interactive 'build' command's results were being written to the sconsign file --- SCons/Script/Interactive.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/SCons/Script/Interactive.py b/SCons/Script/Interactive.py index ad021fd..1015b00 100644 --- a/SCons/Script/Interactive.py +++ b/SCons/Script/Interactive.py @@ -266,7 +266,12 @@ version Prints SCons version information. # from SCons.Debug import Trace # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) - SCons.SConsign.Reset() + # TODO: REMOVE WPD DEBUG 02/14/2022 + # This call was clearing the list of sconsign files to be written, so it would + # only write the results of the first build command. All others wouldn't be written + # to .SConsign. + # Pretty sure commenting this out is the correct fix. + # SCons.SConsign.Reset() SCons.Script.Main.progress_display("scons: done clearing node information.") def do_clean(self, argv): -- cgit v0.12 From 01717fbf21708be8b794aeacebe6664eea8aa138 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 12:00:14 -0700 Subject: [ci skip] Address sider complaints --- SCons/Tool/ninja/NinjaState.py | 2 +- SCons/Tool/ninja/ninja_daemon_build.py | 15 ++++++---- SCons/Tool/ninja/ninja_run_daemon.py | 32 ++++++++++++-------- SCons/Tool/ninja/ninja_scons_daemon.py | 54 +++++++++++++++++++++------------- 4 files changed, 64 insertions(+), 39 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index f2da87a..b6de65c 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -537,7 +537,7 @@ class NinjaState: rule="SCONS_DAEMON", ) - daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) pidfile = None if os.path.exists(scons_daemon_dirty): pidfile = scons_daemon_dirty diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index 4acd328..ff0e2bf 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -9,7 +9,9 @@ import hashlib import traceback ninja_builddir = pathlib.Path(sys.argv[2]) -daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) os.makedirs(daemon_dir, exist_ok=True) logging.basicConfig( @@ -22,7 +24,9 @@ logging.basicConfig( while True: try: logging.debug(f"Sending request: {sys.argv[3]}") - conn = http.client.HTTPConnection("127.0.0.1", port=int(sys.argv[1]), timeout=60) + conn = http.client.HTTPConnection( + "127.0.0.1", port=int(sys.argv[1]), timeout=60 + ) conn.request("GET", "/?build=" + sys.argv[3]) response = None @@ -31,14 +35,14 @@ while True: response = conn.getresponse() except (http.client.RemoteDisconnected, http.client.ResponseNotReady): time.sleep(0.01) - except http.client.HTTPException as e: + except http.client.HTTPException: logging.debug(f"Error: {traceback.format_exc()}") exit(1) else: msg = response.read() status = response.status if status != 200: - print(msg.decode('utf-8')) + print(msg.decode("utf-8")) exit(1) logging.debug(f"Request Done: {sys.argv[3]}") exit(0) @@ -47,6 +51,5 @@ while True: logging.debug(f"Server not ready: {traceback.format_exc()}") time.sleep(1) except ConnectionResetError: - logging.debug(f"Server ConnectionResetError") + logging.debug("Server ConnectionResetError") exit(1) - diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 2ab70e6..442bcad 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -10,7 +10,9 @@ import http.client import traceback ninja_builddir = pathlib.Path(sys.argv[2]) -daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) os.makedirs(daemon_dir, exist_ok=True) logging.basicConfig( @@ -21,10 +23,15 @@ logging.basicConfig( ) if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): - cmd = [sys.executable, str(pathlib.Path(__file__).parent / "ninja_scons_daemon.py")] + sys.argv[1:] + cmd = [ + sys.executable, + str(pathlib.Path(__file__).parent / "ninja_scons_daemon.py"), + ] + sys.argv[1:] logging.debug(f"Starting daemon with {' '.join(cmd)}") - p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False) + p = subprocess.Popen( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False + ) with open(daemon_dir / "pidfile", "w") as f: f.write(str(p.pid)) @@ -34,9 +41,11 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): error_msg = f"ERROR: Failed to connect to scons daemon.\n Check {daemon_dir / 'scons_daemon.log'} for more info.\n" while True: - try: - logging.debug(f"Attempting to connect scons daemon") - conn = http.client.HTTPConnection("127.0.0.1", port=int(sys.argv[1]), timeout=60) + try: + logging.debug("Attempting to connect scons daemon") + conn = http.client.HTTPConnection( + "127.0.0.1", port=int(sys.argv[1]), timeout=60 + ) conn.request("GET", "/?ready=true") response = None @@ -44,7 +53,7 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): response = conn.getresponse() except (http.client.RemoteDisconnected, http.client.ResponseNotReady): time.sleep(0.01) - except http.client.HTTPException as e: + except http.client.HTTPException: logging.debug(f"Error: {traceback.format_exc()}") sys.stderr.write(error_msg) exit(1) @@ -52,20 +61,19 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): msg = response.read() status = response.status if status != 200: - print(msg.decode('utf-8')) + print(msg.decode("utf-8")) exit(1) logging.debug(f"Request Done: {sys.argv[3]}") break - + except ConnectionRefusedError: logging.debug(f"Server not ready: {sys.argv[3]}") time.sleep(1) except ConnectionResetError: - logging.debug(f"Server ConnectionResetError") + logging.debug("Server ConnectionResetError") sys.stderr.write(error_msg) exit(1) - except: + except Exception: logging.debug(f"Error: {traceback.format_exc()}") sys.stderr.write(error_msg) exit(1) - diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 7b839fc..bc14f9e 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -19,7 +19,9 @@ ninja_builddir = pathlib.Path(sys.argv[2]) daemon_keep_alive = int(sys.argv[3]) args = sys.argv[4:] -daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest())) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) os.makedirs(daemon_dir, exist_ok=True) logging.basicConfig( filename=daemon_dir / "scons_daemon.log", @@ -27,9 +29,12 @@ logging.basicConfig( format="%(asctime)s %(message)s", level=logging.DEBUG, ) + + def daemon_log(message): logging.debug(message) + def custom_readlines(handle, line_separator="\n", chunk_size=1): buf = "" while not handle.closed: @@ -46,6 +51,7 @@ def custom_readlines(handle, line_separator="\n", chunk_size=1): yield buf buf = "" + def custom_readerr(handle, line_separator="\n", chunk_size=1): buf = "" while not handle.closed: @@ -58,17 +64,20 @@ def custom_readerr(handle, line_separator="\n", chunk_size=1): buf = chunks.pop() for chunk in chunks: yield chunk + line_separator - + + def enqueue_output(out, queue): for line in iter(custom_readlines(out)): queue.put(line) out.close() - + + def enqueue_error(err, queue): for line in iter(custom_readerr(err)): queue.put(line) err.close() + input_q = queue.Queue() output_q = queue.Queue() error_q = queue.Queue() @@ -81,6 +90,7 @@ error_cv = Condition() thread_error = False + def daemon_thread_func(): global thread_error global finished_building @@ -120,12 +130,12 @@ def daemon_thread_func(): error_output += error_q.get(block=False, timeout=0.01) except queue.Empty: break - error_nodes += [{'node': building_node, 'error': error_output}] + error_nodes += [{"node": building_node, "error": error_output}] daemon_ready = True building_node = None with building_cv: building_cv.notify() - + elif line == "scons>>>": with error_q.mutex: error_q.queue.clear() @@ -159,7 +169,7 @@ def daemon_thread_func(): daemon_ready = False time.sleep(0.01) - except: + except Exception: thread_error = True daemon_log("SERVER ERROR: " + traceback.format_exc()) raise @@ -169,14 +179,15 @@ daemon_thread = threading.Thread(target=daemon_thread_func) daemon_thread.daemon = True daemon_thread.start() -logging.debug(f"Starting request server on port {port}, keep alive: {daemon_keep_alive}") +logging.debug( + f"Starting request server on port {port}, keep alive: {daemon_keep_alive}" +) keep_alive_timer = timer() httpd = None def server_thread_func(): - class S(http.server.BaseHTTPRequestHandler): def do_GET(self): global thread_error @@ -197,15 +208,15 @@ def server_thread_func(): with building_cv: building_cv.wait_for(pred) - + for error_node in error_nodes: - if error_node['node'] == build[0]: + if error_node["node"] == build[0]: self.send_response(500) self.send_header("Content-type", "text/html") self.end_headers() - self.wfile.write(error_node['error'].encode()) + self.wfile.write(error_node["error"].encode()) return - + self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() @@ -213,12 +224,12 @@ def server_thread_func(): exitbuild = gets.get("exit") if exitbuild: - input_q.put('exit') - + input_q.put("exit") + self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() - + except: thread_error = True daemon_log("SERVER ERROR: " + traceback.format_exc()) @@ -230,6 +241,7 @@ def server_thread_func(): httpd = socketserver.TCPServer(("127.0.0.1", port), S) httpd.serve_forever() + server_thread = threading.Thread(target=server_thread_func) server_thread.daemon = True server_thread.start() @@ -240,20 +252,22 @@ while timer() - keep_alive_timer < daemon_keep_alive and not thread_error: if thread_error: daemon_log(f"Shutting server on port {port} down because thread error.") else: - daemon_log(f"Shutting server on port {port} down because timed out: {daemon_keep_alive}") + daemon_log( + f"Shutting server on port {port} down because timed out: {daemon_keep_alive}" + ) -# if there are errors, don't immediatly shutdown the daemon +# if there are errors, don't immediately shut down the daemon # the process which started the server is attempt to connect to # the daemon before allowing jobs to start being sent. If the daemon -# shutsdown too fast, the launchs script will think it has not +# shuts down too fast, the launch script will think it has not # started yet and sit and wait. If the launch script is able to connect -# and then the connection is dropped, it will immediatly exit with fail. +# and then the connection is dropped, it will immediately exit with fail. time.sleep(5) if os.path.exists(ninja_builddir / "scons_daemon_dirty"): os.unlink(ninja_builddir / "scons_daemon_dirty") if os.path.exists(daemon_dir / "pidfile"): os.unlink(daemon_dir / "pidfile") - + httpd.shutdown() server_thread.join() -- cgit v0.12 From f3dc33eb69f0f2fc32aa7aa9caaa40d1cb8b658f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 12:07:11 -0700 Subject: [ci skip] Add standard SCons license blurb,and empty docstring --- SCons/Tool/ninja/ninja_daemon_build.py | 34 ++++++++++++++++++++++++++++++++++ SCons/Tool/ninja/ninja_run_daemon.py | 33 +++++++++++++++++++++++++++++++++ SCons/Tool/ninja/ninja_scons_daemon.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index ff0e2bf..b16c039 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -1,3 +1,30 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" + +""" + import http.client import sys import time @@ -53,3 +80,10 @@ while True: except ConnectionResetError: logging.debug("Server ConnectionResetError") exit(1) + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 442bcad..542198a 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -1,3 +1,30 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" + +""" + import subprocess import sys import os @@ -77,3 +104,9 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): logging.debug(f"Error: {traceback.format_exc()}") sys.stderr.write(error_msg) exit(1) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index bc14f9e..00923fb 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -1,3 +1,30 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" + +""" + import http.server import socketserver from urllib.parse import urlparse, parse_qs @@ -271,3 +298,10 @@ if os.path.exists(daemon_dir / "pidfile"): httpd.shutdown() server_thread.join() + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file -- cgit v0.12 From 247d0ba437342088ee6de073a60e3753527a9946 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 12:13:24 -0700 Subject: [ci skip] Address sider complaints --- SCons/Tool/ninja/ninja_daemon_build.py | 3 ++- SCons/Tool/ninja/ninja_run_daemon.py | 2 +- SCons/Tool/ninja/ninja_scons_daemon.py | 8 +++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index b16c039..af0e6b4 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -86,4 +86,5 @@ while True: # tab-width:4 # indent-tabs-mode:nil # End: -# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file +# vim: set expandtab tabstop=4 shiftwidth=4: + diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 542198a..965e50b 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -109,4 +109,4 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): # tab-width:4 # indent-tabs-mode:nil # End: -# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 00923fb..d644fe9 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -33,7 +33,8 @@ from threading import Condition from subprocess import PIPE, Popen import sys import os -import threading, queue +import threading +import queue import pathlib import logging from timeit import default_timer as timer @@ -257,7 +258,7 @@ def server_thread_func(): self.send_header("Content-type", "text/html") self.end_headers() - except: + except Exception: thread_error = True daemon_log("SERVER ERROR: " + traceback.format_exc()) raise @@ -304,4 +305,5 @@ server_thread.join() # tab-width:4 # indent-tabs-mode:nil # End: -# vim: set expandtab tabstop=4 shiftwidth=4: \ No newline at end of file +# vim: set expandtab tabstop=4 shiftwidth=4: + -- cgit v0.12 From 898d87dc5e026b154ac689d8ddd7400ce07a37bc Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 13:11:39 -0700 Subject: [ci skip] Added NINJA_SCONS_DAEMON_KEEP_ALIVE to docs. --- SCons/Tool/ninja/ninja.xml | 10 ++++++++++ SCons/Tool/ninja/ninja_daemon_build.py | 1 - SCons/Tool/ninja/ninja_scons_daemon.py | 1 - 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index c5ee15b..b023e86 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -73,6 +73,7 @@ See its __doc__ string for a discussion of the format. _NINJA_REGENERATE_DEPS_FUNC __NINJA_NO IMPLICIT_COMMAND_DEPENDENCIES + NINJA_SCONS_DAEMON_KEEP_ALIVE @@ -335,5 +336,14 @@ See its __doc__ string for a discussion of the format. + + + + The number of seconds for the SCons deamon launched by ninja to stay alive. + (Default: 180000) + + + + diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index af0e6b4..6e1c7be 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -87,4 +87,3 @@ while True: # indent-tabs-mode:nil # End: # vim: set expandtab tabstop=4 shiftwidth=4: - diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index d644fe9..2b13993 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -306,4 +306,3 @@ server_thread.join() # indent-tabs-mode:nil # End: # vim: set expandtab tabstop=4 shiftwidth=4: - -- cgit v0.12 From 3c7a7395d80f97f4e0b2f8d038b2a09bcbeb9e60 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 14 Mar 2022 15:24:16 -0500 Subject: remove scons-ninja-daemon option in favor of interactive --- SCons/Node/__init__.py | 1 - SCons/Script/Interactive.py | 161 ++++++++++++++++----------------- SCons/Script/Main.py | 4 +- SCons/Script/SConsOptions.py | 5 - SCons/Tool/ninja/NinjaState.py | 8 +- SCons/Tool/ninja/ninja_scons_daemon.py | 2 +- 6 files changed, 82 insertions(+), 99 deletions(-) diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index d0539b4..ec742a6 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -105,7 +105,6 @@ SConscriptNodes = set() # currently used to release parts of a target's info during # clean builds and update runs (see release_target_info). interactive = False -ninja_scons_daemon = False def is_derived_none(node): raise NotImplementedError diff --git a/SCons/Script/Interactive.py b/SCons/Script/Interactive.py index 1015b00..d694740 100644 --- a/SCons/Script/Interactive.py +++ b/SCons/Script/Interactive.py @@ -29,7 +29,7 @@ # of its own, which might or might not be a good thing. Nevertheless, # here are some enhancements that will probably be requested some day # and are worth keeping in mind (assuming this takes off): -# +# # - A command to re-read / re-load the SConscript files. This may # involve allowing people to specify command-line options (e.g. -f, # -I, --no-site-dir) that affect how the SConscript files are read. @@ -188,91 +188,82 @@ version Prints SCons version information. x.extend(n.alter_targets()[0]) nodes.extend(x) - if SCons.Node.ninja_scons_daemon: - for n in nodes: - print(f"Node: {n}, State: {SCons.Node.failed}") - if n.get_state() == SCons.Node.failed: - print(n) - n.clear() - n.set_state(SCons.Node.no_state) - n.implicit = None - else: - # Clean up so that we can perform the next build correctly. - # - # We do this by walking over all the children of the targets, - # and clearing their state. - # - # We currently have to re-scan each node to find their - # children, because built nodes have already been partially - # cleared and don't remember their children. (In scons - # 0.96.1 and earlier, this wasn't the case, and we didn't - # have to re-scan the nodes.) - # - # Because we have to re-scan each node, we can't clear the - # nodes as we walk over them, because we may end up rescanning - # a cleared node as we scan a later node. Therefore, only - # store the list of nodes that need to be cleared as we walk - # the tree, and clear them in a separate pass. - # - # XXX: Someone more familiar with the inner workings of scons - # may be able to point out a more efficient way to do this. - - SCons.Script.Main.progress_display("scons: Clearing cached node information ...") - - seen_nodes = {} - - def get_unseen_children(node, parent, seen_nodes=seen_nodes): - def is_unseen(node, seen_nodes=seen_nodes): - return node not in seen_nodes - return [child for child in node.children(scan=1) if is_unseen(child)] - - def add_to_seen_nodes(node, parent, seen_nodes=seen_nodes): - seen_nodes[node] = 1 - - # If this file is in a VariantDir and has a - # corresponding source file in the source tree, remember the - # node in the source tree, too. This is needed in - # particular to clear cached implicit dependencies on the - # source file, since the scanner will scan it if the - # VariantDir was created with duplicate=0. - try: - rfile_method = node.rfile - except AttributeError: - return - else: - rfile = rfile_method() - if rfile != node: - seen_nodes[rfile] = 1 - - for node in nodes: - walker = SCons.Node.Walker(node, - kids_func=get_unseen_children, - eval_func=add_to_seen_nodes) + # Clean up so that we can perform the next build correctly. + # + # We do this by walking over all the children of the targets, + # and clearing their state. + # + # We currently have to re-scan each node to find their + # children, because built nodes have already been partially + # cleared and don't remember their children. (In scons + # 0.96.1 and earlier, this wasn't the case, and we didn't + # have to re-scan the nodes.) + # + # Because we have to re-scan each node, we can't clear the + # nodes as we walk over them, because we may end up rescanning + # a cleared node as we scan a later node. Therefore, only + # store the list of nodes that need to be cleared as we walk + # the tree, and clear them in a separate pass. + # + # XXX: Someone more familiar with the inner workings of scons + # may be able to point out a more efficient way to do this. + + SCons.Script.Main.progress_display("scons: Clearing cached node information ...") + + seen_nodes = {} + + def get_unseen_children(node, parent, seen_nodes=seen_nodes): + def is_unseen(node, seen_nodes=seen_nodes): + return node not in seen_nodes + return [child for child in node.children(scan=1) if is_unseen(child)] + + def add_to_seen_nodes(node, parent, seen_nodes=seen_nodes): + seen_nodes[node] = 1 + + # If this file is in a VariantDir and has a + # corresponding source file in the source tree, remember the + # node in the source tree, too. This is needed in + # particular to clear cached implicit dependencies on the + # source file, since the scanner will scan it if the + # VariantDir was created with duplicate=0. + try: + rfile_method = node.rfile + except AttributeError: + return + else: + rfile = rfile_method() + if rfile != node: + seen_nodes[rfile] = 1 + + for node in nodes: + walker = SCons.Node.Walker(node, + kids_func=get_unseen_children, + eval_func=add_to_seen_nodes) + n = walker.get_next() + while n: n = walker.get_next() - while n: - n = walker.get_next() - - for node in seen_nodes.keys(): - # Call node.clear() to clear most of the state - node.clear() - # node.clear() doesn't reset node.state, so call - # node.set_state() to reset it manually - node.set_state(SCons.Node.no_state) - node.implicit = None - - # Debug: Uncomment to verify that all Taskmaster reference - # counts have been reset to zero. - #if node.ref_count != 0: - # from SCons.Debug import Trace - # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) - - # TODO: REMOVE WPD DEBUG 02/14/2022 - # This call was clearing the list of sconsign files to be written, so it would - # only write the results of the first build command. All others wouldn't be written - # to .SConsign. - # Pretty sure commenting this out is the correct fix. - # SCons.SConsign.Reset() - SCons.Script.Main.progress_display("scons: done clearing node information.") + + for node in seen_nodes.keys(): + # Call node.clear() to clear most of the state + node.clear() + # node.clear() doesn't reset node.state, so call + # node.set_state() to reset it manually + node.set_state(SCons.Node.no_state) + node.implicit = None + + # Debug: Uncomment to verify that all Taskmaster reference + # counts have been reset to zero. + #if node.ref_count != 0: + # from SCons.Debug import Trace + # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) + + # TODO: REMOVE WPD DEBUG 02/14/2022 + # This call was clearing the list of sconsign files to be written, so it would + # only write the results of the first build command. All others wouldn't be written + # to .SConsign. + # Pretty sure commenting this out is the correct fix. + # SCons.SConsign.Reset() + SCons.Script.Main.progress_display("scons: done clearing node information.") def do_clean(self, argv): """\ diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index 87d57bc..ab2dc0e 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -987,8 +987,6 @@ def _main(parser): # This would then cause subtle bugs, as already happened in #2971. if options.interactive: SCons.Node.interactive = True - if options.ninja_scons_daemon: - SCons.Node.ninja_scons_daemon = True # That should cover (most of) the options. # Next, set up the variables that hold command-line arguments, # so the SConscript files that we read and execute have access to them. @@ -1126,7 +1124,7 @@ def _main(parser): platform = SCons.Platform.platform_module() - if options.interactive or options.ninja_scons_daemon: + if options.interactive: SCons.Script.Interactive.interact(fs, OptionsParser, options, targets, target_top) diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 133d9ab..e2631fb 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -858,11 +858,6 @@ def Parser(version): action="store_true", help="Run in interactive mode") - op.add_option('--ninja-scons-daemon', - dest='ninja_scons_daemon', default=False, - action="store_true", - help="A special interactive mode to support a scons daemon for ninja builds. Intended for use only by the ninja tool.") - op.add_option('-j', '--jobs', nargs=1, type="int", dest="num_jobs", default=1, diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index b6de65c..3ef3bb2 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -63,7 +63,7 @@ class NinjaState: os.pardir, 'data', 'bin', - ninja_bin)) + ninja_bin)) if not os.path.exists(self.ninja_bin_path): # couldn't find it, just give the bin name and hope # its in the path later @@ -219,7 +219,7 @@ class NinjaState: # output. "restat": 1, }, - + "SCONS_DAEMON": { "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {PORT} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", "description": "Starting scons daemon...", @@ -551,8 +551,8 @@ class NinjaState: os.kill(pid, signal.SIGINT) except OSError: pass - - if os.path.exists(scons_daemon_dirty): + + if os.path.exists(scons_daemon_dirty): os.unlink(scons_daemon_dirty) diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index d644fe9..69d984d 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -124,7 +124,7 @@ def daemon_thread_func(): global finished_building global error_nodes try: - args_list = args + ["--ninja-scons-daemon"] + args_list = args + ["--interactive"] daemon_log(f"Starting daemon with args: {' '.join(args_list)}") daemon_log(f"cwd: {os.getcwd()}") -- cgit v0.12 From d382175ba00943cbc488a648e973f3e38162397e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 13:27:49 -0700 Subject: [ci skip] Add comment/TODO to code to handle Alias() name conflicts with real files/directories --- SCons/Tool/ninja/NinjaState.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 3ef3bb2..b1d7fdd 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -290,6 +290,10 @@ class NinjaState: node_string = str(node) if node_string in self.builds: + # TODO: If we work out a way to handle Alias() with same name as file this logic can be removed + # This works around adding Alias with the same name as a Node. + # It's not great way to workaround because it force renames the alias, + # but the alternative is broken ninja support. warn_msg = f"Alias {node_string} name the same as File node, ninja does not support this. Renaming Alias {node_string} to {node_string}_alias." if isinstance(node, SCons.Node.Alias.Alias): for i, output in enumerate(build["outputs"]): -- cgit v0.12 From e3cbfd6ae3986513148872308e2ddd0584364da4 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 14:01:46 -0700 Subject: Allow explicitly setting the daemon port with NINJA_SCONS_DAEMON_PORT --- SCons/Tool/ninja/NinjaState.py | 9 +++------ SCons/Tool/ninja/__init__.py | 2 ++ SCons/Tool/ninja/ninja.xml | 11 +++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index b1d7fdd..11e928d 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -87,10 +87,7 @@ class NinjaState: # to make the SCONS_INVOCATION variable properly quoted for things # like CCFLAGS scons_escape = env.get("ESCAPE", lambda x: x) - - import random - - PORT = str(random.randint(10000, 60000)) + scons_daemon_port = int(env.get('NINJA_SCONS_DAEMON_PORT',-1)) # if SCons was invoked from python, we expect the first arg to be the scons.py # script, otherwise scons was invoked from the scons script @@ -192,7 +189,7 @@ class NinjaState: "restat": 1, }, "TEMPLATE": { - "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_daemon_build.py'} {PORT} {get_path(env.get('NINJA_DIR'))} $out", + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_daemon_build.py'} {scons_daemon_port} {get_path(env.get('NINJA_DIR'))} $out", "description": "Defer to SCons to build $out", "pool": "local_pool", "restat": 1 @@ -221,7 +218,7 @@ class NinjaState: }, "SCONS_DAEMON": { - "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {PORT} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {scons_daemon_port} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", "description": "Starting scons daemon...", "pool": "local_pool", # restat diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 3d14faf..9fe9dd4 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -26,6 +26,7 @@ import importlib import os +import random import subprocess import sys @@ -188,6 +189,7 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_DIR'] = env.Dir(env.get("NINJA_DIR", '#/.ninja')) env["NINJA_SCONS_DAEMON_KEEP_ALIVE"] = env.get("NINJA_SCONS_DAEMON_KEEP_ALIVE", 180000) + env["NINJA_SCONS_DAEMON_PORT"] = env.get('NINJA_SCONS_DAEMON_PORT', random.randint(10000,60000)) if GetOption("disable_ninja"): env.SConsignFile(os.path.join(str(env['NINJA_DIR']),'.ninja.sconsign')) diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index b023e86..b497156 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -74,6 +74,7 @@ See its __doc__ string for a discussion of the format. __NINJA_NO IMPLICIT_COMMAND_DEPENDENCIES NINJA_SCONS_DAEMON_KEEP_ALIVE + NINJA_SCONS_DAEMON_PORT @@ -345,5 +346,15 @@ See its __doc__ string for a discussion of the format. + + + + The TCP/IP port for the SCons daemon to listen on. + NOTE: You cannot use a port already being listened to on your build machine. + (Default: random number between 10000,60000) + + + + -- cgit v0.12 From 53a9b7cc192bc589a28ef2f73ee6b8428c1b1549 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 14:27:24 -0700 Subject: update test/ninja/force_scons_callback.py to test using NINJA_SCONS_DAEMON_PORT to explicitly set the daemon port --- SCons/Tool/ninja/__init__.py | 2 +- test/ninja/force_scons_callback.py | 57 ++++++++++++---------- .../sconstruct_force_scons_callback | 11 +++-- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 9fe9dd4..913738a 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -189,7 +189,7 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_DIR'] = env.Dir(env.get("NINJA_DIR", '#/.ninja')) env["NINJA_SCONS_DAEMON_KEEP_ALIVE"] = env.get("NINJA_SCONS_DAEMON_KEEP_ALIVE", 180000) - env["NINJA_SCONS_DAEMON_PORT"] = env.get('NINJA_SCONS_DAEMON_PORT', random.randint(10000,60000)) + env["NINJA_SCONS_DAEMON_PORT"] = env.get('NINJA_SCONS_DAEMON_PORT', random.randint(10000, 60000)) if GetOption("disable_ninja"): env.SConsignFile(os.path.join(str(env['NINJA_DIR']),'.ninja.sconsign')) diff --git a/test/ninja/force_scons_callback.py b/test/ninja/force_scons_callback.py index 59b54a0..c99ed58 100644 --- a/test/ninja/force_scons_callback.py +++ b/test/ninja/force_scons_callback.py @@ -37,47 +37,50 @@ except ImportError: _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +ninja_bin = os.path.abspath( + os.path.join(ninja.__file__, os.pardir, "data", "bin", "ninja" + _exe) +) -test.dir_fixture('ninja-fixture') +test.dir_fixture("ninja-fixture") -test.file_fixture('ninja_test_sconscripts/sconstruct_force_scons_callback', 'SConstruct') +test.file_fixture( + "ninja_test_sconscripts/sconstruct_force_scons_callback", "SConstruct" +) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) -if test.stdout().count('Defer to SCons to build') != 1: +test.must_contain_all_lines(test.stdout(), ["Generating: build.ninja"]) +test.must_contain_all(test.stdout(), "Executing:") +test.must_contain_all(test.stdout(), "ninja%(_exe)s -f" % locals()) +if test.stdout().count("Defer to SCons to build") != 1: test.fail_test() -test.must_match('out.txt', 'foo.c' + os.linesep) -test.must_match('out2.txt', "test2.cpp" + os.linesep) +test.must_match("out.txt", "foo.c" + os.linesep) +test.must_match("out2.txt", "test2.cpp" + os.linesep) # clean build and ninja files -test.run(arguments='-c', stdout=None) -test.must_contain_all_lines(test.stdout(), [ - 'Removed out.txt', - 'Removed out2.txt', - 'Removed build.ninja']) +test.run(arguments="-c", stdout=None) +test.must_contain_all_lines( + test.stdout(), ["Removed out.txt", "Removed out2.txt", "Removed build.ninja"] +) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('out.txt')) -test.must_not_exist(test.workpath('out2.txt')) +test.run(arguments="--disable-execute-ninja", stdout=None) +test.must_contain_all_lines(test.stdout(), ["Generating: build.ninja"]) +test.must_not_exist(test.workpath("out.txt")) +test.must_not_exist(test.workpath("out2.txt")) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +program = test.workpath("run_ninja_env.bat") if IS_WINDOWS else ninja_bin test.run(program=program, stdout=None) -if test.stdout().count('Defer to SCons to build') != 1: +if test.stdout().count("Defer to SCons to build") != 1: test.fail_test() -test.must_match('out.txt', 'foo.c' + os.linesep) -test.must_match('out2.txt', "test2.cpp" + os.linesep) +test.must_match("out.txt", "foo.c" + os.linesep) +test.must_match("out2.txt", "test2.cpp" + os.linesep) + +# only generate the ninja file with specific NINJA_SCONS_DAEMON_PORT +test.run(arguments="PORT=9999 --disable-execute-ninja", stdout=None) +# Verify that port # propagates to call to ninja_run_daemon.py +test.must_contain(test.workpath("build.ninja"), "ninja_run_daemon.py 9999") test.pass_test() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback b/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback index 55729a6..ef3562b 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback +++ b/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback @@ -1,8 +1,11 @@ -SetOption('experimental','ninja') +SetOption("experimental", "ninja") DefaultEnvironment(tools=[]) env = Environment(tools=[]) -env.Tool('ninja') +daemon_port = ARGUMENTS.get("PORT", False) +if daemon_port: + env["NINJA_SCONS_DAEMON_PORT"] = int(daemon_port) +env.Tool("ninja") -env.Command('out.txt', 'foo.c', 'echo $SOURCE> $TARGET', NINJA_FORCE_SCONS_BUILD=True) -env.Command('out2.txt', 'test2.cpp', 'echo $SOURCE> $TARGET') \ No newline at end of file +env.Command("out.txt", "foo.c", "echo $SOURCE> $TARGET", NINJA_FORCE_SCONS_BUILD=True) +env.Command("out2.txt", "test2.cpp", "echo $SOURCE> $TARGET") -- cgit v0.12 From d19f74f8be81e8b1a18033afbb66726674311f6f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 14 Mar 2022 14:38:46 -0700 Subject: [ci skip] Updated CHANGES/RELEASE --- CHANGES.txt | 6 ++++++ RELEASE.txt | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 149de73..e2b26e4 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -29,6 +29,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Daniel Moody: - Add cache-debug messages for push failures. + - Ninja: Changed generated build.ninja file to run SCons only build Actions via + a SCons Deamon. Added logic for starting and connecting to SCons daemon (currently + only used for ninja) + - Ninja: Fix issue where Configure files weren't being properly processed when build run + via ninja. + From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index aeffddf..becceaa 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -34,13 +34,18 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY which are still listed in the manpage. - Help is now sensitive to the size of the terminal window: the width of the help text will scale to wider (or narrower) terminals than 80 characters. - +- Ninja: Changed generated build.ninja file to run SCons only build Actions via + a SCons Deamon. Added logic for starting and connecting to SCons daemon (currently + only used for ninja) FIXES ----- - Fix a number of Python ResourceWarnings which are issued when running SCons and/or it's tests with python 3.9 (or higher) +- Ninja: Fix issue where Configure files weren't being properly processed when build run + via ninja. + IMPROVEMENTS ------------ @@ -70,4 +75,4 @@ Thanks to the following contributors listed below for their contributions to thi ========================================================================================== .. code-block:: text - git shortlog --no-merges -ns 4.0.1..HEAD + git shortlog --no-merges -ns 4.3.0..HEAD -- cgit v0.12 From ff81eebb73825091051e5ee846d0d89c60798112 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 16 Aug 2019 08:19:20 -0600 Subject: Use super call instead of direct class call - super used where direct call to superclass existed - convert a few older-style super() (two-argument) uses - in a few places, where there was an intersection with a super change, variables that override a builtin (e.g. "dict") were renamed. Signed-off-by: Mats Wichmann --- CHANGES.txt | 3 + SCons/Action.py | 4 +- SCons/Builder.py | 13 ++-- SCons/Environment.py | 109 +++++++++++++++++--------------- SCons/Errors.py | 6 +- SCons/Job.py | 2 +- SCons/JobTests.py | 6 +- SCons/Memoize.py | 2 +- SCons/Node/Alias.py | 2 +- SCons/Node/FS.py | 12 ++-- SCons/Node/FSTests.py | 4 +- SCons/Node/NodeTests.py | 6 +- SCons/Node/Python.py | 2 +- SCons/SConf.py | 6 +- SCons/SConsign.py | 6 +- SCons/Subst.py | 4 +- SCons/SubstTests.py | 2 +- SCons/Tool/GettextCommon.py | 2 +- SCons/Tool/MSCommon/sdk.py | 4 +- SCons/Tool/msvs.py | 6 +- SCons/UtilTests.py | 2 +- SCons/cppTests.py | 2 +- bin/SConsDoc.py | 2 +- src/test_setup.py | 2 +- testing/framework/TestCommon.py | 2 +- testing/framework/TestRuntest.py | 2 +- testing/framework/TestSCons.py | 4 +- testing/framework/TestSCons_time.py | 2 +- testing/framework/TestSConsign.py | 2 +- testing/framework/TestUnit/taprunner.py | 14 ++-- 30 files changed, 122 insertions(+), 113 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d36e911..d19fb06 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -56,6 +56,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER processors for testing threads. - Fixed crash in C scanner's dictify_CPPDEFINES() function which happens if AppendUnique is called on CPPPATH. (Issue #4108). + - As "code modernization" all of SCons now uses the current super() + zero-argument syntax instead of direct calls to a parent class method + or the super() two-argument syntax. From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/SCons/Action.py b/SCons/Action.py index b7c6bb7..81dc033 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -848,7 +848,7 @@ class CommandAction(_ActionAction): # variables. if SCons.Debug.track_instances: logInstanceCreation(self, 'Action.CommandAction') - _ActionAction.__init__(self, **kw) + super().__init__(**kw) if is_List(cmd): if [c for c in cmd if is_List(c)]: raise TypeError("CommandAction should be given only " @@ -1231,7 +1231,7 @@ class FunctionAction(_ActionAction): # This is weird, just do the best we can. self.funccontents = _object_contents(execfunction) - _ActionAction.__init__(self, **kw) + super().__init__(**kw) def function_name(self): try: diff --git a/SCons/Builder.py b/SCons/Builder.py index 41475ac..ab51c32 100644 --- a/SCons/Builder.py +++ b/SCons/Builder.py @@ -130,8 +130,8 @@ class DictCmdGenerator(SCons.Util.Selector): to return the proper action based on the file suffix of the source file.""" - def __init__(self, dict=None, source_ext_match=1): - SCons.Util.Selector.__init__(self, dict) + def __init__(self, mapping=None, source_ext_match=True): + super().__init__(mapping) self.source_ext_match = source_ext_match def src_suffixes(self): @@ -222,10 +222,11 @@ class OverrideWarner(UserDict): can actually invoke multiple builders. This class only emits the warnings once, no matter how many Builders are invoked. """ - def __init__(self, dict): - UserDict.__init__(self, dict) + def __init__(self, mapping): + super().__init__(mapping) if SCons.Debug.track_instances: logInstanceCreation(self, 'Builder.OverrideWarner') self.already_warned = None + def warn(self): if self.already_warned: return @@ -245,7 +246,7 @@ def Builder(**kw): kw['action'] = SCons.Action.CommandGeneratorAction(kw['generator'], {}) del kw['generator'] elif 'action' in kw: - source_ext_match = kw.get('source_ext_match', 1) + source_ext_match = kw.get('source_ext_match', True) if 'source_ext_match' in kw: del kw['source_ext_match'] if SCons.Util.is_Dict(kw['action']): @@ -876,7 +877,7 @@ class CompositeBuilder(SCons.Util.Proxy): def __init__(self, builder, cmdgen): if SCons.Debug.track_instances: logInstanceCreation(self, 'Builder.CompositeBuilder') - SCons.Util.Proxy.__init__(self, builder) + super().__init__(builder) # cmdgen should always be an instance of DictCmdGenerator. self.cmdgen = cmdgen diff --git a/SCons/Environment.py b/SCons/Environment.py index d0a4baf..34e4968 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -275,12 +275,12 @@ class BuilderDict(UserDict): the Builders. We need to do this because every time someone changes the Builders in the Environment's BUILDERS dictionary, we must update the Environment's attributes.""" - def __init__(self, dict, env): + def __init__(self, mapping, env): # Set self.env before calling the superclass initialization, # because it will end up calling our other methods, which will # need to point the values in this dictionary to self.env. self.env = env - UserDict.__init__(self, dict) + super().__init__(mapping) def __semi_deepcopy__(self): # These cannot be copied since they would both modify the same builder object, and indeed @@ -294,15 +294,15 @@ class BuilderDict(UserDict): pass else: self.env.RemoveMethod(method) - UserDict.__setitem__(self, item, val) + super().__setitem__(item, val) BuilderWrapper(self.env, val, item) def __delitem__(self, item): - UserDict.__delitem__(self, item) + super().__delitem__(item) delattr(self.env, item) - def update(self, dict): - for i, v in dict.items(): + def update(self, mapping): + for i, v in mapping.items(): self.__setitem__(i, v) @@ -637,7 +637,7 @@ class SubstitutionEnvironment: it is assumed to be a command and the rest of the string is executed; the result of that evaluation is then added to the dict. """ - dict = { + mapping = { 'ASFLAGS' : CLVar(''), 'CFLAGS' : CLVar(''), 'CCFLAGS' : CLVar(''), @@ -667,12 +667,12 @@ class SubstitutionEnvironment: arg = self.backtick(arg[1:]) # utility function to deal with -D option - def append_define(name, dict = dict): + def append_define(name, mapping=mapping): t = name.split('=') if len(t) == 1: - dict['CPPDEFINES'].append(name) + mapping['CPPDEFINES'].append(name) else: - dict['CPPDEFINES'].append([t[0], '='.join(t[1:])]) + mapping['CPPDEFINES'].append([t[0], '='.join(t[1:])]) # Loop through the flags and add them to the appropriate option. # This tries to strike a balance between checking for all possible @@ -702,67 +702,67 @@ class SubstitutionEnvironment: append_define(arg) elif append_next_arg_to == '-include': t = ('-include', self.fs.File(arg)) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-imacros': t = ('-imacros', self.fs.File(arg)) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-isysroot': t = ('-isysroot', arg) - dict['CCFLAGS'].append(t) - dict['LINKFLAGS'].append(t) + mapping['CCFLAGS'].append(t) + mapping['LINKFLAGS'].append(t) elif append_next_arg_to == '-isystem': t = ('-isystem', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-iquote': t = ('-iquote', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-idirafter': t = ('-idirafter', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-arch': t = ('-arch', arg) - dict['CCFLAGS'].append(t) - dict['LINKFLAGS'].append(t) + mapping['CCFLAGS'].append(t) + mapping['LINKFLAGS'].append(t) elif append_next_arg_to == '--param': t = ('--param', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) else: - dict[append_next_arg_to].append(arg) + mapping[append_next_arg_to].append(arg) append_next_arg_to = None elif not arg[0] in ['-', '+']: - dict['LIBS'].append(self.fs.File(arg)) + mapping['LIBS'].append(self.fs.File(arg)) elif arg == '-dylib_file': - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) append_next_arg_to = 'LINKFLAGS' elif arg[:2] == '-L': if arg[2:]: - dict['LIBPATH'].append(arg[2:]) + mapping['LIBPATH'].append(arg[2:]) else: append_next_arg_to = 'LIBPATH' elif arg[:2] == '-l': if arg[2:]: - dict['LIBS'].append(arg[2:]) + mapping['LIBS'].append(arg[2:]) else: append_next_arg_to = 'LIBS' elif arg[:2] == '-I': if arg[2:]: - dict['CPPPATH'].append(arg[2:]) + mapping['CPPPATH'].append(arg[2:]) else: append_next_arg_to = 'CPPPATH' elif arg[:4] == '-Wa,': - dict['ASFLAGS'].append(arg[4:]) - dict['CCFLAGS'].append(arg) + mapping['ASFLAGS'].append(arg[4:]) + mapping['CCFLAGS'].append(arg) elif arg[:4] == '-Wl,': if arg[:11] == '-Wl,-rpath=': - dict['RPATH'].append(arg[11:]) + mapping['RPATH'].append(arg[11:]) elif arg[:7] == '-Wl,-R,': - dict['RPATH'].append(arg[7:]) + mapping['RPATH'].append(arg[7:]) elif arg[:6] == '-Wl,-R': - dict['RPATH'].append(arg[6:]) + mapping['RPATH'].append(arg[6:]) else: - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg[:4] == '-Wp,': - dict['CPPFLAGS'].append(arg) + mapping['CPPFLAGS'].append(arg) elif arg[:2] == '-D': if arg[2:]: append_define(arg[2:]) @@ -771,32 +771,32 @@ class SubstitutionEnvironment: elif arg == '-framework': append_next_arg_to = 'FRAMEWORKS' elif arg[:14] == '-frameworkdir=': - dict['FRAMEWORKPATH'].append(arg[14:]) + mapping['FRAMEWORKPATH'].append(arg[14:]) elif arg[:2] == '-F': if arg[2:]: - dict['FRAMEWORKPATH'].append(arg[2:]) + mapping['FRAMEWORKPATH'].append(arg[2:]) else: append_next_arg_to = 'FRAMEWORKPATH' - elif arg in [ + elif arg in ( '-mno-cygwin', '-pthread', '-openmp', '-fmerge-all-constants', '-fopenmp', - ]: - dict['CCFLAGS'].append(arg) - dict['LINKFLAGS'].append(arg) + ): + mapping['CCFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg == '-mwindows': - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg[:5] == '-std=': if '++' in arg[5:]: - key='CXXFLAGS' + key = 'CXXFLAGS' else: - key='CFLAGS' - dict[key].append(arg) + key = 'CFLAGS' + mapping[key].append(arg) elif arg[0] == '+': - dict['CCFLAGS'].append(arg) - dict['LINKFLAGS'].append(arg) + mapping['CCFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg in [ '-include', '-imacros', @@ -809,11 +809,11 @@ class SubstitutionEnvironment: ]: append_next_arg_to = arg else: - dict['CCFLAGS'].append(arg) + mapping['CCFLAGS'].append(arg) for arg in flags: do_parse(arg) - return dict + return mapping def MergeFlags(self, args, unique=True): """Merge flags into construction variables. @@ -2375,7 +2375,7 @@ class OverrideEnvironment(Base): if SCons.Debug.track_instances: logInstanceCreation(self, 'Environment.OverrideEnvironment') self.__dict__['__subject'] = subject if overrides is None: - self.__dict__['overrides'] = dict() + self.__dict__['overrides'] = {} else: self.__dict__['overrides'] = overrides @@ -2472,6 +2472,11 @@ class OverrideEnvironment(Base): self.__dict__['overrides'].update(other) def _update_onlynew(self, other): + """Update a dict with new keys. + + Unlike the .update method, if the key is already present, + it is not replaced. + """ for k, v in other.items(): if k not in self.__dict__['overrides']: self.__dict__['overrides'][k] = v @@ -2526,14 +2531,14 @@ def NoSubstitutionProxy(subject): del kwdict['executor'] else: kwdict['lvars'] = {} - def raw_to_mode(self, dict): + def raw_to_mode(self, mapping): try: - raw = dict['raw'] + raw = mapping['raw'] except KeyError: pass else: - del dict['raw'] - dict['mode'] = raw + del mapping['raw'] + mapping['mode'] = raw def subst(self, string, *args, **kwargs): return string def subst_kw(self, kw, *args, **kwargs): diff --git a/SCons/Errors.py b/SCons/Errors.py index 42db072..04cea38 100644 --- a/SCons/Errors.py +++ b/SCons/Errors.py @@ -99,8 +99,8 @@ class BuildError(Exception): self.action = action self.command = command - Exception.__init__(self, node, errstr, status, exitstatus, filename, - executor, action, command, exc_info) + super().__init__(node, errstr, status, exitstatus, filename, + executor, action, command, exc_info) def __str__(self): if self.filename: @@ -128,7 +128,7 @@ class ExplicitExit(Exception): self.node = node self.status = status self.exitstatus = status - Exception.__init__(self, *args) + super().__init__(*args) def convert_to_BuildError(status, exc_info=None): """Convert a return code to a BuildError Exception. diff --git a/SCons/Job.py b/SCons/Job.py index bcdb88a..b398790 100644 --- a/SCons/Job.py +++ b/SCons/Job.py @@ -238,7 +238,7 @@ else: and a boolean indicating whether the task executed successfully. """ def __init__(self, requestQueue, resultsQueue, interrupted): - threading.Thread.__init__(self) + super().__init__() self.daemon = True self.requestQueue = requestQueue self.resultsQueue = resultsQueue diff --git a/SCons/JobTests.py b/SCons/JobTests.py index 5b5a590..54d7fa4 100644 --- a/SCons/JobTests.py +++ b/SCons/JobTests.py @@ -410,13 +410,13 @@ class DummyNodeInfo: class testnode (SCons.Node.Node): def __init__(self): - SCons.Node.Node.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.executed self.ninfo = DummyNodeInfo() class goodnode (testnode): def __init__(self): - SCons.Node.Node.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.up_to_date self.ninfo = DummyNodeInfo() @@ -430,7 +430,7 @@ class slowgoodnode (goodnode): class badnode (goodnode): def __init__(self): - goodnode.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.failed def build(self, **kw): raise Exception('badnode exception') diff --git a/SCons/Memoize.py b/SCons/Memoize.py index 8c3303f..b02c144 100644 --- a/SCons/Memoize.py +++ b/SCons/Memoize.py @@ -159,7 +159,7 @@ class CountDict(Counter): def __init__(self, cls_name, method_name, keymaker): """ """ - Counter.__init__(self, cls_name, method_name) + super().__init__(cls_name, method_name) self.keymaker = keymaker def count(self, *args, **kw): """ Counts whether the computed key value is already present diff --git a/SCons/Node/Alias.py b/SCons/Node/Alias.py index b5e4eb4..1125c22 100644 --- a/SCons/Node/Alias.py +++ b/SCons/Node/Alias.py @@ -99,7 +99,7 @@ class Alias(SCons.Node.Node): BuildInfo = AliasBuildInfo def __init__(self, name): - SCons.Node.Node.__init__(self) + super().__init__() self.name = name self.changed_since_last_build = 1 self.store_info = 0 diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 5f05a86..c1bfd6a 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -82,7 +82,7 @@ class EntryProxyAttributeError(AttributeError): of the underlying Entry involved in an AttributeError exception. """ def __init__(self, entry_proxy, attribute): - AttributeError.__init__(self) + super().__init__() self.entry_proxy = entry_proxy self.attribute = attribute def __str__(self): @@ -579,7 +579,7 @@ class Base(SCons.Node.Node): signatures.""" if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.Base') - SCons.Node.Node.__init__(self) + super().__init__() # Filenames and paths are probably reused and are intern'ed to save some memory. # Filename with extension as it was specified when the object was @@ -982,7 +982,7 @@ class Entry(Base): 'contentsig'] def __init__(self, name, directory, fs): - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._func_exists = 3 self._func_get_contents = 1 @@ -1574,7 +1574,7 @@ class Dir(Base): def __init__(self, name, directory, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.Dir') - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._morph() def _morph(self): @@ -2563,7 +2563,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): if key != 'dependency_map' and hasattr(self, 'dependency_map'): del self.dependency_map - return super(FileBuildInfo, self).__setattr__(key, value) + return super().__setattr__(key, value) def convert_to_sconsign(self): """ @@ -2674,7 +2674,7 @@ class File(Base): def __init__(self, name, directory, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.File') - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._morph() def Entry(self, name): diff --git a/SCons/Node/FSTests.py b/SCons/Node/FSTests.py index d78940e..b93efc8 100644 --- a/SCons/Node/FSTests.py +++ b/SCons/Node/FSTests.py @@ -2568,7 +2568,7 @@ class FileTestCase(_tempdirTestCase): class ChangedNode(SCons.Node.FS.File): def __init__(self, name, directory=None, fs=None): - SCons.Node.FS.File.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self.name = name self.Tag('found_includes', []) self.stored_info = None @@ -2608,7 +2608,7 @@ class FileTestCase(_tempdirTestCase): class ChangedEnvironment(SCons.Environment.Base): def __init__(self): - SCons.Environment.Base.__init__(self) + super().__init__() self.decide_source = self._changed_timestamp_then_content class FakeNodeInfo: diff --git a/SCons/Node/NodeTests.py b/SCons/Node/NodeTests.py index b36c2e9..ee4d080 100644 --- a/SCons/Node/NodeTests.py +++ b/SCons/Node/NodeTests.py @@ -160,7 +160,7 @@ class NoneBuilder(Builder): class ListBuilder(Builder): def __init__(self, *nodes): - Builder.__init__(self) + super().__init__() self.nodes = nodes def execute(self, target, source, env): if hasattr(self, 'status'): @@ -200,7 +200,7 @@ class MyNode(SCons.Node.Node): simulate a real, functional Node subclass. """ def __init__(self, name): - SCons.Node.Node.__init__(self) + super().__init__() self.name = name self.Tag('found_includes', []) def __str__(self): @@ -1226,7 +1226,7 @@ class NodeTestCase(unittest.TestCase): """Test the get_string() method.""" class TestNode(MyNode): def __init__(self, name, sig): - MyNode.__init__(self, name) + super().__init__(name) self.sig = sig def for_signature(self): diff --git a/SCons/Node/Python.py b/SCons/Node/Python.py index c6850ab..80d2762 100644 --- a/SCons/Node/Python.py +++ b/SCons/Node/Python.py @@ -83,7 +83,7 @@ class Value(SCons.Node.Node): BuildInfo = ValueBuildInfo def __init__(self, value, built_value=None, name=None): - SCons.Node.Node.__init__(self) + super().__init__() self.value = value self.changed_since_last_build = 6 self.store_info = 0 diff --git a/SCons/SConf.py b/SCons/SConf.py index 0ad712d..ad29153 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -142,7 +142,7 @@ SCons.Warnings.enableWarningClass(SConfWarning) # some error definitions class SConfError(SCons.Errors.UserError): def __init__(self,msg): - SCons.Errors.UserError.__init__(self,msg) + super().__init__(msg) class ConfigureDryRunError(SConfError): """Raised when a file or directory needs to be updated during a Configure @@ -152,13 +152,13 @@ class ConfigureDryRunError(SConfError): msg = 'Cannot create configure directory "%s" within a dry-run.' % str(target) else: msg = 'Cannot update configure test "%s" within a dry-run.' % str(target) - SConfError.__init__(self,msg) + super().__init__(msg) class ConfigureCacheError(SConfError): """Raised when a use explicitely requested the cache feature, but the test is run the first time.""" def __init__(self,target): - SConfError.__init__(self, '"%s" is not yet built and cache is forced.' % str(target)) + super().__init__('"%s" is not yet built and cache is forced.' % str(target)) # define actions for building text files diff --git a/SCons/SConsign.py b/SCons/SConsign.py index 5b78855..ecca391 100644 --- a/SCons/SConsign.py +++ b/SCons/SConsign.py @@ -248,7 +248,7 @@ class DB(Base): determined by the database module. """ def __init__(self, dir): - Base.__init__(self) + super().__init__() self.dir = dir @@ -319,7 +319,7 @@ class Dir(Base): """ fp - file pointer to read entries from """ - Base.__init__(self) + super().__init__() if not fp: return @@ -352,7 +352,7 @@ class DirFile(Dir): fp = None try: - Dir.__init__(self, fp, dir) + super().__init__(fp, dir) except KeyboardInterrupt: raise except Exception: diff --git a/SCons/Subst.py b/SCons/Subst.py index 298df38..7535772 100644 --- a/SCons/Subst.py +++ b/SCons/Subst.py @@ -132,7 +132,7 @@ class CmdStringHolder(collections.UserString): proper escape sequences inserted. """ def __init__(self, cmd, literal=None): - collections.UserString.__init__(self, cmd) + super().__init__(cmd) self.literal = literal def is_literal(self): @@ -490,7 +490,7 @@ class ListSubber(collections.UserList): internally. """ def __init__(self, env, mode, conv, gvars): - collections.UserList.__init__(self, []) + super().__init__([]) self.env = env self.mode = mode self.conv = conv diff --git a/SCons/SubstTests.py b/SCons/SubstTests.py index b956a25..1a203a0 100644 --- a/SCons/SubstTests.py +++ b/SCons/SubstTests.py @@ -121,7 +121,7 @@ class SubstTestCase(unittest.TestCase): class MyNode(DummyNode): """Simple node work-alike with some extra stuff for testing.""" def __init__(self, name): - DummyNode.__init__(self, name) + super().__init__(name) class Attribute: pass self.attribute = Attribute() diff --git a/SCons/Tool/GettextCommon.py b/SCons/Tool/GettextCommon.py index 16900e0..058b997 100644 --- a/SCons/Tool/GettextCommon.py +++ b/SCons/Tool/GettextCommon.py @@ -206,7 +206,7 @@ class _POFileBuilder(BuilderBase): del kw['target_alias'] if 'target_factory' not in kw: kw['target_factory'] = _POTargetFactory(env, alias=alias).File - BuilderBase.__init__(self, **kw) + super().__init__(**kw) def _execute(self, env, target, source, *args, **kw): """ Execute builder's actions. diff --git a/SCons/Tool/MSCommon/sdk.py b/SCons/Tool/MSCommon/sdk.py index d95df91..aa94f4d 100644 --- a/SCons/Tool/MSCommon/sdk.py +++ b/SCons/Tool/MSCommon/sdk.py @@ -131,7 +131,7 @@ class WindowsSDK(SDKDefinition): """ HKEY_FMT = r'Software\Microsoft\Microsoft SDKs\Windows\v%s\InstallationFolder' def __init__(self, *args, **kw): - SDKDefinition.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.hkey_data = self.version class PlatformSDK(SDKDefinition): @@ -140,7 +140,7 @@ class PlatformSDK(SDKDefinition): """ HKEY_FMT = r'Software\Microsoft\MicrosoftSDK\InstalledSDKS\%s\Install Dir' def __init__(self, *args, **kw): - SDKDefinition.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.hkey_data = self.uuid # diff --git a/SCons/Tool/msvs.py b/SCons/Tool/msvs.py index 7a827f4..887cb59 100644 --- a/SCons/Tool/msvs.py +++ b/SCons/Tool/msvs.py @@ -320,7 +320,7 @@ class _GenerateV7User(_UserGenerator): self.usrhead = V9UserHeader self.usrconf = V9UserConfiguration self.usrdebg = V9DebugSettings - _UserGenerator.__init__(self, dspfile, source, env) + super().__init__(dspfile, source, env) def UserProject(self): confkeys = sorted(self.configs.keys()) @@ -395,7 +395,7 @@ class _GenerateV10User(_UserGenerator): self.usrhead = V10UserHeader self.usrconf = V10UserConfiguration self.usrdebg = V10DebugSettings - _UserGenerator.__init__(self, dspfile, source, env) + super().__init__(dspfile, source, env) def UserProject(self): confkeys = sorted(self.configs.keys()) @@ -1487,7 +1487,7 @@ class _DSWGenerator: class _GenerateV7DSW(_DSWGenerator): """Generates a Solution file for MSVS .NET""" def __init__(self, dswfile, source, env): - _DSWGenerator.__init__(self, dswfile, source, env) + super().__init__(dswfile, source, env) self.file = None self.version = self.env['MSVS_VERSION'] diff --git a/SCons/UtilTests.py b/SCons/UtilTests.py index 5ed31cb..0cfb70f 100644 --- a/SCons/UtilTests.py +++ b/SCons/UtilTests.py @@ -899,7 +899,7 @@ class HashTestCase(unittest.TestCase): class FIPSHashTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): - super(FIPSHashTestCase, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) ############################### # algorithm mocks, can check if we called with usedforsecurity=False for python >= 3.9 diff --git a/SCons/cppTests.py b/SCons/cppTests.py index 99fa6cf..a9aef9d 100644 --- a/SCons/cppTests.py +++ b/SCons/cppTests.py @@ -853,7 +853,7 @@ class fileTestCase(unittest.TestCase): """) class MyPreProcessor(cpp.DumbPreProcessor): def __init__(self, *args, **kw): - cpp.DumbPreProcessor.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.files = [] def __call__(self, file): self.files.append(file) diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index a465057..82e2c72 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -499,7 +499,7 @@ class Function(Item): class Tool(Item): def __init__(self, name): - Item.__init__(self, name) + super().__init__(name) self.entity = self.name.replace('+', 'X') diff --git a/src/test_setup.py b/src/test_setup.py index 8410be3..5db21a1 100644 --- a/src/test_setup.py +++ b/src/test_setup.py @@ -81,7 +81,7 @@ class MyTestSCons(TestSCons.TestSCons): ] def __init__(self): - TestSCons.TestSCons.__init__(self) + super().__init__() self.root = self.workpath('root') self.prefix = self.root + os.path.splitdrive(sys.prefix)[1] diff --git a/testing/framework/TestCommon.py b/testing/framework/TestCommon.py index 2eeaad0..1999f74 100644 --- a/testing/framework/TestCommon.py +++ b/testing/framework/TestCommon.py @@ -261,7 +261,7 @@ class TestCommon(TestCmd): calling the base class initialization, and then changing directory to the workdir. """ - TestCmd.__init__(self, **kw) + super().__init__(**kw) os.chdir(self.workdir) def options_arguments(self, options, arguments): diff --git a/testing/framework/TestRuntest.py b/testing/framework/TestRuntest.py index 18dcb94..9368953 100644 --- a/testing/framework/TestRuntest.py +++ b/testing/framework/TestRuntest.py @@ -146,7 +146,7 @@ class TestRuntest(TestCommon): del kw['things_to_copy'] orig_cwd = os.getcwd() - TestCommon.__init__(self, **kw) + super().__init__(**kw) dirs = [os.environ.get('SCONS_RUNTEST_DIR', orig_cwd)] diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index 0bf6abd..079e17d 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -321,7 +321,7 @@ class TestSCons(TestCommon): if kw.get('ignore_python_version', -1) != -1: del kw['ignore_python_version'] - TestCommon.__init__(self, **kw) + super().__init__(**kw) if not self.external: import SCons.Node.FS @@ -1755,7 +1755,7 @@ class TimeSCons(TestSCons): if 'verbose' not in kw and not self.calibrate: kw['verbose'] = True - TestSCons.__init__(self, *args, **kw) + super().__init__(*args, **kw) # TODO(sgk): better way to get the script dir than sys.argv[0] self.test_dir = os.path.dirname(sys.argv[0]) diff --git a/testing/framework/TestSCons_time.py b/testing/framework/TestSCons_time.py index a57ca88..e647fe2 100644 --- a/testing/framework/TestSCons_time.py +++ b/testing/framework/TestSCons_time.py @@ -199,7 +199,7 @@ class TestSCons_time(TestCommon): if 'workdir' not in kw: kw['workdir'] = '' - TestCommon.__init__(self, **kw) + super().__init__(**kw) def archive_split(self, path): if path[-7:] == '.tar.gz': diff --git a/testing/framework/TestSConsign.py b/testing/framework/TestSConsign.py index 699e929..b0562bf 100644 --- a/testing/framework/TestSConsign.py +++ b/testing/framework/TestSConsign.py @@ -64,7 +64,7 @@ class TestSConsign(TestSCons): os.chdir(script_dir) self.script_dir = os.getcwd() - TestSCons.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.my_kw = { 'interpreter' : python, # imported from TestSCons diff --git a/testing/framework/TestUnit/taprunner.py b/testing/framework/TestUnit/taprunner.py index 0dde327..001db5c 100644 --- a/testing/framework/TestUnit/taprunner.py +++ b/testing/framework/TestUnit/taprunner.py @@ -43,29 +43,29 @@ class TAPTestResult(TextTestResult): self.stream.flush() def addSuccess(self, test): - super(TextTestResult, self).addSuccess(test) + super().addSuccess(test) self._process(test, "ok") def addFailure(self, test, err): - super(TextTestResult, self).addFailure(test, err) + super().addFailure(test, err) self._process(test, "not ok", "FAIL") # [ ] add structured data about assertion def addError(self, test, err): - super(TextTestResult, self).addError(test, err) + super().addError(test, err) self._process(test, "not ok", "ERROR") # [ ] add structured data about exception def addSkip(self, test, reason): - super(TextTestResult, self).addSkip(test, reason) + super().addSkip(test, reason) self._process(test, "ok", directive=(" # SKIP %s" % reason)) def addExpectedFailure(self, test, err): - super(TextTestResult, self).addExpectedFailure(test, err) + super().addExpectedFailure(test, err) self._process(test, "not ok", directive=" # TODO") def addUnexpectedSuccess(self, test): - super(TextTestResult, self).addUnexpectedSuccess(test) + super().addUnexpectedSuccess(test) self._process(test, "not ok", "FAIL (unexpected success)") """ @@ -90,7 +90,7 @@ class TAPTestRunner(TextTestRunner): for case in test: case.suite = test - return super(TAPTestRunner, self).run(test) + return super().run(test) if __name__ == "__main__": -- cgit v0.12 From 5b0b9b89151f9abb0ed0ae7f47adfe3718088c3c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 15 Mar 2022 08:52:06 -0600 Subject: Fix sider complaint It wants blanks between method definitions, this was one of the compressed proxy classes which was "touched" (not changed). Signed-off-by: Mats Wichmann --- SCons/Environment.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/SCons/Environment.py b/SCons/Environment.py index 34e4968..3507263 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -2521,16 +2521,20 @@ def NoSubstitutionProxy(subject): class _NoSubstitutionProxy(Environment): def __init__(self, subject): self.__dict__['__subject'] = subject + def __getattr__(self, name): return getattr(self.__dict__['__subject'], name) + def __setattr__(self, name, value): return setattr(self.__dict__['__subject'], name, value) + def executor_to_lvars(self, kwdict): if 'executor' in kwdict: kwdict['lvars'] = kwdict['executor'].get_lvars() del kwdict['executor'] else: kwdict['lvars'] = {} + def raw_to_mode(self, mapping): try: raw = mapping['raw'] @@ -2539,10 +2543,13 @@ def NoSubstitutionProxy(subject): else: del mapping['raw'] mapping['mode'] = raw + def subst(self, string, *args, **kwargs): return string + def subst_kw(self, kw, *args, **kwargs): return kw + def subst_list(self, string, *args, **kwargs): nargs = (string, self,) + args nkw = kwargs.copy() @@ -2550,6 +2557,7 @@ def NoSubstitutionProxy(subject): self.executor_to_lvars(nkw) self.raw_to_mode(nkw) return SCons.Subst.scons_subst_list(*nargs, **nkw) + def subst_target_source(self, string, *args, **kwargs): nargs = (string, self,) + args nkw = kwargs.copy() @@ -2557,6 +2565,7 @@ def NoSubstitutionProxy(subject): self.executor_to_lvars(nkw) self.raw_to_mode(nkw) return SCons.Subst.scons_subst(*nargs, **nkw) + return _NoSubstitutionProxy(subject) # Local Variables: -- cgit v0.12 From 17c7e6a59b311b1f3b8c5daedc749f815b2f9d74 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 15 Mar 2022 15:13:50 -0500 Subject: updated startup sequence to check if server process died during start. --- SCons/Tool/ninja/NinjaState.py | 2 +- SCons/Tool/ninja/ninja_run_daemon.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 11e928d..9fb697e 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -218,7 +218,7 @@ class NinjaState: }, "SCONS_DAEMON": { - "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {scons_daemon_port} {get_path(env.get('NINJA_DIR'))} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {scons_daemon_port} {env.get('NINJA_DIR').abspath} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", "description": "Starting scons daemon...", "pool": "local_pool", # restat diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 965e50b..fbeaf67 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -90,12 +90,15 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): if status != 200: print(msg.decode("utf-8")) exit(1) - logging.debug(f"Request Done: {sys.argv[3]}") + logging.debug(f"Server Responded it was ready!") break except ConnectionRefusedError: - logging.debug(f"Server not ready: {sys.argv[3]}") + logging.debug(f"Server not ready, server PID: {p.pid}") time.sleep(1) + if p.poll is not None: + logging.debug(f"Server process died, aborting: {p.returncode}") + sys.exit(p.returncode) except ConnectionResetError: logging.debug("Server ConnectionResetError") sys.stderr.write(error_msg) -- cgit v0.12 From 0a1a6f14c8c954fa968b6d911fe1a3ad6d4881f8 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 15 Mar 2022 16:12:07 -0500 Subject: adjust iterative speedup to pre-launch scons daemon before taking timing --- SCons/Tool/ninja/NinjaState.py | 3 ++- SCons/Tool/ninja/__init__.py | 4 ++++ test/ninja/iterative_speedup.py | 10 ++++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 9fb697e..fff60be 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -534,10 +534,11 @@ class NinjaState: ) ninja.build( - ["run_scons_daemon", scons_daemon_dirty], + ["run_ninja_scons_daemon_phony", scons_daemon_dirty], rule="SCONS_DAEMON", ) + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) pidfile = None if os.path.exists(scons_daemon_dirty): diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 913738a..9734ffc 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -206,6 +206,7 @@ def generate(env): SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] + def ninja_generate_deps(env): """Return a list of SConscripts TODO: Should we also include files loaded from site_scons/*** @@ -456,3 +457,6 @@ def generate(env): env['TEMPFILEDIR'] = "$NINJA_DIR/.response_files" env["TEMPFILE"] = NinjaNoResponseFiles + + + env.Alias('run-ninja-scons-daemon', 'run_ninja_scons_daemon_phony') diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 05e372c..e5673b0 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -109,7 +109,7 @@ def generate_source(parent_source, current_source): test.write('source_{}.h'.format(current_source), """ #include #include - + int print_function%(current_source)s(); """ % locals()) @@ -125,7 +125,7 @@ def mod_source_return(test_num): int print_function%(test_num)s() - { + { int test = 5 + 5; print_function%(parent_source)s(); return test; @@ -143,7 +143,7 @@ def mod_source_orig(test_num): int print_function%(test_num)s() - { + { return print_function%(parent_source)s(); } """ % locals()) @@ -190,13 +190,15 @@ jobs = '-j' + str(get_num_cpus()) ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] -start = time.perf_counter() test.run(arguments='--disable-execute-ninja', stdout=None) +test.run(program=ninja_program, arguments='run-ninja-scons-daemon', stdout=None) +start = time.perf_counter() test.run(program=ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] test.run(program=test.workpath('print_bin'), stdout="main print") + for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() -- cgit v0.12 From 77ba2d3b905b284350d5773b6fb06d6abcdf5236 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 16 Mar 2022 01:02:07 -0500 Subject: Added doc strings for ninja scripts --- SCons/Tool/ninja/ninja_daemon_build.py | 15 +++++++++------ SCons/Tool/ninja/ninja_run_daemon.py | 13 ++++++++++++- SCons/Tool/ninja/ninja_scons_daemon.py | 11 ++++++++++- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index 6e1c7be..f34935b 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# # MIT License # # Copyright The SCons Foundation @@ -22,7 +24,10 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - +This script is intended to execute a single build target. This script should be +called by ninja, passing the port, ninja dir, and build target via arguments. +The script then executes a simple get request to the scons daemon which is listening +on from localhost on the set port. """ import http.client @@ -74,14 +79,12 @@ while True: logging.debug(f"Request Done: {sys.argv[3]}") exit(0) - except ConnectionRefusedError: - logging.debug(f"Server not ready: {traceback.format_exc()}") - time.sleep(1) - except ConnectionResetError: - logging.debug("Server ConnectionResetError") + except Exception: + logging.debug(f"Failed to send command: {traceback.format_exc()}") exit(1) + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index fbeaf67..68d9893 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# # MIT License # # Copyright The SCons Foundation @@ -22,7 +24,16 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +This script is intended to be called by ninja to start up the scons daemon process. It will +launch the server and attempt to connect to it. This process needs to completely detach +from the spawned process so ninja can consider the build edge completed. It should be passed +the args which should be forwarded to the scons daemon process which could be any number of +# arguments. However the first few arguements are required to be port, ninja dir, and keep alive +timeout in seconds. +The scons_daemon_dirty file acts as a pidfile marker letting this script quickly skip over +restarting the server if the server is running. The assumption here is the pidfile should only +exist if the server is running. """ import subprocess @@ -90,7 +101,7 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): if status != 200: print(msg.decode("utf-8")) exit(1) - logging.debug(f"Server Responded it was ready!") + logging.debug("Server Responded it was ready!") break except ConnectionRefusedError: diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 99fcbb9..88f3b22 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 +# # MIT License # # Copyright The SCons Foundation @@ -22,7 +24,14 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - +This script primarily consists of two threads, the http server thread and the scons interactive +process thread. The http server thread will listen on the passed port for http get request +which should indicate some action for the scons interactive process to take. + +The daemon will keep log files in a tmp directory correlated to the hash of the absolute path +of the ninja build dir passed. The daemon will also use a keep alive time to know when to shut +itself down after the passed timeout of no activity. Any time the server receives a get request, +the keep alive time will be reset. """ import http.server -- cgit v0.12 From 77a1e99d356990b03c2be014658a072a5d2be4c1 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 16 Mar 2022 01:08:36 -0500 Subject: fix spelling mistake --- SCons/Tool/ninja/ninja_run_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 68d9893..057537a 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -28,7 +28,7 @@ This script is intended to be called by ninja to start up the scons daemon proce launch the server and attempt to connect to it. This process needs to completely detach from the spawned process so ninja can consider the build edge completed. It should be passed the args which should be forwarded to the scons daemon process which could be any number of -# arguments. However the first few arguements are required to be port, ninja dir, and keep alive +# arguments. However the first few arguments are required to be port, ninja dir, and keep alive timeout in seconds. The scons_daemon_dirty file acts as a pidfile marker letting this script quickly skip over -- cgit v0.12 From c47a00687a2753c00af8682f00bfe671671074b5 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 17 Mar 2022 12:36:44 -0600 Subject: Add simple unittest for -include Missing test was flagged by code coverage because the line was "touched" due to a variable rename. Signed-off-by: Mats Wichmann --- SCons/EnvironmentTests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index c505422..8359405 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -810,11 +810,13 @@ sys.exit(0) "-fmerge-all-constants " "-fopenmp " "-mno-cygwin -mwindows " - "-arch i386 -isysroot /tmp " + "-arch i386 " + "-isysroot /tmp " "-iquote /usr/include/foo1 " "-isystem /usr/include/foo2 " "-idirafter /usr/include/foo3 " "-imacros /usr/include/foo4 " + "-include /usr/include/foo5 " "--param l1-cache-size=32 --param l2-cache-size=6144 " "+DD64 " "-DFOO -DBAR=value -D BAZ " @@ -832,6 +834,7 @@ sys.exit(0) ('-isystem', '/usr/include/foo2'), ('-idirafter', '/usr/include/foo3'), ('-imacros', env.fs.File('/usr/include/foo4')), + ('-include', env.fs.File('/usr/include/foo5')), ('--param', 'l1-cache-size=32'), ('--param', 'l2-cache-size=6144'), '+DD64'], repr(d['CCFLAGS']) assert d['CXXFLAGS'] == ['-std=c++0x'], repr(d['CXXFLAGS']) -- cgit v0.12 From 915bfb3a55619175e03c3dc3573ad5ba83611e75 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 18 Mar 2022 17:19:04 -0700 Subject: flake8 fixes --- test/ninja/ninja_handle_control_c_rebuild.py | 2 +- test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/ninja/ninja_handle_control_c_rebuild.py b/test/ninja/ninja_handle_control_c_rebuild.py index c2c8c8d..9f6b413 100644 --- a/test/ninja/ninja_handle_control_c_rebuild.py +++ b/test/ninja/ninja_handle_control_c_rebuild.py @@ -22,7 +22,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # """ -This test ensures if ninja get's a control-c (or other interrupting signal) while +This test ensures if ninja gets a control-c (or other interrupting signal) while regenerating the build.ninja, it doesn't remove the build.ninja leaving it in an unworkable state. """ diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja b/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja index 0124576..34d7872 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja +++ b/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja @@ -1,7 +1,7 @@ import os import signal -SetOption('experimental','ninja') +SetOption('experimental', 'ninja') DefaultEnvironment(tools=[]) env = Environment() @@ -9,4 +9,4 @@ env.Tool('ninja') env.Program(target='foo', source='foo.c') if ARGUMENTS.get('NINJA_DISABLE_AUTO_RUN', 0): - os.kill(os.getppid(),signal.SIGINT ) + os.kill(os.getppid(), signal.SIGINT) -- cgit v0.12 From e521c1b0fec2064166fe5f78ad0ccc3dd2b22551 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 20 Mar 2022 08:00:10 -0600 Subject: Update CHANGES for dict -> mapping rename [ci skip] Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index d19fb06..1f09c5b 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -59,6 +59,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - As "code modernization" all of SCons now uses the current super() zero-argument syntax instead of direct calls to a parent class method or the super() two-argument syntax. + - Renamed ParseFlag's internal data structure to "mapping" instead of + "dict" (avoid redefining builtin) From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. -- cgit v0.12 From fb49becabc0cc641eef911473fdabc6c3b8e4788 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 25 Mar 2022 12:39:07 -0600 Subject: Add a bit more info to tool "generate" function [skip appveyor] Seemed useful to actually record the suggestion that tool modules should not unconditionally set the values of all contstruction variables - some could be considered user-settable and if so, existing values should be respected. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 47aba1f..e3466e2 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -7219,23 +7219,38 @@ A tool specification module must include two functions: generate(env, **kwargs) -Modifies the &consenv; env -to set up necessary &consvars; so that the facilities represented by -the tool can be executed. -It may use any keyword arguments -that the user supplies in kwargs +Modify the &consenv; env +to set up necessary &consvars;, Builders, Emitters, etc., +so the facilities represented by the tool can be executed. +Care should be taken not to overwrite &consvars; intended +to be settable by the user. For example: + + +def generate(env): + ... + if 'MYTOOL' not in env: + env['MYTOOL'] = env.Detect("mytool") + if 'MYTOOLFLAGS' not in env: + env['MYTOOLFLAGS'] = SCons.Util.CLVar('--myarg') + ... + + +The generate function +may use any keyword arguments +that the user supplies via kwargs to vary its initialization. exists(env) -Returns True if the tool can +Return a true value if the tool can be called in the context of env. +else false. Usually this means looking up one or more known programs using the PATH from the supplied env, but the tool can -make the "exists" decision in any way it chooses. +make the exists decision in any way it chooses. -- cgit v0.12 From 2b673e30e81f83304746e7fec9aa4ddb0fd66f88 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 22 Mar 2022 14:11:13 -0600 Subject: Fix some more tests to use context-mgr open Fiddled formatting a bit. Signed-off-by: Mats Wichmann --- test/Actions/append.py | 16 ++--- test/Actions/function.py | 64 +++++++++----------- test/Actions/pre-post.py | 31 +++++----- test/option/option--.py | 24 ++++---- test/option/option--Q.py | 27 ++++----- test/option/option-i.py | 51 ++++++++-------- test/option/option-k.py | 152 +++++++++++++++++++++++++---------------------- test/option/option-s.py | 23 ++++--- 8 files changed, 197 insertions(+), 191 deletions(-) diff --git a/test/Actions/append.py b/test/Actions/append.py index b5d4c3a..42a414b 100644 --- a/test/Actions/append.py +++ b/test/Actions/append.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,13 +22,13 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# This test exercises the addition operator of Action objects. -# Using Environment.Prepend() and Environment.Append(), you should be -# able to add new actions to existing ones, effectively adding steps -# to a build process. -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +This test exercises the addition operator of Action objects. +Using Environment.Prepend() and Environment.Append(), you should be +able to add new actions to existing ones, effectively adding steps +to a build process. +""" import os import stat diff --git a/test/Actions/function.py b/test/Actions/function.py index 7f292cf..699cd17 100644 --- a/test/Actions/function.py +++ b/test/Actions/function.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import sys @@ -59,6 +58,7 @@ options.AddVariables( ('literal_in_listcomp', 'Literal inside list comprehension', '2'), ) +DefaultEnvironment(tools=[]) optEnv = Environment(options=options, tools=[]) r = re.compile(optEnv['regexp']) @@ -67,25 +67,24 @@ withClosure = \ r''' def toto(header='%(header)s', trailer='%(trailer)s'): xxx = %(closure_cell_value)s - def writeDeps(target, source, env, b=%(b)s, r=r %(extraarg)s , - header=header, trailer=trailer): + def writeDeps(target, source, env, b=%(b)s, r=r %(extraarg)s , header=header, trailer=trailer): """+'"""%(docstring)s"""'+""" def foo(b=b): return %(nestedfuncexp)s - f = open(str(target[0]),'wb') - f.write(bytearray(header,'utf-8')) - for d in env['ENVDEPS']: - f.write(bytearray(d+'%(separator)s','utf-8')) - f.write(bytearray(trailer+'\\n','utf-8')) - f.write(bytearray(str(foo())+'\\n','utf-8')) - f.write(bytearray(r.match('aaaa').group(1)+'\\n','utf-8')) - f.write(bytearray(str(sum([x*%(literal_in_listcomp)s for x in [1,2]]))+'\\n', 'utf-8')) - %(extracode)s - try: - f.write(bytearray(str(xarg),'utf-8')+b'\\n') - except NameError: - pass - f.close() + + with open(str(target[0]), 'wb') as f: + f.write(bytearray(header, 'utf-8')) + for d in env['ENVDEPS']: + f.write(bytearray(d+'%(separator)s', 'utf-8')) + f.write(bytearray(trailer+'\\n', 'utf-8')) + f.write(bytearray(str(foo())+'\\n', 'utf-8')) + f.write(bytearray(r.match('aaaa').group(1)+'\\n', 'utf-8')) + f.write(bytearray(str(sum([x*%(literal_in_listcomp)s for x in [1, 2]]))+'\\n', 'utf-8')) + %(extracode)s + try: + f.write(bytearray(str(xarg), 'utf-8')+b'\\n') + except NameError: + pass return writeDeps ''' @@ -93,13 +92,9 @@ def toto(header='%(header)s', trailer='%(trailer)s'): exec(withClosure % optEnv) genHeaderBld = SCons.Builder.Builder( - action = SCons.Action.Action( - toto(), - 'Generating $TARGET', - varlist=['ENVDEPS'] - ), - suffix = '.gen.h' - ) + action=SCons.Action.Action(toto(), 'Generating $TARGET', varlist=['ENVDEPS']), + suffix='.gen.h', +) DefaultEnvironment(tools=[]) env = Environment(tools=[]) @@ -128,23 +123,22 @@ scons: done building targets. """ def runtest(arguments, expectedOutFile, expectedRebuild=True, stderr=""): - test.run(arguments=arguments, - stdout=expectedRebuild and rebuildstr or nobuildstr, - stderr="") + test.run( + arguments=arguments, + stdout=expectedRebuild and rebuildstr or nobuildstr, + stderr="", + ) sys.stdout.write('First Build.\n') test.must_match('Out.gen.h', expectedOutFile, message="First Build") - # Should not be rebuild when run a second time with the same - # arguments. - + # Should not be rebuild when run a second time with the same arguments. sys.stdout.write('Rebuild.\n') - test.run(arguments = arguments, stdout=nobuildstr, stderr="") + test.run(arguments=arguments, stdout=nobuildstr, stderr="") test.must_match('Out.gen.h', expectedOutFile, message="Should not rebuild") - # We're making this script chatty to prevent timeouts on really really # slow buildbot slaves (*cough* Solaris *cough*). diff --git a/test/Actions/pre-post.py b/test/Actions/pre-post.py index 358aa43..ce8cbdc 100644 --- a/test/Actions/pre-post.py +++ b/test/Actions/pre-post.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,12 +22,11 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# This test exercises the AddPreAction() and AddPostAction() API -# functions, which add pre-build and post-build actions to nodes. -# -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +This test exercises the AddPreAction() and AddPostAction() API +functions, which add pre-build and post-build actions to nodes. +""" import os @@ -56,11 +57,8 @@ def before(env, target, source): def after(env, target, source): t = str(target[0]) a = "after_" + t - fin = open(t, "rb") - fout = open(a, "wb") - fout.write(fin.read()) - fout.close() - fin.close() + with open(t, "rb") as fin, open(a, "wb") as fout: + fout.write(fin.read()) os.chmod(a, os.stat(a)[stat.ST_MODE] | stat.S_IXUSR) foo = env.Program(source='foo.c', target='foo') @@ -103,19 +101,22 @@ test.must_match(['work3', 'dir', 'file'], "build()\n") # work4 start test.write(['work4', 'SConstruct'], """\ - DefaultEnvironment(tools=[]) def pre_action(target, source, env): with open(str(target[0]), 'ab') as f: f.write(('pre %%s\\n' %% source[0]).encode()) + def post_action(target, source, env): with open(str(target[0]), 'ab') as f: f.write(('post %%s\\n' %% source[0]).encode()) + env = Environment(tools=[]) -o = env.Command(['pre-post', 'file.out'], - 'file.in', - r'%(_python_)s build.py ${TARGETS[1]} $SOURCE') +o = env.Command( + ['pre-post', 'file.out'], + 'file.in', + r'%(_python_)s build.py ${TARGETS[1]} $SOURCE' +) env.AddPreAction(o, pre_action) env.AddPostAction(o, post_action) """ % locals()) diff --git a/test/option/option--.py b/test/option/option--.py index 8e06260..dbfb3c9 100644 --- a/test/option/option--.py +++ b/test/option/option--.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -34,16 +33,17 @@ test = TestSCons.TestSCons() test.write('build.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("build.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("build.py: %s\n" % sys.argv[1]) +sys.exit(0) """) test.write('SConstruct', """ -MyBuild = Builder(action = r'%(_python_)s build.py $TARGETS') -env = Environment(BUILDERS = { 'MyBuild' : MyBuild }) -env.MyBuild(target = '-f1.out', source = 'f1.in') -env.MyBuild(target = '-f2.out', source = 'f2.in') +MyBuild = Builder(action=r'%(_python_)s build.py $TARGETS') +env = Environment(BUILDERS={'MyBuild': MyBuild}) +env.MyBuild(target='-f1.out', source='f1.in') +env.MyBuild(target='-f2.out', source='f2.in') """ % locals()) test.write('f1.in', "f1.in\n") @@ -51,7 +51,7 @@ test.write('f2.in', "f2.in\n") expect = test.wrap_stdout('%(_python_)s build.py -f1.out\n%(_python_)s build.py -f2.out\n' % locals()) -test.run(arguments = '-- -f1.out -f2.out', stdout = expect) +test.run(arguments='-- -f1.out -f2.out', stdout=expect) test.fail_test(not os.path.exists(test.workpath('-f1.out'))) test.fail_test(not os.path.exists(test.workpath('-f2.out'))) diff --git a/test/option/option--Q.py b/test/option/option--Q.py index f3b82f9..ac56d34 100644 --- a/test/option/option--Q.py +++ b/test/option/option--Q.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -35,24 +34,24 @@ test = TestSCons.TestSCons() test.write('build.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("build.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("build.py: %s\n" % sys.argv[1]) +sys.exit(0) """) test.write('SConstruct', """ - AddOption('--use_SetOption', action='store_true', dest='setoption', default=False) -use_setoption=GetOption('setoption') +use_setoption = GetOption('setoption') if use_setoption: SetOption('no_progress', True) - -MyBuild = Builder(action = r'%(_python_)s build.py $TARGET') -env = Environment(BUILDERS = { 'MyBuild' : MyBuild }) -env.MyBuild(target = 'f1.out', source = 'f1.in') -env.MyBuild(target = 'f2.out', source = 'f2.in') + +MyBuild = Builder(action=r'%(_python_)s build.py $TARGET') +env = Environment(BUILDERS={'MyBuild': MyBuild}) +env.MyBuild(target='f1.out', source='f1.in') +env.MyBuild(target='f2.out', source='f2.in') """ % locals()) test.write('f1.in', "f1.in\n") diff --git a/test/option/option-i.py b/test/option/option-i.py index 9b5212d..e426e1f 100644 --- a/test/option/option-i.py +++ b/test/option/option-i.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -34,43 +33,46 @@ test = TestSCons.TestSCons() test.write('succeed.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("succeed.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("succeed.py: %s\n" % sys.argv[1]) sys.exit(0) """) test.write('fail.py', r""" import sys + sys.exit(1) """) test.write('SConstruct', """ -Succeed = Builder(action = r'%(_python_)s succeed.py $TARGETS') -Fail = Builder(action = r'%(_python_)s fail.py $TARGETS') -env = Environment(BUILDERS = { 'Succeed' : Succeed, 'Fail' : Fail }) -env.Fail(target = 'aaa.1', source = 'aaa.in') -env.Succeed(target = 'aaa.out', source = 'aaa.1') -env.Fail(target = 'bbb.1', source = 'bbb.in') -env.Succeed(target = 'bbb.out', source = 'bbb.1') +Succeed = Builder(action=r'%(_python_)s succeed.py $TARGETS') +Fail = Builder(action=r'%(_python_)s fail.py $TARGETS') +env = Environment(BUILDERS={'Succeed': Succeed, 'Fail': Fail}) +env.Fail(target='aaa.1', source='aaa.in') +env.Succeed(target='aaa.out', source='aaa.1') +env.Fail(target='bbb.1', source='bbb.in') +env.Succeed(target='bbb.out', source='bbb.1') """ % locals()) test.write('aaa.in', "aaa.in\n") test.write('bbb.in', "bbb.in\n") -test.run(arguments = 'aaa.1 aaa.out bbb.1 bbb.out', - stderr = 'scons: *** [aaa.1] Error 1\n', - status = 2) +test.run( + arguments='aaa.1 aaa.out bbb.1 bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.fail_test(os.path.exists(test.workpath('aaa.1'))) test.fail_test(os.path.exists(test.workpath('aaa.out'))) test.fail_test(os.path.exists(test.workpath('bbb.1'))) test.fail_test(os.path.exists(test.workpath('bbb.out'))) -test.run(arguments = '-i aaa.1 aaa.out bbb.1 bbb.out', - stderr = - 'scons: *** [aaa.1] Error 1\n' - 'scons: *** [bbb.1] Error 1\n') +test.run( + arguments='-i aaa.1 aaa.out bbb.1 bbb.out', + stderr='scons: *** [aaa.1] Error 1\nscons: *** [bbb.1] Error 1\n', +) test.fail_test(os.path.exists(test.workpath('aaa.1'))) test.fail_test(test.read('aaa.out',mode='r') != "succeed.py: aaa.out\n") @@ -80,9 +82,10 @@ test.fail_test(test.read('bbb.out',mode='r') != "succeed.py: bbb.out\n") test.unlink("aaa.out") test.unlink("bbb.out") -test.run(arguments='--ignore-errors aaa.1 aaa.out bbb.1 bbb.out', - stderr='scons: *** [aaa.1] Error 1\n' - 'scons: *** [bbb.1] Error 1\n') +test.run( + arguments='--ignore-errors aaa.1 aaa.out bbb.1 bbb.out', + stderr='scons: *** [aaa.1] Error 1\nscons: *** [bbb.1] Error 1\n', +) test.fail_test(os.path.exists(test.workpath('aaa.1'))) test.fail_test(test.read('aaa.out', mode='r') != "succeed.py: aaa.out\n") diff --git a/test/option/option-k.py b/test/option/option-k.py index d6c81ea..6a7cfcb 100644 --- a/test/option/option-k.py +++ b/test/option/option-k.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import TestSCons @@ -33,18 +32,17 @@ test = TestSCons.TestSCons() test.subdir('work1', 'work2', 'work3') - - test.write('succeed.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("succeed.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("succeed.py: %s\n" % sys.argv[1]) sys.exit(0) """) test.write('fail.py', r""" import sys + sys.exit(1) """) @@ -66,19 +64,23 @@ env.Succeed(target='bbb.out', source='bbb.in') test.write(['work1', 'aaa.in'], "aaa.in\n") test.write(['work1', 'bbb.in'], "bbb.in\n") -test.run(chdir='work1', - arguments='aaa.out bbb.out', - stderr='scons: *** [aaa.1] Error 1\n', - status=2) +test.run( + chdir='work1', + arguments='aaa.out bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.must_not_exist(test.workpath('work1', 'aaa.1')) test.must_not_exist(test.workpath('work1', 'aaa.out')) test.must_not_exist(test.workpath('work1', 'bbb.out')) -test.run(chdir='work1', - arguments='-k aaa.out bbb.out', - stderr='scons: *** [aaa.1] Error 1\n', - status=2) +test.run( + chdir='work1', + arguments='-k aaa.out bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.must_not_exist(test.workpath('work1', 'aaa.1')) test.must_not_exist(test.workpath('work1', 'aaa.out')) @@ -86,10 +88,12 @@ test.must_match(['work1', 'bbb.out'], "succeed.py: bbb.out\n", mode='r') test.unlink(['work1', 'bbb.out']) -test.run(chdir = 'work1', - arguments='--keep-going aaa.out bbb.out', - stderr='scons: *** [aaa.1] Error 1\n', - status=2) +test.run( + chdir='work1', + arguments='--keep-going aaa.out bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.must_not_exist(test.workpath('work1', 'aaa.1')) test.must_not_exist(test.workpath('work1', 'aaa.out')) @@ -131,11 +135,12 @@ env.Succeed('ddd.out', 'ccc.in') test.write(['work2', 'aaa.in'], "aaa.in\n") test.write(['work2', 'ccc.in'], "ccc.in\n") -test.run(chdir='work2', - arguments='-k .', - status=2, - stderr=None, - stdout="""\ +test.run( + chdir='work2', + arguments='-k .', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -143,7 +148,9 @@ scons: Building targets ... %(_python_)s ../succeed.py ccc.out %(_python_)s ../succeed.py ddd.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work2', 'aaa.out']) test.must_not_exist(['work2', 'bbb.out']) @@ -175,19 +182,19 @@ test.must_match(['work2', 'ddd.out'], "succeed.py: ddd.out\n", mode='r') test.write(['work3', 'SConstruct'], """\ DefaultEnvironment(tools=[]) -Succeed = Builder(action = r'%(_python_)s ../succeed.py $TARGETS') -Fail = Builder(action = r'%(_python_)s ../fail.py $TARGETS') -env = Environment(BUILDERS = {'Succeed': Succeed, 'Fail': Fail}, tools=[]) +Succeed = Builder(action=r'%(_python_)s ../succeed.py $TARGETS') +Fail = Builder(action=r'%(_python_)s ../fail.py $TARGETS') +env = Environment(BUILDERS={'Succeed': Succeed, 'Fail': Fail}, tools=[]) a = env.Fail('aaa.out', 'aaa.in') b = env.Succeed('bbb.out', 'bbb.in') c = env.Succeed('ccc.out', 'ccc.in') -a1 = Alias( 'a1', a ) -a2 = Alias( 'a2', a+b) -a4 = Alias( 'a4', c) -a3 = Alias( 'a3', a4+c) +a1 = Alias('a1', a) +a2 = Alias('a2', a + b) +a4 = Alias('a4', c) +a3 = Alias('a3', a4 + c) -Alias('all', a1+a2+a3) +Alias('all', a1 + a2 + a3) """ % locals()) test.write(['work3', 'aaa.in'], "aaa.in\n") @@ -196,36 +203,37 @@ test.write(['work3', 'ccc.in'], "ccc.in\n") # Test tegular build (i.e. without -k) -test.run(chdir = 'work3', - arguments = '.', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='.', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... %(_python_)s ../fail.py aaa.out scons: building terminated because of errors. -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) - -test.run(chdir = 'work3', - arguments = '-c .') +test.run(chdir='work3', arguments='-c .') test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) - # Current directory -test.run(chdir = 'work3', - arguments = '-k .', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='-k .', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -233,26 +241,27 @@ scons: Building targets ... %(_python_)s ../succeed.py bbb.out %(_python_)s ../succeed.py ccc.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_exist(['work3', 'bbb.out']) test.must_exist(['work3', 'ccc.out']) - -test.run(chdir = 'work3', - arguments = '-c .') +test.run(chdir='work3', arguments='-c .') test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) # Single target -test.run(chdir = 'work3', - arguments = '--keep-going all', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='--keep-going all', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -260,26 +269,26 @@ scons: Building targets ... %(_python_)s ../succeed.py bbb.out %(_python_)s ../succeed.py ccc.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_exist(['work3', 'bbb.out']) test.must_exist(['work3', 'ccc.out']) - -test.run(chdir = 'work3', - arguments = '-c .') +test.run(chdir='work3', arguments='-c .') test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) - # Separate top-level targets -test.run(chdir = 'work3', - arguments = '-k a1 a2 a3', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='-k a1 a2 a3', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -287,13 +296,14 @@ scons: Building targets ... %(_python_)s ../succeed.py bbb.out %(_python_)s ../succeed.py ccc.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_exist(['work3', 'bbb.out']) test.must_exist(['work3', 'ccc.out']) - test.pass_test() # Local Variables: diff --git a/test/option/option-s.py b/test/option/option-s.py index 89a0c62..359c295 100644 --- a/test/option/option-s.py +++ b/test/option/option-s.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -34,18 +33,19 @@ test = TestSCons.TestSCons() test.write('build.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("build.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("build.py: %s\n" % sys.argv[1]) +sys.exit(0) """) test.write('SConstruct', """ DefaultEnvironment(tools=[]) -MyBuild = Builder(action = r'%(_python_)s build.py $TARGET') +MyBuild = Builder(action=r'%(_python_)s build.py $TARGET') -silent = ARGUMENTS.get('QUIET',0) +silent = ARGUMENTS.get('QUIET', 0) if silent: - SetOption('silent',True) + SetOption('silent', True) env = Environment(BUILDERS={'MyBuild': MyBuild}, tools=[]) env.MyBuild(target='f1.out', source='f1.in') @@ -86,10 +86,7 @@ test.run(arguments='QUIET=1 f1.out f2.out', test.fail_test(not os.path.exists(test.workpath('f1.out'))) test.fail_test(not os.path.exists(test.workpath('f2.out'))) - - test.pass_test() - # Local Variables: # tab-width:4 -- cgit v0.12 From c0469d3b29141e49ff00b7ddcb5aba5660c37b8f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 3 Apr 2022 10:52:42 -0600 Subject: Fix a ref-before-assign issue in tex tool Also some formatting fiddles. Fixes #2888 Signed-off-by: Mats Wichmann --- CHANGES.txt | 1 + SCons/Tool/tex.py | 52 +++++++++++++++++++++++++++++----------------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 9e77793..c1fa7d2 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -72,6 +72,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER or the super() two-argument syntax. - Renamed ParseFlag's internal data structure to "mapping" instead of "dict" (avoid redefining builtin) + - Fix an old use-before-set bug in tex tool (issue #2888) From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/SCons/Tool/tex.py b/SCons/Tool/tex.py index d8b694e..6d0a5fb 100644 --- a/SCons/Tool/tex.py +++ b/SCons/Tool/tex.py @@ -492,21 +492,23 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None return result -def LaTeXAuxAction(target = None, source= None, env=None): - result = InternalLaTeXAuxAction( LaTeXAction, target, source, env ) +def LaTeXAuxAction(target=None, source=None, env=None): + result = InternalLaTeXAuxAction(LaTeXAction, target, source, env) return result + LaTeX_re = re.compile("\\\\document(style|class)") -def is_LaTeX(flist,env,abspath): + +def is_LaTeX(flist, env, abspath) -> bool: """Scan a file list to decide if it's TeX- or LaTeX-flavored.""" # We need to scan files that are included in case the # \documentclass command is in them. # get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS'] - savedpath = modify_env_var(env, 'TEXINPUTS', abspath) - paths = env['ENV']['TEXINPUTS'] + savedpath = modify_env_var(env, "TEXINPUTS", abspath) + paths = env["ENV"]["TEXINPUTS"] if SCons.Util.is_List(paths): pass else: @@ -516,54 +518,58 @@ def is_LaTeX(flist,env,abspath): # now that we have the path list restore the env if savedpath is _null: try: - del env['ENV']['TEXINPUTS'] + del env["ENV"]["TEXINPUTS"] except KeyError: - pass # was never set + pass # was never set else: - env['ENV']['TEXINPUTS'] = savedpath + env["ENV"]["TEXINPUTS"] = savedpath if Verbose: - print("is_LaTeX search path ",paths) - print("files to search :",flist) + print("is_LaTeX search path ", paths) + print("files to search: ", flist) # Now that we have the search path and file list, check each one + file_test = False for f in flist: if Verbose: - print(" checking for Latex source ",str(f)) + print(f" checking for Latex source {f}") content = f.get_text_contents() if LaTeX_re.search(content): if Verbose: - print("file %s is a LaTeX file" % str(f)) - return 1 + print(f"file {f} is a LaTeX file") + return True if Verbose: - print("file %s is not a LaTeX file" % str(f)) + print(f"file {f} is not a LaTeX file") # now find included files - inc_files = [ ] - inc_files.extend( include_re.findall(content) ) + inc_files = [] + inc_files.extend(include_re.findall(content)) if Verbose: - print("files included by '%s': "%str(f),inc_files) + print(f"files included by '{f}': ", inc_files) # inc_files is list of file names as given. need to find them # using TEXINPUTS paths. # search the included files for src in inc_files: - srcNode = FindFile(src,['.tex','.ltx','.latex'],paths,env,requireExt=False) + srcNode = FindFile( + src, [".tex", ".ltx", ".latex"], paths, env, requireExt=False + ) # make this a list since is_LaTeX takes a list. - fileList = [srcNode,] + fileList = [srcNode] if Verbose: - print("FindFile found ",srcNode) + print("FindFile found ", srcNode) if srcNode is not None: file_test = is_LaTeX(fileList, env, abspath) # return on first file that finds latex is needed. if file_test: - return file_test + return True if Verbose: - print(" done scanning ",str(f)) + print(f" done scanning {f}") + + return False - return 0 def TeXLaTeXFunction(target = None, source= None, env=None): """A builder for TeX and LaTeX that scans the source file to -- cgit v0.12 From f0f5657f3214bf50f56cf076c9ecbe37c1fd0d75 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 5 Apr 2022 12:29:32 -0600 Subject: test framework: fix exception on timeout If the framework wait_for() method actually times out, it tries to return stdout and stderr by calling the framework methods of those names. The stdout() method was protected against the message not having been captured (as is the case on timeout). Updated the stderr() method to use the same technique. Signed-off-by: Mats Wichmann --- CHANGES.txt | 1 + testing/framework/TestCmd.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c1fa7d2..8390f00 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -73,6 +73,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Renamed ParseFlag's internal data structure to "mapping" instead of "dict" (avoid redefining builtin) - Fix an old use-before-set bug in tex tool (issue #2888) + - Fix a test harness exception returning stderr if a wait_for timed out. From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 79b16f5..5759121 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -311,6 +311,7 @@ import time import traceback from collections import UserList, UserString from subprocess import PIPE, STDOUT +from typing import Optional IS_WINDOWS = sys.platform == 'win32' IS_MACOS = sys.platform == 'darwin' @@ -1641,7 +1642,7 @@ class TestCmd: """ time.sleep(seconds) - def stderr(self, run=None): + def stderr(self, run=None) -> Optional[str]: """Returns the error output from the specified run number. If there is no specified run number, then returns the error @@ -1653,10 +1654,13 @@ class TestCmd: run = len(self._stderr) elif run < 0: run = len(self._stderr) + run - run = run - 1 - return self._stderr[run] + run -= 1 + try: + return self._stderr[run] + except IndexError: + return None - def stdout(self, run=None): + def stdout(self, run=None) -> Optional[str]: """Returns the stored standard output from a given run. Args: @@ -1673,7 +1677,7 @@ class TestCmd: run = len(self._stdout) elif run < 0: run = len(self._stdout) + run - run = run - 1 + run -= 1 try: return self._stdout[run] except IndexError: -- cgit v0.12