From de04bf85a087d2c914b30aaa4e3d563e53b06454 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 26 Oct 2022 10:34:34 -0700 Subject: Fixed taskmaster trace tests. Previously there was an extra line at the end of the file. It's no longer there. Added TestCommon.detailed_diff() function which can be used to diff large text blobs expected vs actual --- SCons/Taskmaster/TaskmasterTests.py | 107 ++++++++++++++++++++---------------- test/Interactive/taskmastertrace.py | 11 ++-- test/option/taskmastertrace.py | 9 +-- testing/framework/TestCommon.py | 16 ++++++ 4 files changed, 84 insertions(+), 59 deletions(-) diff --git a/SCons/Taskmaster/TaskmasterTests.py b/SCons/Taskmaster/TaskmasterTests.py index fad028e..9d7f959 100644 --- a/SCons/Taskmaster/TaskmasterTests.py +++ b/SCons/Taskmaster/TaskmasterTests.py @@ -26,10 +26,10 @@ import SCons.compat import sys import unittest - import SCons.Taskmaster import SCons.Errors +import TestCommon built_text = None cache_text = [] @@ -37,8 +37,9 @@ visited_nodes = [] executed = None scan_called = 0 + class Node: - def __init__(self, name, kids = [], scans = []): + def __init__(self, name, kids=[], scans=[]): self.name = name self.kids = kids self.scans = scans @@ -47,9 +48,11 @@ class Node: self.scanner = None self.targets = [self] self.prerequisites = None + class Builder: def targets(self, node): return node.targets + self.builder = Builder() self.bsig = None self.csig = None @@ -83,7 +86,7 @@ class Node: def prepare(self): self.prepared = 1 - self.get_binfo() + self.get_binfo() def build(self): global built_text @@ -119,7 +122,7 @@ class Node: self.binfo = binfo return binfo - + def clear(self): # The del_binfo() call here isn't necessary for normal execution, # but is for interactive mode, where we might rebuild the same @@ -130,14 +133,14 @@ class Node: global built_text if not self.cached: built_text = built_text + " really" - + # Clear the implicit dependency caches of any Nodes # waiting for this Node to be built. for parent in self.waiting_parents: parent.implicit = None self.clear() - + def release_target_info(self): pass @@ -199,7 +202,7 @@ class Node: def is_up_to_date(self): return self._current_val - + def depends_on(self, nodes): for node in nodes: if node in self.kids: @@ -218,26 +221,34 @@ class Node: class Executor: def prepare(self): pass + def get_action_targets(self): return self.targets + def get_all_targets(self): return self.targets + def get_all_children(self): result = [] for node in self.targets: result.extend(node.children()) return result + def get_all_prerequisites(self): return [] + def get_action_side_effects(self): return [] + self.executor = Executor() self.executor.targets = self.targets return self.executor + class OtherError(Exception): pass + class MyException(Exception): pass @@ -306,7 +317,7 @@ class TaskmasterTestCase(unittest.TestCase): n2._current_val = 1 n3.set_state(SCons.Node.no_state) n3._current_val = 1 - tm = SCons.Taskmaster.Taskmaster(targets = [n3], tasker = MyTask) + tm = SCons.Taskmaster.Taskmaster(targets=[n3], tasker=MyTask) t = tm.next_task() t.prepare() @@ -331,7 +342,6 @@ class TaskmasterTestCase(unittest.TestCase): assert tm.next_task() is None - n1 = Node("n1") n2 = Node("n2") n3 = Node("n3", [n1, n2]) @@ -366,7 +376,6 @@ class TaskmasterTestCase(unittest.TestCase): assert tm.next_task() is None - n4 = Node("n4") n4.set_state(SCons.Node.executed) tm = SCons.Taskmaster.Taskmaster([n4]) @@ -374,14 +383,13 @@ class TaskmasterTestCase(unittest.TestCase): n1 = Node("n1") n2 = Node("n2", [n1]) - tm = SCons.Taskmaster.Taskmaster([n2,n2]) + tm = SCons.Taskmaster.Taskmaster([n2, n2]) t = tm.next_task() t.executed() t.postprocess() t = tm.next_task() assert tm.next_task() is None - n1 = Node("n1") n2 = Node("n2") n3 = Node("n3", [n1], [n2]) @@ -440,11 +448,11 @@ class TaskmasterTestCase(unittest.TestCase): n1 = Node("n1") n2 = Node("n2") n3 = Node("n3") - n4 = Node("n4", [n1,n2,n3]) + n4 = Node("n4", [n1, n2, n3]) n5 = Node("n5", [n4]) n3.side_effect = 1 n1.side_effects = n2.side_effects = n3.side_effects = [n4] - tm = SCons.Taskmaster.Taskmaster([n1,n2,n3,n4,n5]) + tm = SCons.Taskmaster.Taskmaster([n1, n2, n3, n4, n5]) t = tm.next_task() assert t.get_target() == n1 assert n4.state == SCons.Node.executing, n4.state @@ -471,10 +479,12 @@ class TaskmasterTestCase(unittest.TestCase): n1 = Node("n1") n2 = Node("n2") n3 = Node("n3") - n4 = Node("n4", [n1,n2,n3]) + n4 = Node("n4", [n1, n2, n3]) + def reverse(dependencies): dependencies.reverse() return dependencies + tm = SCons.Taskmaster.Taskmaster([n4], order=reverse) t = tm.next_task() assert t.get_target() == n3, t.get_target() @@ -534,11 +544,11 @@ class TaskmasterTestCase(unittest.TestCase): s = n2.get_state() assert s == SCons.Node.executed, s - def test_make_ready_out_of_date(self): """Test the Task.make_ready() method's list of out-of-date Nodes """ ood = [] + def TaskGen(tm, targets, top, node, ood=ood): class MyTask(SCons.Taskmaster.AlwaysTask): def make_ready(self): @@ -558,8 +568,8 @@ class TaskmasterTestCase(unittest.TestCase): a5 = Node("a5") a5._current_val = 1 a5.always_build = 1 - tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4, a5], - tasker = TaskGen) + tm = SCons.Taskmaster.Taskmaster(targets=[n1, c2, n3, c4, a5], + tasker=TaskGen) del ood[:] t = tm.next_task() @@ -584,12 +594,13 @@ class TaskmasterTestCase(unittest.TestCase): def test_make_ready_exception(self): """Test handling exceptions from Task.make_ready() """ + class MyTask(SCons.Taskmaster.AlwaysTask): def make_ready(self): raise MyException("from make_ready()") n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster(targets = [n1], tasker = MyTask) + tm = SCons.Taskmaster.Taskmaster(targets=[n1], tasker=MyTask) t = tm.next_task() exc_type, exc_value, exc_tb = t.exception assert exc_type == MyException, repr(exc_type) @@ -601,6 +612,7 @@ class TaskmasterTestCase(unittest.TestCase): We should be getting: TypeError: Can't instantiate abstract class MyTask with abstract methods needs_execute """ + class MyTask(SCons.Taskmaster.Task): pass @@ -611,6 +623,7 @@ class TaskmasterTestCase(unittest.TestCase): def test_make_ready_all(self): """Test the make_ready_all() method""" + class MyTask(SCons.Taskmaster.AlwaysTask): make_ready = SCons.Taskmaster.Task.make_ready_all @@ -621,7 +634,7 @@ class TaskmasterTestCase(unittest.TestCase): c4 = Node("c4") c4._current_val = 1 - tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4]) + tm = SCons.Taskmaster.Taskmaster(targets=[n1, c2, n3, c4]) t = tm.next_task() target = t.get_target() @@ -647,8 +660,8 @@ class TaskmasterTestCase(unittest.TestCase): n3 = Node("n3") c4 = Node("c4") - tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4], - tasker = MyTask) + tm = SCons.Taskmaster.Taskmaster(targets=[n1, c2, n3, c4], + tasker=MyTask) t = tm.next_task() target = t.get_target() @@ -669,13 +682,14 @@ class TaskmasterTestCase(unittest.TestCase): t = tm.next_task() assert t is None - def test_children_errors(self): """Test errors when fetching the children of a node. """ + class StopNode(Node): def children(self): raise SCons.Errors.StopError("stop!") + class ExitNode(Node): def children(self): sys.exit(77) @@ -879,8 +893,8 @@ class TaskmasterTestCase(unittest.TestCase): n9 = Node("n9") n10 = Node("n10") - n6.side_effects = [ n8 ] - n7.side_effects = [ n9, n10 ] + n6.side_effects = [n8] + n7.side_effects = [n9, n10] tm = SCons.Taskmaster.Taskmaster([n6, n7]) t = tm.next_task() @@ -897,15 +911,19 @@ class TaskmasterTestCase(unittest.TestCase): class ExceptionExecutor: def prepare(self): raise Exception("Executor.prepare() exception") + def get_all_targets(self): return self.nodes + def get_all_children(self): result = [] for node in self.nodes: result.extend(node.children()) return result + def get_all_prerequisites(self): return [] + def get_action_side_effects(self): return [] @@ -935,6 +953,7 @@ class TaskmasterTestCase(unittest.TestCase): def raise_UserError(): raise SCons.Errors.UserError + n2 = Node("n2") n2.build = raise_UserError tm = SCons.Taskmaster.Taskmaster([n2]) @@ -948,6 +967,7 @@ class TaskmasterTestCase(unittest.TestCase): def raise_BuildError(): raise SCons.Errors.BuildError + n3 = Node("n3") n3.build = raise_BuildError tm = SCons.Taskmaster.Taskmaster([n3]) @@ -964,6 +984,7 @@ class TaskmasterTestCase(unittest.TestCase): # args set to the exception value, instance, and traceback. def raise_OtherError(): raise OtherError + n4 = Node("n4") n4.build = raise_OtherError tm = SCons.Taskmaster.Taskmaster([n4]) @@ -1066,7 +1087,7 @@ class TaskmasterTestCase(unittest.TestCase): """ n1 = Node("n1") tm = SCons.Taskmaster.Taskmaster([n1]) - t = tm.next_task() + t = tm.next_task() t.exception_set((1, 2)) exc_type, exc_value = t.exception @@ -1076,25 +1097,26 @@ class TaskmasterTestCase(unittest.TestCase): t.exception_set(3) assert t.exception == 3 - try: 1//0 + try: + 1 // 0 except: # Moved from below t.exception_set(None) - #pass + # pass -# import pdb; pdb.set_trace() + # import pdb; pdb.set_trace() # Having this here works for python 2.x, # but it is a tuple (None, None, None) when called outside # an except statement # t.exception_set(None) - + exc_type, exc_value, exc_tb = t.exception - assert exc_type is ZeroDivisionError, "Expecting ZeroDevisionError got:%s"%exc_type + assert exc_type is ZeroDivisionError, "Expecting ZeroDevisionError got:%s" % exc_type exception_values = [ "integer division or modulo", "integer division or modulo by zero", - "integer division by zero", # PyPy2 + "integer division by zero", # PyPy2 ] assert str(exc_value) in exception_values, exc_value @@ -1108,7 +1130,7 @@ class TaskmasterTestCase(unittest.TestCase): except: exc_type, exc_value = sys.exc_info()[:2] assert exc_type == Exception1, exc_type - assert str(exc_value) == '', "Expecting empty string got:%s (type %s)"%(exc_value,type(exc_value)) + assert str(exc_value) == '', "Expecting empty string got:%s (type %s)" % (exc_value, type(exc_value)) else: assert 0, "did not catch expected exception" @@ -1129,7 +1151,7 @@ class TaskmasterTestCase(unittest.TestCase): pass try: - 1//0 + 1 // 0 except: tb = sys.exc_info()[2] t.exception_set((Exception3, "arg", tb)) @@ -1242,18 +1264,11 @@ Task.postprocess(): node Taskmaster: Looking for a node to evaluate Taskmaster: No candidate anymore. """ - v_split=value.split('\n') - e_split=expect.split('\n') - if len(v_split) != len(e_split): - print("different number of lines:%d %d" % (len(v_split), len(e_split))) - - # breakpoint() - for v, e in zip(v_split, e_split): - # print("%s:%s"%(v,e)) - if v != e: - print("\n[%s]\n[%s]" % (v, e)) - - assert value == expect, "Expected:\n%s\nGot:\n%s" % (expect, value) + + if value != expect: + TestCommon.TestCommon.detailed_diff(value, expect) + + assert value == expect, "Expected taskmaster trace contents didn't match. See above" if __name__ == "__main__": diff --git a/test/Interactive/taskmastertrace.py b/test/Interactive/taskmastertrace.py index 93ee068..04e95fd 100644 --- a/test/Interactive/taskmastertrace.py +++ b/test/Interactive/taskmastertrace.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,8 +22,7 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + """ Verify use of the --taskmastertrace= option to the "build" command @@ -42,7 +43,6 @@ Command('2', [], Touch('$TARGET')) test.write('foo.in', "foo.in 1\n") - scons = test.start(arguments = '-Q --interactive') scons.send("build foo.out 1\n") @@ -101,7 +101,6 @@ Task.postprocess(): node Taskmaster: Looking for a node to evaluate Taskmaster: No candidate anymore. - scons>>> Touch("2") scons>>> scons: `foo.out' is up to date. scons>>> @@ -109,8 +108,6 @@ scons>>> test.finish(scons, stdout = expect_stdout) - - test.pass_test() # Local Variables: diff --git a/test/option/taskmastertrace.py b/test/option/taskmastertrace.py index fc8522c..9d0a606 100644 --- a/test/option/taskmastertrace.py +++ b/test/option/taskmastertrace.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Simple tests of the --taskmastertrace= option. @@ -123,7 +122,6 @@ Task.postprocess(): node Taskmaster: Looking for a node to evaluate Taskmaster: No candidate anymore. - """) test.run(arguments='--taskmastertrace=- .', stdout=expect_stdout) @@ -212,7 +210,6 @@ Task.postprocess(): node Taskmaster: Looking for a node to evaluate Taskmaster: No candidate anymore. - """ test.must_match('trace.out', expect_trace, mode='r') diff --git a/testing/framework/TestCommon.py b/testing/framework/TestCommon.py index f45b856..df005b3 100644 --- a/testing/framework/TestCommon.py +++ b/testing/framework/TestCommon.py @@ -796,6 +796,22 @@ class TestCommon(TestCmd): # so this is an Aegis invocation; pass the test (exit 0). self.pass_test() + @staticmethod + def detailed_diff(value, expect): + v_split = value.split('\n') + e_split = expect.split('\n') + if len(v_split) != len(e_split): + print("different number of lines:%d %d" % (len(v_split), len(e_split))) + + # breakpoint() + for v, e in zip(v_split, e_split): + # print("%s:%s"%(v,e)) + if v != e: + print("\n[%s]\n[%s]" % (v, e)) + + return "Expected:\n%s\nGot:\n%s" % (expect, value) + + # Local Variables: # tab-width:4 # indent-tabs-mode:nil -- cgit v0.12 From 2c791e4d7f4ab304fda9a03eabc3916056b0e36f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 2 Nov 2022 23:21:57 -0400 Subject: Move execution environment sanitation from Action._suproc to SCons.Util.sanitize_shell_env(ENV). Have ninja's spawning logic use this same function to clean it's execution environment --- CHANGES.txt | 5 +++++ RELEASE.txt | 3 +++ SCons/Action.py | 19 +------------------ SCons/Platform/darwin.py | 1 + SCons/Tool/ninja/Utils.py | 16 ++-------------- SCons/Tool/ninja/__init__.py | 8 +++++++- SCons/Util.py | 27 +++++++++++++++++++++++++++ test/ninja/shell_command.py | 6 ++++++ 8 files changed, 52 insertions(+), 33 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0a9f697..146a861 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -22,6 +22,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER NOTE: If you hook into SCons.Jobs, you'll have to change that to use SCons.Taskmaster.Jobs - Changed the Taskmaster trace logic to use python's logging module. The output formatting should remain (mostly) the same. Minor update to unittest for this to adjust for 1 less newline. + - Ninja: Fix execution environment sanitation for launching ninja. Previously if you set an + execution environment variable set to a python list it would crash. Now it + will create a string joining the list with os.pathsep + - Move execution environment sanitation from Action._subproc() to + SCons.Util.sanitize_shell_env(ENV) diff --git a/RELEASE.txt b/RELEASE.txt index 94ba72e..b93c449 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -50,6 +50,9 @@ FIXES use: env["JAVACLASSPATH"] = env.Split("foo bar baz") There is no change in how JAVACLASSPATH gets turned into the -classpath argument passed to the JDK tools. +- Ninja: Fix execution environment sanitation for launching ninja. Previously if you set an + execution environment variable set to a python list it would crash. Now it + will create a string joining the list with os.pathsep IMPROVEMENTS ------------ diff --git a/SCons/Action.py b/SCons/Action.py index ccb3a2c..8151b2a 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -806,24 +806,7 @@ def _subproc(scons_env, cmd, error='ignore', **kw): ENV = kw.get('env', None) if ENV is None: ENV = get_default_ENV(scons_env) - # Ensure that the ENV values are all strings: - new_env = {} - for key, value in ENV.items(): - if is_List(value): - # If the value is a list, then we assume it is a path list, - # because that's a pretty common list-like value to stick - # in an environment variable: - value = SCons.Util.flatten_sequence(value) - new_env[key] = os.pathsep.join(map(str, value)) - else: - # It's either a string or something else. If it's a string, - # we still want to call str() because it might be a *Unicode* - # string, which makes subprocess.Popen() gag. If it isn't a - # string or a list, then we just coerce it to a string, which - # is the proper way to handle Dir and File instances and will - # produce something reasonable for just about everything else: - new_env[key] = str(value) - kw['env'] = new_env + kw['env'] = SCons.Util.sanitize_shell_env(ENV) try: pobj = subprocess.Popen(cmd, **kw) diff --git a/SCons/Platform/darwin.py b/SCons/Platform/darwin.py index f997a7d..d57f810 100644 --- a/SCons/Platform/darwin.py +++ b/SCons/Platform/darwin.py @@ -40,6 +40,7 @@ def generate(env): # env['ENV']['PATH'] = '/opt/local/bin:/opt/local/sbin:' + env['ENV']['PATH'] + ':/sw/bin' # Store extra system paths in env['ENV']['PATHOSX'] + env['ENV']['PATHOSX'] = '/opt/local/bin' filelist = ['/etc/paths',] # make sure this works on Macs with Tiger or earlier diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index 583301e..7269fb2 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -285,6 +285,7 @@ def ninja_sorted_build(ninja, **build): sorted_dict = ninja_recursive_sorted_dict(build) ninja.build(**sorted_dict) + def get_command_env(env, target, source): """ Return a string that sets the environment for any environment variables that @@ -311,21 +312,8 @@ def get_command_env(env, target, source): windows = env["PLATFORM"] == "win32" command_env = "" + scons_specified_env = SCons.Util.sanitize_shell_env(scons_specified_env) for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - if windows: command_env += "set '{}={}' && ".format(key, value) else: diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index c4023c3..2622641 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -34,6 +34,7 @@ import SCons import SCons.Script import SCons.Tool.ninja.Globals from SCons.Script import GetOption +from SCons.Util import sanitize_shell_env from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS, NINJA_CMDLINE_TARGETS from .Methods import register_custom_handler, register_custom_rule_mapping, register_custom_rule, register_custom_pool, \ @@ -100,11 +101,16 @@ def ninja_builder(env, target, source): # reproduce the output like a ninja build would def execute_ninja(): + if env['PLATFORM'] == 'win32': + spawn_env = os.environ + else: + spawn_env = sanitize_shell_env(env['ENV']) + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] + env=spawn_env ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line diff --git a/SCons/Util.py b/SCons/Util.py index 7eb2005..c914af0 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -2138,6 +2138,33 @@ class DispatchingFormatter(Formatter): formatter = self._formatters.get(record.name, self._default_formatter) return formatter.format(record) + +def sanitize_shell_env(env): + """ + Sanitize all values from env['ENV'] which will be propagated to the shell + :param env: The shell environment variables to be propagated to spawned shell + :return: sanitize dictionary of env variables (similar to what you'd get from os.environ) + """ + + # Ensure that the ENV values are all strings: + new_env = {} + for key, value in env.items(): + if is_List(value): + # If the value is a list, then we assume it is a path list, + # because that's a pretty common list-like value to stick + # in an environment variable: + value = flatten_sequence(value) + new_env[key] = os.pathsep.join(map(str, value)) + else: + # It's either a string or something else. If it's a string, + # we still want to call str() because it might be a *Unicode* + # string, which makes subprocess.Popen() gag. If it isn't a + # string or a list, then we just coerce it to a string, which + # is the proper way to handle Dir and File instances and will + # produce something reasonable for just about everything else: + new_env[key] = str(value) + return new_env + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index a6926c7..a6c48c4 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -53,6 +53,12 @@ SetOption('experimental','ninja') DefaultEnvironment(tools=[]) env = Environment() + +# Added to verify that SCons Ninja tool is sanitizing the shell environment +# before it spawns a new shell +env['ENV']['ZPATH']=['/a/b/c','/c/d/e'] + + env.Tool('ninja') prog = env.Program(target = 'foo', source = 'foo.c') env.Command('foo.out', prog, '%(shell)sfoo%(_exe)s > foo.out') -- cgit v0.12 From ef9879e9d8de55631e07a7e05510ca33aaf61185 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 3 Nov 2022 10:35:17 -0400 Subject: Changed argument name for sanitize_shell_env from env -> execution_env per feedback from mwichmann --- SCons/Util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/SCons/Util.py b/SCons/Util.py index c914af0..a7a6307 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -2139,16 +2139,16 @@ class DispatchingFormatter(Formatter): return formatter.format(record) -def sanitize_shell_env(env): +def sanitize_shell_env(execution_env): """ - Sanitize all values from env['ENV'] which will be propagated to the shell - :param env: The shell environment variables to be propagated to spawned shell + Sanitize all values in execution_env (typically this is env['ENV']) which will be propagated to the shell + :param execution_env: The shell environment variables to be propagated to spawned shell :return: sanitize dictionary of env variables (similar to what you'd get from os.environ) """ # Ensure that the ENV values are all strings: new_env = {} - for key, value in env.items(): + for key, value in execution_env.items(): if is_List(value): # If the value is a list, then we assume it is a path list, # because that's a pretty common list-like value to stick -- cgit v0.12 From c309375eb9972bab48cc8e84187a40c33a59e713 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 5 Nov 2022 07:36:08 -0600 Subject: Maintenance: fix some fiddly checker errors A rawstring to avoid an invalid escape warning. A couple of instances where code is comparing type () calls, advice is to is isinstance() instead. Missing backslashes on escapes in framewok test (rarely run) Updating expected traceback msg in framework test after the code which forces the exception changed in a previous revision. Signed-off-by: Mats Wichmann --- .../docbook/docbook-xsl-1.76.1/extensions/docbook.py | 6 +++--- testing/framework/TestCommonTests.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/extensions/docbook.py b/SCons/Tool/docbook/docbook-xsl-1.76.1/extensions/docbook.py index d79ece3..17241bb 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/extensions/docbook.py +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/extensions/docbook.py @@ -157,7 +157,7 @@ def convertLength(length): global pixelsPerInch global unitHash - m = re.search('([+-]?[\d.]+)(\S+)', length) + m = re.search(r'([+-]?[\d.]+)(\S+)', length) if m is not None and m.lastindex > 1: unit = pixelsPerInch if m.group(2) in unitHash: @@ -204,11 +204,11 @@ def lookupVariable(tctxt, varName, default): return default # If it's a list, get the first element - if type(varString) == type([]): + if isinstance(varString, list): varString = varString[0] # If it's not a string, it must be a node, get its content - if type(varString) != type(""): + if not isinstance(varString, str): varString = varString.content return varString diff --git a/testing/framework/TestCommonTests.py b/testing/framework/TestCommonTests.py index 525f263..68f52cf 100644 --- a/testing/framework/TestCommonTests.py +++ b/testing/framework/TestCommonTests.py @@ -1880,7 +1880,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*fail \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) @@ -1940,13 +1940,13 @@ class run_TestCase(TestCommonTestCase): Traceback \\(most recent call last\\): File "", line \\d+, in (\\?|) File "[^"]+TestCommon.py", line \\d+, in run - TestCmd.run\\(self, \\*\\*kw\\) + super().run\\(\\*\\*kw\\) File "[^"]+TestCmd.py", line \\d+, in run - .* + p = self.start(program=program, File "[^"]+TestCommon.py", line \\d+, in start raise e File "[^"]+TestCommon.py", line \\d+, in start - return TestCmd.start\\(self, program, interpreter, arguments, + return super().start\\(program, interpreter, arguments, File "", line \\d+, in raise_exception TypeError: forced TypeError """ % re.escape(repr(sys.executable))) @@ -2066,7 +2066,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*pass \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) @@ -2096,7 +2096,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*fail \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) @@ -2128,7 +2128,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*pass \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) @@ -2162,7 +2162,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*stderr \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) -- cgit v0.12 From e9fe1258201d598ce1feec35345d08bf524238fc Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 11 Nov 2022 11:05:19 -0700 Subject: Update setup/installation chapter in User Guide [skip appveyor] No longer tell people to install via "python setup.py install", possibly with arguments on library directories and paths. Simplify, and mention use of virtualenvs as the current way to have multiple SCons versions installed. Fixes #4259 Signed-off-by: Mats Wichmann --- CHANGES.txt | 4 + RELEASE.txt | 3 + doc/user/build-install.xml | 428 ++++++++++++++++++--------------------------- 3 files changed, 180 insertions(+), 255 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0a9f697..e7ba3f3 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -52,6 +52,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Fixes #4243. - Cleanup: make sure BoolVariable usage in tests and examples uses Python boolean values instead of 0/1. + - Stop telling people to run "python setup.py install" in the User Guide. + Adds new content on using virtualenvs to be able to have multiple + different SCons versions available on one system. + From Andrew Morrow - Avoid returning UniqueList for `children` and other `Executor` APIs. This type diff --git a/RELEASE.txt b/RELEASE.txt index 94ba72e..a796668 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -79,6 +79,9 @@ DOCUMENTATION Added note on the possibly surprising feature that SCons always passes -sourcepath when calling javac, which affects how the class path is used when finding sources. +- Updated the User Guide chapter on installation: modernized the notes + on Python installs, SCons installs, and having multiple SCons versions + present on a single system. DEVELOPMENT ----------- diff --git a/doc/user/build-install.xml b/doc/user/build-install.xml index c106dc5..697f5a2 100644 --- a/doc/user/build-install.xml +++ b/doc/user/build-install.xml @@ -1,4 +1,10 @@ + + @@ -22,46 +28,19 @@ xmlns="http://www.scons.org/dbxsd/v1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> -Building and Installing &SCons; - - +Building and Installing &SCons; This chapter will take you through the basic steps - of installing &SCons; on your system, - and building &SCons; if you don't have a - pre-built package available - (or simply prefer the flexibility of building it yourself). + of installing &SCons; so you can use it for your your projects. Before that, however, this chapter will also describe the basic steps - involved in installing Python on your system, + involved in installing &Python; on your system, in case that is necessary. - Fortunately, both &SCons; and Python - are very easy to install on almost any system, - and Python already comes installed on many systems. + Fortunately, both &SCons; and &Python; + are easy to install on almost any system, + and &Python; already comes installed on many systems. @@ -70,31 +49,31 @@ Lastly, this chapter also contains a section that - provides a brief overview of the Python programming language, + provides a brief overview of the &Python; programming language, which is the language used to implement &SCons;, and which forms the basis of the &SCons; configuration files. - Becoming familiar with some Python concepts will make it easier + Becoming familiar with some &Python; concepts will make it easier to understand many of the examples in this User's Guide. Nevertheless, it is possible - to configure simple &SCons; builds without knowing Python, + to configure simple &SCons; builds without knowing &Python;, so you can skip this section if you want to dive in and pick up things by example- -or, of course, if you are - already familiar with Python. + already familiar with &Python;. --> -
+
Installing Python - Because &SCons; is written in Python, - you need to have Python installed on your system + Because &SCons; is written in the &Python; programming language, + you need to have a &Python; interpreter available on your system to use &SCons;. - Before you try to install Python, - you should check to see if Python is already + Before you try to install &Python;, + check to see if &Python; is already available on your system by typing python -V (capital 'V') @@ -106,28 +85,34 @@ $ python -V -Python 3.7.1 +Python 3.9.15 - Note to Windows users: there are a number of different ways Python + If you get a version like 2.7.x, you may need to try using the + name python3 - current &SCons; no longer + works with &Python; 2. + + + + Note to Windows users: there are a number of different ways &Python; can be installed or invoked on Windows, it is beyond the scope - of this guide to unravel all of them. Many will have an additional + of this guide to unravel all of them. Some have an additional program called the Python launcher (described, somewhat technically, in PEP 397): try using the command name py instead of python, if that is not available drop - back to trying python. + back to trying python C:\>py -V -Python 3.7.1 +Python 3.9.15 - If Python is not installed on your system, + If &Python; is not installed on your system, or is not findable in the current search path, you will see an error message stating something like "command not found" @@ -135,14 +120,14 @@ Python 3.7.1 or "'python' is not recognized as an internal or external command, operable progam or batch file" (on Windows cmd). - In that case, you need to either install Python + In that case, you need to either install &Python; or fix the search path before you can install &SCons;. - The canonical location for downloading Python - from Python's own website is: + The link for downloading &Python; installers (Windows and Mac) + from the project's own website is: https://www.python.org/download. There are useful system-specific entries on setup and usage to be found at: @@ -150,34 +135,56 @@ Python 3.7.1 - For Linux systems, Python is - almost certainly available as a supported package, possibly + For Linux systems, &Python; is + almost certainly available as a supported package, probably installed by default; this is often preferred over installing - by other means, and is easier than installing from source code. + by other means as the system package will be built with + carefully chosen optimizations, and will be kept up to date + with bug fixes and security patches. In fact, the &Python; + project itself does not build installers for Linux for this reason. Many such systems have separate packages for - Python 2 and Python 3 - make sure the Python 3 package is + &Python; 2 and &Python; 3 - make sure the &Python; 3 package is installed, as the latest &SCons; requires it. Building from source may still be a - useful option if you need a version that is not offered by + useful option if you need a specific version that is not offered by the distribution you are using. - &SCons; will work with Python 3.5 or later. - If you need to install Python and have a choice, - we recommend using the most recent Python version available. - Newer Pythons have significant improvements + Recent versions of the Mac no longer come with &Python; + pre-installed; older versions came with a rather out of date + version (based on &Python; 2.7) which is insufficient to run + current &SCons;. + The python.org installer can be used on the Mac, but there are + also other sources such as MacPorts and Homebrew. + The Anaconda installation also comes with a bundled &Python;. + + + + Windows has even more choices. The Python.org installer is + a traditional .exe style; + the same software is also released as a Windows application through + the Microsoft Store. Several alternative builds also exist + such as Chocolatey and ActiveState, and, again, + a version of Python comes with Anaconda. + + + + &SCons; will work with &Python; 3.6 or later. + If you need to install &Python; and have a choice, + we recommend using the most recent &Python; version available. + Newer &Python; versions have significant improvements that help speed up the performance of &SCons;.
-
+
Installing &SCons; - The canonical way to install &SCons; is from the Python Package - Index (PyPi): + The recommended way to install &SCons; is from the &Python; Package + Index (PyPI): @@ -185,9 +192,9 @@ Python 3.7.1 - If you prefer not to install to the Python system location, - or do not have privileges to do so, you can add a flag to - install to a location specific to your own account: + If you prefer not to install to the &Python; system location, + or do not have privileges to do so, you can add a flag to install + to a location specific to your own account and &Python; version: @@ -197,7 +204,7 @@ Python 3.7.1 For those users using Anaconda or Miniconda, use the conda installer instead, so the &scons; - install location will match the version of Python that + install location will match the version of &Python; that system will be using. For example: @@ -206,43 +213,61 @@ Python 3.7.1 - &SCons; comes pre-packaged for installation on many Linux systems. - Check your package installation system - to see if there is an &SCons; package available. - Many people prefer to install distribution-native packages if available, - as they provide a central point for management and updating. - During the still-ongoing Python 2 to 3 transition, - some distributions may still have two &SCons; packages available, - one which uses Python 2 and one which uses Python 3. Since - the latest &scons; only runs on Python 3, to get the current version - you should choose the Python 3 package. + If you need a specific + version of &SCons; that is different from the current version, + pip has a version option + (e.g. python -m pip install scons==3.1.2), + or you can follow the instructions in the following sections. - If you need a specific - version of &SCons; that is different from the package available, - pip has a version option or you can follow - the instructions in the next section. + &SCons; does comes pre-packaged for installation on many Linux systems. + Check your package installation system + to see if there is an up-to-date &SCons; package available. + Many people prefer to install distribution-native packages if available, + as they provide a central point for management and updating; + however not all distributions update in a timely fashion. + During the still-ongoing &Python; 2 to 3 transition, + some distributions may still have two &SCons; packages available, + one which uses &Python; 2 and one which uses &Python; 3. Since + the latest &scons; only runs on &Python; 3, to get the current version + you should choose the &Python; 3 package.
-
- Building and Installing &SCons; on Any System +
+ Using &SCons; Without Installing - If a pre-built &SCons; package is not available for your system, - and installing using pip is not suitable, - then you can still easily build and install &SCons; using the native - Python setuptools package. + You don't actually need to "install" &SCons; to use it. + Nor do you need to "build" it, unless you are interested in + producing the &SCons; documentation, which does use several + tools to produce HTML, PDF and other output formats from + files in the source tree. + All you need to do is + call the scons.py driver script in a + location that contains an &SCons; tree, and it will figure out + the rest. You can test that like this: + +$ python /path/to/unpacked/scripts/scons.py --version + + - The first step is to download either the + To make use of an uninstalled &SCons;, + the first step is to download either the scons-&buildversion;.tar.gz or scons-&buildversion;.zip, which are available from the SCons download page at https://scons.org/pages/download.html. + There is also a scons-local bundle you can make + use of. It is arranged a little bit differently, with the idea + that you can include it with your own project if you want people + to be able to do builds without having to download or install &SCons;. + Finally, you can also use a checkout of the git tree from GitHub + at a location to point to. @@ -252,195 +277,88 @@ Python 3.7.1 or WinZip on Windows. This will create a directory called scons-&buildversion;, - usually in your local directory. - Then change your working directory to that directory - and install &SCons; by executing the following commands: + usually in your local directory. The driver script + will be in a subdirectory named scripts, + unless you are using scons-local, + in which case it will be in the top directory. + Now you only need to call scons.py by + giving a full or relative path to it in order to use that + &SCons; version. - -# cd scons-&buildversion; -# python setup.py install - - - - This will build &SCons;, - install the &scons; script - in the python which is used to run the setup.py's scripts directory - (/usr/local/bin or - C:\Python37\Scripts), - and will install the &SCons; build engine - in the corresponding library directory for the python used - (/usr/local/lib/scons or - C:\Python37\scons). - Because these are system directories, - you may need root (on Linux or UNIX) or Administrator (on Windows) - privileges to install &SCons; like this. - + Note that instructions for older versions may have suggested + running python setup.py install to + "build and install" &SCons;. This is no longer recommended + (in fact, it is not recommended by the wider &Python; packaging + community for any end-user installations + of &Python; software). There is a setup.py file, + but it is only tested and used for the automated procedure which + prepares an &SCons; bundle for making a release on PyPI, + and even that is not guaranteed to work in future. - - -
- Building and Installing Multiple Versions of &SCons; Side-by-Side - - +
- The &SCons; setup.py script - has some extensions that support - easy installation of multiple versions of &SCons; - in side-by-side locations. - This makes it easier to download and - experiment with different versions of &SCons; - before moving your official build process to a new version, - for example. - - +
+ Running Multiple Versions of &SCons; Side-by-Side - - To install &SCons; in a version-specific location, - add the option - when you call setup.py: - + In some cases you may need several versions of &SCons; + present on a system at the same time - perhaps you have + an older project to build that has not yet been "ported" + to a newer &SCons; version, or maybe you want to test a + new &SCons; release side-by-side with a previous one + before switching over. + The use of an "uninstalled" package as described in the + previous section can be of use for this purpose. - -# python setup.py install --version-lib - - - - This will install the &SCons; build engine - in the - /usr/lib/scons-&buildversion; - or - C:\Python27\scons-&buildversion; - directory, for example. - + Another approach to multiple versions is to create + &Python; virtualenvs, and install different &SCons; versions in each. + A Python virtual environment + is a directory with an isolated set of Python packages, + where packages you install/upgrade/remove inside the + environment do not affect anything outside it, + and those you install/upgrade/remove outside of it + do not affect anything inside it. + In other words, anything you do with pip + in the environment stays in that environment. + The &Python; standard library provides a module called + venv for creating these + (), + although there are also other tools which provide more precise + control of the setup. - - If you use the option - the first time you install &SCons;, - you do not need to specify it each time you install - a new version. - The &SCons; setup.py script - will detect the version-specific directory name(s) - and assume you want to install all versions - in version-specific directories. - You can override that assumption in the future - by explicitly specifying the option. - + Using a virtualenv can be useful even for a single version of + &SCons;, to gain the advantages of having an isolated environment. + It also gets around the problem of not having administrative + privileges on a particular system to install a distribution + package or use pip to install to a + system location, as the virtualenv is completely under your control. -
- -
- Installing &SCons; in Other Locations - - - You can install &SCons; in locations other than - the default by specifying the option: - + The following outline shows how this could be set up + on a Linux/POSIX system (the syntax will be a bit different + on Windows): -# python setup.py install --prefix=/opt/scons +$ create virtualenv named scons3 +$ create virtualenv named scons4 +$ source scons3/bin/activate +$ pip install scons==3.1.2 +$ deactivate +$ source scons4/bin/activate +$ pip install scons +$ deactivate +$ activate a virtualenv and run 'scons' to use that version - - - This would - install the scons script in - /opt/scons/bin - and the build engine in - /opt/scons/lib/scons, - - - - - - Note that you can specify both the - and the options - at the same type, - in which case setup.py - will install the build engine - in a version-specific directory - relative to the specified prefix. - Adding to the - above example would install the build engine in - /opt/scons/lib/scons-&buildversion;. - - - -
- -
- Building and Installing &SCons; Without Administrative Privileges - - - - If you don't have the right privileges to install &SCons; - in a system location, - simply use the --prefix= option - to install it in a location of your choosing. - For example, - to install &SCons; in appropriate locations - relative to the user's $HOME directory, - the &scons; script in - $HOME/bin - and the build engine in - $HOME/lib/scons, - simply type: - - - - -$ python setup.py install --prefix=$HOME - - - - - You may, of course, specify any other location you prefer, - and may use the option - if you would like to install version-specific directories - relative to the specified prefix. - - - - - - This can also be used to experiment with a newer - version of &SCons; than the one installed - in your system locations. - Of course, the location in which you install the - newer version of the &scons; script - ($HOME/bin in the above example) - must be configured in your &PATH; variable - before the directory containing - the system-installed version - of the &scons; script. - - - -
-