diff options
| author | William Deegan <bill@baddogconsulting.com> | 2019-04-23 17:11:36 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-04-23 17:11:36 (GMT) |
| commit | c46a38c0a326938832edda77575ef3da00db75d5 (patch) | |
| tree | c72c94a5ae1c277a443a45862cc1b7ee675952b8 /src/engine | |
| parent | b9859740b1253ec58e5ab43ae345432dbeadd3ae (diff) | |
| parent | 9d992057717b2a35f7cccbaecfadc609ec5e3583 (diff) | |
| download | SCons-c46a38c0a326938832edda77575ef3da00db75d5.zip SCons-c46a38c0a326938832edda77575ef3da00db75d5.tar.gz SCons-c46a38c0a326938832edda77575ef3da00db75d5.tar.bz2 | |
Merge branch 'master' into tool-add
Diffstat (limited to 'src/engine')
40 files changed, 451 insertions, 319 deletions
diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py index efe6e98..042d3dd 100644 --- a/src/engine/SCons/ActionTests.py +++ b/src/engine/SCons/ActionTests.py @@ -1554,19 +1554,19 @@ class CommandGeneratorActionTestCase(unittest.TestCase): assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected \n" + repr( func_matches[sys.version_info[:2]]) - def f_global(target, source, env, for_signature): + def f_global2(target, source, env, for_signature): return SCons.Action.Action(GlobalFunc, varlist=['XYZ']) - def f_local(target, source, env, for_signature): + def f_local2(target, source, env, for_signature): return SCons.Action.Action(LocalFunc, varlist=['XYZ']) matches_foo = func_matches[sys.version_info[:2]] + b'foo' - a = self.factory(f_global) + a = self.factory(f_global2) c = a.get_contents(target=[], source=[], env=env) assert c in matches_foo, repr(c) - a = self.factory(f_local) + a = self.factory(f_local2) c = a.get_contents(target=[], source=[], env=env) assert c in matches_foo, repr(c) diff --git a/src/engine/SCons/BuilderTests.py b/src/engine/SCons/BuilderTests.py index 847e30a..b03a425 100644 --- a/src/engine/SCons/BuilderTests.py +++ b/src/engine/SCons/BuilderTests.py @@ -409,10 +409,6 @@ class BuilderTestCase(unittest.TestCase): builder = SCons.Builder.Builder(generator=generator) assert builder.action.generator == generator - def test_get_name(self): - """Test the get_name() method - """ - def test_cmp(self): """Test simple comparisons of Builder objects """ @@ -552,7 +548,7 @@ class BuilderTestCase(unittest.TestCase): def test_src_suffix(self): """Test Builder creation with a specified source file suffix - + Make sure that the '.' separator is appended to the beginning if it isn't already present. """ @@ -685,7 +681,7 @@ class BuilderTestCase(unittest.TestCase): pass if (len(source) == 1 and len(target) == 1): env['CNT'][0] = env['CNT'][0] + 1 - + env = Environment() infiles = [] outfiles = [] @@ -732,8 +728,8 @@ class BuilderTestCase(unittest.TestCase): pass else: assert 0 - - + + def test_lists(self): """Testing handling lists of targets and source""" def function2(target, source, env, tlist = [outfile, outfile2], **kw): @@ -881,7 +877,7 @@ class BuilderTestCase(unittest.TestCase): def func(self): pass - + scanner = SCons.Scanner.Base(func, name='fooscan') b1 = SCons.Builder.Builder(action='bld', target_scanner=scanner) @@ -890,8 +886,8 @@ class BuilderTestCase(unittest.TestCase): assert b1 == b2 assert b1 != b3 - - def test_src_scanner(slf): + + def test_src_scanner(self): """Testing ability to set a source file scanner through a builder.""" class TestScanner(object): def key(self, env): @@ -1245,7 +1241,7 @@ class BuilderTestCase(unittest.TestCase): for t in target: t.builder = nb return [nn], source - + builder=SCons.Builder.Builder(action='foo', emitter=emit, target_factory=MyNode, @@ -1321,7 +1317,7 @@ class BuilderTestCase(unittest.TestCase): builder2 = SCons.Builder.Builder(action='foo', emitter='$EMITTERLIST', node_factory=MyNode) - + env = Environment(EMITTERLIST = [emit2a, emit2b]) tgts = builder2(env, target='target-2', source='aaa.2') @@ -1419,7 +1415,7 @@ class BuilderTestCase(unittest.TestCase): b6 = SCons.Builder.Builder(action='foo') assert isinstance(b4, SCons.Builder.CompositeBuilder) assert isinstance(b4.action, SCons.Action.CommandGeneratorAction) - + env = Environment(BUILDERS={'bldr1': b1, 'bldr2': b2, 'bldr3': b3, @@ -1481,7 +1477,7 @@ class CompositeBuilderTestCase(unittest.TestCase): tgt = builder(env, source=[]) assert tgt == [], tgt - + assert isinstance(builder, SCons.Builder.CompositeBuilder) assert isinstance(builder.action, SCons.Action.CommandGeneratorAction) diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index ab23f31..3d8e7bd 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -111,7 +111,7 @@ def CachePushFunc(target, source, env): # has beaten us creating the directory. if not fs.isdir(cachedir): msg = errfmt % (str(target), cachefile) - raise SCons.Errors.EnvironmentError(msg) + raise SCons.Errors.SConsEnvironmentError(msg) try: if fs.islink(t.get_internal_path()): @@ -177,8 +177,8 @@ class CacheDir(object): # If someone else is trying to create the directory at # the same time as me, bad things will happen msg = "Failed to create cache directory " + path - raise SCons.Errors.EnvironmentError(msg) - + raise SCons.Errors.SConsEnvironmentError(msg) + self.config['prefix_len'] = 2 if not os.path.exists(config_file): try: @@ -186,15 +186,15 @@ class CacheDir(object): json.dump(self.config, config) except: msg = "Failed to write cache configuration for " + path - raise SCons.Errors.EnvironmentError(msg) + raise SCons.Errors.SConsEnvironmentError(msg) else: try: with open(config_file) as config: self.config = json.load(config) except ValueError: msg = "Failed to read cache configuration for " + path - raise SCons.Errors.EnvironmentError(msg) - + raise SCons.Errors.SConsEnvironmentError(msg) + def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py index 479ef7e..aa7dd2f 100644 --- a/src/engine/SCons/Defaults.py +++ b/src/engine/SCons/Defaults.py @@ -582,6 +582,7 @@ ConstructionEnvironment = { '__DSHLIBVERSIONFLAGS' : '${__libversionflags(__env__,"DSHLIBVERSION","_DSHLIBVERSIONFLAGS")}', 'TEMPFILE' : NullCmdGenerator, + 'TEMPFILEARGJOIN': ' ', 'Dir' : Variable_Method_Caller('TARGET', 'Dir'), 'Dirs' : Variable_Method_Caller('TARGET', 'Dirs'), 'File' : Variable_Method_Caller('TARGET', 'File'), diff --git a/src/engine/SCons/DefaultsTests.py b/src/engine/SCons/DefaultsTests.py index e04d1eb..2cbad70 100644 --- a/src/engine/SCons/DefaultsTests.py +++ b/src/engine/SCons/DefaultsTests.py @@ -69,11 +69,11 @@ class DefaultsTestCase(unittest.TestCase): test.write(file, "test\n") try: mkdir_func(file) - except os.error as e: + except OSError as e: pass else: - fail("expected os.error") - + self.fail("expected OSError") + if __name__ == "__main__": unittest.main() diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py index 2525e0f..347c410 100644 --- a/src/engine/SCons/EnvironmentTests.py +++ b/src/engine/SCons/EnvironmentTests.py @@ -74,7 +74,7 @@ def diff_dict(d1, d2): s2 = s2 + " " + repr(k) + " : " + repr(d2[k]) + "\n" else: s1 = s1 + " " + repr(k) + " : " + repr(d1[k]) + "\n" - elif k in env2: + elif k in d2: s2 = s2 + " " + repr(k) + " : " + repr(d2[k]) + "\n" s1 = s1 + "}\n" s2 = s2 + "}\n" @@ -263,39 +263,39 @@ class SubstitutionTestCase(unittest.TestCase): nodes = env.arg2nodes("Util.py UtilTests.py", Factory) assert len(nodes) == 1, nodes assert isinstance(nodes[0], X) - assert nodes[0].name == "Util.py UtilTests.py" + assert nodes[0].name == "Util.py UtilTests.py", nodes[0].name nodes = env.arg2nodes(u"Util.py UtilTests.py", Factory) assert len(nodes) == 1, nodes assert isinstance(nodes[0], X) - assert nodes[0].name == u"Util.py UtilTests.py" + assert nodes[0].name == u"Util.py UtilTests.py", nodes[0].name nodes = env.arg2nodes(["Util.py", "UtilTests.py"], Factory) assert len(nodes) == 2, nodes assert isinstance(nodes[0], X) assert isinstance(nodes[1], X) - assert nodes[0].name == "Util.py" - assert nodes[1].name == "UtilTests.py" + assert nodes[0].name == "Util.py", nodes[0].name + assert nodes[1].name == "UtilTests.py", nodes[1].name n1 = Factory("Util.py") nodes = env.arg2nodes([n1, "UtilTests.py"], Factory) assert len(nodes) == 2, nodes assert isinstance(nodes[0], X) assert isinstance(nodes[1], X) - assert nodes[0].name == "Util.py" - assert nodes[1].name == "UtilTests.py" + assert nodes[0].name == "Util.py", nodes[0].name + assert nodes[1].name == "UtilTests.py", nodes[1].name class SConsNode(SCons.Node.Node): pass nodes = env.arg2nodes(SConsNode()) assert len(nodes) == 1, nodes - assert isinstance(nodes[0], SConsNode), node + assert isinstance(nodes[0], SConsNode), nodes[0] class OtherNode(object): pass nodes = env.arg2nodes(OtherNode()) assert len(nodes) == 1, nodes - assert isinstance(nodes[0], OtherNode), node + assert isinstance(nodes[0], OtherNode), nodes[0] def lookup_a(str, F=Factory): if str[0] == 'a': @@ -484,7 +484,7 @@ class SubstitutionTestCase(unittest.TestCase): env = SubstitutionEnvironment(AAA = '$BBB', BBB = '$CCC', CCC = 'c') l = env.subst_list("$AAA ${AAA}A ${AAA}B $BBB") - assert l == [["c", "cA", "cB", "c"]], mystr + assert l == [["c", "cA", "cB", "c"]], l env = SubstitutionEnvironment(AAA = '$BBB', BBB = '$CCC', CCC = [ 'a', 'b\nc' ]) lst = env.subst_list([ "$AAA", "B $CCC" ]) @@ -1195,7 +1195,7 @@ env4.builder1.env, env3) test_it('foo.bar') test_it('foo-bar') - def test_autogenerate(dict): + def test_autogenerate(self): """Test autogenerating variables in a dictionary.""" drive, p = os.path.splitdrive(os.getcwd()) @@ -1206,9 +1206,9 @@ env4.builder1.env, env3) drive, path = os.path.splitdrive(path) return drive.lower() + path - env = dict.TestEnvironment(LIBS = [ 'foo', 'bar', 'baz' ], - LIBLINKPREFIX = 'foo', - LIBLINKSUFFIX = 'bar') + env = self.TestEnvironment(LIBS = [ 'foo', 'bar', 'baz' ], + LIBLINKPREFIX = 'foo', + LIBLINKSUFFIX = 'bar') def RDirs(pathlist, fs=env.fs): return fs.Dir('xx').Rfindalldirs(pathlist) @@ -2433,16 +2433,16 @@ f5: \ exc_caught = None try: env.Tool('does_not_exist') - except SCons.Errors.EnvironmentError: + except SCons.Errors.SConsEnvironmentError: exc_caught = 1 - assert exc_caught, "did not catch expected EnvironmentError" + assert exc_caught, "did not catch expected SConsEnvironmentError" exc_caught = None try: env.Tool('$NONE') - except SCons.Errors.EnvironmentError: + except SCons.Errors.SConsEnvironmentError: exc_caught = 1 - assert exc_caught, "did not catch expected EnvironmentError" + assert exc_caught, "did not catch expected SConsEnvironmentError" # Use a non-existent toolpath directory just to make sure we # can call Tool() with the keyword argument. @@ -3272,11 +3272,11 @@ def generate(env): s = e.src_builder() assert s is None, s - def test_SourceSignatures(type): + def test_SourceSignatures(self): """Test the SourceSignatures() method""" import SCons.Errors - env = type.TestEnvironment(M = 'MD5', T = 'timestamp') + env = self.TestEnvironment(M = 'MD5', T = 'timestamp') exc_caught = None try: @@ -3312,7 +3312,7 @@ def generate(env): def test_Split(self): """Test the Split() method""" - env = self.TestEnvironment(FOO='fff', BAR='bbb') + env = self.TestEnvironment(FOO = 'fff', BAR = 'bbb') s = env.Split("foo bar") assert s == ["foo", "bar"], s s = env.Split("$FOO bar") @@ -3326,11 +3326,11 @@ def generate(env): s = env.Split("$FOO$BAR") assert s == ["fffbbb"], s - def test_TargetSignatures(type): + def test_TargetSignatures(self): """Test the TargetSignatures() method""" import SCons.Errors - env = type.TestEnvironment(B = 'build', C = 'content') + env = self.TestEnvironment(B='build', C='content') exc_caught = None try: @@ -3397,7 +3397,7 @@ def generate(env): - def test_Environment_global_variable(type): + def test_Environment_global_variable(self): """Test setting Environment variable to an Environment.Base subclass""" class MyEnv(SCons.Environment.Base): def xxx(self, string): diff --git a/src/engine/SCons/Errors.py b/src/engine/SCons/Errors.py index 3746d5d..a3a891f 100644 --- a/src/engine/SCons/Errors.py +++ b/src/engine/SCons/Errors.py @@ -124,7 +124,7 @@ class UserError(Exception): class StopError(Exception): pass -class EnvironmentError(Exception): +class SConsEnvironmentError(Exception): pass class MSVCError(IOError): @@ -184,7 +184,7 @@ def convert_to_BuildError(status, exc_info=None): filename=filename, exc_info=exc_info) - elif isinstance(status, (EnvironmentError, OSError, IOError)): + elif isinstance(status, (SConsEnvironmentError, OSError, IOError)): # If an IOError/OSError happens, raise a BuildError. # Report the name of the file or directory that caused the # error, which might be different from the target being built diff --git a/src/engine/SCons/ErrorsTests.py b/src/engine/SCons/ErrorsTests.py index 5c16160..d777ba1 100644 --- a/src/engine/SCons/ErrorsTests.py +++ b/src/engine/SCons/ErrorsTests.py @@ -101,10 +101,10 @@ class ErrorsTestCase(unittest.TestCase): assert e.node == "node" def test_convert_EnvironmentError_to_BuildError(self): - """Test the convert_to_BuildError function on EnvironmentError + """Test the convert_to_BuildError function on SConsEnvironmentError exceptions. """ - ee = SCons.Errors.EnvironmentError("test env error") + ee = SCons.Errors.SConsEnvironmentError("test env error") be = SCons.Errors.convert_to_BuildError(ee) assert be.errstr == "test env error" assert be.status == 2 diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index 01d01cd..6c68e09 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -41,10 +41,10 @@ from SCons.compat import with_metaclass, NoSlotsPyPy class Batch(object): """Remembers exact association between targets and sources of executor.""" - + __slots__ = ('targets', 'sources') - + def __init__(self, targets=[], sources=[]): self.targets = targets self.sources = sources @@ -127,13 +127,13 @@ def execute_action_list(obj, target, kw): status = act(*args, **kw) if isinstance(status, SCons.Errors.BuildError): status.executor = obj - raise status + raise status # TODO pylint E0702: raising int not allowed elif status: msg = "Error %s" % status raise SCons.Errors.BuildError( - errstr=msg, + errstr=msg, node=obj.batches[0].targets, - executor=obj, + executor=obj, action=act) return status @@ -597,7 +597,7 @@ class Null(object, with_metaclass(NoSlotsPyPy)): disassociate Builders from Nodes entirely, so we're not going to worry about unit tests for this--at least for now. """ - + __slots__ = ('pre_actions', 'post_actions', 'env', @@ -613,7 +613,7 @@ class Null(object, with_metaclass(NoSlotsPyPy)): 'action_list', '_do_execute', '_execute_str') - + def __init__(self, *args, **kw): if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Null') self.batches = [Batch(kw['targets'][:], [])] @@ -649,7 +649,7 @@ class Null(object, with_metaclass(NoSlotsPyPy)): """Morph this Null executor to a real Executor object.""" batches = self.batches self.__class__ = Executor - self.__init__([]) + self.__init__([]) self.batches = batches # The following methods require morphing this Null Executor to a diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py index 325c0e1..26e3d37 100644 --- a/src/engine/SCons/JobTests.py +++ b/src/engine/SCons/JobTests.py @@ -73,7 +73,7 @@ class DummyLock(object): def release(self): pass -class NoThreadsException(object): +class NoThreadsException(Exception): "raised by the ParallelTestCase if threads are not supported" def __str__(self): @@ -206,7 +206,7 @@ class Taskmaster(object): self.parallel_list = [0] * (n+1) self.found_parallel = False self.Task = Task - + # 'guard' guards 'task_begin_list' and 'task_end_list' try: import threading diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 23ec48e..d09e6da 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -1072,7 +1072,7 @@ class FSTestCase(_tempdirTestCase): def Dir_test(lpath, path_, abspath_, up_path_, sep=sep, func=_do_Dir_test): return func(lpath, path_, abspath_, up_path_, sep) - + Dir_test('/', '/', '/', '/') Dir_test('', './', sub_dir, sub) Dir_test('foo', 'foo/', sub_dir_foo, './') @@ -1834,7 +1834,7 @@ class FSTestCase(_tempdirTestCase): # Should be a normalized Windows UNC path as below. assert str(f) == r'\\servername\C$\foo', \ 'UNC path %s got looked up as %s'%(path, f) - + def test_unc_drive_letter(self): """Test drive-letter lookup for windows UNC-style directories""" if sys.platform not in ('win32',): @@ -3534,11 +3534,11 @@ class clearTestCase(unittest.TestCase): e = fs.Entry('e') assert not e.exists() assert not e.rexists() - assert str(e) == 'e', str(d) + assert str(e) == 'e', str(e) e.clear() assert not e.exists() assert not e.rexists() - assert str(e) == 'e', str(d) + assert str(e) == 'e', str(e) d = fs.Dir(test.workpath('d')) test.subdir('d') diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index 7d347ac..678e03e 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -271,7 +271,7 @@ class NodeInfoBaseTestCase(unittest.TestCase): f = ni1.format() assert f == ['x', 'y', 'z'], f - + field_list = ['xxx', 'zzz', 'aaa'] f = ni1.format(field_list) @@ -751,7 +751,7 @@ class NodeTestCase(unittest.TestCase): e = node.exists() assert e == 1, e - def test_exists(self): + def test_exists_repo(self): """Test evaluating whether a Node exists locally or in a repository. """ node = SCons.Node.Node() diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 131953b..b9aca96 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -103,9 +103,9 @@ implicit_deps_changed = 0 # A variable that can be set to an interface-specific function be called # to annotate a Node with information about its creation. -def do_nothing(node): pass +def do_nothing_node(node): pass -Annotate = do_nothing +Annotate = do_nothing_node # Gets set to 'True' if we're running in interactive mode. Is # currently used to release parts of a target's info during diff --git a/src/engine/SCons/Platform/PlatformTests.py b/src/engine/SCons/Platform/PlatformTests.py index 3f42eae..ae070f8 100644 --- a/src/engine/SCons/Platform/PlatformTests.py +++ b/src/engine/SCons/Platform/PlatformTests.py @@ -27,12 +27,14 @@ import SCons.compat import collections import unittest +import os import SCons.Errors import SCons.Platform import SCons.Environment import SCons.Action + class Environment(collections.UserDict): def Detect(self, cmd): return cmd @@ -40,6 +42,7 @@ class Environment(collections.UserDict): def AppendENVPath(self, key, value): pass + class PlatformTestCase(unittest.TestCase): def test_Platform(self): """Test the Platform() function""" @@ -110,13 +113,14 @@ class PlatformTestCase(unittest.TestCase): p = SCons.Platform.Platform('_does_not_exist_') except SCons.Errors.UserError: pass - else: + else: # TODO pylint E0704: bare raise not inside except raise env = Environment() SCons.Platform.Platform()(env) assert env != {}, env + class TempFileMungeTestCase(unittest.TestCase): def test_MAXLINELENGTH(self): """ Test different values for MAXLINELENGTH with the same @@ -138,22 +142,57 @@ class TempFileMungeTestCase(unittest.TestCase): env['OVERSIMPLIFIED'] = 'command' expanded_cmd = env.subst(defined_cmd) # Call the tempfile munger - cmd = t(None,None,env,0) + cmd = t(None, None, env, 0) assert cmd == defined_cmd, cmd # Let MAXLINELENGTH equal the string's length env['MAXLINELENGTH'] = len(expanded_cmd) - cmd = t(None,None,env,0) + cmd = t(None, None, env, 0) assert cmd == defined_cmd, cmd # Finally, let the actual tempfile mechanism kick in # Disable printing of actions... old_actions = SCons.Action.print_actions SCons.Action.print_actions = 0 env['MAXLINELENGTH'] = len(expanded_cmd)-1 - cmd = t(None,None,env,0) + cmd = t(None, None, env, 0) # ...and restoring its setting. SCons.Action.print_actions = old_actions assert cmd != defined_cmd, cmd + def test_TEMPFILEARGJOINBYTE(self): + """ + Test argument join byte TEMPFILEARGJOINBYTE + """ + + # Init class with cmd, such that the fully expanded + # string reads "a test command line". + # Note, how we're using a command string here that is + # actually longer than the substituted one. This is to ensure + # that the TempFileMunge class internally really takes the + # length of the expanded string into account. + defined_cmd = "a $VERY $OVERSIMPLIFIED line" + t = SCons.Platform.TempFileMunge(defined_cmd) + env = SCons.Environment.SubstitutionEnvironment(tools=[]) + # Setting the line length high enough... + env['MAXLINELENGTH'] = 1024 + env['VERY'] = 'test' + env['OVERSIMPLIFIED'] = 'command' + env['TEMPFILEARGJOINBYTE'] = os.linesep + expanded_cmd = env.subst(defined_cmd) + + # For tempfilemunge to operate. + old_actions = SCons.Action.print_actions + SCons.Action.print_actions = 0 + env['MAXLINELENGTH'] = len(expanded_cmd)-1 + cmd = t(None, None, env, 0) + # print("CMD is:%s"%cmd) + + file_content = open(cmd[-1],'rb').read() + # print("Content is:[%s]"%file_content) + # ...and restoring its setting. + SCons.Action.print_actions = old_actions + assert file_content != env['TEMPFILEARGJOINBYTE'].join(['test','command','line']) + + def test_tempfilecreation_once(self): # Init class with cmd, such that the fully expanded # string reads "a test command line". @@ -173,9 +212,11 @@ class TempFileMungeTestCase(unittest.TestCase): old_actions = SCons.Action.print_actions SCons.Action.print_actions = 0 # Create an instance of object derived class to allow setattrb - class Node(object) : + + class Node(object): class Attrs(object): pass + def __init__(self): self.attributes = self.Attrs() target = [Node()] @@ -185,6 +226,7 @@ class TempFileMungeTestCase(unittest.TestCase): assert cmd != defined_cmd, cmd assert cmd == getattr(target[0].attributes, 'tempfile_cmdlist', None) + class PlatformEscapeTestCase(unittest.TestCase): def test_posix_escape(self): """ Check that paths with parens are escaped properly diff --git a/src/engine/SCons/Platform/__init__.py b/src/engine/SCons/Platform/__init__.py index e06eb0f..7d959b7 100644 --- a/src/engine/SCons/Platform/__init__.py +++ b/src/engine/SCons/Platform/__init__.py @@ -226,8 +226,10 @@ class TempFileMunge(object): prefix = '@' args = list(map(SCons.Subst.quote_spaces, cmd[1:])) - os.write(fd, bytearray(" ".join(args) + "\n",'utf-8')) + join_char = env.get('TEMPFILEARGJOIN',' ') + os.write(fd, bytearray(join_char.join(args) + "\n",'utf-8')) os.close(fd) + # XXX Using the SCons.Action.print_actions value directly # like this is bogus, but expedient. This class should # really be rewritten as an Action that defines the diff --git a/src/engine/SCons/Platform/__init__.xml b/src/engine/SCons/Platform/__init__.xml index f113278..7ea895e 100644 --- a/src/engine/SCons/Platform/__init__.xml +++ b/src/engine/SCons/Platform/__init__.xml @@ -257,4 +257,15 @@ The default is '.lnk'. </summary> </cvar> +<cvar name="TEMPFILEARGJOIN"> +<summary> +<para> +The string (or character) to be used to join the arguments passed to TEMPFILE when command line exceeds the limit set by &cv-MAXLINELENGTH;. +The default value is a space. However for MSVC, MSLINK the default is a line seperator characters as defined by os.linesep. +Note this value is used literally and not expanded by the subst logic. +</para> +</summary> +</cvar> + + </sconsdoc> diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py index b123c11..c3f93db 100644 --- a/src/engine/SCons/SConf.py +++ b/src/engine/SCons/SConf.py @@ -247,6 +247,7 @@ class SConfBuildTask(SCons.Taskmaster.AlwaysTask): # ConfigureCacheError and if yes, reraise the exception exc_type = self.exc_info()[0] if issubclass(exc_type, SConfError): + # TODO pylint E0704: bare raise not inside except raise elif issubclass(exc_type, SCons.Errors.BuildError): # we ignore Build Errors (occurs, when a test doesn't pass) diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index f3475f2..fdcd252 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -68,6 +68,20 @@ import SCons.Warnings import SCons.Script.Interactive +# Global variables +first_command_start = None +last_command_end = None +print_objects = 0 +print_memoizer = 0 +print_stacktrace = 0 +print_time = 0 +sconscript_time = 0 +cumulative_command_time = 0 +exit_status = 0 # final exit status, assume success by default +this_build_status = 0 # "exit status" of an individual build +num_jobs = None +delayed_warnings = [] + def fetch_win32_parallel_msg(): # A subsidiary function that exists solely to isolate this import @@ -87,15 +101,14 @@ def revert_io(): sys.stderr = sys.__stderr__ sys.stdout = sys.__stdout__ + class SConsPrintHelpException(Exception): pass + display = SCons.Util.display progress_display = SCons.Util.DisplayEngine() -first_command_start = None -last_command_end = None - class Progressor(object): prev = '' @@ -443,19 +456,6 @@ def python_version_deprecated(version=sys.version_info): return version < deprecated_python_version -# Global variables - -print_objects = 0 -print_memoizer = 0 -print_stacktrace = 0 -print_time = 0 -sconscript_time = 0 -cumulative_command_time = 0 -exit_status = 0 # final exit status, assume success by default -this_build_status = 0 # "exit status" of an individual build -num_jobs = None -delayed_warnings = [] - class FakeOptionParser(object): """ A do-nothing option parser, used for the initial OptionsParser variable. diff --git a/src/engine/SCons/Script/SConsOptions.py b/src/engine/SCons/Script/SConsOptions.py index 37dd644..01d365d 100644 --- a/src/engine/SCons/Script/SConsOptions.py +++ b/src/engine/SCons/Script/SConsOptions.py @@ -226,39 +226,8 @@ class SConsOption(optparse.Option): fmt = "option %s: nargs='?' is incompatible with short options" raise SCons.Errors.UserError(fmt % self._short_opts[0]) - try: - _orig_CONST_ACTIONS = optparse.Option.CONST_ACTIONS - - _orig_CHECK_METHODS = optparse.Option.CHECK_METHODS - - except AttributeError: - # optparse.Option had no CONST_ACTIONS before Python 2.5. - - _orig_CONST_ACTIONS = ("store_const",) - - def _check_const(self): - if self.action not in self.CONST_ACTIONS and self.const is not None: - raise OptionError( - "'const' must not be supplied for action %r" % self.action, - self) - - # optparse.Option collects its list of unbound check functions - # up front. This sucks because it means we can't just override - # the _check_const() function like a normal method, we have to - # actually replace it in the list. This seems to be the most - # straightforward way to do that. - - _orig_CHECK_METHODS = [optparse.Option._check_action, - optparse.Option._check_type, - optparse.Option._check_choice, - optparse.Option._check_dest, - _check_const, - optparse.Option._check_nargs, - optparse.Option._check_callback] - - CHECK_METHODS = _orig_CHECK_METHODS + [_check_nargs_optional] - - CONST_ACTIONS = _orig_CONST_ACTIONS + optparse.Option.TYPED_ACTIONS + CHECK_METHODS = optparse.Option.CHECK_METHODS + [_check_nargs_optional] + CONST_ACTIONS = optparse.Option.CONST_ACTIONS + optparse.Option.TYPED_ACTIONS class SConsOptionGroup(optparse.OptionGroup): """ @@ -364,7 +333,7 @@ class SConsOptionParser(optparse.OptionParser): in self.largs, so that any value overridden on the command line is immediately available if the user turns around and does a GetOption() right away. - + We mimic the processing of the single args in the original OptionParser._process_args(), but here we allow exact matches for long-opts only (no partial @@ -375,7 +344,7 @@ class SConsOptionParser(optparse.OptionParser): command-line arguments that 1. haven't been processed so far (self.largs), but 2. are possibly not added to the list of options yet. - + So, when we only have a value for "--myargument" yet, a command-line argument of "--myarg=test" would set it. Responsible for this behaviour is the method @@ -384,7 +353,7 @@ class SConsOptionParser(optparse.OptionParser): be unique. This would lead to further confusion, because we might want to add another option "--myarg" later on (see issue #2929). - + """ rargs = [] largs_restore = [] @@ -401,7 +370,7 @@ class SConsOptionParser(optparse.OptionParser): if "=" in l: # Split into option and value lopt = l.split("=", 1) - + if lopt[0] in self._long_opt: # Argument is already known rargs.append('='.join(lopt)) @@ -416,7 +385,7 @@ class SConsOptionParser(optparse.OptionParser): skip = True else: rargs.append(l) - + # Parse the filtered list self.parse_args(rargs, self.values) # Restore the list of remaining arguments for the diff --git a/src/engine/SCons/SubstTests.py b/src/engine/SCons/SubstTests.py index a111e4b..f6fe1ec 100644 --- a/src/engine/SCons/SubstTests.py +++ b/src/engine/SCons/SubstTests.py @@ -132,6 +132,7 @@ class SubstTestCase(unittest.TestCase): def __str__(self): return self.value + # only use of this is currently commented out below def function_foo(arg): pass @@ -1046,7 +1047,7 @@ class scons_subst_list_TestCase(SubstTestCase): node = scons_subst_list("$NODE", env, mode=SUBST_SIG, conv=s, gvars=gvars) assert node == [[n1]], node - def test_subst_list_overriding_gvars(self): + def test_subst_list_overriding_gvars2(self): """Test scons_subst_list(): supplying an overriding gvars dictionary""" env = DummyEnv({'XXX' : 'xxx'}) result = scons_subst_list('$XXX', env, gvars=env.Dictionary()) diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 1522ca2..4892026 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -27,10 +27,10 @@ import sys __doc__ = """ Generic Taskmaster module for the SCons build engine. ===================================================== - + This module contains the primary interface(s) between a wrapping user interface and the SCons build engine. There are two key classes here: - + Taskmaster ---------- This is the main engine for walking the dependency graph and @@ -477,7 +477,7 @@ class Task(object): for s in t.side_effects: if s.get_state() == NODE_EXECUTING: s.set_state(NODE_NO_STATE) - + # The side-effects may have been transferred to # NODE_NO_STATE by executed_with{,out}_callbacks, but was # not taken out of the waiting parents/pending children @@ -551,7 +551,7 @@ class Task(object): try: exc_type, exc_value, exc_traceback = exc except ValueError: - exc_type, exc_value = exc + exc_type, exc_value = exc # pylint: disable=unbalanced-tuple-unpacking exc_traceback = None # raise exc_type(exc_value).with_traceback(exc_traceback) diff --git a/src/engine/SCons/Tool/JavaCommon.py b/src/engine/SCons/Tool/JavaCommon.py index 853f7f2..f1c1b4f 100644 --- a/src/engine/SCons/Tool/JavaCommon.py +++ b/src/engine/SCons/Tool/JavaCommon.py @@ -42,6 +42,33 @@ default_java_version = '1.4' # anonymous inner class parsing. scopeStateVersions = ('1.8') +# Glob patterns for use in finding where the JDK is. +# These are pairs, *dir_glob used in the general case, +# *version_dir_glob if matching only a specific version. +# For now only used for Windows. +java_win32_dir_glob = 'C:/Program Files*/Java/jdk*/bin' +# On windows, since Java 9, there is a dash between 'jdk' and the version +# string that wasn't there before. this glob should catch either way. +java_win32_version_dir_glob = 'C:/Program Files*/Java/jdk*%s*/bin' + +# Glob patterns for use in finding where the JDK headers are. +# These are pairs, *dir_glob used in the general case, +# *version_dir_glob if matching only a specific version. +java_macos_include_dir_glob = '/System/Library/Frameworks/JavaVM.framework/Headers/' +java_macos_version_include_dir_glob = '/System/Library/Frameworks/JavaVM.framework/Versions/%s*/Headers/' + +java_linux_include_dirs_glob = [ + '/usr/lib/jvm/default-java/include', + '/usr/lib/jvm/java-*/include' +] +# Need to match path like below (from Centos 7) +# /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.191.b12-0.el7_5.x86_64/include/ +java_linux_version_include_dirs_glob = [ + '/usr/lib/jvm/java-*-sun-%s*/include', + '/usr/lib/jvm/java-%s*-openjdk*/include', + '/usr/java/jdk%s*/include' +] + if java_parsing: # Parse Java files for class names. # @@ -64,14 +91,15 @@ if java_parsing: r'\d*\.\d*|[A-Za-z_][\w\$\.]*|<[A-Za-z_]\w+>|' + r'/\*|\*/|\[\])') + class OuterState(object): """The initial state for parsing a Java file for classes, interfaces, and anonymous inner classes.""" + def __init__(self, version=default_java_version): if not version in ('1.1', '1.2', '1.3', '1.4', '1.5', '1.6', '1.7', - '1.8', '5', '6', '9.0', '10.0', '11.0'): - + '1.8', '5', '6', '9.0', '10.0', '11.0', '12.0'): msg = "Java version %s not supported" % version raise NotImplementedError(msg) @@ -131,15 +159,15 @@ if java_parsing: def closeBracket(self): self.brackets = self.brackets - 1 if len(self.stackBrackets) and \ - self.brackets == self.stackBrackets[-1]: + self.brackets == self.stackBrackets[-1]: self.listOutputs.append('$'.join(self.listClasses)) self.localClasses.pop() self.listClasses.pop() self.anonStacksStack.pop() self.stackBrackets.pop() if len(self.stackAnonClassBrackets) and \ - self.brackets == self.stackAnonClassBrackets[-1] and \ - self.version not in scopeStateVersions: + self.brackets == self.stackAnonClassBrackets[-1] and \ + self.version not in scopeStateVersions: self._getAnonStack().pop() self.stackAnonClassBrackets.pop() @@ -152,13 +180,13 @@ if java_parsing: self.openBracket() elif token == '}': self.closeBracket() - elif token in [ '"', "'" ]: + elif token in ['"', "'"]: return IgnoreState(token, self) elif token == "new": # anonymous inner class if len(self.listClasses) > 0: return self.__getAnonClassState() - return self.__getSkipState() # Skip the class name + return self.__getSkipState() # Skip the class name elif token in ['class', 'interface', 'enum']: if len(self.listClasses) == 0: self.nextAnon = 1 @@ -178,7 +206,7 @@ if java_parsing: if self.version in ('1.1', '1.2', '1.3', '1.4'): clazz = self.listClasses[0] self.listOutputs.append('%s$%d' % (clazz, self.nextAnon)) - elif self.version in ('1.5', '1.6', '1.7', '1.8', '5', '6', '9.0', '10.0', '11.0'): + elif self.version in ('1.5', '1.6', '1.7', '1.8', '5', '6', '9.0', '10.0', '11.0', '12.0'): self.stackAnonClassBrackets.append(self.brackets) className = [] className.extend(self.listClasses) @@ -193,11 +221,13 @@ if java_parsing: def setPackage(self, package): self.package = package + class ScopeState(object): """ A state that parses code within a scope normally, within the confines of a scope. """ + def __init__(self, old_state): self.outer_state = old_state.outer_state self.old_state = old_state @@ -259,13 +289,16 @@ if java_parsing: return self.__getSkipState() return self + class AnonClassState(object): """A state that looks for anonymous inner classes.""" + def __init__(self, old_state): # outer_state is always an instance of OuterState self.outer_state = old_state.outer_state self.old_state = old_state self.brace_level = 0 + def parseToken(self, token): # This is an anonymous class if and only if the next # non-whitespace token is a bracket. Everything between @@ -293,26 +326,32 @@ if java_parsing: if token == '{': self.outer_state.addAnonClass() if self.outer_state.version in scopeStateVersions: - return ScopeState(old_state = self.old_state).parseToken(token) + return ScopeState(old_state=self.old_state).parseToken(token) return self.old_state.parseToken(token) + class SkipState(object): """A state that will skip a specified number of tokens before reverting to the previous state.""" + def __init__(self, tokens_to_skip, old_state): self.tokens_to_skip = tokens_to_skip self.old_state = old_state + def parseToken(self, token): self.tokens_to_skip = self.tokens_to_skip - 1 if self.tokens_to_skip < 1: return self.old_state return self + class ClassState(object): """A state we go into when we hit a class or interface keyword.""" + def __init__(self, outer_state): # outer_state is always an instance of OuterState self.outer_state = outer_state + def parseToken(self, token): # the next non-whitespace token should be the name of the class if token == '\n': @@ -322,12 +361,12 @@ if java_parsing: # 'Foo$1Inner' # https://github.com/SCons/scons/issues/2087 if self.outer_state.localClasses and \ - self.outer_state.stackBrackets[-1] > \ - self.outer_state.stackBrackets[-2]+1: + self.outer_state.stackBrackets[-1] > \ + self.outer_state.stackBrackets[-2] + 1: locals = self.outer_state.localClasses[-1] try: idx = locals[token] - locals[token] = locals[token]+1 + locals[token] = locals[token] + 1 except KeyError: locals[token] = 1 token = str(locals[token]) + token @@ -336,32 +375,40 @@ if java_parsing: self.outer_state.anonStacksStack.append([0]) return self.outer_state + class IgnoreState(object): """A state that will ignore all tokens until it gets to a specified token.""" + def __init__(self, ignore_until, old_state): self.ignore_until = ignore_until self.old_state = old_state + def parseToken(self, token): if self.ignore_until == token: return self.old_state return self + class PackageState(object): """The state we enter when we encounter the package keyword. We assume the next token will be the package name.""" + def __init__(self, outer_state): # outer_state is always an instance of OuterState self.outer_state = outer_state + def parseToken(self, token): self.outer_state.setPackage(token) return self.outer_state + def parse_java_file(fn, version=default_java_version): with open(fn, 'r') as f: data = f.read() return parse_java(data, version) + def parse_java(contents, version=default_java_version, trace=None): """Parse a .java file and return a double of package directory, plus a list of .class files that compiling that .java file will @@ -395,81 +442,68 @@ else: return os.path.split(fn) - -java_win32_version_dir_glob = 'C:/Program Files*/Java/jdk%s*/bin' -java_win32_dir_glob = 'C:/Program Files*/Java/jdk*/bin' - -java_macos_include_dir = '/System/Library/Frameworks/JavaVM.framework/Headers/' -java_macos_version_include_dir = '/System/Library/Frameworks/JavaVM.framework/Versions/%s*/Headers/' - -java_linux_include_dirs = ['/usr/lib/jvm/default-java/include', - '/usr/lib/jvm/java-*/include'] -# Need to match path like below (from Centos 7) -# /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.191.b12-0.el7_5.x86_64/include/ -java_linux_version_include_dirs = ['/usr/lib/jvm/java-*-sun-%s*/include', - '/usr/lib/jvm/java-%s*-openjdk*/include', - '/usr/java/jdk%s*/include'] - - - def get_java_install_dirs(platform, version=None): """ - Using patterns above find the java jdk install dir - :param platform: + Find the java jdk installation directories. + + This list is intended to supply as "default paths" for use when looking + up actual java binaries. + + :param platform: selector for search algorithm. :param version: If specified, only look for java sdk's of this version :return: list of default paths for java. """ + paths = [] if platform == 'win32': if version: - paths = glob.glob(java_win32_version_dir_glob%version) + paths = glob.glob(java_win32_version_dir_glob % version) else: paths = glob.glob(java_win32_dir_glob) else: - # do nothing for now + # other platforms, do nothing for now pass - paths=sorted(paths) + return sorted(paths) - return paths def get_java_include_paths(env, javac, version): """ - Return java include paths - :param platform: - :param javac: - :return: + Find java include paths for JNI building. + + :param env: construction environment, used to extract platform. + :param javac: path to detected javac. + :return: list of paths. """ + paths = [] if not javac: # there are no paths if we've not detected javac. pass elif env['PLATFORM'] == 'win32': + # on Windows, we have the right path to javac, so look locally javac_bin_dir = os.path.dirname(javac) java_inc_dir = os.path.normpath(os.path.join(javac_bin_dir, '..', 'include')) paths = [java_inc_dir, os.path.join(java_inc_dir, 'win32')] elif env['PLATFORM'] == 'darwin': if not version: - paths = [java_macos_include_dir] + paths = [java_macos_include_dir_glob] else: - paths = sorted(glob.glob(java_macos_version_include_dir%version)) + paths = sorted(glob.glob(java_macos_version_include_dir_glob % version)) else: - base_paths=[] + base_paths = [] if not version: - for p in java_linux_include_dirs: + for p in java_linux_include_dirs_glob: base_paths.extend(glob.glob(p)) else: - for p in java_linux_version_include_dirs: - base_paths.extend(glob.glob(p%version)) + for p in java_linux_version_include_dirs_glob: + base_paths.extend(glob.glob(p % version)) for p in base_paths: - paths.extend([p, os.path.join(p,'linux')]) - - #print("PATHS:%s"%paths) - return paths - - + paths.extend([p, os.path.join(p, 'linux')]) + # print("PATHS:%s"%paths) + return paths # Local Variables: # tab-width:4 diff --git a/src/engine/SCons/Tool/JavaCommonTests.py b/src/engine/SCons/Tool/JavaCommonTests.py index 10be671..9242624 100644 --- a/src/engine/SCons/Tool/JavaCommonTests.py +++ b/src/engine/SCons/Tool/JavaCommonTests.py @@ -26,10 +26,12 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path import sys import unittest +import fnmatch import SCons.Scanner.IDL import SCons.Tool.JavaCommon +import TestSCons # Adding trace=trace to any of the parse_jave() calls below will cause # the parser to spit out trace messages of the tokens it sees and the @@ -607,6 +609,76 @@ public class AnonDemo { pkg_dir, classes = SCons.Tool.JavaCommon.parse_java(input, '1.8') assert expect == classes, (expect, classes) + def test_jdk_globs(self): + """ + Verify that the java path globs work with specific examples. + :return: + """ + from SCons.Tool.JavaCommon import java_linux_include_dirs_glob, java_linux_version_include_dirs_glob, java_win32_dir_glob, java_win32_version_dir_glob, java_macos_include_dir_glob, java_macos_version_include_dir_glob + + # Test windows globs + win_java_dirs = [ + ('C:/Program Files/Java/jdk1.8.0_201/bin', '1.8.0'), + ('C:/Program Files/Java/jdk-11.0.2/bin', '11.0.2'), + ('C:/Program Files/Java/jdk1.7.0_80/bin', '1.7.0') + ] + + for (wjd, version) in win_java_dirs: + if not fnmatch.fnmatch(wjd, java_win32_dir_glob): + self.fail("Didn't properly match %s with pattern %s" % (wjd, java_win32_dir_glob)) + if not fnmatch.fnmatch(wjd, java_win32_version_dir_glob % version): + self.fail("Didn't properly match %s with version (%s) specific pattern %s" % ( + wjd, version, java_win32_version_dir_glob % version)) + + non_win_java_include_dirs = [ + ('/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.191.b12-0.el7_5.x86_64/include', '1.8.0'), + ('/usr/lib/jvm/java-1.8.0-openjdk-amd64/include', '1.8.0'), + ('/usr/lib/jvm/java-8-openjdk-amd64/include', '8'), + ] + + # Test non-windows/non-macos globs + for (wjd, version) in non_win_java_include_dirs: + match = False + globs_tried =[] + for jlig in java_linux_include_dirs_glob: + globs_tried.append(jlig) + + if fnmatch.fnmatch(wjd, jlig): + match = True + break + + if not match: + self.fail("Didn't properly match %s with pattern %s" % (wjd, globs_tried)) + + match = False + globs_tried = [] + for jlvig in java_linux_version_include_dirs_glob: + globs_tried.append(jlvig%version) + if fnmatch.fnmatch(wjd, jlvig % version): + match = True + break + + if not match: + self.fail("Didn't properly match %s with version (%s) specific pattern %s" % ( + wjd, version, globs_tried)) + + # Test macos globs + # Test windows globs + macos_java_dirs = [ + # ('/System/Library/Frameworks/JavaVM.framework/Headers/', None), + ('/System/Library/Frameworks/JavaVM.framework/Versions/11.0.2/Headers/', '11.0.2'), + ] + + if not fnmatch.fnmatch('/System/Library/Frameworks/JavaVM.framework/Headers/', java_macos_include_dir_glob): + self.fail("Didn't properly match %s with pattern %s" % ('/System/Library/Frameworks/JavaVM.framework/Headers/', java_macos_include_dir_glob)) + + for (wjd, version) in macos_java_dirs: + if not fnmatch.fnmatch(wjd, java_macos_version_include_dir_glob % version): + self.fail("Didn't properly match %s with version (%s) specific pattern %s" % ( + wjd, version, java_macos_version_include_dir_glob % version)) + + + if __name__ == "__main__": unittest.main() diff --git a/src/engine/SCons/Tool/MSCommon/netframework.py b/src/engine/SCons/Tool/MSCommon/netframework.py index 787d008..b40576a 100644 --- a/src/engine/SCons/Tool/MSCommon/netframework.py +++ b/src/engine/SCons/Tool/MSCommon/netframework.py @@ -68,7 +68,7 @@ def query_versions(): # sequence comparison in python is lexicographical # which is exactly what we want. # Note we sort backwards so the highest version is first. - return cmp(bbl,aal) + return (aal > bbl) - (aal < bbl) versions.sort(versrt) else: diff --git a/src/engine/SCons/Tool/ToolTests.py b/src/engine/SCons/Tool/ToolTests.py index a4353b1..f005143 100644 --- a/src/engine/SCons/Tool/ToolTests.py +++ b/src/engine/SCons/Tool/ToolTests.py @@ -67,14 +67,14 @@ class ToolTestCase(unittest.TestCase): SCons.Tool.Tool() except TypeError: pass - else: + else: # TODO pylint E0704: bare raise not inside except raise try: p = SCons.Tool.Tool('_does_not_exist_') - except SCons.Errors.EnvironmentError: + except SCons.Errors.SConsEnvironmentError: pass - else: + else: # TODO pylint E0704: bare raise not inside except raise diff --git a/src/engine/SCons/Tool/__init__.py b/src/engine/SCons/Tool/__init__.py index 152e186..166c3b0 100644 --- a/src/engine/SCons/Tool/__init__.py +++ b/src/engine/SCons/Tool/__init__.py @@ -149,7 +149,7 @@ class Tool(object): except ImportError as e: splitname = self.name.split('.') if str(e) != "No module named %s" % splitname[0]: - raise SCons.Errors.EnvironmentError(e) + raise SCons.Errors.SConsEnvironmentError(e) try: import zipimport except ImportError: @@ -211,13 +211,13 @@ class Tool(object): if spec is None: error_string = "No module named %s" % self.name - raise SCons.Errors.EnvironmentError(error_string) + raise SCons.Errors.SConsEnvironmentError(error_string) module = importlib.util.module_from_spec(spec) if module is None: if debug: print("MODULE IS NONE:%s" % self.name) error_string = "No module named %s" % self.name - raise SCons.Errors.EnvironmentError(error_string) + raise SCons.Errors.SConsEnvironmentError(error_string) # Don't reload a tool we already loaded. sys_modules_value = sys.modules.get(found_name, False) @@ -258,7 +258,7 @@ class Tool(object): return module except ImportError as e: if str(e) != "No module named %s" % self.name: - raise SCons.Errors.EnvironmentError(e) + raise SCons.Errors.SConsEnvironmentError(e) try: import zipimport importer = zipimport.zipimporter(sys.modules['SCons.Tool'].__path__[0]) @@ -267,10 +267,10 @@ class Tool(object): return module except ImportError as e: m = "No tool named '%s': %s" % (self.name, e) - raise SCons.Errors.EnvironmentError(m) + raise SCons.Errors.SConsEnvironmentError(m) except ImportError as e: m = "No tool named '%s': %s" % (self.name, e) - raise SCons.Errors.EnvironmentError(m) + raise SCons.Errors.SConsEnvironmentError(m) def __call__(self, env, *args, **kw): if self.init_kw is not None: diff --git a/src/engine/SCons/Tool/intelc.py b/src/engine/SCons/Tool/intelc.py index 9fc0bf7..1f3fcea 100644 --- a/src/engine/SCons/Tool/intelc.py +++ b/src/engine/SCons/Tool/intelc.py @@ -37,7 +37,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import math, sys, os.path, glob, string, re is_windows = sys.platform == 'win32' -is_win64 = is_windows and (os.environ['PROCESSOR_ARCHITECTURE'] == 'AMD64' or +is_win64 = is_windows and (os.environ['PROCESSOR_ARCHITECTURE'] == 'AMD64' or ('PROCESSOR_ARCHITEW6432' in os.environ and os.environ['PROCESSOR_ARCHITEW6432'] == 'AMD64')) is_linux = sys.platform.startswith('linux') @@ -113,11 +113,6 @@ def check_abi(abi): (abi, list(valid_abis.keys()))) return abi -def vercmp(a, b): - """Compare strings as floats, - but Intel changed Linux naming convention at 9.0""" - return cmp(linux_ver_normalize(b), linux_ver_normalize(a)) - def get_version_from_list(v, vlist): """See if we can match v (string) in vlist (list of strings) Linux has to match in a fuzzy way.""" @@ -221,7 +216,7 @@ def get_all_compiler_versions(): versions = [] try: while i < 100: - subkey = SCons.Util.RegEnumKey(k, i) # raises EnvironmentError + subkey = SCons.Util.RegEnumKey(k, i) # raises SConsEnvironmentError # Check that this refers to an existing dir. # This is not 100% perfect but should catch common # installation issues like when the compiler was installed @@ -293,7 +288,7 @@ def get_all_compiler_versions(): m = re.search(r'([0-9]{0,4})(?:_sp\d*)?\.([0-9][0-9.]*)$', d) if m: versions.append("%s.%s"%(m.group(1), m.group(2))) - + def keyfunc(str): """Given a dot-separated version string, return a tuple of ints representing it.""" return [int(x) for x in str.split('.')] @@ -383,7 +378,7 @@ def get_intel_compiler_top(version, abi): top = d break return top - + top = find_in_2016style_dir(version) or find_in_2011style_dir(version) or find_in_2010style_dir(version) or find_in_2008style_dir(version) # print "INTELC: top=",top if not top: diff --git a/src/engine/SCons/Tool/jar.py b/src/engine/SCons/Tool/jar.py index e0a6a69..02bda57 100644 --- a/src/engine/SCons/Tool/jar.py +++ b/src/engine/SCons/Tool/jar.py @@ -209,9 +209,9 @@ def generate(env): env.AddMethod(Jar) if env['PLATFORM'] == 'win32': - # Ensure that we have a proper path for clang - jar = SCons.Tool.find_program_path(env, 'jar', - default_paths=get_java_install_dirs(env['PLATFORM'])) + # Ensure that we have a proper path for jar + paths = get_java_install_dirs('win32') + jar = SCons.Tool.find_program_path(env, 'jar', default_paths=paths) if jar: jar_bin_dir = os.path.dirname(jar) env.AppendENVPath('PATH', jar_bin_dir) diff --git a/src/engine/SCons/Tool/javac.py b/src/engine/SCons/Tool/javac.py index 8d98b54..537913f 100644 --- a/src/engine/SCons/Tool/javac.py +++ b/src/engine/SCons/Tool/javac.py @@ -210,15 +210,15 @@ def generate(env): version = env.get('JAVAVERSION', None) - javac = SCons.Tool.find_program_path(env, 'javac') if env['PLATFORM'] == 'win32': # Ensure that we have a proper path for javac - paths=get_java_install_dirs(env['PLATFORM'], version=version) - javac = SCons.Tool.find_program_path(env, 'javac', - default_paths=paths) + paths = get_java_install_dirs('win32', version=version) + javac = SCons.Tool.find_program_path(env, 'javac', default_paths=paths) if javac: javac_bin_dir = os.path.dirname(javac) env.AppendENVPath('PATH', javac_bin_dir) + else: + javac = SCons.Tool.find_program_path(env, 'javac') env['JAVAINCLUDES'] = get_java_include_paths(env, javac, version) diff --git a/src/engine/SCons/Tool/javah.py b/src/engine/SCons/Tool/javah.py index f514479..80f8a6b 100644 --- a/src/engine/SCons/Tool/javah.py +++ b/src/engine/SCons/Tool/javah.py @@ -123,9 +123,9 @@ def generate(env): java_javah.emitter = emit_java_headers if env['PLATFORM'] == 'win32': - # Ensure that we have a proper path for clang - javah = SCons.Tool.find_program_path(env, 'javah', - default_paths=get_java_install_dirs(env['PLATFORM'])) + # Ensure that we have a proper path for javah + paths = get_java_install_dirs('win32') + javah = SCons.Tool.find_program_path(env, 'javah', default_paths=paths) if javah: javah_bin_dir = os.path.dirname(javah) env.AppendENVPath('PATH', javah_bin_dir) diff --git a/src/engine/SCons/Tool/mslib.py b/src/engine/SCons/Tool/mslib.py index c901a75..354f5cf 100644 --- a/src/engine/SCons/Tool/mslib.py +++ b/src/engine/SCons/Tool/mslib.py @@ -33,6 +33,8 @@ selection method. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import os + import SCons.Defaults import SCons.Tool import SCons.Tool.msvs @@ -54,6 +56,13 @@ def generate(env): env['LIBPREFIX'] = '' env['LIBSUFFIX'] = '.lib' + # Issue #3350 + # Change tempfile argument joining character from a space to a newline + # mslink will fail if any single line is too long, but is fine with many lines + # in a tempfile + env['TEMPFILEARGJOIN'] = os.linesep + + def exists(env): return msvc_exists(env) diff --git a/src/engine/SCons/Tool/mslink.py b/src/engine/SCons/Tool/mslink.py index c8b00d2..eae2951 100644 --- a/src/engine/SCons/Tool/mslink.py +++ b/src/engine/SCons/Tool/mslink.py @@ -34,6 +34,7 @@ from __future__ import print_function __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import os import os.path import SCons.Action @@ -327,6 +328,12 @@ def generate(env): env['LDMODULEEMITTER'] = [ldmodEmitter] env['LDMODULECOM'] = compositeLdmodAction + # Issue #3350 + # Change tempfile argument joining character from a space to a newline + # mslink will fail if any single line is too long, but is fine with many lines + # in a tempfile + env['TEMPFILEARGJOIN'] = os.linesep + def exists(env): return msvc_exists(env) diff --git a/src/engine/SCons/Tool/msvc.py b/src/engine/SCons/Tool/msvc.py index 6cfa245..dd7d0ec 100644 --- a/src/engine/SCons/Tool/msvc.py +++ b/src/engine/SCons/Tool/msvc.py @@ -34,6 +34,7 @@ selection method. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path +import os import re import sys @@ -283,6 +284,12 @@ def generate(env): msvc_set_PCHPDBFLAGS(env) + # Issue #3350 + # Change tempfile argument joining character from a space to a newline + # mslink will fail if any single line is too long, but is fine with many lines + # in a tempfile + env['TEMPFILEARGJOIN'] = os.linesep + env['PCHCOM'] = '$CXX /Fo${TARGETS[1]} $CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS /c $SOURCES /Yc$PCHSTOP /Fp${TARGETS[0]} $CCPDBFLAGS $PCHPDBFLAGS' env['BUILDERS']['PCH'] = pch_builder diff --git a/src/engine/SCons/Tool/packaging/__init__.py b/src/engine/SCons/Tool/packaging/__init__.py index 174ab8c..5795396 100644 --- a/src/engine/SCons/Tool/packaging/__init__.py +++ b/src/engine/SCons/Tool/packaging/__init__.py @@ -38,8 +38,8 @@ import os import importlib __all__ = [ - 'src_targz', 'src_tarbz2', 'src_xz', 'src_zip', - 'targz', 'tarbz2', 'xz', 'zip', + 'src_targz', 'src_tarbz2', 'src_tarxz', 'src_zip', + 'targz', 'tarbz2', 'tarxz', 'zip', 'rpm', 'msi', 'ipk', ] @@ -125,7 +125,7 @@ def Package(env, target=None, source=None, **kw): # the specific packager is a relative import return importlib.import_module("." + type, __name__) except ImportError as e: - raise EnvironmentError("packager %s not available: %s"%(type,str(e))) + raise SConsEnvironmentError("packager %s not available: %s" % (type, str(e))) packagers = list(map(load_packager, PACKAGETYPE)) diff --git a/src/engine/SCons/Tool/packaging/ipk.py b/src/engine/SCons/Tool/packaging/ipk.py index 2ecaa9b..fe3f49b 100644 --- a/src/engine/SCons/Tool/packaging/ipk.py +++ b/src/engine/SCons/Tool/packaging/ipk.py @@ -3,7 +3,7 @@ # # __COPYRIGHT__ -# +# # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including @@ -117,17 +117,17 @@ def build_specfiles(source, target, env): # # opened_files={} - def open_file(needle, haystack): + def open_file(needle, haystack=None): try: return opened_files[needle] except KeyError: files = filter(lambda x: x.get_path().rfind(needle) != -1, haystack) # Py3: filter returns an iterable, not a list file = list(files)[0] - opened_files[needle]=open(file.get_abspath(), 'w') + opened_files[needle] = open(file.get_abspath(), 'w') return opened_files[needle] - control_file=open_file('control', target) + control_file = open_file('control', target) if 'X_IPK_DESCRIPTION' not in env: env['X_IPK_DESCRIPTION']="%s\n %s"%(env['SUMMARY'], @@ -149,7 +149,7 @@ Description: $X_IPK_DESCRIPTION control_file.write(env.subst(content)) # - # now handle the various other files, which purpose it is to set post-, + # now handle the various other files, which purpose it is to set post-, # pre-scripts and mark files as config files. # # We do so by filtering the source files for files which are marked with @@ -161,14 +161,14 @@ Description: $X_IPK_DESCRIPTION # into the same named file. # for f in [x for x in source if 'PACKAGING_CONFIG' in dir(x)]: - config=open_file('conffiles') + config = open_file('conffiles') config.write(f.PACKAGING_INSTALL_LOCATION) config.write('\n') for str in 'POSTRM PRERM POSTINST PREINST'.split(): name="PACKAGING_X_IPK_%s"%str for f in [x for x in source if name in dir(x)]: - file=open_file(name) + file = open_file(name) file.write(env[str]) # diff --git a/src/engine/SCons/Tool/rmic.py b/src/engine/SCons/Tool/rmic.py index 173ef5f..5c7a040 100644 --- a/src/engine/SCons/Tool/rmic.py +++ b/src/engine/SCons/Tool/rmic.py @@ -110,11 +110,9 @@ def generate(env): if env['PLATFORM'] == 'win32': version = env.get('JAVAVERSION', None) - default_paths=get_java_install_dirs(env['PLATFORM'], version=version) - # Ensure that we have a proper path for rmic - rmic = SCons.Tool.find_program_path(env, 'rmic', default_paths=default_paths) - + paths = get_java_install_dirs('win32', version=version) + rmic = SCons.Tool.find_program_path(env, 'rmic', default_paths=paths) # print("RMIC: %s"%rmic) if rmic: rmic_bin_dir = os.path.dirname(rmic) diff --git a/src/engine/SCons/Tool/xgettext.py b/src/engine/SCons/Tool/xgettext.py index e4a17ca..936924b 100644 --- a/src/engine/SCons/Tool/xgettext.py +++ b/src/engine/SCons/Tool/xgettext.py @@ -26,6 +26,23 @@ Tool specific initialization of `xgettext` tool. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import os +import re +import subprocess +import sys + +import SCons.Action +import SCons.Node.FS +import SCons.Tool +import SCons.Util +from SCons.Builder import BuilderBase +from SCons.Environment import _null +from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS +from SCons.Platform.mingw import MINGW_DEFAULT_PATHS +from SCons.Tool.GettextCommon import _POTargetFactory +from SCons.Tool.GettextCommon import RPaths, _detect_xgettext +from SCons.Tool.GettextCommon import _xgettext_exists + ############################################################################# class _CmdRunner(object): @@ -41,10 +58,6 @@ class _CmdRunner(object): self.commandstr = commandstr def __call__(self, target, source, env): - import SCons.Action - import subprocess - import os - import sys kw = { 'stdin': 'devnull', 'stdout': subprocess.PIPE, @@ -57,11 +70,10 @@ class _CmdRunner(object): self.out, self.err = proc.communicate() self.status = proc.wait() if self.err: - sys.stderr.write(unicode(self.err)) + sys.stderr.write(SCons.Util.UnicodeType(self.err)) return self.status def strfunction(self, target, source, env): - import os comstr = self.commandstr if env.subst(comstr, target=target, source=source) == "": comstr = self.command @@ -74,9 +86,6 @@ class _CmdRunner(object): ############################################################################# def _update_pot_file(target, source, env): """ Action function for `POTUpdate` builder """ - import re - import os - import SCons.Action nop = lambda target, source, env: 0 # Save scons cwd and os cwd (NOTE: they may be different. After the job, we @@ -154,10 +163,6 @@ def _update_pot_file(target, source, env): ############################################################################# ############################################################################# -from SCons.Builder import BuilderBase - - -############################################################################# class _POTBuilder(BuilderBase): def _execute(self, env, target, source, *args): if not target: @@ -175,10 +180,6 @@ class _POTBuilder(BuilderBase): def _scan_xgettext_from_files(target, source, env, files=None, path=None): """ Parses `POTFILES.in`-like file and returns list of extracted file names. """ - import re - import SCons.Util - import SCons.Node.FS - if files is None: return 0 if not SCons.Util.is_List(files): @@ -230,10 +231,6 @@ def _scan_xgettext_from_files(target, source, env, files=None, path=None): ############################################################################# def _pot_update_emitter(target, source, env): """ Emitter function for `POTUpdate` builder """ - from SCons.Tool.GettextCommon import _POTargetFactory - import SCons.Util - import SCons.Node.FS - if 'XGETTEXTFROM' in env: xfrom = env['XGETTEXTFROM'] else: @@ -261,10 +258,6 @@ def _pot_update_emitter(target, source, env): ############################################################################# ############################################################################# -from SCons.Environment import _null - - -############################################################################# def _POTUpdateBuilderWrapper(env, target=None, source=_null, **kw): return env._POTUpdateBuilder(target, source, **kw) @@ -274,8 +267,6 @@ def _POTUpdateBuilderWrapper(env, target=None, source=_null, **kw): ############################################################################# def _POTUpdateBuilder(env, **kw): """ Creates `POTUpdate` builder object """ - import SCons.Action - from SCons.Tool.GettextCommon import _POTargetFactory kw['action'] = SCons.Action.Action(_update_pot_file, None) kw['suffix'] = '$POTSUFFIX' kw['target_factory'] = _POTargetFactory(env, alias='$POTUPDATE_ALIAS').File @@ -288,13 +279,6 @@ def _POTUpdateBuilder(env, **kw): ############################################################################# def generate(env, **kw): """ Generate `xgettext` tool """ - import sys - import os - import SCons.Util - import SCons.Tool - from SCons.Tool.GettextCommon import RPaths, _detect_xgettext - from SCons.Platform.mingw import MINGW_DEFAULT_PATHS - from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS if sys.platform == 'win32': xgettext = SCons.Tool.find_program_path(env, 'xgettext', default_paths=MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) @@ -359,7 +343,6 @@ def generate(env, **kw): ############################################################################# def exists(env): """ Check, whether the tool exists """ - from SCons.Tool.GettextCommon import _xgettext_exists try: return _xgettext_exists(env) except: diff --git a/src/engine/SCons/Util.py b/src/engine/SCons/Util.py index 07f62ea..baf972f 100644 --- a/src/engine/SCons/Util.py +++ b/src/engine/SCons/Util.py @@ -33,6 +33,7 @@ import re import types import codecs import pprint +import hashlib PY3 = sys.version_info[0] == 3 @@ -656,13 +657,15 @@ except ImportError: pass RegError = _NoError -WinError = None + # Make sure we have a definition of WindowsError so we can # run platform-independent tests of Windows functionality on # platforms other than Windows. (WindowsError is, in fact, an # OSError subclass on Windows.) + class PlainWindowsError(OSError): pass + try: WinError = WindowsError except NameError: @@ -1176,10 +1179,13 @@ def unique(s): # ASPN: Python Cookbook: Remove duplicates from a sequence # First comment, dated 2001/10/13. # (Also in the printed Python Cookbook.) +# This not currently used, in favor of the next function... def uniquer(seq, idfun=None): - if idfun is None: - def idfun(x): return x + def default_idfun(x): + return x + if not idfun: + idfun = default_idfun seen = {} result = [] for item in seq: @@ -1444,56 +1450,54 @@ def RenameFunction(function, name): function.__defaults__) -md5 = False +if hasattr(hashlib, 'md5'): + md5 = True + def MD5signature(s): + """ + Generate md5 signature of a string -def MD5signature(s): - return str(s) + :param s: either string or bytes. Normally should be bytes + :return: String of hex digits representing the signature + """ + m = hashlib.md5() + try: + m.update(to_bytes(s)) + except TypeError as e: + m.update(to_bytes(str(s))) -def MD5filesignature(fname, chunksize=65536): - with open(fname, "rb") as f: - result = f.read() - return result + return m.hexdigest() -try: - import hashlib -except ImportError: - pass + def MD5filesignature(fname, chunksize=65536): + """ + Generate the md5 signature of a file + + :param fname: file to hash + :param chunksize: chunk size to read + :return: String of Hex digits representing the signature + """ + m = hashlib.md5() + f = open(fname, "rb") + while True: + blck = f.read(chunksize) + if not blck: + break + m.update(to_bytes(blck)) + f.close() + return m.hexdigest() else: - if hasattr(hashlib, 'md5'): - md5 = True + # if md5 algorithm not available, just return data unmodified + # could add alternative signature scheme here + md5 = False - def MD5signature(s): - """ - Generate a String of Hex digits representing the md5 signature of the string - :param s: either string or bytes. Normally should be bytes - :return: String of hex digits - """ - m = hashlib.md5() + def MD5signature(s): + return str(s) - try: - m.update(to_bytes(s)) - except TypeError as e: - m.update(to_bytes(str(s))) - - return m.hexdigest() - - def MD5filesignature(fname, chunksize=65536): - """ - :param fname: - :param chunksize: - :return: String of Hex digits - """ - m = hashlib.md5() - f = open(fname, "rb") - while True: - blck = f.read(chunksize) - if not blck: - break - m.update(to_bytes(blck)) - f.close() - return m.hexdigest() + def MD5filesignature(fname, chunksize=65536): + with open(fname, "rb") as f: + result = f.read() + return result def MD5collect(signatures): diff --git a/src/engine/SCons/compat/__init__.py b/src/engine/SCons/compat/__init__.py index 83b084a..d34243e 100644 --- a/src/engine/SCons/compat/__init__.py +++ b/src/engine/SCons/compat/__init__.py @@ -113,28 +113,28 @@ except AttributeError: # intern into the sys package sys.intern = intern -# Preparing for 3.x. UserDict, UserList, UserString are in -# collections for 3.x, but standalone in 2.7.x +# UserDict, UserList, UserString are in # collections for 3.x, +# but standalone in 2.7.x. Monkey-patch into collections for 2.7. import collections try: collections.UserDict except AttributeError: - exec ('from UserDict import UserDict as _UserDict') + from UserDict import UserDict as _UserDict collections.UserDict = _UserDict del _UserDict try: collections.UserList except AttributeError: - exec ('from UserList import UserList as _UserList') + from UserList import UserList as _UserList collections.UserList = _UserList del _UserList try: collections.UserString except AttributeError: - exec ('from UserString import UserString as _UserString') + from UserString import UserString as _UserString collections.UserString = _UserString del _UserString diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index a413690..c1172aa 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -165,7 +165,7 @@ def CPP_to_Python(s): """ s = CPP_to_Python_Ops_Expression.sub(CPP_to_Python_Ops_Sub, s) for expr, repl in CPP_to_Python_Eval_List: - s = expr.sub(repl, s) + s = re.sub(expr, repl, s) return s |