From 29a46d09fdb0bc3e8d7b18dca6ac8495e96c5eb4 Mon Sep 17 00:00:00 2001 From: Alexey Klimkin Date: Thu, 26 May 2016 11:03:00 -0700 Subject: Optimize implicit dependency scan When calculating path, performance spent on two things: - Variable expansion, if CPPPATH contains any variables - CPPPATH flattening Use memoization to optimize PATH evaluation across all dependencies per node. --- src/engine/SCons/Node/__init__.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 1a76b60..d3cea53 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -926,9 +926,9 @@ class Node(object): scanner's recursive flag says that we should. """ nodes = [self] - seen = {} - seen[self] = 1 + seen = set(nodes) dependencies = [] + path_memo = {} root_node_scanner = self._get_scanner(env, initial_scanner, None, kw) @@ -936,30 +936,32 @@ class Node(object): node = nodes.pop(0) scanner = node._get_scanner(env, initial_scanner, root_node_scanner, kw) - if not scanner: continue - path = path_func(scanner) + try: + path = path_memo[scanner] + except KeyError: + path = path_func(scanner) + path_memo[scanner] = path included_deps = [x for x in node.get_found_includes(env, scanner, path) if x not in seen] if included_deps: dependencies.extend(included_deps) - for dep in included_deps: - seen[dep] = 1 + seen.update(included_deps) nodes.extend(scanner.recurse_nodes(included_deps)) return dependencies def _get_scanner(self, env, initial_scanner, root_node_scanner, kw): - if not initial_scanner: + if initial_scanner: + # handle explicit scanner case + scanner = initial_scanner.select(self) + else: # handle implicit scanner case scanner = self.get_env_scanner(env, kw) if scanner: scanner = scanner.select(self) - else: - # handle explicit scanner case - scanner = initial_scanner.select(self) if not scanner: # no scanner could be found for the given node's scanner key; -- cgit v0.12 From e4b799e8dc348159c5f2d70bdf49819c4d8270bc Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sun, 5 Jun 2016 20:23:12 -0400 Subject: avoid using __slots__ on Node and Executor (but only on PyPy) --- src/engine/SCons/Executor.py | 6 ++--- src/engine/SCons/Node/__init__.py | 5 +++- src/engine/SCons/compat/__init__.py | 47 +++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index 7107fde..dd5088d 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -35,7 +35,7 @@ import SCons.Debug from SCons.Debug import logInstanceCreation import SCons.Errors import SCons.Memoize - +from SCons.compat import with_metaclass, NoSlotsPyPy class Batch(object): """Remembers exact association between targets @@ -154,7 +154,7 @@ _execute_str_map = {0 : execute_null_str, 1 : execute_actions_str} -class Executor(object): +class Executor(object, with_metaclass(NoSlotsPyPy)): """A class for controlling instances of executing an action. This largely exists to hold a single association of an action, @@ -580,7 +580,7 @@ def get_NullEnvironment(): nullenv = NullEnvironment() return nullenv -class Null(object): +class Null(object, with_metaclass(NoSlotsPyPy)): """A null Executor, with a null build Environment, that does nothing when the rest of the methods call it. diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 1a76b60..e714172 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -57,6 +57,8 @@ import SCons.Util from SCons.Debug import Trace +from SCons.compat import with_metaclass, NoSlotsPyPy + print_duplicate = 0 def classname(obj): @@ -489,7 +491,8 @@ class BuildInfoBase(object): if key not in ('__weakref__',): setattr(self, key, value) -class Node(object): + +class Node(object, with_metaclass(NoSlotsPyPy)): """The base Node class, for entities that we know how to build, or use to build other Nodes. """ diff --git a/src/engine/SCons/compat/__init__.py b/src/engine/SCons/compat/__init__.py index 0ddbdd5..285c4ac 100644 --- a/src/engine/SCons/compat/__init__.py +++ b/src/engine/SCons/compat/__init__.py @@ -63,6 +63,8 @@ import os import sys import imp # Use the "imp" module to protect imports from fixers. +PYPY = hasattr(sys, 'pypy_translation_info') + def import_as(module, name): """ Imports the specified module (from our local directory) as the @@ -135,6 +137,51 @@ except AttributeError: del _UserString +def with_metaclass(meta, *bases): + """ + Function from jinja2/_compat.py. License: BSD. + + Use it like this:: + + class BaseForm(object): + pass + + class FormType(type): + pass + + class Form(with_metaclass(FormType, BaseForm)): + pass + + This requires a bit of explanation: the basic idea is to make a + dummy metaclass for one level of class instantiation that replaces + itself with the actual metaclass. Because of internal type checks + we also need to make sure that we downgrade the custom metaclass + for one level to something closer to type (that's why __call__ and + __init__ comes back from type etc.). + + This has the advantage over six.with_metaclass of not introducing + dummy classes into the final MRO. + """ + class metaclass(meta): + __call__ = type.__call__ + __init__ = type.__init__ + def __new__(cls, name, this_bases, d): + if this_bases is None: + return type.__new__(cls, name, (), d) + return meta(name, bases, d) + return metaclass('temporary_class', None, {}) + + +class NoSlotsPyPy(type): + """ + Workaround for PyPy not working well with __slots__ and __class__ assignment. + """ + def __new__(meta, name, bases, dct): + if PYPY and '__slots__' in dct: + dct['__slats__'] = dct.pop('__slots__') + return super(NoSlotsPyPy, meta).__new__(meta, name, bases, dct) + + # Local Variables: # tab-width:4 # indent-tabs-mode:nil -- cgit v0.12 From 3128ce66f69ee2be54080caea8d86f817885616b Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sun, 5 Jun 2016 22:57:26 -0400 Subject: fix some tests for pypy The major category of broken test is that which depends on garbage collection to flush and close an open file. Also, the empty list is apparently not a singleton in pypy. --- QMTest/TestCmd.py | 11 ++++++----- src/engine/SCons/ActionTests.py | 30 ++++++++++++++++++------------ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/QMTest/TestCmd.py b/QMTest/TestCmd.py index 2fba0a0..b0a456b 100644 --- a/QMTest/TestCmd.py +++ b/QMTest/TestCmd.py @@ -1731,11 +1731,12 @@ class TestCmd(object): file = self.canonicalize(file) if mode[0] != 'w': raise ValueError("mode must begin with 'w'") - try: - open(file, mode).write(content) - except TypeError as e: - # python 3 default strings are not bytes, but unicode - open(file, mode).write(bytes(content,'utf-8')) + with open(file, mode) as f: + try: + f.write(content) + except TypeError as e: + # python 3 default strings are not bytes, but unicode + f.write(bytes(content,'utf-8')) # Local Variables: # tab-width:4 diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py index 3eee4b6..9007183 100644 --- a/src/engine/SCons/ActionTests.py +++ b/src/engine/SCons/ActionTests.py @@ -231,7 +231,7 @@ def test_positional_args(pos_callback, cmd, **kw): """ act = SCons.Action.Action(cmd, **kw) pos_callback(act) - assert act.varlist is (), act.varlist + assert act.varlist == (), act.varlist if not isinstance(act, SCons.Action._ActionAction): # only valid cmdstrfunc is None @@ -316,7 +316,7 @@ class ActionTestCase(unittest.TestCase): """ a1 = SCons.Action.Action(["x", "y", "z", [ "a", "b", "c"]]) assert isinstance(a1, SCons.Action.ListAction), a1 - assert a1.varlist is (), a1.varlist + assert a1.varlist == (), a1.varlist assert isinstance(a1.list[0], SCons.Action.CommandAction), a1.list[0] assert a1.list[0].cmd_list == "x", a1.list[0].cmd_list assert isinstance(a1.list[1], SCons.Action.CommandAction), a1.list[1] @@ -328,7 +328,7 @@ class ActionTestCase(unittest.TestCase): a2 = SCons.Action.Action("x\ny\nz") assert isinstance(a2, SCons.Action.ListAction), a2 - assert a2.varlist is (), a2.varlist + assert a2.varlist == (), a2.varlist assert isinstance(a2.list[0], SCons.Action.CommandAction), a2.list[0] assert a2.list[0].cmd_list == "x", a2.list[0].cmd_list assert isinstance(a2.list[1], SCons.Action.CommandAction), a2.list[1] @@ -341,7 +341,7 @@ class ActionTestCase(unittest.TestCase): a3 = SCons.Action.Action(["x", foo, "z"]) assert isinstance(a3, SCons.Action.ListAction), a3 - assert a3.varlist is (), a3.varlist + assert a3.varlist == (), a3.varlist assert isinstance(a3.list[0], SCons.Action.CommandAction), a3.list[0] assert a3.list[0].cmd_list == "x", a3.list[0].cmd_list assert isinstance(a3.list[1], SCons.Action.FunctionAction), a3.list[1] @@ -351,7 +351,7 @@ class ActionTestCase(unittest.TestCase): a4 = SCons.Action.Action(["x", "y"], strfunction=foo) assert isinstance(a4, SCons.Action.ListAction), a4 - assert a4.varlist is (), a4.varlist + assert a4.varlist == (), a4.varlist assert isinstance(a4.list[0], SCons.Action.CommandAction), a4.list[0] assert a4.list[0].cmd_list == "x", a4.list[0].cmd_list assert a4.list[0].strfunction == foo, a4.list[0].strfunction @@ -361,7 +361,7 @@ class ActionTestCase(unittest.TestCase): a5 = SCons.Action.Action("x\ny", strfunction=foo) assert isinstance(a5, SCons.Action.ListAction), a5 - assert a5.varlist is (), a5.varlist + assert a5.varlist == (), a5.varlist assert isinstance(a5.list[0], SCons.Action.CommandAction), a5.list[0] assert a5.list[0].cmd_list == "x", a5.list[0].cmd_list assert a5.list[0].strfunction == foo, a5.list[0].strfunction @@ -1533,7 +1533,8 @@ class FunctionActionTestCase(unittest.TestCase): global count count = count + 1 for t in target: - open(t, 'w').write("function1\n") + with open(t, 'w') as f: + f.write("function1\n") return 1 act = SCons.Action.FunctionAction(function1, {}) @@ -1548,7 +1549,8 @@ class FunctionActionTestCase(unittest.TestCase): class class1a(object): def __init__(self, target, source, env): - open(env['out'], 'w').write("class1a\n") + with open(env['out'], 'w') as f: + f.write("class1a\n") act = SCons.Action.FunctionAction(class1a, {}) r = act([], [], Environment(out = outfile)) @@ -1558,7 +1560,8 @@ class FunctionActionTestCase(unittest.TestCase): class class1b(object): def __call__(self, target, source, env): - open(env['out'], 'w').write("class1b\n") + with open(env['out'], 'w') as f: + f.write("class1b\n") return 2 act = SCons.Action.FunctionAction(class1b(), {}) @@ -1710,17 +1713,20 @@ class ListActionTestCase(unittest.TestCase): cmd2 = r'%s %s %s syzygy' % (_python_, act_py, outfile) def function2(target, source, env): - open(env['out'], 'a').write("function2\n") + with open(env['out'], 'a') as f: + f.write("function2\n") return 0 class class2a(object): def __call__(self, target, source, env): - open(env['out'], 'a').write("class2a\n") + with open(env['out'], 'a') as f: + f.write("class2a\n") return 0 class class2b(object): def __init__(self, target, source, env): - open(env['out'], 'a').write("class2b\n") + with open(env['out'], 'a') as f: + f.write("class2b\n") act = SCons.Action.ListAction([cmd2, function2, class2a(), class2b]) r = act([], [], Environment(out = outfile)) assert isinstance(r.status, class2b), r.status -- cgit v0.12 From 88958d1f4beae35315b26ec1b32b38eeb41946f8 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Mon, 6 Jun 2016 20:05:23 -0400 Subject: just throw away __slots__ on pypy --- src/engine/SCons/compat/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/compat/__init__.py b/src/engine/SCons/compat/__init__.py index 285c4ac..6f20b73 100644 --- a/src/engine/SCons/compat/__init__.py +++ b/src/engine/SCons/compat/__init__.py @@ -178,7 +178,7 @@ class NoSlotsPyPy(type): """ def __new__(meta, name, bases, dct): if PYPY and '__slots__' in dct: - dct['__slats__'] = dct.pop('__slots__') + dct.pop('__slots__') return super(NoSlotsPyPy, meta).__new__(meta, name, bases, dct) -- cgit v0.12 From f8fbe351d2677bc2e4e3094e905c579825c1582a Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Wed, 15 Jun 2016 19:46:00 -0400 Subject: wrap several frequently-used open() calls in with statements --- src/engine/SCons/Node/FS.py | 4 ++-- src/engine/SCons/Node/__init__.py | 3 ++- src/engine/SCons/Scanner/C.py | 5 ++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index d19afa9..71511b5 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -1211,9 +1211,9 @@ class FS(LocalFS): def getcwd(self): if hasattr(self, "_cwd"): - return self._cwd + return self._cwd else: - return "" + return "" def chdir(self, dir, change_os_dir=0): """Change the current working directory for lookups. diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index e714172..51762a3 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -214,7 +214,8 @@ def get_contents_file(node): return '' fname = node.rfile().get_abspath() try: - contents = open(fname, "rb").read() + with open(fname, "rb") as fp: + contents = fp.read() except EnvironmentError as e: if not e.filename: e.filename = fname diff --git a/src/engine/SCons/Scanner/C.py b/src/engine/SCons/Scanner/C.py index 3b34b88..c2e50a6 100644 --- a/src/engine/SCons/Scanner/C.py +++ b/src/engine/SCons/Scanner/C.py @@ -58,12 +58,11 @@ class SConsCPPScanner(SCons.cpp.PreProcessor): return result def read_file(self, file): try: - fp = open(str(file.rfile())) + with open(str(file.rfile())) as fp: + return fp.read() except EnvironmentError as e: self.missing.append((file, self.current_file)) return '' - else: - return fp.read() def dictify_CPPDEFINES(env): cppdefines = env.get('CPPDEFINES', {}) -- cgit v0.12 From d31cff4a15d3adf3842826c3af8317a2ba050038 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 16 Jun 2016 22:46:03 -0400 Subject: add __main__.py for 'python -m SCons' --- src/engine/SCons/__main__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/engine/SCons/__main__.py diff --git a/src/engine/SCons/__main__.py b/src/engine/SCons/__main__.py new file mode 100644 index 0000000..5a3aadb --- /dev/null +++ b/src/engine/SCons/__main__.py @@ -0,0 +1,5 @@ +if __name__ == "__main__": + import SCons.Script + # this does all the work, and calls sys.exit + # with the proper exit status when done. + SCons.Script.main() -- cgit v0.12 From 16c3e45d786b939f78cf0169be160fd65c679150 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 16 Jun 2016 22:49:38 -0400 Subject: no need for a __main__ guard in __main__.py --- src/engine/SCons/__main__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/__main__.py b/src/engine/SCons/__main__.py index 5a3aadb..0dfbb9d 100644 --- a/src/engine/SCons/__main__.py +++ b/src/engine/SCons/__main__.py @@ -1,5 +1,4 @@ -if __name__ == "__main__": - import SCons.Script - # this does all the work, and calls sys.exit - # with the proper exit status when done. - SCons.Script.main() +import SCons.Script +# this does all the work, and calls sys.exit +# with the proper exit status when done. +SCons.Script.main() -- cgit v0.12 From 707a453c9c9db84449c4d3bf214b9ec65f2807db Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sun, 19 Jun 2016 22:48:06 -0400 Subject: update CHANGES.txt --- src/CHANGES.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index aab73b4..db52cf0 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -10,6 +10,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Whatever John Doe did. + From Daniel Holth: + - Add basic support for PyPy (by deleting __slots__ from Node with a + metaclass on PyPy); wrap most-used open() calls in 'with' statements to + avoid too many open files. + - Add __main__.py for `python -m SCons` in case it is on PYTHONPATH. From Paweł Tomulik: - Fixed the issue with LDMODULEVERSIONFLAGS reported by Tim Jennes -- cgit v0.12 From 1d93e3871ee6858e3bc362aa4e9822c29aadb1a3 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sun, 19 Jun 2016 22:50:06 -0400 Subject: 'with open' in own SConstruct --- SConstruct | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/SConstruct b/SConstruct index 5eded92..c5734af 100644 --- a/SConstruct +++ b/SConstruct @@ -356,7 +356,8 @@ try: if os.path.isfile(dest) or os.path.islink(dest): os.unlink(dest) if not os.path.isdir(dest): - open(dest, 'wb').write(zf.read(name)) + with open(dest, 'wb') as fp: + fp.write(zf.read(name)) except ImportError: if unzip and zip: @@ -371,7 +372,8 @@ def SCons_revision(target, source, env): """ t = str(target[0]) s = source[0].rstr() - contents = open(s, 'rb').read() + with open(s, 'rb') as fp: + contents = fp.read() # Note: We construct the __*__ substitution strings here # so that they don't get replaced when this file gets # copied into the tree for packaging. -- cgit v0.12 From 3d4767acc37e29665fd43ac79981495b17253421 Mon Sep 17 00:00:00 2001 From: Alexey Klimkin Date: Wed, 13 Jul 2016 14:42:15 -0700 Subject: Update CHANGES.txt --- src/CHANGES.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index aab73b4..38f29c0 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -20,6 +20,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added LoadableModule to the list of global functions (DefaultEnvironment builders). + From Alexey Klimkin: + - Use memoization to optimize PATH evaluation across all dependencies per + node. (PR #345) + RELEASE 2.5.0 - Mon, 09 Apr 2016 11:27:42 -0700 From Dirk Baechle: -- cgit v0.12