Regain lost performance improvements by using paths instead of targets for scanner calls and re-using Binder objects for identical paths.

4 files changed, 49 insertions, 38 deletions

diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 50e3818..789d5c3 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -612,11 +612,11 @@ class Entry(Base):
         self.clear()
         return File.rfile(self)
 
-    def get_found_includes(self, env, scanner, target):
+    def get_found_includes(self, env, scanner, path):
         """If we're looking for included files, it's because this Entry
         is really supposed to be a File itself."""
         node = self.rfile()
-        return node.get_found_includes(env, scanner, target)
+        return node.get_found_includes(env, scanner, path)
 
     def scanner_key(self):
         return self.get_suffix()
@@ -1456,29 +1456,13 @@ class File(Base):
         except AttributeError:
             return None
 
-    def get_found_includes(self, env, scanner, target):
+    def get_found_includes(self, env, scanner, path):
         """Return the included implicit dependencies in this file.
-        Cache results so we only scan the file once regardless of
-        how many times this information is requested."""
+        Cache results so we only scan the file once per path
+        regardless of how many times this information is requested.
+        __cacheable__"""
         if not scanner:
             return []
-
-        try:
-            path = target.scanner_paths[scanner]
-        except AttributeError:
-            # The target had no scanner_paths attribute, which means
-            # it's an Alias or some other node that's not actually a
-            # file.  In that case, back off and use the path for this
-            # node itself.
-            try:
-                path = self.scanner_paths[scanner]
-            except KeyError:
-                path = scanner.path(env, self.cwd, target)
-                self.scanner_paths[scanner] = path
-        except KeyError:
-            path = scanner.path(env, target.cwd, target)
-            target.scanner_paths[scanner] = path
-
         return scanner(self, env, path)
 
     def _createDir(self):
diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py
index e65c10b..1a3236e 100644
--- a/src/engine/SCons/Node/FSTests.py
+++ b/src/engine/SCons/Node/FSTests.py
@@ -54,7 +54,7 @@ class Scanner:
         scanner_count = scanner_count + 1
         self.hash = scanner_count
         self.node = node
-    def path(self, env, dir, target=None):
+    def path(self, env, dir, target=None, source=None):
         return ()
     def __call__(self, node, env, path):
         return [self.node]
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index 3e5a9f0..f801963 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -169,6 +169,8 @@ class Scanner:
     def __call__(self, node):
         self.called = 1
         return node.found_includes
+    def path(self, env, dir, target=None, source=None):
+        return ()
     def select(self, node):
         return self
 
@@ -309,8 +311,19 @@ class NodeTestCase(unittest.TestCase):
             assert str(act.built_target[0]) == "xxx", str(act.built_target[0])
             assert act.built_source == ["yyy", "zzz"], act.built_source
 
+    def test_get_build_scanner_path(self):
+        """Test the get_build_scanner_path() method"""
+        n = SCons.Node.Node()
+        class MyExecutor:
+            def get_build_scanner_path(self, scanner):
+                return 'executor would call %s' % scanner
+        x = MyExecutor()
+        n.set_executor(x)
+        p = n.get_build_scanner_path('fake_scanner')
+        assert p == "executor would call fake_scanner", p
+
     def test_get_executor(self):
-        """Test the reset_executor() method"""
+        """Test the get_executor() method"""
         n = SCons.Node.Node()
 
         try:
@@ -337,7 +350,7 @@ class NodeTestCase(unittest.TestCase):
         assert x.env == 'env2', x.env
 
     def test_set_executor(self):
-        """Test the reset_executor() method"""
+        """Test the set_executor() method"""
         n = SCons.Node.Node()
         n.set_executor(1)
         assert n.executor == 1, n.executor
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index 7ffec0e..c20d657 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -148,6 +148,11 @@ class Node:
         __cacheable__"""
         return self.get_executor().get_build_env()
 
+    def get_build_scanner_path(self, scanner):
+        """Fetch the appropriate Environment to build this node.
+        __cacheable__"""
+        return self.get_executor().get_build_scanner_path(scanner)
+
     def set_executor(self, executor):
         """Set the action executor for this node."""
         self.executor = executor
@@ -315,8 +320,7 @@ class Node:
         This allows an internal Builder created by SCons to be marked
         non-explicit, so that it can be overridden by an explicit
         builder that the user supplies (the canonical example being
-        directories).
-        __cacheable__"""
+        directories)."""
         return self.has_builder() and self.builder.is_explicit
 
     def get_builder(self, default_builder=None):
@@ -354,7 +358,7 @@ class Node:
         """
         return [], None
 
-    def get_found_includes(self, env, scanner, target):
+    def get_found_includes(self, env, scanner, path):
         """Return the scanned include lines (implicit dependencies)
         found in this node.
 
@@ -364,7 +368,7 @@ class Node:
         """
         return []
 
-    def get_implicit_deps(self, env, scanner, target):
+    def get_implicit_deps(self, env, scanner, path):
         """Return a list of implicit dependencies for this node.
 
         This method exists to handle recursive invocation of the scanner
@@ -390,7 +394,7 @@ class Node:
         while nodes:
            n = nodes.pop(0)
            d = filter(lambda x, seen=seen: not seen.has_key(x),
-                      n.get_found_includes(env, scanner, target))
+                      n.get_found_includes(env, scanner, path))
            if d:
                deps.extend(d)
                for n in d:
@@ -421,7 +425,6 @@ class Node:
 
         This function may be called very often; it attempts to cache
         the scanner found to improve performance.
-        __cacheable__
         """
         # Called from scan() for each child (node) of this node
         # (self).  The scan() may be called multiple times, so this
@@ -434,10 +437,9 @@ class Node:
         if not self.has_builder():
             return None
         
+        scanner = None
         try:
             scanner = self.builder.source_scanner
-            if scanner:
-                return scanner
         except AttributeError:
             pass
 
@@ -445,7 +447,10 @@ class Node:
         # based on the node's scanner key (usually the file
         # extension).
         
-        scanner = self.get_build_env().get_scanner(node.scanner_key())
+        if not scanner:
+            scanner = self.get_build_env().get_scanner(node.scanner_key())
+        if scanner:
+            scanner = scanner.select(node)
         return scanner
 
     def scan(self):
@@ -481,16 +486,24 @@ class Node:
                     self._children_reset()
                     self.del_binfo()
 
+        # Potential optimization for the N^2 problem if we can tie
+        # scanning to the Executor in some way so that we can scan
+        # source files onces and then spread the implicit dependencies
+        # to all of the targets at once.
+        #kids = self.children(scan=0)
+        #for child in filter(lambda n: n.implicit is None, kids):
         for child in self.children(scan=0):
             scanner = self.get_source_scanner(child)
             if scanner:
-                deps = child.get_implicit_deps(build_env, scanner, self)
+                path = self.get_build_scanner_path(scanner)
+                deps = child.get_implicit_deps(build_env, scanner, path)
                 self._add_child(self.implicit, self.implicit_dict, deps)
 
         # scan this node itself for implicit dependencies
         scanner = self.builder.target_scanner
         if scanner:
-            deps = self.get_implicit_deps(build_env, scanner, self)
+            path = self.get_build_scanner_path(scanner)
+            deps = self.get_implicit_deps(build_env, scanner, path)
             self._add_child(self.implicit, self.implicit_dict, deps)
 
         # XXX See note above re: --implicit-cache.
@@ -836,8 +849,9 @@ class Node:
             env = self.get_build_env()
             for s in self.sources:
                 scanner = self.get_source_scanner(s)
-                def f(node, env=env, scanner=scanner, target=self):
-                    return node.get_found_includes(env, scanner, target)
+                path = self.get_build_scanner_path(scanner)
+                def f(node, env=env, scanner=scanner, path=path):
+                    return node.get_found_includes(env, scanner, path)
                 return SCons.Util.render_tree(s, f, 1)
         else:
             return None