Remove Node scanner storage. (Kevin Quick)

8 files changed, 139 insertions, 67 deletions

diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py
index 1144dfe..04dd1e5 100644
--- a/src/engine/SCons/Builder.py
+++ b/src/engine/SCons/Builder.py
@@ -301,9 +301,6 @@ def _init_nodes(builder, env, overrides, executor_kw, tlist, slist):
             elif t.overrides != overrides:
                 raise UserError, "Two different sets of overrides were specified for the same target: %s"%str(t)
 
-            elif builder.target_scanner and t.target_scanner and builder.target_scanner != t.target_scanner:
-                raise UserError, "Two different scanners were specified for the same target: %s"%str(t)
-
             if builder.multi:
                 if t.builder != builder:
                     if isinstance(t.builder, ListBuilder) and isinstance(builder, ListBuilder) and t.builder.builder == builder.builder:
@@ -348,22 +345,6 @@ def _init_nodes(builder, env, overrides, executor_kw, tlist, slist):
         t.env_set(env)
         t.add_source(slist)
         t.set_executor(executor)
-        if builder.target_scanner:
-            t.target_scanner = builder.target_scanner
-        if t.source_scanner is None:
-            t.source_scanner = builder.source_scanner
-
-    # Add backup source scanners from the environment to the source
-    # nodes.  This may not be necessary if the node will have a real
-    # source scanner added later (which is why these are the "backup"
-    # source scanners, not the real ones), but because source nodes may
-    # be used multiple times for different targets, it ends up being
-    # more efficient to do this calculation once here, as opposed to
-    # delaying it until later when we potentially have to calculate it
-    # over and over and over.
-    for s in slist:
-        if s.source_scanner is None and s.backup_source_scanner is None:
-            s.backup_source_scanner = env.get_scanner(s.scanner_key())
 
 class EmitterProxy:
     """This is a callable class that can act as a
diff --git a/src/engine/SCons/BuilderTests.py b/src/engine/SCons/BuilderTests.py
index 4a18182..0ce451c 100644
--- a/src/engine/SCons/BuilderTests.py
+++ b/src/engine/SCons/BuilderTests.py
@@ -83,6 +83,14 @@ class Environment:
         except IndexError:
             pass
         return self.d.get(s, s)
+    def subst_target_source(self, string, raw=0, target=None,
+                            source=None, dict=None, conv=None):
+        return SCons.Util.scons_subst(string, self, raw, target,
+                                      source, dict, conv)
+    def subst_list(self, string, raw=0, target=None,
+                   source=None, dict=None, conv=None):
+        return SCons.Util.scons_subst_list(string, self, raw, target,
+                                           source, dict, conv)
     def arg2nodes(self, args, factory):
         global env_arg2nodes_called
         env_arg2nodes_called = 1
@@ -113,6 +121,7 @@ class Environment:
     def Override(self, overrides):
         env = apply(Environment, (), self.d)
         env.d.update(overrides)
+        env.scanner = self.scanner
         return env
     def _update(self, dict):
         self.d.update(dict)
@@ -126,6 +135,8 @@ class Environment:
         d['SOURCES'] = ['__s1__', '__s2__', '__s3__', '__s4__', '__s5__', '__s6__']
         d['SOURCE'] = d['SOURCES'][0]
         return d
+    def __cmp__(self, other):
+        return cmp(self.scanner, other.scanner) or cmp(self.d, other.d)
 
 class MyNode_without_target_from_source:
     def __init__(self, name):
@@ -133,8 +144,6 @@ class MyNode_without_target_from_source:
         self.sources = []
         self.builder = None
         self.side_effect = 0
-        self.source_scanner = None
-        self.backup_source_scanner = None
     def __str__(self):
         return self.name
     def builder_set(self, builder):
@@ -886,8 +895,8 @@ class BuilderTestCase(unittest.TestCase):
                                         source_scanner=sscan,
                                         action='')
         tgt = builder(env, target='foo2', source='bar')[0]
-        assert tgt.target_scanner == tscan, tgt.target_scanner
-        assert tgt.source_scanner == sscan, tgt.source_scanner
+        assert tgt.builder.target_scanner == tscan, tgt.builder.target_scanner
+        assert tgt.builder.source_scanner == sscan, tgt.builder.source_scanner
 
         builder1 = SCons.Builder.Builder(action='foo',
                                          src_suffix='.bar',
@@ -897,8 +906,8 @@ class BuilderTestCase(unittest.TestCase):
                                          target_scanner = tscan,
                                          source_scanner = tscan)
         tgt = builder2(env, target='baz2', source='test.bar test2.foo test3.txt')[0]
-        assert tgt.target_scanner == tscan, tgt.target_scanner
-        assert tgt.source_scanner == tscan, tgt.source_scanner
+        assert tgt.builder.target_scanner == tscan, tgt.builder.target_scanner
+        assert tgt.builder.source_scanner == tscan, tgt.builder.source_scanner
 
     def test_actual_scanner(self):
         """Test usage of actual Scanner objects."""
@@ -924,28 +933,82 @@ class BuilderTestCase(unittest.TestCase):
                  return 'TestScannerkey'
             def instance(self, env):
                  return self
+            name = 'TestScanner'
+            def __str__(self):
+                return self.name
 
         scanner = TestScanner()
         builder = SCons.Builder.Builder(action='action')
 
         # With no scanner specified, source_scanner and
         # backup_source_scanner are None.
+        bar_y = MyNode('bar.y')
         env1 = Environment()
         tgt = builder(env1, target='foo1.x', source='bar.y')[0]
         src = tgt.sources[0]
-        assert tgt.target_scanner != scanner, tgt.target_scanner
-        assert src.source_scanner is None, src.source_scanner
-        assert src.backup_source_scanner is None, src.backup_source_scanner
-
-        # Later use of the same source file with an environment that
-        # has a scanner must still set the scanner.
-        env2 = Environment()
-        env2.scanner = scanner
-        tgt = builder(env2, target='foo2.x', source='bar.y')[0]
+        assert tgt.builder.target_scanner != scanner, tgt.builder.target_scanner
+        assert tgt.builder.source_scanner is None, tgt.builder.source_scanner
+        assert tgt.get_source_scanner(bar_y) is None, tgt.get_source_scanner(bar_y)
+        assert not src.has_builder(), src.has_builder()
+        assert src.get_source_scanner(bar_y) is None, src.get_source_scanner(bar_y)
+
+        # An Environment that has suffix-specified SCANNERS should
+        # provide a source scanner to the target.
+        class EnvTestScanner:
+            def key(self, env):
+                 return '.y'
+            def instance(self, env):
+                 return self
+            name = 'EnvTestScanner'
+            def __str__(self):
+                return self.name
+            def select(self, scanner):
+                return self
+            def path(self, env, dir=None):
+                return ()
+            def __call__(self, node, env, path):
+                return []
+        env3 = Environment(SCANNERS = [EnvTestScanner()])
+        env3.scanner = EnvTestScanner() # test env's version of SCANNERS
+        tgt = builder(env3, target='foo2.x', source='bar.y')[0]
+        src = tgt.sources[0]
+        assert tgt.builder.target_scanner != scanner, tgt.builder.target_scanner
+        assert not tgt.builder.source_scanner, tgt.builder.source_scanner
+        assert tgt.get_source_scanner(bar_y), tgt.get_source_scanner(bar_y)
+        assert str(tgt.get_source_scanner(bar_y)) == 'EnvTestScanner', tgt.get_source_scanner(bar_y)
+        assert not src.has_builder(), src.has_builder()
+        assert src.get_source_scanner(bar_y) is None, src.get_source_scanner(bar_y)
+
+        # Can't simply specify the scanner as a builder argument; it's
+        # global to all invocations of this builder.
+        tgt = builder(env3, target='foo3.x', source='bar.y', source_scanner = scanner)[0]
         src = tgt.sources[0]
-        assert tgt.target_scanner != scanner, tgt.target_scanner
-        assert src.source_scanner is None, src.source_scanner
-        assert src.backup_source_scanner == scanner, src.backup_source_scanner
+        assert tgt.builder.target_scanner != scanner, tgt.builder.target_scanner
+        assert not tgt.builder.source_scanner, tgt.builder.source_scanner
+        assert tgt.get_source_scanner(bar_y), tgt.get_source_scanner(bar_y)
+        assert str(tgt.get_source_scanner(bar_y)) == 'EnvTestScanner', tgt.get_source_scanner(bar_y)
+        assert not src.has_builder(), src.has_builder()
+        assert src.get_source_scanner(bar_y) is None, src.get_source_scanner(bar_y)
+
+        # Now use a builder that actually has scanners and ensure that
+        # the target is set accordingly (using the specified scanner
+        # instead of the Environment's scanner)
+        builder = SCons.Builder.Builder(action='action',
+                                        source_scanner=scanner,
+                                        target_scanner=scanner)
+        tgt = builder(env3, target='foo4.x', source='bar.y')[0]
+        src = tgt.sources[0]
+        assert tgt.builder.target_scanner == scanner, tgt.builder.target_scanner
+        assert tgt.builder.source_scanner, tgt.builder.source_scanner
+        assert tgt.builder.source_scanner == scanner, tgt.builder.source_scanner
+        assert str(tgt.builder.source_scanner) == 'TestScanner', str(tgt.builder.source_scanner)
+        assert tgt.get_source_scanner(bar_y), tgt.get_source_scanner(bar_y)
+        assert tgt.get_source_scanner(bar_y) == scanner, tgt.get_source_scanner(bar_y)
+        assert str(tgt.get_source_scanner(bar_y)) == 'TestScanner', tgt.get_source_scanner(bar_y)
+        assert not src.has_builder(), src.has_builder()
+        assert src.get_source_scanner(bar_y) is None, src.get_source_scanner(bar_y)
+
+
 
     def test_Builder_API(self):
         """Test Builder interface.
diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py
index ca543bd..d973a01 100644
--- a/src/engine/SCons/Environment.py
+++ b/src/engine/SCons/Environment.py
@@ -1141,8 +1141,12 @@ class Base:
         source files using the supplied action.  Action may
         be any type that the Builder constructor will accept
         for an action."""
-        bld = SCons.Builder.Builder(action = action,
-                                    source_factory = self.fs.Entry)
+        builder_kw = {
+            'action' : action,
+            'source_factory' : self.fs.Entry,
+        }
+        builder_kw.update(kw)
+        bld = apply(SCons.Builder.Builder, (), builder_kw)
         return apply(bld, (self, target, source), kw)
 
     def Depends(self, target, dependency):
diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py
index 71383e7..56c42cb 100644
--- a/src/engine/SCons/EnvironmentTests.py
+++ b/src/engine/SCons/EnvironmentTests.py
@@ -2001,6 +2001,13 @@ f5: \
         assert str(t) == 'xxx.out', str(t)
         assert 'xxx.in' in map(lambda x: x.path, t.sources)
 
+        # Make sure we can use Builder keyword arguments
+        # on Command() calls.
+        env.Command(target='mmm.out', source='mmm.1.in',
+                    action='multibuild', multi=1)
+        env.Command(target='mmm.out', source='mmm.2.in',
+                    action='multibuild', multi=1)
+
     def test_Configure(self):
         """Test the Configure() method"""
         # Configure() will write to a local temporary file.
diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py
index 4031d3f..747fdf1 100644
--- a/src/engine/SCons/Node/FSTests.py
+++ b/src/engine/SCons/Node/FSTests.py
@@ -93,6 +93,7 @@ class Builder:
         self.env = Environment()
         self.overrides = {}
         self.action = action
+        self.target_scanner = None
 
     def targets(self, t):
         return [t]
@@ -868,7 +869,7 @@ class FSTestCase(unittest.TestCase):
         f1.builder_set(Builder(fs.File))
         f1.env_set(Environment())
         xyz = fs.File("xyz")
-        f1.target_scanner = Scanner(xyz)
+        f1.builder.target_scanner = Scanner(xyz)
 
         f1.scan()
         assert f1.implicit[0].path == "xyz"
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index 7282f80..12224d3 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -117,10 +117,13 @@ class Environment:
         self._dict.update(dict)
     def get_calculator(self):
         return SCons.Sig.default_calc
+    def get_scanner(self, scanner_key):
+        return self._dict['SCANNERS'][0]
 
 class Builder:
-    def __init__(self, is_explicit=1):
-        self.env = Environment()
+    def __init__(self, env=None, is_explicit=1):
+        if env is None: env = Environment()
+        self.env = env
         self.overrides = {}
         self.action = MyAction()
         self.source_factory = MyNode
@@ -796,8 +799,6 @@ class NodeTestCase(unittest.TestCase):
     def test_get_source_scanner(self):
         """Test fetching the source scanner for a Node
         """
-        class Builder:
-            pass
         target = SCons.Node.Node()
         source = SCons.Node.Node()
         s = target.get_source_scanner(source)
@@ -807,19 +808,34 @@ class NodeTestCase(unittest.TestCase):
         ts2 = Scanner()
         ts3 = Scanner()
 
-        source.backup_source_scanner = ts1
-        s = target.get_source_scanner(source)
+        class Builder1(Builder):
+            def __call__(self, source):
+                r = SCons.Node.Node()
+                r.builder = self
+                return [r]
+        class Builder2(Builder1):
+            def __init__(self, scanner):
+                self.source_scanner = scanner
+
+        builder = Builder2(ts1)
+            
+        targets = builder([source])
+        s = targets[0].get_source_scanner(source)
         assert s is ts1, s
 
-        target.builder = Builder()
+        target.builder_set(Builder2(ts1))
         target.builder.source_scanner = ts2
         s = target.get_source_scanner(source)
         assert s is ts2, s
 
-        target.source_scanner = ts3
-        s = target.get_source_scanner(source)
+        builder = Builder1(env=Environment(SCANNERS = [ts3]))
+
+        targets = builder([source])
+        
+        s = targets[0].get_source_scanner(source)
         assert s is ts3, s
 
+
     def test_scan(self):
         """Test Scanner functionality
         """
@@ -831,8 +847,7 @@ class NodeTestCase(unittest.TestCase):
         d = MyNode("ddd")
         node.found_includes = [d]
 
-        assert node.target_scanner == None, node.target_scanner
-        node.target_scanner = s
+        node.builder.target_scanner = s
         assert node.implicit is None
 
         node.scan()
@@ -866,7 +881,7 @@ class NodeTestCase(unittest.TestCase):
             sn = StoredNode("eee")
             sn._children = ['fake']
             sn.builder_set(Builder())
-            sn.target_scanner = s
+            sn.builder.target_scanner = s
 
             sn.scan()
 
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index 3b0d3a4..24ea2a9 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -68,7 +68,7 @@ executed = 4
 failed = 5
 stack = 6 # nodes that are in the current Taskmaster execution stack
 
-# controls whether implicit depedencies are cached:
+# controls whether implicit dependencies are cached:
 implicit_cache = 0
 
 # controls whether implicit dep changes are ignored:
@@ -119,9 +119,6 @@ class Node:
         self.implicit = None    # implicit (scanned) dependencies (None means not scanned yet)
         self.waiting_parents = []
         self.wkids = None       # Kids yet to walk, when it's an array
-        self.target_scanner = None      # explicit scanner from this node's Builder
-        self.source_scanner = None
-        self.backup_source_scanner = None
 
         self.env = None
         self.state = None
@@ -431,15 +428,17 @@ class Node:
         NOTE:  "self" is the target being built, "node" is
         the source file for which we want to fetch the scanner.
         """
-        if self.source_scanner:
-            return self.source_scanner
+        if not self.has_builder():
+            return None  # if not buildable, can't have sources...
         try:
             scanner = self.builder.source_scanner
             if scanner:
                 return scanner
         except AttributeError:
             pass
-        return node.backup_source_scanner or None
+
+        # No scanner specified by builder, try env['SCANNERS']
+        return self.get_build_env().get_scanner(node.scanner_key())
 
     def scan(self):
         """Scan this node's dependents for implicit dependencies."""
@@ -481,8 +480,10 @@ class Node:
                 self._add_child(self.implicit, self.implicit_dict, deps)
 
         # scan this node itself for implicit dependencies
-        deps = self.get_implicit_deps(build_env, self.target_scanner, self)
-        self._add_child(self.implicit, self.implicit_dict, deps)
+        scanner = self.builder.target_scanner
+        if scanner:
+            deps = self.get_implicit_deps(build_env, scanner, self)
+            self._add_child(self.implicit, self.implicit_dict, deps)
 
         # XXX See note above re: --implicit-cache.
         #if implicit_cache:
diff --git a/test/Scanner.py b/test/Scanner.py
index 2814759..828d198 100644
--- a/test/Scanner.py
+++ b/test/Scanner.py
@@ -88,7 +88,6 @@ k2scan = env.Scanner(name = 'k2',
 
 ##########################################################
 # Test scanner as found automatically from the environment
-# (backup_source_scanner)
 
 env = Environment()
 env.Append(SCANNERS = kscan)
@@ -103,14 +102,15 @@ env2.Append(SCANNERS = [k2scan])
 env2.Command('junk', 'junk.k2', r'%(python)s build.py $SOURCES $TARGET')
 
 ##########################################################
-# Test specifying a specific source scanner for a target Node
+# Test specifying a specific source scanner for a Builder
 
-bar = env.Command('bar', 'bar.in', r'%(python)s build.py $SOURCES  $TARGET')
-bar[0].source_scanner = kscan
+bar = env.Command('bar', 'bar.in',
+                  r'%(python)s build.py $SOURCES  $TARGET',
+                  source_scanner=kscan)
 
 ##########################################################
-# Test specifying a source scanner for a Builder that gets
-# automatically applied to targets generated from that Builder
+# Test specifying a source scanner for an intermediary Builder to
+# ensure that the right scanner gets used for the right nodes.
 
 import string