Handle scanning of the in-memory entries for a Dir with a scanner, not a hard-coded Python method.

13 files changed, 161 insertions, 80 deletions

diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py
index 6b9b115..cb628b8 100644
--- a/src/engine/SCons/Defaults.py
+++ b/src/engine/SCons/Defaults.py
@@ -109,6 +109,7 @@ ProgScan = SCons.Tool.ProgramScanner
 # should go.  Leave it here for now.
 import SCons.Scanner.Dir
 DirScanner = SCons.Scanner.Dir.DirScanner()
+DirEntryScanner = SCons.Scanner.Dir.DirEntryScanner()
 
 # Actions for common languages.
 CAction = SCons.Action.Action("$CCCOM", "$CCCOMSTR")
diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py
index ad67152..d95fd81 100644
--- a/src/engine/SCons/Executor.py
+++ b/src/engine/SCons/Executor.py
@@ -182,7 +182,8 @@ class Executor:
         self.scan(scanner, self.targets)
 
     def scan_sources(self, scanner):
-        self.scan(scanner, self.sources)
+        if self.sources:
+            self.scan(scanner, self.sources)
 
     def scan(self, scanner, node_list):
         """Scan a list of this Executor's files (targets or sources) for
@@ -191,16 +192,16 @@ class Executor:
         each individual target, which is a hell of a lot more efficient.
         """
         env = self.get_build_env()
-        select_specific_scanner = lambda t: (t[0], t[0].select_scanner(t[1]))
+        select_specific_scanner = lambda t: (t[0], t[1].select(t[0]))
         remove_null_scanners = lambda t: not t[1] is None
         add_scanner_path = lambda t, s=self: \
                                   (t[0], t[1], s.get_build_scanner_path(t[1]))
         if scanner:
-            scanner_list = map(lambda src, s=scanner: (src, s), node_list)
+            scanner_list = map(lambda n, s=scanner: (n, s), node_list)
         else:
             kw = self.get_kw()
-            get_initial_scanners = lambda src, e=env, kw=kw: \
-                                          (src, src.get_scanner(e, kw))
+            get_initial_scanners = lambda n, e=env, kw=kw: \
+                                          (n, n.get_env_scanner(e, kw))
             scanner_list = map(get_initial_scanners, node_list)
             scanner_list = filter(remove_null_scanners, scanner_list)
 
diff --git a/src/engine/SCons/ExecutorTests.py b/src/engine/SCons/ExecutorTests.py
index a5ac9a6..44086d7 100644
--- a/src/engine/SCons/ExecutorTests.py
+++ b/src/engine/SCons/ExecutorTests.py
@@ -78,7 +78,7 @@ class MyNode:
                                            [self],
                                            ['s1', 's2'])
         apply(executor, (self, errfunc), {})
-    def get_scanner(self, env, kw):
+    def get_env_scanner(self, env, kw):
         return MyScanner('dep-')
     def get_implicit_deps(self, env, scanner, path):
         return [scanner.prefix + str(self)]
@@ -88,8 +88,6 @@ class MyNode:
         return self.missing_val
     def calc_signature(self, calc):
         return 'cs-'+calc+'-'+self.name
-    def select_scanner(self, scanner):
-        return scanner.select(self)
 
 class MyScanner:
     def __init__(self, prefix):
diff --git a/src/engine/SCons/Node/Alias.py b/src/engine/SCons/Node/Alias.py
index bc7e53b..e023ab7 100644
--- a/src/engine/SCons/Node/Alias.py
+++ b/src/engine/SCons/Node/Alias.py
@@ -76,7 +76,7 @@ class Alias(SCons.Node.Node):
     def get_contents(self):
         """The contents of an alias is the concatenation
         of all the contents of its sources"""
-        contents = map(lambda n: n.get_contents(), self.children(None))
+        contents = map(lambda n: n.get_contents(), self.children())
         return string.join(contents, '')
 
     def sconsign(self):
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 59ad707..69936d2 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -198,12 +198,14 @@ def get_MkdirBuilder():
     global MkdirBuilder
     if MkdirBuilder is None:
         import SCons.Builder
+        import SCons.Defaults
         # "env" will get filled in by Executor.get_build_env()
         # calling SCons.Defaults.DefaultEnvironment() when necessary.
         MkdirBuilder = SCons.Builder.Builder(action = Mkdir,
                                              env = None,
                                              explain = None,
                                              is_explicit = None,
+                                             target_scanner = SCons.Defaults.DirEntryScanner,
                                              name = "MkdirBuilder")
     return MkdirBuilder
 
@@ -1287,21 +1289,11 @@ class Dir(Base):
              
         return string.join(path_elems, os.sep)
 
-    def scan(self):
-        if not self.implicit is None:
-            return
-        self.implicit = []
-        self.implicit_dict = {}
-        self._children_reset()
+    def get_env_scanner(self, env, kw={}):
+        return SCons.Defaults.DirEntryScanner
 
-        dont_scan = lambda k: k not in ['.', '..', '.sconsign']
-        deps = filter(dont_scan, self.entries.keys())
-        # keys() is going to give back the entries in an internal,
-        # unsorted order.  Sort 'em so the order is deterministic.
-        deps.sort()
-        entries = map(lambda n, e=self.entries: e[n], deps)
-
-        self._add_child(self.implicit, self.implicit_dict, entries)
+    def get_target_scanner(self):
+        return SCons.Defaults.DirEntryScanner
 
     def get_found_includes(self, env, scanner, path):
         """Return the included implicit dependencies in this file.
@@ -1310,7 +1302,7 @@ class Dir(Base):
         __cacheable__"""
         if not scanner:
             return []
-        # Clear cached info for this Node.  If we already visited this
+        # Clear cached info for this Dir.  If we already visited this
         # directory on our walk down the tree (because we didn't know at
         # that point it was being used as the source for another Node)
         # then we may have calculated build signature before realizing
diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py
index ba6cea8..8c95f87 100644
--- a/src/engine/SCons/Node/FSTests.py
+++ b/src/engine/SCons/Node/FSTests.py
@@ -966,22 +966,6 @@ class FSTestCase(_tempdirTestCase):
                 dir = fs.Dir(drive)
                 assert str(dir) == drive + os.sep, str(dir)
 
-            # Test Dir.scan()
-            dir = fs.Dir('ddd')
-            fs.File(string.join(['ddd', 'f1'], sep))
-            fs.File(string.join(['ddd', 'f2'], sep))
-            fs.File(string.join(['ddd', 'f3'], sep))
-            fs.Dir(string.join(['ddd', 'd1'], sep))
-            fs.Dir(string.join(['ddd', 'd1', 'f4'], sep))
-            fs.Dir(string.join(['ddd', 'd1', 'f5'], sep))
-            dir.scan()
-            kids = map(lambda x: x.path, dir.children(None))
-            kids.sort()
-            assert kids == [os.path.join('ddd', 'd1'),
-                            os.path.join('ddd', 'f1'),
-                            os.path.join('ddd', 'f2'),
-                            os.path.join('ddd', 'f3')], kids
-
         # Test for a bug in 0.04 that did not like looking up
         # dirs with a trailing slash on Win32.
         d=fs.Dir('./')
@@ -1366,7 +1350,7 @@ class FSTestCase(_tempdirTestCase):
         assert str(t) == 'pre-z-suf', str(t)
 
     def test_same_name(self):
-        """Test that a local same-named file isn't found for # Dir lookup"""
+        """Test that a local same-named file isn't found for a Dir lookup"""
         test = self.test
         fs = self.fs
 
@@ -1485,6 +1469,42 @@ class DirTestCase(_tempdirTestCase):
         assert a[0] == 'pre', a
         assert a[2] == 'post', a
 
+    def test_get_env_scanner(self):
+        """Test the Dir.get_env_scanner() method
+        """
+        import SCons.Defaults
+        d = self.fs.Dir('ddd')
+        s = d.get_env_scanner(Environment())
+        assert s is SCons.Defaults.DirEntryScanner, s
+
+    def test_get_target_scanner(self):
+        """Test the Dir.get_target_scanner() method
+        """
+        import SCons.Defaults
+        d = self.fs.Dir('ddd')
+        s = d.get_target_scanner()
+        assert s is SCons.Defaults.DirEntryScanner, s
+
+    def test_scan(self):
+        """Test scanning a directory for in-memory entries
+        """
+        fs = self.fs
+
+        dir = fs.Dir('ddd')
+        fs.File(os.path.join('ddd', 'f1'))
+        fs.File(os.path.join('ddd', 'f2'))
+        fs.File(os.path.join('ddd', 'f3'))
+        fs.Dir(os.path.join('ddd', 'd1'))
+        fs.Dir(os.path.join('ddd', 'd1', 'f4'))
+        fs.Dir(os.path.join('ddd', 'd1', 'f5'))
+        dir.scan()
+        kids = map(lambda x: x.path, dir.children(None))
+        kids.sort()
+        assert kids == [os.path.join('ddd', 'd1'),
+                        os.path.join('ddd', 'f1'),
+                        os.path.join('ddd', 'f2'),
+                        os.path.join('ddd', 'f3')], kids
+
     def test_entry_exists_on_disk(self):
         """Test the Dir.entry_exists_on_disk() method
         """
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index 3e2fd5c..1cd5201 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -930,17 +930,28 @@ class NodeTestCase(unittest.TestCase):
         deps = node.get_implicit_deps(env, s, target)
         assert deps == [d1, d2], map(str, deps)
 
-    def test_get_scanner(self):
+    def test_get_env_scanner(self):
         """Test fetching the environment scanner for a Node
         """
         node = SCons.Node.Node()
         scanner = Scanner()
         env = Environment(SCANNERS = [scanner])
-        s = node.get_scanner(env)
+        s = node.get_env_scanner(env)
         assert s == scanner, s
-        s = node.get_scanner(env, {'X':1})
+        s = node.get_env_scanner(env, {'X':1})
         assert s == scanner, s
 
+    def test_get_target_scanner(self):
+        """Test fetching the target scanner for a Node
+        """
+        s = Scanner()
+        b = Builder()
+        b.target_scanner = s
+        n = SCons.Node.Node()
+        n.builder = b
+        x = n.get_target_scanner()
+        assert x is s, x
+
     def test_get_source_scanner(self):
         """Test fetching the source scanner for a Node
         """
@@ -1044,12 +1055,6 @@ class NodeTestCase(unittest.TestCase):
         """Test that a scanner_key() method exists"""
         assert SCons.Node.Node().scanner_key() == None
 
-    def test_select_scanner(self):
-        """Test the base select_scanner() method returns its scanner"""
-        scanner = Scanner()
-        s = SCons.Node.Node().select_scanner(scanner)
-        assert scanner is s, s
-
     def test_children(self):
         """Test fetching the non-ignored "children" of a Node.
         """
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index 7a29f95..250c714 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -474,9 +474,12 @@ class Node:
 
         return deps
 
-    def get_scanner(self, env, kw={}):
+    def get_env_scanner(self, env, kw={}):
         return env.get_scanner(self.scanner_key())
 
+    def get_target_scanner(self):
+        return self.builder.target_scanner
+
     def get_source_scanner(self, node):
         """Fetch the source scanner for the specified node
 
@@ -498,7 +501,7 @@ class Node:
             # The builder didn't have an explicit scanner, so go look up
             # a scanner from env['SCANNERS'] based on the node's scanner
             # key (usually the file extension).
-            scanner = self.get_scanner(self.get_build_env())
+            scanner = self.get_env_scanner(self.get_build_env())
         if scanner:
             scanner = scanner.select(node)
         return scanner
@@ -563,7 +566,7 @@ class Node:
 
         # If there's a target scanner, have the executor scan the target
         # node itself and associated targets that might be built.
-        scanner = self.builder.target_scanner
+        scanner = self.get_target_scanner()
         if scanner:
             executor.scan_targets(scanner)
 
diff --git a/src/engine/SCons/Scanner/Dir.py b/src/engine/SCons/Scanner/Dir.py
index 3da1661..fb23d1b 100644
--- a/src/engine/SCons/Scanner/Dir.py
+++ b/src/engine/SCons/Scanner/Dir.py
@@ -43,15 +43,31 @@ def DirEntryScanner(**kw):
     """Return a prototype Scanner instance for "scanning"
     directory Nodes for their in-memory entries"""
     kw['node_factory'] = SCons.Node.FS.Entry
-    kw['recursive'] = only_dirs
+    kw['recursive'] = None
     return apply(SCons.Scanner.Base, (scan_in_memory, "DirEntryScanner"), kw)
 
-skip_entry = {
-   '.' : 1,
-   '..' : 1,
-   '.sconsign' : 1,
-   '.sconsign.dblite' : 1,
-}
+skip_entry = {}
+
+skip_entry_list = [
+   '.',
+   '..',
+   '.sconsign',
+   # Used by the native dblite.py module.
+   '.sconsign.dblite',
+   # Used by dbm and dumbdbm.
+   '.sconsign.dir',
+   # Used by dbm.
+   '.sconsign.pag',
+   # Used by dumbdbm.
+   '.sconsign.dat',
+   '.sconsign.bak',
+   # Used by some dbm emulations using Berkeley DB.
+   '.sconsign.db',
+]
+
+for skip in skip_entry_list:
+    skip_entry[skip] = 1
+    skip_entry[SCons.Node.FS._my_normcase(skip)] = 1
 
 do_not_scan = lambda k: not skip_entry.has_key(k)
 
diff --git a/src/engine/SCons/Scanner/DirTests.py b/src/engine/SCons/Scanner/DirTests.py
index e4e59a3..0dde95e 100644
--- a/src/engine/SCons/Scanner/DirTests.py
+++ b/src/engine/SCons/Scanner/DirTests.py
@@ -62,11 +62,20 @@ class DirScannerTestBase(unittest.TestCase):
         self.test.write(['dir', 'f1'], "dir/f1\n")
         self.test.write(['dir', 'f2'], "dir/f2\n")
         self.test.write(['dir', '.sconsign'], "dir/.sconsign\n")
+        self.test.write(['dir', '.sconsign.bak'], "dir/.sconsign.bak\n")
+        self.test.write(['dir', '.sconsign.dat'], "dir/.sconsign.dat\n")
+        self.test.write(['dir', '.sconsign.db'], "dir/.sconsign.db\n")
         self.test.write(['dir', '.sconsign.dblite'], "dir/.sconsign.dblite\n")
+        self.test.write(['dir', '.sconsign.dir'], "dir/.sconsign.dir\n")
+        self.test.write(['dir', '.sconsign.pag'], "dir/.sconsign.pag\n")
         self.test.write(['dir', 'sub', 'f3'], "dir/sub/f3\n")
         self.test.write(['dir', 'sub', 'f4'], "dir/sub/f4\n")
         self.test.write(['dir', 'sub', '.sconsign'], "dir/.sconsign\n")
+        self.test.write(['dir', 'sub', '.sconsign.bak'], "dir/.sconsign.bak\n")
+        self.test.write(['dir', 'sub', '.sconsign.dat'], "dir/.sconsign.dat\n")
         self.test.write(['dir', 'sub', '.sconsign.dblite'], "dir/.sconsign.dblite\n")
+        self.test.write(['dir', 'sub', '.sconsign.dir'], "dir/.sconsign.dir\n")
+        self.test.write(['dir', 'sub', '.sconsign.pag'], "dir/.sconsign.pag\n")
 
 class DirScannerTestCase1(DirScannerTestBase):
     def runTest(self):
@@ -74,13 +83,22 @@ class DirScannerTestCase1(DirScannerTestBase):
 
         s = SCons.Scanner.Dir.DirScanner()
 
+        expect = [
+            os.path.join('dir', 'f1'),
+            os.path.join('dir', 'f2'),
+            os.path.join('dir', 'sub'),
+        ]
         deps = s(env.Dir('dir'), env, ())
         sss = map(str, deps)
-        assert sss == ['dir/f1', 'dir/f2', 'dir/sub'], sss
+        assert sss == expect, sss
 
+        expect = [
+            os.path.join('dir', 'sub', 'f3'),
+            os.path.join('dir', 'sub', 'f4'),
+        ]
         deps = s(env.Dir('dir/sub'), env, ())
         sss = map(str, deps)
-        assert sss == ['dir/sub/f3', 'dir/sub/f4'], sss
+        assert sss == expect, sss
 
 class DirScannerTestCase2(DirScannerTestBase):
     def runTest(self):
diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py
index cc7bf13..16e790e 100644
--- a/src/engine/SCons/Script/Main.py
+++ b/src/engine/SCons/Script/Main.py
@@ -1236,7 +1236,7 @@ def _main(args, parser):
     if options.taskmastertrace_file == '-':
         tmtrace = sys.stdout
     elif options.taskmastertrace_file:
-        tmtrace = open(options.taskmastertrace_file, 'w')
+        tmtrace = open(options.taskmastertrace_file, 'wb')
     else:
         tmtrace = None
     taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, tmtrace)
diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py
index cbb3f89..5d68989 100644
--- a/src/engine/SCons/Taskmaster.py
+++ b/src/engine/SCons/Taskmaster.py
@@ -320,29 +320,28 @@ class Taskmaster:
             else:
                 S = None
 
-            if T: T.write('Taskmaster: %s' % repr(str(node)))
+            if T: T.write('Taskmaster: %s:' % repr(str(node)))
 
             # Skip this node if it has already been handled:
             if not state in [ SCons.Node.no_state, SCons.Node.stack ]:
                 if S: S.already_handled = S.already_handled + 1
-                if T: T.write(': already handled\n')
+                if T: T.write(' already handled\n')
                 continue
 
             # Mark this node as being on the execution stack:
             node.set_state(SCons.Node.stack)
 
             try:
-                childinfo = map(lambda N: (N.get_state(),
-                                           N.is_derived() or N.is_pseudo_derived(),
-                                           N), node.children())
+                children = node.children()
             except SystemExit:
                 exc_value = sys.exc_info()[1]
                 e = SCons.Errors.ExplicitExit(node, exc_value.code)
                 self.ready_exc = (SCons.Errors.ExplicitExit, e)
                 self.ready = node
-                if T: T.write(': SystemExit\n')
+                if T: T.write(' SystemExit\n')
                 break
             except KeyboardInterrupt:
+                if T: T.write(' KeyboardInterrupt\n')
                 raise
             except:
                 # We had a problem just trying to figure out the
@@ -352,8 +351,16 @@ class Taskmaster:
                 self.ready_exc = sys.exc_info()
                 self.ready = node
                 if S: S.problem = S.problem + 1
-                if T: T.write(': exception problem\n')
+                if T: T.write(' exception\n')
                 break
+            else:
+                c = map(str, children)
+                c.sort()
+                if T: T.write(' children:\n    %s\n   ' % c)
+
+            childinfo = map(lambda N: (N.get_state(),
+                                       N.is_derived() or N.is_pseudo_derived(),
+                                       N), children)
 
             # Skip this node if any of its children have failed.  This
             # catches the case where we're descending a top-level target
@@ -364,7 +371,10 @@ class Taskmaster:
             if failed_children:
                 node.set_state(SCons.Node.failed)
                 if S: S.child_failed = S.child_failed + 1
-                if T: T.write(': children failed:\n    %s\n' % map(str, failed_children))
+                if T:
+                    c = map(str, failed_children)
+                    c.sort()
+                    T.write(' children failed:\n    %s\n' % c)
                 continue
 
             # Detect dependency cycles:
@@ -379,7 +389,7 @@ class Taskmaster:
                                map(lambda I: I[2], cycle)
                 nodes.reverse()
                 desc = "Dependency cycle: " + string.join(map(str, nodes), " -> ")
-                if T: T.write(': dependency cycle\n')
+                if T: T.write(' dependency cycle\n')
                 raise SCons.Errors.UserError, desc
 
             # Select all of the dependencies that are derived targets
@@ -413,7 +423,10 @@ class Taskmaster:
                 not_started.reverse()
                 self.candidates.extend(self.order(not_started))
                 if S: S.not_started = S.not_started + 1
-                if T: T.write(': waiting on unstarted children:\n    %s\n' % map(str, not_started))
+                if T:
+                    c = map(str, not_started)
+                    c.sort()
+                    T.write(' waiting on unstarted children:\n    %s\n' % c)
                 continue
 
             not_built = filter(lambda I: I[0] <= SCons.Node.executing, derived_children)
@@ -433,7 +446,10 @@ class Taskmaster:
                 self.pending.append(node)
                 node.set_state(SCons.Node.pending)
                 if S: S.not_built = S.not_built + 1
-                if T: T.write(': waiting on unfinished children:\n    %s\n' % map(str, not_built))
+                if T:
+                    c = map(str, not_built)
+                    c.sort()
+                    T.write(' waiting on unfinished children:\n    %s\n' % c)
                 continue
 
             # Skip this node if it has side-effects that are
@@ -446,14 +462,17 @@ class Taskmaster:
                 self.pending.append(node)
                 node.set_state(SCons.Node.pending)
                 if S: S.side_effects = S.side_effects + 1
-                if T: T.write(': waiting on side effects:\n    %s\n' % map(str, side_effects))
+                if T:
+                    c = map(str, side_effects)
+                    c.sort()
+                    T.write(' waiting on side effects:\n    %s\n' % c)
                 continue
 
             # The default when we've gotten through all of the checks above:
             # this node is ready to be built.
             self.ready = node
             if S: S.build = S.build + 1
-            if T: T.write(': building\n')
+            if T: T.write(' evaluating\n')
             break
 
     def next_task(self):
diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py
index dc23159..c35a89c 100644
--- a/src/engine/SCons/TaskmasterTests.py
+++ b/src/engine/SCons/TaskmasterTests.py
@@ -1052,12 +1052,20 @@ class TaskmasterTestCase(unittest.TestCase):
 
         value = trace.getvalue()
         expect = """\
-Taskmaster: 'n1': building
+Taskmaster: 'n1': children:
+    []
+    evaluating
 Taskmaster: 'n1': already handled
-Taskmaster: 'n3': waiting on unstarted children:
+Taskmaster: 'n3': children:
+    ['n1', 'n2']
+    waiting on unstarted children:
     ['n2']
-Taskmaster: 'n2': building
-Taskmaster: 'n3': waiting on unfinished children:
+Taskmaster: 'n2': children:
+    []
+    evaluating
+Taskmaster: 'n3': children:
+    ['n1', 'n2']
+    waiting on unfinished children:
     ['n2']
 """
         assert value == expect, value