Don't read up entire directories to decide if an Alias is up-to-date.

20 files changed, 473 insertions, 97 deletions

diff --git a/doc/man/scons.1 b/doc/man/scons.1
index db1da8c..690803f 100644
--- a/doc/man/scons.1
+++ b/doc/man/scons.1
@@ -2107,6 +2107,12 @@ for a given target;
 each additional call
 adds to the list of entries
 that will be built into the archive.
+Any source directories will
+be scanned for changes to
+any on-disk files,
+regardless of whether or not
+.B scons
+knows about them from other Builder or function calls.
 
 .ES
 env.Tar('src.tar', 'src')
@@ -2173,6 +2179,12 @@ for a given target;
 each additional call
 adds to the list of entries
 that will be built into the archive.
+Any source directories will
+be scanned for changes to
+any on-disk files,
+regardless of whether or not
+.B scons
+knows about them from other Builder or function calls.
 
 .ES
 env.Zip('src.zip', 'src')
@@ -2720,7 +2732,22 @@ to build a target file or files.
 This is more convenient
 than defining a separate Builder object
 for a single special-case build.
-Any keyword arguments specified override any
+
+As a special case, the
+.B source_scanner
+keyword argument can
+be used to specify
+a Scanner object
+that will be used to scan the sources.
+(The global
+.B DirScanner
+object can be used
+if any of the sources will be directories
+that must be scanned on-disk for
+changes to files that aren't
+already specified in other Builder of function calls.)
+
+Any other keyword arguments specified override any
 same-named existing construction variables.
 
 Note that an action can be an external command,
@@ -8157,8 +8184,17 @@ specify a scanner to
 find things like
 .B #include
 lines in source files.
+The pre-built
+.B DirScanner
+Scanner object may be used to
+indicate that this Builder
+should scan directory trees
+for on-disk changes to files
+that
+.B scons
+does not know about from other Builder or function calls.
 (See the section "Scanner Objects," below,
-for information about creating Scanner objects.)
+for information about creating your own Scanner objects.)
 
 .IP target_factory
 A factory function that the Builder will use
@@ -9222,6 +9258,14 @@ only invoke the scanner on the file being scanned,
 and not (for example) also on the files
 specified by the #include lines
 in the file being scanned.
+.I recursive
+may be a callable function,
+in which case it will be called with a list of
+Nodes found and
+should return a list of Nodes
+that should be scanned recursively;
+this can be used to select a specific subset of
+Nodes for additional scanning.
 
 Note that
 .B scons
diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index dbd0685..2dbb6bf 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -198,6 +198,15 @@ RELEASE 0.97 - XXX
     part of the public interface.  Keep the old SCons.Defaults.*Scan names
     around for a while longer since some people were already using them.
 
+  - By default, don't scan directories for on-disk files.  Add a
+    DirScanner global scanner that can be used in Builders or Command()
+    calls that want source directory trees scanned for on-disk changes.
+    Have the Tar() and Zip() Builders use the new DirScanner to preserve
+    the behavior of rebuilding a .tar or .zip file if any file or
+    directory under a source tree changes.  Add Command() support for
+    a source_scanner keyword argument to Command() that can be set to
+    DirScanner to get this behavior.
+
   From Wayne Lee:
 
   - Avoid "maximum recursion limit" errors when removing $(-$) pairs
diff --git a/src/RELEASE.txt b/src/RELEASE.txt
index 2592f5c..4089d5a 100644
--- a/src/RELEASE.txt
+++ b/src/RELEASE.txt
@@ -37,6 +37,28 @@ RELEASE 0.97 - XXX
       entries to construction variables.  The old behavior may be
       specified using a new "unique=0" keyword argument.
 
+    - Custom builders that accept directories as source arguments no
+      longer scan entire directory trees by default.  This means that
+      their targets will not be automatically rebuilt if a file that
+      SCons does *not* already know about changes on disk.  Note that
+      the targets *will* still be rebuilt correctly if a file changes
+      that SCons already knows about due to a Builder or other call.
+
+      The existing behavior of scanning directory trees for any changed
+      file on-disk can be maintained by passing the new DirScanner global
+      directory scanner as the source_scanner keyword argument to the
+      Builder call:
+
+          bld = Builder("build < $SOURCE > $TARGET",
+                        source_scanner = DirScanner)
+
+      The same keyword argument can also be supplied to any Command()
+      calls that need to scan directory trees on-disk for changed files:
+
+          env.Command("archive.out", "directory",
+                      "archiver -o $TARGET $SOURCE",
+                      source_scanner = DirScanner)
+
     - When compiling with Microsoft Visual Studio, SCons no longer
       adds the ATL and MFC directories to the INCLUDE and LIB
       environment variables by default.  If you want these directories
diff --git a/src/engine/MANIFEST.in b/src/engine/MANIFEST.in
index 5c10cae..4888736 100644
--- a/src/engine/MANIFEST.in
+++ b/src/engine/MANIFEST.in
@@ -38,6 +38,7 @@ SCons/Platform/win32.py
 SCons/Scanner/__init__.py
 SCons/Scanner/C.py
 SCons/Scanner/D.py
+SCons/Scanner/Dir.py
 SCons/Scanner/Fortran.py
 SCons/Scanner/IDL.py
 SCons/Scanner/Prog.py
diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py
index 032a067..8ade792 100644
--- a/src/engine/SCons/Defaults.py
+++ b/src/engine/SCons/Defaults.py
@@ -102,6 +102,12 @@ DScan = SCons.Tool.DScanner
 ObjSourceScan = SCons.Tool.SourceFileScanner
 ProgScan = SCons.Tool.ProgramScanner
 
+# This isn't really a tool scanner, so it doesn't quite belong with
+# the rest of those in Tool/__init__.py, but I'm not sure where else it
+# should go.  Leave it here for now.
+import SCons.Scanner.Dir
+DirScanner = SCons.Scanner.Dir.DirScanner()
+
 # Actions for common languages.
 CAction = SCons.Action.Action("$CCCOM", "$CCCOMSTR")
 ShCAction = SCons.Action.Action("$SHCCCOM", "$SHCCCOMSTR")
diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py
index 419e4e1..9470623 100644
--- a/src/engine/SCons/Environment.py
+++ b/src/engine/SCons/Environment.py
@@ -1202,8 +1202,14 @@ class Base(SubstitutionEnvironment):
         source files using the supplied action.  Action may
         be any type that the Builder constructor will accept
         for an action."""
-        bld = SCons.Builder.Builder(action = action,
-                                    source_factory = self.fs.Entry)
+        bkw = {
+            'action' : action,
+            'source_factory' : self.fs.Entry,
+        }
+        try: bkw['source_scanner'] = kw['source_scanner']
+        except KeyError: pass
+        else: del kw['source_scanner']
+        bld = apply(SCons.Builder.Builder, (), bkw)
         return apply(bld, (self, target, source), kw)
 
     def Depends(self, target, dependency):
diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py
index 3ef5e73..bfd1262 100644
--- a/src/engine/SCons/EnvironmentTests.py
+++ b/src/engine/SCons/EnvironmentTests.py
@@ -2171,6 +2171,12 @@ f5: \
         assert str(t) == 'xxx.out', str(t)
         assert 'xxx.in' in map(lambda x: x.path, t.sources)
 
+        env = Environment(source_scanner = 'should_not_find_this')
+        t = env.Command(target='file.out', source='file.in',
+                        action = 'foo',
+                        source_scanner = 'fake')[0]
+        assert t.builder.source_scanner == 'fake', t.builder.source_scanner
+
     def test_Configure(self):
         """Test the Configure() method"""
         # Configure() will write to a local temporary file.
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 314faf8..cc0fe95 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -599,6 +599,16 @@ class Entry(Base):
     time comes, and then call the same-named method in the transformed
     class."""
 
+    def disambiguate(self):
+        if self.fs.isdir(self.abspath):
+            self.__class__ = Dir
+            self._morph()
+        else:
+            self.__class__ = File
+            self._morph()
+            self.clear()
+        return self
+
     def rfile(self):
         """We're a generic Entry, but the caller is actually looking for
         a File at this point, so morph into one."""
@@ -610,8 +620,7 @@ class Entry(Base):
     def get_found_includes(self, env, scanner, path):
         """If we're looking for included files, it's because this Entry
         is really supposed to be a File itself."""
-        node = self.rfile()
-        return node.get_found_includes(env, scanner, path)
+        return self.disambiguate().get_found_includes(env, scanner, path)
 
     def scanner_key(self):
         return self.get_suffix()
@@ -638,29 +647,13 @@ class Entry(Base):
         """Return if the Entry exists.  Check the file system to see
         what we should turn into first.  Assume a file if there's no
         directory."""
-        if self.fs.isdir(self.abspath):
-            self.__class__ = Dir
-            self._morph()
-            return Dir.exists(self)
-        else:
-            self.__class__ = File
-            self._morph()
-            self.clear()
-            return File.exists(self)
+        return self.disambiguate().exists()
 
     def calc_signature(self, calc=None):
         """Return the Entry's calculated signature.  Check the file
         system to see what we should turn into first.  Assume a file if
         there's no directory."""
-        if self.fs.isdir(self.abspath):
-            self.__class__ = Dir
-            self._morph()
-            return Dir.calc_signature(self, calc)
-        else:
-            self.__class__ = File
-            self._morph()
-            self.clear()
-            return File.calc_signature(self, calc)
+        return self.disambiguate().calc_signature(calc)
 
     def must_be_a_Dir(self):
         """Called to make sure a Node is a Dir.  Since we're an
@@ -1180,6 +1173,9 @@ class Dir(Base):
         self._sconsign = None
         self.build_dirs = []
 
+    def disambiguate(self):
+        return self
+
     def __clearRepositoryCache(self, duplicate=None):
         """Called when we change the repository(ies) for a directory.
         This clears any cached information that is invalidated by changing
@@ -1256,19 +1252,33 @@ class Dir(Base):
         self.implicit = []
         self.implicit_dict = {}
         self._children_reset()
-        try:
-            for filename in self.fs.listdir(self.abspath):
-                if filename != '.sconsign':
-                    self.Entry(filename)
-        except OSError:
-            # Directory does not exist.  No big deal
-            pass
-        keys = filter(lambda k: k != '.' and k != '..', self.entries.keys())
-        kids = map(lambda x, s=self: s.entries[x], keys)
-        def c(one, two):
-            return cmp(one.abspath, two.abspath)
-        kids.sort(c)
-        self._add_child(self.implicit, self.implicit_dict, kids)
+
+        dont_scan = lambda k: k not in ['.', '..', '.sconsign']
+        deps = filter(dont_scan, self.entries.keys())
+        # keys() is going to give back the entries in an internal,
+        # unsorted order.  Sort 'em so the order is deterministic.
+        deps.sort()
+        entries = map(lambda n, e=self.entries: e[n], deps)
+
+        self._add_child(self.implicit, self.implicit_dict, entries)
+
+    def get_found_includes(self, env, scanner, path):
+        """Return the included implicit dependencies in this file.
+        Cache results so we only scan the file once per path
+        regardless of how many times this information is requested.
+        __cacheable__"""
+        if not scanner:
+            return []
+        # Clear cached info for this Node.  If we already visited this
+        # directory on our walk down the tree (because we didn't know at
+        # that point it was being used as the source for another Node)
+        # then we may have calculated build signature before realizing
+        # we had to scan the disk.  Now that we have to, though, we need
+        # to invalidate the old calculated signature so that any node
+        # dependent on our directory structure gets one that includes
+        # info about everything on disk.
+        self.clear()
+        return scanner(self, env, path)
 
     def build(self, **kw):
         """A null "builder" for directories."""
@@ -1295,7 +1305,7 @@ class Dir(Base):
         for kid in self.children():
             contents.write(kid.get_contents())
         return contents.getvalue()
-    
+
     def prepare(self):
         pass
 
@@ -1464,6 +1474,9 @@ class File(Base):
         if not hasattr(self, '_local'):
             self._local = 0
 
+    def disambiguate(self):
+        return self
+
     def root(self):
         return self.dir.root()
 
diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py
index 2846f64..99a95b6 100644
--- a/src/engine/SCons/Node/FSTests.py
+++ b/src/engine/SCons/Node/FSTests.py
@@ -62,6 +62,8 @@ class Scanner:
         return self.hash
     def select(self, node):
         return self
+    def recurse_nodes(self, nodes):
+        return nodes
 
 class Environment:
     def __init__(self):
@@ -1876,6 +1878,32 @@ class clearTestCase(unittest.TestCase):
         assert not f.rexists()
         assert str(f) == test.workpath('f'), str(f)
 
+class disambiguateTestCase(unittest.TestCase):
+    def runTest(self):
+        """Test calling the disambiguate() method."""
+        test = TestCmd(workdir='')
+
+        fs = SCons.Node.FS.FS()
+
+        ddd = fs.Dir('ddd')
+        d = ddd.disambiguate()
+        assert d is ddd, d
+
+        fff = fs.File('fff')
+        f = fff.disambiguate()
+        assert f is fff, f
+
+        test.subdir('edir')
+        test.write('efile', "efile\n")
+
+        edir = fs.Entry(test.workpath('edir'))
+        d = edir.disambiguate()
+        assert d.__class__ is ddd.__class__, d.__class__
+
+        efile = fs.Entry(test.workpath('efile'))
+        f = efile.disambiguate()
+        assert f.__class__ is fff.__class__, f.__class__
+
 class postprocessTestCase(unittest.TestCase):
     def runTest(self):
         """Test calling the postprocess() method."""
@@ -2108,6 +2136,7 @@ if __name__ == "__main__":
     suite.addTest(SConstruct_dirTestCase())
     suite.addTest(CacheDirTestCase())
     suite.addTest(clearTestCase())
+    suite.addTest(disambiguateTestCase())
     suite.addTest(postprocessTestCase())
     suite.addTest(SpecialAttrTestCase())
     suite.addTest(SaveStringsTestCase())
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index 281b5f2..90bb332 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -181,6 +181,8 @@ class Scanner:
         return ()
     def select(self, node):
         return self
+    def recurse_nodes(self, nodes):
+        return nodes
 
 class MyNode(SCons.Node.Node):
     """The base Node class contains a number of do-nothing methods that
@@ -807,28 +809,31 @@ class NodeTestCase(unittest.TestCase):
         deps = node.get_implicit_deps(env, s, target)
         assert deps == [d], deps
 
-        # No "recursive" attribute on scanner doesn't recurse
+        # By default, our fake scanner recurses
         e = MyNode("eee")
-        d.found_includes = [e]
+        f = MyNode("fff")
+        g = MyNode("ggg")
+        d.found_includes = [e, f]
+        f.found_includes = [g]
         deps = node.get_implicit_deps(env, s, target)
-        assert deps == [d], map(str, deps)
+        assert deps == [d, e, f, g], map(str, deps)
 
-        # Explicit "recursive" attribute on scanner doesn't recurse
-        s.recursive = None
+        # Recursive scanning eliminates duplicates
+        e.found_includes = [f]
         deps = node.get_implicit_deps(env, s, target)
-        assert deps == [d], map(str, deps)
+        assert deps == [d, e, f, g], map(str, deps)
 
-        # Explicit "recursive" attribute on scanner which does recurse
-        s.recursive = 1
+        # Scanner method can select specific nodes to recurse
+        def no_fff(nodes):
+            return filter(lambda n: str(n)[0] != 'f', nodes)
+        s.recurse_nodes = no_fff
         deps = node.get_implicit_deps(env, s, target)
-        assert deps == [d, e], map(str, deps)
+        assert deps == [d, e, f], map(str, deps)
 
-        # Recursive scanning eliminates duplicates
-        f = MyNode("fff")
-        d.found_includes = [e, f]
-        e.found_includes = [f]
+        # Scanner method can short-circuit recursing entirely
+        s.recurse_nodes = lambda nodes: []
         deps = node.get_implicit_deps(env, s, target)
-        assert deps == [d, e, f], map(str, deps)
+        assert deps == [d], map(str, deps)
 
     def test_get_scanner(self):
         """Test fetching the environment scanner for a Node
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index 96a78ca..3c0ce99 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -394,25 +394,19 @@ class Node:
         # for this Node.
         scanner = scanner.select(self)
 
-        try:
-            recurse = scanner.recursive
-        except AttributeError:
-            recurse = None
-
         nodes = [self]
         seen = {}
         seen[self] = 1
         deps = []
         while nodes:
-           n = nodes.pop(0)
-           d = filter(lambda x, seen=seen: not seen.has_key(x),
-                      n.get_found_includes(env, scanner, path))
-           if d:
-               deps.extend(d)
-               for n in d:
-                   seen[n] = 1
-               if recurse:
-                   nodes.extend(d)
+            n = nodes.pop(0)
+            d = filter(lambda x, seen=seen: not seen.has_key(x),
+                       n.get_found_includes(env, scanner, path))
+            if d:
+                deps.extend(d)
+                for n in d:
+                    seen[n] = 1
+                nodes.extend(scanner.recurse_nodes(d))
 
         return deps
 
diff --git a/src/engine/SCons/Scanner/Dir.py b/src/engine/SCons/Scanner/Dir.py
new file mode 100644
index 0000000..6161059
--- /dev/null
+++ b/src/engine/SCons/Scanner/Dir.py
@@ -0,0 +1,62 @@
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+import string
+
+import SCons.Node.FS
+import SCons.Scanner
+
+def DirScanner(fs = SCons.Node.FS.default_fs, **kw):
+    """Return a prototype Scanner instance for scanning
+    directories for on-disk files"""
+    def only_dirs(nodes, fs=fs):
+        return filter(lambda n: isinstance(n.disambiguate(), SCons.Node.FS.Dir), nodes)
+    kw['node_factory'] = fs.Entry
+    kw['recursive'] = only_dirs
+    ds = apply(SCons.Scanner.Base, [scan, "DirScanner"], kw)
+    return ds
+
+skip_entry = {
+   '.' : 1,
+   '..' : 1,
+   '.sconsign' : 1,
+   '.sconsign.dblite' : 1,
+}
+
+def scan(node, env, path=()):
+    """
+    This scanner scans program files for static-library
+    dependencies.  It will search the LIBPATH environment variable
+    for libraries specified in the LIBS variable, returning any
+    files it finds as dependencies.
+    """
+    try:
+        flist = node.fs.listdir(node.abspath)
+    except OSError:
+        return []
+    dont_scan = lambda k: not skip_entry.has_key(k)
+    flist = filter(dont_scan, flist)
+    flist.sort()
+    return map(node.Entry, flist)
diff --git a/src/engine/SCons/Scanner/DirTests.py b/src/engine/SCons/Scanner/DirTests.py
new file mode 100644
index 0000000..e735ca2
--- /dev/null
+++ b/src/engine/SCons/Scanner/DirTests.py
@@ -0,0 +1,80 @@
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+import os.path
+import string
+import sys
+import types
+import unittest
+
+import TestCmd
+import SCons.Node.FS
+import SCons.Scanner.Dir
+
+test = TestCmd.TestCmd(workdir = '')
+
+test.subdir('dir', ['dir', 'sub'])
+
+test.write(['dir', 'f1'], "dir/f1\n")
+test.write(['dir', 'f2'], "dir/f2\n")
+test.write(['dir', '.sconsign'], "dir/.sconsign\n")
+test.write(['dir', '.sconsign.dblite'], "dir/.sconsign.dblite\n")
+test.write(['dir', 'sub', 'f3'], "dir/sub/f3\n")
+test.write(['dir', 'sub', 'f4'], "dir/sub/f4\n")
+test.write(['dir', 'sub', '.sconsign'], "dir/.sconsign\n")
+test.write(['dir', 'sub', '.sconsign.dblite'], "dir/.sconsign.dblite\n")
+
+class DummyNode:
+    def __init__(self, name):
+        self.name = name
+        self.abspath = test.workpath(name)
+        self.fs = SCons.Node.FS.default_fs
+    def __str__(self):
+        return self.name
+    def Entry(self, name):
+        return self.fs.Entry(name)
+
+class DirScannerTestCase1(unittest.TestCase):
+    def runTest(self):
+        s = SCons.Scanner.Dir.DirScanner()
+
+        deps = s(DummyNode('dir'), {}, ())
+        sss = map(str, deps)
+        assert sss == ['f1', 'f2', 'sub'], sss
+
+        deps = s(DummyNode('dir/sub'), {}, ())
+        sss = map(str, deps)
+        assert sss == ['f3', 'f4'], sss
+
+def suite():
+    suite = unittest.TestSuite()
+    suite.addTest(DirScannerTestCase1())
+    return suite
+
+if __name__ == "__main__":
+    runner = unittest.TextTestRunner()
+    result = runner.run(suite())
+    if not result.wasSuccessful():
+        sys.exit(1)
diff --git a/src/engine/SCons/Scanner/ScannerTests.py b/src/engine/SCons/Scanner/ScannerTests.py
index c38dc84..ce5411c 100644
--- a/src/engine/SCons/Scanner/ScannerTests.py
+++ b/src/engine/SCons/Scanner/ScannerTests.py
@@ -221,15 +221,29 @@ class BaseTestCase(unittest.TestCase):
 
     def test_recursive(self):
         """Test the Scanner.Base class recursive flag"""
+        nodes = [1, 2, 3, 4]
+
         s = SCons.Scanner.Base(function = self.func)
-        self.failUnless(s.recursive == None,
-                        "incorrect default recursive value")
+        n = s.recurse_nodes(nodes)
+        self.failUnless(n == [],
+                        "default behavior returned nodes: %s" % n)
+
         s = SCons.Scanner.Base(function = self.func, recursive = None)
-        self.failUnless(s.recursive == None,
-                        "did not set recursive flag to None")
+        n = s.recurse_nodes(nodes)
+        self.failUnless(n == [],
+                        "recursive = None returned nodes: %s" % n)
+
         s = SCons.Scanner.Base(function = self.func, recursive = 1)
-        self.failUnless(s.recursive == 1,
-                        "did not set recursive flag to 1")
+        n = s.recurse_nodes(nodes)
+        self.failUnless(n == n,
+                        "recursive = 1 didn't return all nodes: %s" % n)
+
+        def odd_only(nodes):
+            return filter(lambda n: n % 2, nodes)
+        s = SCons.Scanner.Base(function = self.func, recursive = odd_only)
+        n = s.recurse_nodes(nodes)
+        self.failUnless(n == [1, 3],
+                        "recursive = 1 didn't return all nodes: %s" % n)
 
     def test_get_skeys(self):
         """Test the Scanner.Base get_skeys() method"""
diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py
index 3f7ead4..cda156c 100644
--- a/src/engine/SCons/Scanner/__init__.py
+++ b/src/engine/SCons/Scanner/__init__.py
@@ -148,8 +148,12 @@ class Base:
         this node really needs to be scanned.
 
         'recursive' - specifies that this scanner should be invoked
-        recursively on the implicit dependencies it returns (the
-        canonical example being #include lines in C source files).
+        recursively on all of the implicit dependencies it returns
+        (the canonical example being #include lines in C source files).
+        May be a callable, which will be called to filter the list
+        of nodes found to select a subset for recursive scanning
+        (the canonical example being only recursively scanning
+        subdirectories within a directory).
 
         The scanner function's first argument will be the a Node that
         should be scanned for dependencies, the second argument will
@@ -182,7 +186,12 @@ class Base:
         self.node_class = node_class
         self.node_factory = node_factory
         self.scan_check = scan_check
-        self.recursive = recursive
+        if callable(recursive):
+            self.recurse_nodes = recursive
+        elif recursive:
+            self.recurse_nodes = self._recurse_all_nodes
+        else:
+            self.recurse_nodes = self._recurse_no_nodes
 
     def path(self, env, dir=None, target=None, source=None):
         if not self.path_function:
@@ -241,6 +250,14 @@ class Base:
     def select(self, node):
         return self
 
+    def _recurse_all_nodes(self, nodes):
+        return nodes
+
+    def _recurse_no_nodes(self, nodes):
+        return []
+
+    recurse_nodes = _recurse_no_nodes
+
 if not SCons.Memoize.has_metaclass:
     _Base = Base
     class Base(SCons.Memoize.Memoizer, _Base):
diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py
index 6e27ab4..6d532d6 100644
--- a/src/engine/SCons/Script/__init__.py
+++ b/src/engine/SCons/Script/__init__.py
@@ -64,6 +64,7 @@ if "--debug=memoizer" in sys.argv + sconsflags:
 import SCons.Action
 import SCons.Builder
 import SCons.Environment
+import SCons.Node.FS
 import SCons.Options
 import SCons.Platform
 import SCons.Scanner
@@ -108,6 +109,7 @@ Touch                   = SCons.Defaults.Touch
 # Pre-made, public scanners.
 CScanner                = SCons.Tool.CScanner
 DScanner                = SCons.Tool.DScanner
+DirScanner              = SCons.Defaults.DirScanner
 ProgramScanner          = SCons.Tool.ProgramScanner
 SourceFileScanner       = SCons.Tool.SourceFileScanner
 
diff --git a/src/engine/SCons/Tool/tar.py b/src/engine/SCons/Tool/tar.py
index 75d2038..079865e 100644
--- a/src/engine/SCons/Tool/tar.py
+++ b/src/engine/SCons/Tool/tar.py
@@ -35,6 +35,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 import SCons.Action
 import SCons.Builder
+import SCons.Defaults
 import SCons.Node.FS
 import SCons.Util
 
@@ -44,7 +45,8 @@ TarAction = SCons.Action.Action('$TARCOM', '$TARCOMSTR')
 
 TarBuilder = SCons.Builder.Builder(action = TarAction,
                                    source_factory = SCons.Node.FS.default_fs.Entry,
-				   suffix = '$TARSUFFIX',
+                                   source_scanner = SCons.Defaults.DirScanner,
+                                   suffix = '$TARSUFFIX',
                                    multi = 1)
 
 
diff --git a/src/engine/SCons/Tool/zip.py b/src/engine/SCons/Tool/zip.py
index b32f024..b67528b 100644
--- a/src/engine/SCons/Tool/zip.py
+++ b/src/engine/SCons/Tool/zip.py
@@ -70,6 +70,7 @@ zipAction = SCons.Action.Action(zip, varlist=['ZIPCOMPRESSION'])
 
 ZipBuilder = SCons.Builder.Builder(action = SCons.Action.Action('$ZIPCOM', '$ZIPCOMSTR'),
                                    source_factory = SCons.Node.FS.default_fs.Entry,
+                                   source_scanner = SCons.Defaults.DirScanner,
                                    suffix = '$ZIPSUFFIX',
                                    multi = 1)
 
diff --git a/test/DirSource.py b/test/DirSource.py
index 84d8185..6d225c6 100644
--- a/test/DirSource.py
+++ b/test/DirSource.py
@@ -38,59 +38,124 @@ import TestSCons
 test = TestSCons.TestSCons()
 
 test.subdir('bsig', [ 'bsig', 'subdir' ],
-            'csig', [ 'csig', 'subdir' ])
-test.write([ 'bsig', 'foo.txt' ], 'foo.txt\n')
-test.write([ 'bsig', 'subdir', 'bar.txt'], 'bar.txt\n')
-test.write([ 'csig', 'foo.txt' ], 'foo.txt\n')
-test.write([ 'csig', 'subdir', 'bar.txt' ], 'bar.txt\n')
-test.write('junk.txt', 'junk.txt\n')
+            'csig', [ 'csig', 'subdir' ],
+            'cmd-bsig', [ 'cmd-bsig', 'subdir' ],
+            'cmd-csig', [ 'cmd-csig', 'subdir' ])
 
-test.write('SConstruct',
-"""def writeTarget(target, source, env):
+test.write('SConstruct', """\
+def writeTarget(target, source, env):
     f=open(str(target[0]), 'wb')
     f.write("stuff\\n")
     f.close()
     return 0
 
-test_bld_dir = Builder(action=writeTarget, source_factory=Dir)
+test_bld_dir = Builder(action=writeTarget,
+                       source_factory=Dir,
+                       source_scanner=DirScanner)
 test_bld_file = Builder(action=writeTarget)
-env_bsig = Environment()
-env_bsig['BUILDERS']['TestDir'] = test_bld_dir
-env_bsig['BUILDERS']['TestFile'] = test_bld_file
+env = Environment()
+env['BUILDERS']['TestDir'] = test_bld_dir
+env['BUILDERS']['TestFile'] = test_bld_file
 
+env_bsig = env.Copy()
 env_bsig.TargetSignatures('build')
 env_bsig.TestFile(source='junk.txt', target='bsig/junk.out')
 env_bsig.TestDir(source='bsig', target='bsig.out')
+env_bsig.Command('cmd-bsig-noscan.out', 'cmd-bsig', writeTarget)
+env_bsig.Command('cmd-bsig.out', 'cmd-bsig', writeTarget,
+                 source_scanner=DirScanner)
 
-env_csig = env_bsig.Copy()
+env_csig = env.Copy()
 env_csig.TargetSignatures('content')
 env_csig.TestFile(source='junk.txt', target='csig/junk.out')
 env_csig.TestDir(source='csig', target='csig.out')
+env_csig.Command('cmd-csig-noscan.out', 'cmd-csig', writeTarget)
+env_csig.Command('cmd-csig.out', 'cmd-csig', writeTarget,
+                 source_scanner=DirScanner)
 """)
 
+test.write([ 'bsig', 'foo.txt' ], 'foo.txt 1\n')
+test.write([ 'bsig', 'subdir', 'bar.txt'], 'bar.txt 1\n')
+test.write([ 'csig', 'foo.txt' ], 'foo.txt 1\n')
+test.write([ 'csig', 'subdir', 'bar.txt' ], 'bar.txt 1\n')
+test.write([ 'cmd-bsig', 'foo.txt' ], 'foo.txt 1\n')
+test.write([ 'cmd-bsig', 'subdir', 'bar.txt' ], 'bar.txt 1\n')
+test.write([ 'cmd-csig', 'foo.txt' ], 'foo.txt 1\n')
+test.write([ 'cmd-csig', 'subdir', 'bar.txt' ], 'bar.txt 1\n')
+test.write('junk.txt', 'junk.txt\n')
+
 test.run(arguments=".", stderr=None)
 test.must_match('bsig.out', 'stuff\n')
 test.must_match('csig.out', 'stuff\n')
+test.must_match('cmd-bsig.out', 'stuff\n')
+test.must_match('cmd-csig.out', 'stuff\n')
+test.must_match('cmd-bsig-noscan.out', 'stuff\n')
+test.must_match('cmd-csig-noscan.out', 'stuff\n')
 
 test.up_to_date(arguments='bsig.out')
 test.up_to_date(arguments='csig.out')
+test.up_to_date(arguments='cmd-bsig.out')
+test.up_to_date(arguments='cmd-csig.out')
+test.up_to_date(arguments='cmd-bsig-noscan.out')
+test.up_to_date(arguments='cmd-csig-noscan.out')
+
+test.write([ 'bsig', 'foo.txt' ], 'foo.txt 2\n')
+test.not_up_to_date(arguments='bsig.out')
 
-test.write([ 'bsig', 'foo.txt' ], 'foo2.txt\n')
+test.write([ 'bsig', 'new.txt' ], 'new.txt\n')
 test.not_up_to_date(arguments='bsig.out')
 
-test.write([ 'csig', 'foo.txt' ], 'foo2.txt\n')
+test.write([ 'csig', 'foo.txt' ], 'foo.txt 2\n')
 test.not_up_to_date(arguments='csig.out')
 
-test.write([ 'bsig', 'foo.txt' ], 'foo3.txt\n')
+test.write([ 'csig', 'new.txt' ], 'new.txt\n')
+test.not_up_to_date(arguments='csig.out')
+
+test.write([ 'cmd-bsig', 'foo.txt' ], 'foo.txt 2\n')
+test.not_up_to_date(arguments='cmd-bsig.out')
+test.up_to_date(arguments='cmd-bsig-noscan.out')
+
+test.write([ 'cmd-bsig', 'new.txt' ], 'new.txt\n')
+test.not_up_to_date(arguments='cmd-bsig.out')
+test.up_to_date(arguments='cmd-bsig-noscan.out')
+
+test.write([ 'cmd-csig', 'foo.txt' ], 'foo.txt 2\n')
+test.not_up_to_date(arguments='cmd-csig.out')
+test.up_to_date(arguments='cmd-csig-noscan.out')
+
+test.write([ 'cmd-csig', 'new.txt' ], 'new.txt\n')
+test.not_up_to_date(arguments='cmd-csig.out')
+test.up_to_date(arguments='cmd-csig-noscan.out')
+
+test.write([ 'bsig', 'subdir', 'bar.txt' ], 'bar.txt 2\n')
 test.not_up_to_date(arguments='bsig.out')
 
-test.write([ 'bsig', 'subdir', 'bar.txt' ], 'bar2.txt\n')
+test.write([ 'bsig', 'subdir', 'new.txt' ], 'new.txt\n')
 test.not_up_to_date(arguments='bsig.out')
 
-test.write([ 'csig', 'subdir', 'bar.txt' ], 'bar2.txt\n')
+test.write([ 'csig', 'subdir', 'bar.txt' ], 'bar.txt 2\n')
+test.not_up_to_date(arguments='csig.out')
+
+test.write([ 'csig', 'subdir', 'new.txt' ], 'new.txt\n')
 test.not_up_to_date(arguments='csig.out')
 
-test.write('junk.txt', 'junk2.txt\n')
+test.write([ 'cmd-bsig', 'subdir', 'bar.txt' ], 'bar.txt 2\n')
+test.not_up_to_date(arguments='cmd-bsig.out')
+test.up_to_date(arguments='cmd-bsig-noscan.out')
+
+test.write([ 'cmd-bsig', 'subdir', 'new.txt' ], 'new.txt\n')
+test.not_up_to_date(arguments='cmd-bsig.out')
+test.up_to_date(arguments='cmd-bsig-noscan.out')
+
+test.write([ 'cmd-csig', 'subdir', 'bar.txt' ], 'bar.txt 2\n')
+test.not_up_to_date(arguments='cmd-csig.out')
+test.up_to_date(arguments='cmd-csig-noscan.out')
+
+test.write([ 'cmd-csig', 'subdir', 'new.txt' ], 'new.txt\n')
+test.not_up_to_date(arguments='cmd-csig.out')
+test.up_to_date(arguments='cmd-csig-noscan.out')
+
+test.write('junk.txt', 'junk.txt 2\n')
 test.not_up_to_date(arguments='bsig.out')
 # XXX For some reason, 'csig' is still reported as up to date.
 # XXX Comment out this test until someone can look at it.
diff --git a/test/option/debug-tree.py b/test/option/debug-tree.py
index 4bb1229..e7847af 100644
--- a/test/option/debug-tree.py
+++ b/test/option/debug-tree.py
@@ -89,13 +89,11 @@ tree2 = """
 +-.
   +-SConstruct
   +-bar.c
-  +-bar.h
   +-bar.ooo
   | +-bar.c
   | +-bar.h
   | +-foo.h
   +-foo.c
-  +-foo.h
   +-foo.ooo
   | +-foo.c
   | +-foo.h