From 08d7c4cd103fb39b6010b980209a777ceea1ead2 Mon Sep 17 00:00:00 2001 From: Steven Knight Date: Mon, 14 Feb 2005 03:22:34 +0000 Subject: Don't read up entire directories to decide if an Alias is up-to-date. --- doc/man/scons.1 | 48 +++++++++++++- src/CHANGES.txt | 9 +++ src/RELEASE.txt | 22 +++++++ src/engine/MANIFEST.in | 1 + src/engine/SCons/Defaults.py | 6 ++ src/engine/SCons/Environment.py | 10 ++- src/engine/SCons/EnvironmentTests.py | 6 ++ src/engine/SCons/Node/FS.py | 81 ++++++++++++++---------- src/engine/SCons/Node/FSTests.py | 29 +++++++++ src/engine/SCons/Node/NodeTests.py | 33 +++++----- src/engine/SCons/Node/__init__.py | 22 +++---- src/engine/SCons/Scanner/Dir.py | 62 +++++++++++++++++++ src/engine/SCons/Scanner/DirTests.py | 80 ++++++++++++++++++++++++ src/engine/SCons/Scanner/ScannerTests.py | 26 ++++++-- src/engine/SCons/Scanner/__init__.py | 23 ++++++- src/engine/SCons/Script/__init__.py | 2 + src/engine/SCons/Tool/tar.py | 4 +- src/engine/SCons/Tool/zip.py | 1 + test/DirSource.py | 103 +++++++++++++++++++++++++------ test/option/debug-tree.py | 2 - 20 files changed, 473 insertions(+), 97 deletions(-) create mode 100644 src/engine/SCons/Scanner/Dir.py create mode 100644 src/engine/SCons/Scanner/DirTests.py diff --git a/doc/man/scons.1 b/doc/man/scons.1 index db1da8c..690803f 100644 --- a/doc/man/scons.1 +++ b/doc/man/scons.1 @@ -2107,6 +2107,12 @@ for a given target; each additional call adds to the list of entries that will be built into the archive. +Any source directories will +be scanned for changes to +any on-disk files, +regardless of whether or not +.B scons +knows about them from other Builder or function calls. .ES env.Tar('src.tar', 'src') @@ -2173,6 +2179,12 @@ for a given target; each additional call adds to the list of entries that will be built into the archive. +Any source directories will +be scanned for changes to +any on-disk files, +regardless of whether or not +.B scons +knows about them from other Builder or function calls. .ES env.Zip('src.zip', 'src') @@ -2720,7 +2732,22 @@ to build a target file or files. This is more convenient than defining a separate Builder object for a single special-case build. -Any keyword arguments specified override any + +As a special case, the +.B source_scanner +keyword argument can +be used to specify +a Scanner object +that will be used to scan the sources. +(The global +.B DirScanner +object can be used +if any of the sources will be directories +that must be scanned on-disk for +changes to files that aren't +already specified in other Builder of function calls.) + +Any other keyword arguments specified override any same-named existing construction variables. Note that an action can be an external command, @@ -8157,8 +8184,17 @@ specify a scanner to find things like .B #include lines in source files. +The pre-built +.B DirScanner +Scanner object may be used to +indicate that this Builder +should scan directory trees +for on-disk changes to files +that +.B scons +does not know about from other Builder or function calls. (See the section "Scanner Objects," below, -for information about creating Scanner objects.) +for information about creating your own Scanner objects.) .IP target_factory A factory function that the Builder will use @@ -9222,6 +9258,14 @@ only invoke the scanner on the file being scanned, and not (for example) also on the files specified by the #include lines in the file being scanned. +.I recursive +may be a callable function, +in which case it will be called with a list of +Nodes found and +should return a list of Nodes +that should be scanned recursively; +this can be used to select a specific subset of +Nodes for additional scanning. Note that .B scons diff --git a/src/CHANGES.txt b/src/CHANGES.txt index dbd0685..2dbb6bf 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -198,6 +198,15 @@ RELEASE 0.97 - XXX part of the public interface. Keep the old SCons.Defaults.*Scan names around for a while longer since some people were already using them. + - By default, don't scan directories for on-disk files. Add a + DirScanner global scanner that can be used in Builders or Command() + calls that want source directory trees scanned for on-disk changes. + Have the Tar() and Zip() Builders use the new DirScanner to preserve + the behavior of rebuilding a .tar or .zip file if any file or + directory under a source tree changes. Add Command() support for + a source_scanner keyword argument to Command() that can be set to + DirScanner to get this behavior. + From Wayne Lee: - Avoid "maximum recursion limit" errors when removing $(-$) pairs diff --git a/src/RELEASE.txt b/src/RELEASE.txt index 2592f5c..4089d5a 100644 --- a/src/RELEASE.txt +++ b/src/RELEASE.txt @@ -37,6 +37,28 @@ RELEASE 0.97 - XXX entries to construction variables. The old behavior may be specified using a new "unique=0" keyword argument. + - Custom builders that accept directories as source arguments no + longer scan entire directory trees by default. This means that + their targets will not be automatically rebuilt if a file that + SCons does *not* already know about changes on disk. Note that + the targets *will* still be rebuilt correctly if a file changes + that SCons already knows about due to a Builder or other call. + + The existing behavior of scanning directory trees for any changed + file on-disk can be maintained by passing the new DirScanner global + directory scanner as the source_scanner keyword argument to the + Builder call: + + bld = Builder("build < $SOURCE > $TARGET", + source_scanner = DirScanner) + + The same keyword argument can also be supplied to any Command() + calls that need to scan directory trees on-disk for changed files: + + env.Command("archive.out", "directory", + "archiver -o $TARGET $SOURCE", + source_scanner = DirScanner) + - When compiling with Microsoft Visual Studio, SCons no longer adds the ATL and MFC directories to the INCLUDE and LIB environment variables by default. If you want these directories diff --git a/src/engine/MANIFEST.in b/src/engine/MANIFEST.in index 5c10cae..4888736 100644 --- a/src/engine/MANIFEST.in +++ b/src/engine/MANIFEST.in @@ -38,6 +38,7 @@ SCons/Platform/win32.py SCons/Scanner/__init__.py SCons/Scanner/C.py SCons/Scanner/D.py +SCons/Scanner/Dir.py SCons/Scanner/Fortran.py SCons/Scanner/IDL.py SCons/Scanner/Prog.py diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py index 032a067..8ade792 100644 --- a/src/engine/SCons/Defaults.py +++ b/src/engine/SCons/Defaults.py @@ -102,6 +102,12 @@ DScan = SCons.Tool.DScanner ObjSourceScan = SCons.Tool.SourceFileScanner ProgScan = SCons.Tool.ProgramScanner +# This isn't really a tool scanner, so it doesn't quite belong with +# the rest of those in Tool/__init__.py, but I'm not sure where else it +# should go. Leave it here for now. +import SCons.Scanner.Dir +DirScanner = SCons.Scanner.Dir.DirScanner() + # Actions for common languages. CAction = SCons.Action.Action("$CCCOM", "$CCCOMSTR") ShCAction = SCons.Action.Action("$SHCCCOM", "$SHCCCOMSTR") diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py index 419e4e1..9470623 100644 --- a/src/engine/SCons/Environment.py +++ b/src/engine/SCons/Environment.py @@ -1202,8 +1202,14 @@ class Base(SubstitutionEnvironment): source files using the supplied action. Action may be any type that the Builder constructor will accept for an action.""" - bld = SCons.Builder.Builder(action = action, - source_factory = self.fs.Entry) + bkw = { + 'action' : action, + 'source_factory' : self.fs.Entry, + } + try: bkw['source_scanner'] = kw['source_scanner'] + except KeyError: pass + else: del kw['source_scanner'] + bld = apply(SCons.Builder.Builder, (), bkw) return apply(bld, (self, target, source), kw) def Depends(self, target, dependency): diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py index 3ef5e73..bfd1262 100644 --- a/src/engine/SCons/EnvironmentTests.py +++ b/src/engine/SCons/EnvironmentTests.py @@ -2171,6 +2171,12 @@ f5: \ assert str(t) == 'xxx.out', str(t) assert 'xxx.in' in map(lambda x: x.path, t.sources) + env = Environment(source_scanner = 'should_not_find_this') + t = env.Command(target='file.out', source='file.in', + action = 'foo', + source_scanner = 'fake')[0] + assert t.builder.source_scanner == 'fake', t.builder.source_scanner + def test_Configure(self): """Test the Configure() method""" # Configure() will write to a local temporary file. diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 314faf8..cc0fe95 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -599,6 +599,16 @@ class Entry(Base): time comes, and then call the same-named method in the transformed class.""" + def disambiguate(self): + if self.fs.isdir(self.abspath): + self.__class__ = Dir + self._morph() + else: + self.__class__ = File + self._morph() + self.clear() + return self + def rfile(self): """We're a generic Entry, but the caller is actually looking for a File at this point, so morph into one.""" @@ -610,8 +620,7 @@ class Entry(Base): def get_found_includes(self, env, scanner, path): """If we're looking for included files, it's because this Entry is really supposed to be a File itself.""" - node = self.rfile() - return node.get_found_includes(env, scanner, path) + return self.disambiguate().get_found_includes(env, scanner, path) def scanner_key(self): return self.get_suffix() @@ -638,29 +647,13 @@ class Entry(Base): """Return if the Entry exists. Check the file system to see what we should turn into first. Assume a file if there's no directory.""" - if self.fs.isdir(self.abspath): - self.__class__ = Dir - self._morph() - return Dir.exists(self) - else: - self.__class__ = File - self._morph() - self.clear() - return File.exists(self) + return self.disambiguate().exists() def calc_signature(self, calc=None): """Return the Entry's calculated signature. Check the file system to see what we should turn into first. Assume a file if there's no directory.""" - if self.fs.isdir(self.abspath): - self.__class__ = Dir - self._morph() - return Dir.calc_signature(self, calc) - else: - self.__class__ = File - self._morph() - self.clear() - return File.calc_signature(self, calc) + return self.disambiguate().calc_signature(calc) def must_be_a_Dir(self): """Called to make sure a Node is a Dir. Since we're an @@ -1180,6 +1173,9 @@ class Dir(Base): self._sconsign = None self.build_dirs = [] + def disambiguate(self): + return self + def __clearRepositoryCache(self, duplicate=None): """Called when we change the repository(ies) for a directory. This clears any cached information that is invalidated by changing @@ -1256,19 +1252,33 @@ class Dir(Base): self.implicit = [] self.implicit_dict = {} self._children_reset() - try: - for filename in self.fs.listdir(self.abspath): - if filename != '.sconsign': - self.Entry(filename) - except OSError: - # Directory does not exist. No big deal - pass - keys = filter(lambda k: k != '.' and k != '..', self.entries.keys()) - kids = map(lambda x, s=self: s.entries[x], keys) - def c(one, two): - return cmp(one.abspath, two.abspath) - kids.sort(c) - self._add_child(self.implicit, self.implicit_dict, kids) + + dont_scan = lambda k: k not in ['.', '..', '.sconsign'] + deps = filter(dont_scan, self.entries.keys()) + # keys() is going to give back the entries in an internal, + # unsorted order. Sort 'em so the order is deterministic. + deps.sort() + entries = map(lambda n, e=self.entries: e[n], deps) + + self._add_child(self.implicit, self.implicit_dict, entries) + + def get_found_includes(self, env, scanner, path): + """Return the included implicit dependencies in this file. + Cache results so we only scan the file once per path + regardless of how many times this information is requested. + __cacheable__""" + if not scanner: + return [] + # Clear cached info for this Node. If we already visited this + # directory on our walk down the tree (because we didn't know at + # that point it was being used as the source for another Node) + # then we may have calculated build signature before realizing + # we had to scan the disk. Now that we have to, though, we need + # to invalidate the old calculated signature so that any node + # dependent on our directory structure gets one that includes + # info about everything on disk. + self.clear() + return scanner(self, env, path) def build(self, **kw): """A null "builder" for directories.""" @@ -1295,7 +1305,7 @@ class Dir(Base): for kid in self.children(): contents.write(kid.get_contents()) return contents.getvalue() - + def prepare(self): pass @@ -1464,6 +1474,9 @@ class File(Base): if not hasattr(self, '_local'): self._local = 0 + def disambiguate(self): + return self + def root(self): return self.dir.root() diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 2846f64..99a95b6 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -62,6 +62,8 @@ class Scanner: return self.hash def select(self, node): return self + def recurse_nodes(self, nodes): + return nodes class Environment: def __init__(self): @@ -1876,6 +1878,32 @@ class clearTestCase(unittest.TestCase): assert not f.rexists() assert str(f) == test.workpath('f'), str(f) +class disambiguateTestCase(unittest.TestCase): + def runTest(self): + """Test calling the disambiguate() method.""" + test = TestCmd(workdir='') + + fs = SCons.Node.FS.FS() + + ddd = fs.Dir('ddd') + d = ddd.disambiguate() + assert d is ddd, d + + fff = fs.File('fff') + f = fff.disambiguate() + assert f is fff, f + + test.subdir('edir') + test.write('efile', "efile\n") + + edir = fs.Entry(test.workpath('edir')) + d = edir.disambiguate() + assert d.__class__ is ddd.__class__, d.__class__ + + efile = fs.Entry(test.workpath('efile')) + f = efile.disambiguate() + assert f.__class__ is fff.__class__, f.__class__ + class postprocessTestCase(unittest.TestCase): def runTest(self): """Test calling the postprocess() method.""" @@ -2108,6 +2136,7 @@ if __name__ == "__main__": suite.addTest(SConstruct_dirTestCase()) suite.addTest(CacheDirTestCase()) suite.addTest(clearTestCase()) + suite.addTest(disambiguateTestCase()) suite.addTest(postprocessTestCase()) suite.addTest(SpecialAttrTestCase()) suite.addTest(SaveStringsTestCase()) diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index 281b5f2..90bb332 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -181,6 +181,8 @@ class Scanner: return () def select(self, node): return self + def recurse_nodes(self, nodes): + return nodes class MyNode(SCons.Node.Node): """The base Node class contains a number of do-nothing methods that @@ -807,28 +809,31 @@ class NodeTestCase(unittest.TestCase): deps = node.get_implicit_deps(env, s, target) assert deps == [d], deps - # No "recursive" attribute on scanner doesn't recurse + # By default, our fake scanner recurses e = MyNode("eee") - d.found_includes = [e] + f = MyNode("fff") + g = MyNode("ggg") + d.found_includes = [e, f] + f.found_includes = [g] deps = node.get_implicit_deps(env, s, target) - assert deps == [d], map(str, deps) + assert deps == [d, e, f, g], map(str, deps) - # Explicit "recursive" attribute on scanner doesn't recurse - s.recursive = None + # Recursive scanning eliminates duplicates + e.found_includes = [f] deps = node.get_implicit_deps(env, s, target) - assert deps == [d], map(str, deps) + assert deps == [d, e, f, g], map(str, deps) - # Explicit "recursive" attribute on scanner which does recurse - s.recursive = 1 + # Scanner method can select specific nodes to recurse + def no_fff(nodes): + return filter(lambda n: str(n)[0] != 'f', nodes) + s.recurse_nodes = no_fff deps = node.get_implicit_deps(env, s, target) - assert deps == [d, e], map(str, deps) + assert deps == [d, e, f], map(str, deps) - # Recursive scanning eliminates duplicates - f = MyNode("fff") - d.found_includes = [e, f] - e.found_includes = [f] + # Scanner method can short-circuit recursing entirely + s.recurse_nodes = lambda nodes: [] deps = node.get_implicit_deps(env, s, target) - assert deps == [d, e, f], map(str, deps) + assert deps == [d], map(str, deps) def test_get_scanner(self): """Test fetching the environment scanner for a Node diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 96a78ca..3c0ce99 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -394,25 +394,19 @@ class Node: # for this Node. scanner = scanner.select(self) - try: - recurse = scanner.recursive - except AttributeError: - recurse = None - nodes = [self] seen = {} seen[self] = 1 deps = [] while nodes: - n = nodes.pop(0) - d = filter(lambda x, seen=seen: not seen.has_key(x), - n.get_found_includes(env, scanner, path)) - if d: - deps.extend(d) - for n in d: - seen[n] = 1 - if recurse: - nodes.extend(d) + n = nodes.pop(0) + d = filter(lambda x, seen=seen: not seen.has_key(x), + n.get_found_includes(env, scanner, path)) + if d: + deps.extend(d) + for n in d: + seen[n] = 1 + nodes.extend(scanner.recurse_nodes(d)) return deps diff --git a/src/engine/SCons/Scanner/Dir.py b/src/engine/SCons/Scanner/Dir.py new file mode 100644 index 0000000..6161059 --- /dev/null +++ b/src/engine/SCons/Scanner/Dir.py @@ -0,0 +1,62 @@ +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import string + +import SCons.Node.FS +import SCons.Scanner + +def DirScanner(fs = SCons.Node.FS.default_fs, **kw): + """Return a prototype Scanner instance for scanning + directories for on-disk files""" + def only_dirs(nodes, fs=fs): + return filter(lambda n: isinstance(n.disambiguate(), SCons.Node.FS.Dir), nodes) + kw['node_factory'] = fs.Entry + kw['recursive'] = only_dirs + ds = apply(SCons.Scanner.Base, [scan, "DirScanner"], kw) + return ds + +skip_entry = { + '.' : 1, + '..' : 1, + '.sconsign' : 1, + '.sconsign.dblite' : 1, +} + +def scan(node, env, path=()): + """ + This scanner scans program files for static-library + dependencies. It will search the LIBPATH environment variable + for libraries specified in the LIBS variable, returning any + files it finds as dependencies. + """ + try: + flist = node.fs.listdir(node.abspath) + except OSError: + return [] + dont_scan = lambda k: not skip_entry.has_key(k) + flist = filter(dont_scan, flist) + flist.sort() + return map(node.Entry, flist) diff --git a/src/engine/SCons/Scanner/DirTests.py b/src/engine/SCons/Scanner/DirTests.py new file mode 100644 index 0000000..e735ca2 --- /dev/null +++ b/src/engine/SCons/Scanner/DirTests.py @@ -0,0 +1,80 @@ +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os.path +import string +import sys +import types +import unittest + +import TestCmd +import SCons.Node.FS +import SCons.Scanner.Dir + +test = TestCmd.TestCmd(workdir = '') + +test.subdir('dir', ['dir', 'sub']) + +test.write(['dir', 'f1'], "dir/f1\n") +test.write(['dir', 'f2'], "dir/f2\n") +test.write(['dir', '.sconsign'], "dir/.sconsign\n") +test.write(['dir', '.sconsign.dblite'], "dir/.sconsign.dblite\n") +test.write(['dir', 'sub', 'f3'], "dir/sub/f3\n") +test.write(['dir', 'sub', 'f4'], "dir/sub/f4\n") +test.write(['dir', 'sub', '.sconsign'], "dir/.sconsign\n") +test.write(['dir', 'sub', '.sconsign.dblite'], "dir/.sconsign.dblite\n") + +class DummyNode: + def __init__(self, name): + self.name = name + self.abspath = test.workpath(name) + self.fs = SCons.Node.FS.default_fs + def __str__(self): + return self.name + def Entry(self, name): + return self.fs.Entry(name) + +class DirScannerTestCase1(unittest.TestCase): + def runTest(self): + s = SCons.Scanner.Dir.DirScanner() + + deps = s(DummyNode('dir'), {}, ()) + sss = map(str, deps) + assert sss == ['f1', 'f2', 'sub'], sss + + deps = s(DummyNode('dir/sub'), {}, ()) + sss = map(str, deps) + assert sss == ['f3', 'f4'], sss + +def suite(): + suite = unittest.TestSuite() + suite.addTest(DirScannerTestCase1()) + return suite + +if __name__ == "__main__": + runner = unittest.TextTestRunner() + result = runner.run(suite()) + if not result.wasSuccessful(): + sys.exit(1) diff --git a/src/engine/SCons/Scanner/ScannerTests.py b/src/engine/SCons/Scanner/ScannerTests.py index c38dc84..ce5411c 100644 --- a/src/engine/SCons/Scanner/ScannerTests.py +++ b/src/engine/SCons/Scanner/ScannerTests.py @@ -221,15 +221,29 @@ class BaseTestCase(unittest.TestCase): def test_recursive(self): """Test the Scanner.Base class recursive flag""" + nodes = [1, 2, 3, 4] + s = SCons.Scanner.Base(function = self.func) - self.failUnless(s.recursive == None, - "incorrect default recursive value") + n = s.recurse_nodes(nodes) + self.failUnless(n == [], + "default behavior returned nodes: %s" % n) + s = SCons.Scanner.Base(function = self.func, recursive = None) - self.failUnless(s.recursive == None, - "did not set recursive flag to None") + n = s.recurse_nodes(nodes) + self.failUnless(n == [], + "recursive = None returned nodes: %s" % n) + s = SCons.Scanner.Base(function = self.func, recursive = 1) - self.failUnless(s.recursive == 1, - "did not set recursive flag to 1") + n = s.recurse_nodes(nodes) + self.failUnless(n == n, + "recursive = 1 didn't return all nodes: %s" % n) + + def odd_only(nodes): + return filter(lambda n: n % 2, nodes) + s = SCons.Scanner.Base(function = self.func, recursive = odd_only) + n = s.recurse_nodes(nodes) + self.failUnless(n == [1, 3], + "recursive = 1 didn't return all nodes: %s" % n) def test_get_skeys(self): """Test the Scanner.Base get_skeys() method""" diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py index 3f7ead4..cda156c 100644 --- a/src/engine/SCons/Scanner/__init__.py +++ b/src/engine/SCons/Scanner/__init__.py @@ -148,8 +148,12 @@ class Base: this node really needs to be scanned. 'recursive' - specifies that this scanner should be invoked - recursively on the implicit dependencies it returns (the - canonical example being #include lines in C source files). + recursively on all of the implicit dependencies it returns + (the canonical example being #include lines in C source files). + May be a callable, which will be called to filter the list + of nodes found to select a subset for recursive scanning + (the canonical example being only recursively scanning + subdirectories within a directory). The scanner function's first argument will be the a Node that should be scanned for dependencies, the second argument will @@ -182,7 +186,12 @@ class Base: self.node_class = node_class self.node_factory = node_factory self.scan_check = scan_check - self.recursive = recursive + if callable(recursive): + self.recurse_nodes = recursive + elif recursive: + self.recurse_nodes = self._recurse_all_nodes + else: + self.recurse_nodes = self._recurse_no_nodes def path(self, env, dir=None, target=None, source=None): if not self.path_function: @@ -241,6 +250,14 @@ class Base: def select(self, node): return self + def _recurse_all_nodes(self, nodes): + return nodes + + def _recurse_no_nodes(self, nodes): + return [] + + recurse_nodes = _recurse_no_nodes + if not SCons.Memoize.has_metaclass: _Base = Base class Base(SCons.Memoize.Memoizer, _Base): diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index 6e27ab4..6d532d6 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -64,6 +64,7 @@ if "--debug=memoizer" in sys.argv + sconsflags: import SCons.Action import SCons.Builder import SCons.Environment +import SCons.Node.FS import SCons.Options import SCons.Platform import SCons.Scanner @@ -108,6 +109,7 @@ Touch = SCons.Defaults.Touch # Pre-made, public scanners. CScanner = SCons.Tool.CScanner DScanner = SCons.Tool.DScanner +DirScanner = SCons.Defaults.DirScanner ProgramScanner = SCons.Tool.ProgramScanner SourceFileScanner = SCons.Tool.SourceFileScanner diff --git a/src/engine/SCons/Tool/tar.py b/src/engine/SCons/Tool/tar.py index 75d2038..079865e 100644 --- a/src/engine/SCons/Tool/tar.py +++ b/src/engine/SCons/Tool/tar.py @@ -35,6 +35,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import SCons.Action import SCons.Builder +import SCons.Defaults import SCons.Node.FS import SCons.Util @@ -44,7 +45,8 @@ TarAction = SCons.Action.Action('$TARCOM', '$TARCOMSTR') TarBuilder = SCons.Builder.Builder(action = TarAction, source_factory = SCons.Node.FS.default_fs.Entry, - suffix = '$TARSUFFIX', + source_scanner = SCons.Defaults.DirScanner, + suffix = '$TARSUFFIX', multi = 1) diff --git a/src/engine/SCons/Tool/zip.py b/src/engine/SCons/Tool/zip.py index b32f024..b67528b 100644 --- a/src/engine/SCons/Tool/zip.py +++ b/src/engine/SCons/Tool/zip.py @@ -70,6 +70,7 @@ zipAction = SCons.Action.Action(zip, varlist=['ZIPCOMPRESSION']) ZipBuilder = SCons.Builder.Builder(action = SCons.Action.Action('$ZIPCOM', '$ZIPCOMSTR'), source_factory = SCons.Node.FS.default_fs.Entry, + source_scanner = SCons.Defaults.DirScanner, suffix = '$ZIPSUFFIX', multi = 1) diff --git a/test/DirSource.py b/test/DirSource.py index 84d8185..6d225c6 100644 --- a/test/DirSource.py +++ b/test/DirSource.py @@ -38,59 +38,124 @@ import TestSCons test = TestSCons.TestSCons() test.subdir('bsig', [ 'bsig', 'subdir' ], - 'csig', [ 'csig', 'subdir' ]) -test.write([ 'bsig', 'foo.txt' ], 'foo.txt\n') -test.write([ 'bsig', 'subdir', 'bar.txt'], 'bar.txt\n') -test.write([ 'csig', 'foo.txt' ], 'foo.txt\n') -test.write([ 'csig', 'subdir', 'bar.txt' ], 'bar.txt\n') -test.write('junk.txt', 'junk.txt\n') + 'csig', [ 'csig', 'subdir' ], + 'cmd-bsig', [ 'cmd-bsig', 'subdir' ], + 'cmd-csig', [ 'cmd-csig', 'subdir' ]) -test.write('SConstruct', -"""def writeTarget(target, source, env): +test.write('SConstruct', """\ +def writeTarget(target, source, env): f=open(str(target[0]), 'wb') f.write("stuff\\n") f.close() return 0 -test_bld_dir = Builder(action=writeTarget, source_factory=Dir) +test_bld_dir = Builder(action=writeTarget, + source_factory=Dir, + source_scanner=DirScanner) test_bld_file = Builder(action=writeTarget) -env_bsig = Environment() -env_bsig['BUILDERS']['TestDir'] = test_bld_dir -env_bsig['BUILDERS']['TestFile'] = test_bld_file +env = Environment() +env['BUILDERS']['TestDir'] = test_bld_dir +env['BUILDERS']['TestFile'] = test_bld_file +env_bsig = env.Copy() env_bsig.TargetSignatures('build') env_bsig.TestFile(source='junk.txt', target='bsig/junk.out') env_bsig.TestDir(source='bsig', target='bsig.out') +env_bsig.Command('cmd-bsig-noscan.out', 'cmd-bsig', writeTarget) +env_bsig.Command('cmd-bsig.out', 'cmd-bsig', writeTarget, + source_scanner=DirScanner) -env_csig = env_bsig.Copy() +env_csig = env.Copy() env_csig.TargetSignatures('content') env_csig.TestFile(source='junk.txt', target='csig/junk.out') env_csig.TestDir(source='csig', target='csig.out') +env_csig.Command('cmd-csig-noscan.out', 'cmd-csig', writeTarget) +env_csig.Command('cmd-csig.out', 'cmd-csig', writeTarget, + source_scanner=DirScanner) """) +test.write([ 'bsig', 'foo.txt' ], 'foo.txt 1\n') +test.write([ 'bsig', 'subdir', 'bar.txt'], 'bar.txt 1\n') +test.write([ 'csig', 'foo.txt' ], 'foo.txt 1\n') +test.write([ 'csig', 'subdir', 'bar.txt' ], 'bar.txt 1\n') +test.write([ 'cmd-bsig', 'foo.txt' ], 'foo.txt 1\n') +test.write([ 'cmd-bsig', 'subdir', 'bar.txt' ], 'bar.txt 1\n') +test.write([ 'cmd-csig', 'foo.txt' ], 'foo.txt 1\n') +test.write([ 'cmd-csig', 'subdir', 'bar.txt' ], 'bar.txt 1\n') +test.write('junk.txt', 'junk.txt\n') + test.run(arguments=".", stderr=None) test.must_match('bsig.out', 'stuff\n') test.must_match('csig.out', 'stuff\n') +test.must_match('cmd-bsig.out', 'stuff\n') +test.must_match('cmd-csig.out', 'stuff\n') +test.must_match('cmd-bsig-noscan.out', 'stuff\n') +test.must_match('cmd-csig-noscan.out', 'stuff\n') test.up_to_date(arguments='bsig.out') test.up_to_date(arguments='csig.out') +test.up_to_date(arguments='cmd-bsig.out') +test.up_to_date(arguments='cmd-csig.out') +test.up_to_date(arguments='cmd-bsig-noscan.out') +test.up_to_date(arguments='cmd-csig-noscan.out') + +test.write([ 'bsig', 'foo.txt' ], 'foo.txt 2\n') +test.not_up_to_date(arguments='bsig.out') -test.write([ 'bsig', 'foo.txt' ], 'foo2.txt\n') +test.write([ 'bsig', 'new.txt' ], 'new.txt\n') test.not_up_to_date(arguments='bsig.out') -test.write([ 'csig', 'foo.txt' ], 'foo2.txt\n') +test.write([ 'csig', 'foo.txt' ], 'foo.txt 2\n') test.not_up_to_date(arguments='csig.out') -test.write([ 'bsig', 'foo.txt' ], 'foo3.txt\n') +test.write([ 'csig', 'new.txt' ], 'new.txt\n') +test.not_up_to_date(arguments='csig.out') + +test.write([ 'cmd-bsig', 'foo.txt' ], 'foo.txt 2\n') +test.not_up_to_date(arguments='cmd-bsig.out') +test.up_to_date(arguments='cmd-bsig-noscan.out') + +test.write([ 'cmd-bsig', 'new.txt' ], 'new.txt\n') +test.not_up_to_date(arguments='cmd-bsig.out') +test.up_to_date(arguments='cmd-bsig-noscan.out') + +test.write([ 'cmd-csig', 'foo.txt' ], 'foo.txt 2\n') +test.not_up_to_date(arguments='cmd-csig.out') +test.up_to_date(arguments='cmd-csig-noscan.out') + +test.write([ 'cmd-csig', 'new.txt' ], 'new.txt\n') +test.not_up_to_date(arguments='cmd-csig.out') +test.up_to_date(arguments='cmd-csig-noscan.out') + +test.write([ 'bsig', 'subdir', 'bar.txt' ], 'bar.txt 2\n') test.not_up_to_date(arguments='bsig.out') -test.write([ 'bsig', 'subdir', 'bar.txt' ], 'bar2.txt\n') +test.write([ 'bsig', 'subdir', 'new.txt' ], 'new.txt\n') test.not_up_to_date(arguments='bsig.out') -test.write([ 'csig', 'subdir', 'bar.txt' ], 'bar2.txt\n') +test.write([ 'csig', 'subdir', 'bar.txt' ], 'bar.txt 2\n') +test.not_up_to_date(arguments='csig.out') + +test.write([ 'csig', 'subdir', 'new.txt' ], 'new.txt\n') test.not_up_to_date(arguments='csig.out') -test.write('junk.txt', 'junk2.txt\n') +test.write([ 'cmd-bsig', 'subdir', 'bar.txt' ], 'bar.txt 2\n') +test.not_up_to_date(arguments='cmd-bsig.out') +test.up_to_date(arguments='cmd-bsig-noscan.out') + +test.write([ 'cmd-bsig', 'subdir', 'new.txt' ], 'new.txt\n') +test.not_up_to_date(arguments='cmd-bsig.out') +test.up_to_date(arguments='cmd-bsig-noscan.out') + +test.write([ 'cmd-csig', 'subdir', 'bar.txt' ], 'bar.txt 2\n') +test.not_up_to_date(arguments='cmd-csig.out') +test.up_to_date(arguments='cmd-csig-noscan.out') + +test.write([ 'cmd-csig', 'subdir', 'new.txt' ], 'new.txt\n') +test.not_up_to_date(arguments='cmd-csig.out') +test.up_to_date(arguments='cmd-csig-noscan.out') + +test.write('junk.txt', 'junk.txt 2\n') test.not_up_to_date(arguments='bsig.out') # XXX For some reason, 'csig' is still reported as up to date. # XXX Comment out this test until someone can look at it. diff --git a/test/option/debug-tree.py b/test/option/debug-tree.py index 4bb1229..e7847af 100644 --- a/test/option/debug-tree.py +++ b/test/option/debug-tree.py @@ -89,13 +89,11 @@ tree2 = """ +-. +-SConstruct +-bar.c - +-bar.h +-bar.ooo | +-bar.c | +-bar.h | +-foo.h +-foo.c - +-foo.h +-foo.ooo | +-foo.c | +-foo.h -- cgit v0.12