diff options
author | Steven Knight <knight@baldmt.com> | 2003-01-06 18:42:37 (GMT) |
---|---|---|
committer | Steven Knight <knight@baldmt.com> | 2003-01-06 18:42:37 (GMT) |
commit | 7fbd5909a526fc1ad282c7e701b0f7832af2e3ed (patch) | |
tree | 04f622a7b8fdab4ca337f20eced35b4d2699beb6 /src | |
parent | 6702e9dce5182eaa012da9dc511fcf85cf205049 (diff) | |
download | SCons-7fbd5909a526fc1ad282c7e701b0f7832af2e3ed.zip SCons-7fbd5909a526fc1ad282c7e701b0f7832af2e3ed.tar.gz SCons-7fbd5909a526fc1ad282c7e701b0f7832af2e3ed.tar.bz2 |
Refactor the Scanner interface to eliminate unnecessary scanning and make it easier to write efficient scanners.
Diffstat (limited to 'src')
-rw-r--r-- | src/CHANGES.txt | 4 | ||||
-rw-r--r-- | src/RELEASE.txt | 9 | ||||
-rw-r--r-- | src/engine/SCons/Defaults.py | 8 | ||||
-rw-r--r-- | src/engine/SCons/EnvironmentTests.py | 12 | ||||
-rw-r--r-- | src/engine/SCons/Node/FS.py | 23 | ||||
-rw-r--r-- | src/engine/SCons/Node/FSTests.py | 33 | ||||
-rw-r--r-- | src/engine/SCons/Node/__init__.py | 24 | ||||
-rw-r--r-- | src/engine/SCons/Scanner/C.py | 122 | ||||
-rw-r--r-- | src/engine/SCons/Scanner/CTests.py | 64 | ||||
-rw-r--r-- | src/engine/SCons/Scanner/Fortran.py | 86 | ||||
-rw-r--r-- | src/engine/SCons/Scanner/FortranTests.py | 95 | ||||
-rw-r--r-- | src/engine/SCons/Scanner/Prog.py | 24 | ||||
-rw-r--r-- | src/engine/SCons/Scanner/ProgTests.py | 23 | ||||
-rw-r--r-- | src/engine/SCons/Scanner/ScannerTests.py | 13 | ||||
-rw-r--r-- | src/engine/SCons/Scanner/__init__.py | 53 |
15 files changed, 348 insertions, 245 deletions
diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 88ee9e8..0c608da 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -28,8 +28,8 @@ RELEASE 0.10 - XXX - Don't create duplicate source files in a BuildDir when the -n option is used. - - Fix SCons not exiting with the appropriate status on build errors - (and probably in other situations). + - Refactor the Scanner interface to eliminate unnecessary Scanner + calls and make it easier to write efficient scanners. - Significant performance improvement from using a more efficient check, throughout the code, for whether a Node has a Builder. diff --git a/src/RELEASE.txt b/src/RELEASE.txt index 0280fed..efb455b 100644 --- a/src/RELEASE.txt +++ b/src/RELEASE.txt @@ -27,11 +27,16 @@ RELEASE 0.10 - XXX Please note the following important changes since release 0.09: + - The Scanner interface has been changed to make it easier to + write user-defined scanners and to eliminate unnecessary + scanner calls. This will require changing your user-defined + SCanner definitions. XXX + - The .sconsign file format has been changed from ASCII to a pickled Python data structure. This improves performance and future extensibility, but means that the first time you execute SCons - 0.10 on an already-existing source tree, for every .sconsign - file in the tree it will report: + 0.10 on an already-existing source tree built with SCons 0.09 or + earlier, SCons will report for every .sconsign file in the tree: SCons warning: Ignoring corrupt .sconsign file: xxx diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py index e50be98..2016694 100644 --- a/src/engine/SCons/Defaults.py +++ b/src/engine/SCons/Defaults.py @@ -111,10 +111,10 @@ class SharedFlagChecker: elif not self.shared and src.attributes.shared: raise SCons.Errors.UserError, "Source file: %s is shared and is not compatible with static target: %s" % (src, target[0]) -SharedCheck = SCons.Action.Action(SharedFlagChecker(1, 0)) -StaticCheck = SCons.Action.Action(SharedFlagChecker(0, 0)) -SharedCheckSet = SCons.Action.Action(SharedFlagChecker(1, 1)) -StaticCheckSet = SCons.Action.Action(SharedFlagChecker(0, 1)) +SharedCheck = SCons.Action.Action(SharedFlagChecker(1, 0), None) +StaticCheck = SCons.Action.Action(SharedFlagChecker(0, 0), None) +SharedCheckSet = SCons.Action.Action(SharedFlagChecker(1, 1), None) +StaticCheckSet = SCons.Action.Action(SharedFlagChecker(0, 1), None) CAction = SCons.Action.Action([ StaticCheckSet, "$CCCOM" ]) ShCAction = SCons.Action.Action([ SharedCheckSet, "$SHCCCOM" ]) diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py index ff89424..0ec3b4e 100644 --- a/src/engine/SCons/EnvironmentTests.py +++ b/src/engine/SCons/EnvironmentTests.py @@ -84,7 +84,7 @@ class Scanner: self.name = name self.skeys = skeys - def scan(self, filename): + def __call__(self, filename): scanned_it[filename] = 1 def __cmp__(self, other): @@ -195,20 +195,20 @@ class EnvironmentTestCase(unittest.TestCase): scanned_it = {} env1 = Environment(SCANNERS = s1) - env1.scanner1.scan(filename = 'out1') + env1.scanner1(filename = 'out1') assert scanned_it['out1'] scanned_it = {} env2 = Environment(SCANNERS = [s1]) - env1.scanner1.scan(filename = 'out1') + env1.scanner1(filename = 'out1') assert scanned_it['out1'] scanned_it = {} env3 = Environment() env3.Replace(SCANNERS = [s1, s2]) - env3.scanner1.scan(filename = 'out1') - env3.scanner2.scan(filename = 'out2') - env3.scanner1.scan(filename = 'out3') + env3.scanner1(filename = 'out1') + env3.scanner2(filename = 'out2') + env3.scanner1(filename = 'out3') assert scanned_it['out1'] assert scanned_it['out2'] assert scanned_it['out3'] diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 826307b..aa7f973 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -790,6 +790,8 @@ class File(Entry): def _morph(self): """Turn a file system node into a File object.""" self.linked = 0 + self.scanner_paths = {} + self.found_includes = {} if not hasattr(self, '_local'): self._local = 0 @@ -856,11 +858,23 @@ class File(Entry): return self.dir.sconsign().get_implicit(self.name) def get_implicit_deps(self, env, scanner, target): - if scanner: - return scanner.scan(self, env, target) - else: + if not scanner: return [] - + + try: + path = target.scanner_paths[scanner] + except KeyError: + path = scanner.path(env, target.cwd) + target.scanner_paths[scanner] = path + + try: + includes = self.found_includes[path] + except KeyError: + includes = scanner(self, env, path) + self.found_includes[path] = includes + + return includes + def scanner_key(self): return os.path.splitext(self.name)[1] @@ -895,6 +909,7 @@ class File(Entry): def built(self): SCons.Node.Node.built(self) + self.found_includes = {} if hasattr(self, '_exists'): delattr(self, '_exists') if hasattr(self, '_rexists'): diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index a624d97..e23178c 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -69,7 +69,9 @@ class Scanner: global scanner_count scanner_count = scanner_count + 1 self.hash = scanner_count - def scan(self, node, env, target): + def path(self, env, target): + return () + def __call__(self, node, env, path): return [node] def __hash__(self): return self.hash @@ -669,6 +671,35 @@ class FSTestCase(unittest.TestCase): f1.store_implicit() assert f1.get_stored_implicit()[0] == os.path.join("d1", "f1") + # Test underlying scanning functionality in get_implicit_deps() + env = Environment() + f12 = fs.File("f12") + t1 = fs.File("t1") + + deps = f12.get_implicit_deps(env, None, t1) + assert deps == [], deps + + class MyScanner(Scanner): + call_count = 0 + def __call__(self, node, env, path): + self.call_count = self.call_count + 1 + return [node] + s = MyScanner() + + deps = f12.get_implicit_deps(env, s, t1) + assert deps == [f12], deps + assert s.call_count == 1, s.call_count + + deps = f12.get_implicit_deps(env, s, t1) + assert deps == [f12], deps + assert s.call_count == 1, s.call_count + + f12.built() + + deps = f12.get_implicit_deps(env, s, t1) + assert deps == [f12], deps + assert s.call_count == 2, s.call_count + # Test building a file whose directory is not there yet... f1 = fs.File(test.workpath("foo/bar/baz/ack")) assert not f1.dir.exists() diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 3bafb9c..16e28e2 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -221,15 +221,17 @@ class Node: self.implicit = [] self.del_bsig() + build_env = self.generate_build_env() + for child in self.children(scan=0): self._add_child(self.implicit, - child.get_implicit_deps(self.generate_build_env(), + child.get_implicit_deps(build_env, child.source_scanner, self)) # scan this node itself for implicit dependencies self._add_child(self.implicit, - self.get_implicit_deps(self.generate_build_env(), + self.get_implicit_deps(build_env, self.target_scanner, self)) @@ -384,7 +386,23 @@ class Node: def all_children(self, scan=1): """Return a list of all the node's direct children.""" - #XXX Need to remove duplicates from this + # The return list may contain duplicate Nodes, especially in + # source trees where there are a lot of repeated #includes + # of a tangle of .h files. Profiling shows, however, that + # eliminating the duplicates with a brute-force approach that + # preserves the order (that is, something like: + # + # u = [] + # for n in list: + # if n not in u: + # u.append(n)" + # + # takes more cycles than just letting the underlying methods + # hand back cached values if a Node's information is requested + # multiple times. (Other methods of removing duplicates, like + # using dictionary keys, lose the order, and the only ordered + # dictionary patterns I found all ended up using "not in" + # internally anyway...) if scan: self.scan() if self.implicit is None: diff --git a/src/engine/SCons/Scanner/C.py b/src/engine/SCons/Scanner/C.py index 6e7db58..cbcf1c6 100644 --- a/src/engine/SCons/Scanner/C.py +++ b/src/engine/SCons/Scanner/C.py @@ -48,10 +48,20 @@ def CScan(fs = SCons.Node.FS.default_fs): cs = SCons.Scanner.Recursive(scan, "CScan", fs, [".c", ".C", ".cxx", ".cpp", ".c++", ".cc", ".h", ".H", ".hxx", ".hpp", ".hh", - ".F", ".fpp", ".FPP"]) + ".F", ".fpp", ".FPP"], + path_function = path) return cs -def scan(node, env, target, fs = SCons.Node.FS.default_fs): +def path(env, dir, fs = SCons.Node.FS.default_fs): + try: + cpppath = env['CPPPATH'] + except KeyError: + return () + return tuple(fs.Rsearchall(SCons.Util.mapPaths(cpppath, dir, env), + clazz = SCons.Node.FS.Dir, + must_exist = 0)) + +def scan(node, env, cpppath = (), fs = SCons.Node.FS.default_fs): """ scan(node, Environment) -> [node] @@ -72,70 +82,54 @@ def scan(node, env, target, fs = SCons.Node.FS.default_fs): dependencies. """ - # This function caches various information in node and target: - # target.cpppath - env['CPPPATH'] converted to nodes - # node.found_includes - include files found by previous call to scan, - # keyed on cpppath - # node.includes - the result of include_re.findall() - - if not hasattr(target, 'cpppath'): - try: - target.cpppath = tuple(fs.Rsearchall(SCons.Util.mapPaths(env['CPPPATH'], target.cwd, env), clazz=SCons.Node.FS.Dir, must_exist=0)) - except KeyError: - target.cpppath = () - - cpppath = target.cpppath - node = node.rfile() - if not node.found_includes.has_key(cpppath): - if node.exists(): - - # cache the includes list in node so we only scan it once: - if node.includes != None: - includes = node.includes - else: - includes = include_re.findall(node.get_contents()) - node.includes = includes - - nodes = [] - source_dir = node.get_dir() - for include in includes: - if include[0] == '"': - n = SCons.Node.FS.find_file(include[1], - (source_dir,) + cpppath, - fs.File) - else: - n = SCons.Node.FS.find_file(include[1], - cpppath + (source_dir,), - fs.File) - - if not n is None: - nodes.append(n) - else: - SCons.Warnings.warn(SCons.Warnings.DependencyWarning, - "No dependency generated for file: %s (included from: %s) -- file not found" % (include[1], node)) - - # Schwartzian transform from the Python FAQ Wizard - def st(List, Metric): - def pairing(element, M = Metric): - return (M(element), element) - def stripit(pair): - return pair[1] - paired = map(pairing, List) - paired.sort() - return map(stripit, paired) - - def normalize(node): - # We don't want the order of includes to be - # modified by case changes on case insensitive OSes, so - # normalize the case of the filename here: - # (see test/win32pathmadness.py for a test of this) - return SCons.Node.FS._my_normcase(str(node)) - node.found_includes[cpppath] = st(nodes, normalize) + # This function caches the following information: + # node.includes - the result of include_re.findall() + if not node.exists(): + return [] + + # cache the includes list in node so we only scan it once: + if node.includes != None: + includes = node.includes + else: + includes = include_re.findall(node.get_contents()) + node.includes = includes + + nodes = [] + source_dir = node.get_dir() + for include in includes: + if include[0] == '"': + n = SCons.Node.FS.find_file(include[1], + (source_dir,) + cpppath, + fs.File) else: + n = SCons.Node.FS.find_file(include[1], + cpppath + (source_dir,), + fs.File) - node.found_includes[cpppath] = [] - - return node.found_includes[cpppath] + if not n is None: + nodes.append(n) + else: + SCons.Warnings.warn(SCons.Warnings.DependencyWarning, + "No dependency generated for file: %s (included from: %s) -- file not found" % (include[1], node)) + + # Schwartzian transform from the Python FAQ Wizard + def st(List, Metric): + def pairing(element, M = Metric): + return (M(element), element) + def stripit(pair): + return pair[1] + paired = map(pairing, List) + paired.sort() + return map(stripit, paired) + + def normalize(node): + # We don't want the order of includes to be + # modified by case changes on case insensitive OSes, so + # normalize the case of the filename here: + # (see test/win32pathmadness.py for a test of this) + return SCons.Node.FS._my_normcase(str(node)) + + return st(nodes, normalize) diff --git a/src/engine/SCons/Scanner/CTests.py b/src/engine/SCons/Scanner/CTests.py index 28a5f52..f02474c 100644 --- a/src/engine/SCons/Scanner/CTests.py +++ b/src/engine/SCons/Scanner/CTests.py @@ -159,10 +159,6 @@ test.write([ 'repository', 'src', 'ddd.h'], "\n") # define some helpers: -class DummyTarget: - def __init__(self, cwd=None): - self.cwd = cwd - class DummyEnvironment: def __init__(self, listCppPath): self.path = listCppPath @@ -178,6 +174,9 @@ class DummyEnvironment: def subst(self, arg): return arg + def has_key(self, key): + return self.Dictionary().has_key(key) + def __getitem__(self,key): return self.Dictionary()[key] @@ -207,7 +206,8 @@ class CScannerTestCase1(unittest.TestCase): def runTest(self): env = DummyEnvironment([]) s = SCons.Scanner.C.CScan() - deps = s.scan(make_node('f1.cpp'), env, DummyTarget()) + path = s.path(env) + deps = s(make_node('f1.cpp'), env, path) headers = ['f1.h', 'f2.h', 'fi.h'] deps_match(self, deps, map(test.workpath, headers)) @@ -215,7 +215,8 @@ class CScannerTestCase2(unittest.TestCase): def runTest(self): env = DummyEnvironment([test.workpath("d1")]) s = SCons.Scanner.C.CScan() - deps = s.scan(make_node('f1.cpp'), env, DummyTarget()) + path = s.path(env) + deps = s(make_node('f1.cpp'), env, path) headers = ['d1/f2.h', 'f1.h'] deps_match(self, deps, map(test.workpath, headers)) @@ -223,7 +224,8 @@ class CScannerTestCase3(unittest.TestCase): def runTest(self): env = DummyEnvironment([test.workpath("d1")]) s = SCons.Scanner.C.CScan() - deps = s.scan(make_node('f2.cpp'), env, DummyTarget()) + path = s.path(env) + deps = s(make_node('f2.cpp'), env, path) headers = ['d1/d2/f1.h', 'd1/f1.h', 'f1.h'] deps_match(self, deps, map(test.workpath, headers)) @@ -231,7 +233,8 @@ class CScannerTestCase4(unittest.TestCase): def runTest(self): env = DummyEnvironment([test.workpath("d1"), test.workpath("d1/d2")]) s = SCons.Scanner.C.CScan() - deps = s.scan(make_node('f2.cpp'), env, DummyTarget()) + path = s.path(env) + deps = s(make_node('f2.cpp'), env, path) headers = ['d1/d2/f1.h', 'd1/d2/f4.h', 'd1/f1.h', 'f1.h'] deps_match(self, deps, map(test.workpath, headers)) @@ -239,6 +242,7 @@ class CScannerTestCase5(unittest.TestCase): def runTest(self): env = DummyEnvironment([]) s = SCons.Scanner.C.CScan() + path = s.path(env) n = make_node('f3.cpp') def my_rexists(s=n): @@ -247,7 +251,7 @@ class CScannerTestCase5(unittest.TestCase): setattr(n, 'old_rexists', n.rexists) setattr(n, 'rexists', my_rexists) - deps = s.scan(n, env, DummyTarget()) + deps = s(n, env, path) # Make sure rexists() got called on the file node being # scanned, essential for cooperation with BuildDir functionality. @@ -261,10 +265,11 @@ class CScannerTestCase6(unittest.TestCase): def runTest(self): env1 = DummyEnvironment([test.workpath("d1")]) env2 = DummyEnvironment([test.workpath("d1/d2")]) - env3 = DummyEnvironment([test.workpath("d1/../d1")]) s = SCons.Scanner.C.CScan() - deps1 = s.scan(make_node('f1.cpp'), env1, DummyTarget()) - deps2 = s.scan(make_node('f1.cpp'), env2, DummyTarget()) + path1 = s.path(env1) + path2 = s.path(env2) + deps1 = s(make_node('f1.cpp'), env1, path1) + deps2 = s(make_node('f1.cpp'), env2, path2) headers1 = ['d1/f2.h', 'f1.h'] headers2 = ['d1/d2/f2.h', 'f1.h'] deps_match(self, deps1, map(test.workpath, headers1)) @@ -275,11 +280,13 @@ class CScannerTestCase8(unittest.TestCase): fs = SCons.Node.FS.FS(test.workpath('')) env = DummyEnvironment(["include"]) s = SCons.Scanner.C.CScan(fs = fs) - deps1 = s.scan(fs.File('fa.cpp'), env, DummyTarget()) + path = s.path(env) + deps1 = s(fs.File('fa.cpp'), env, path) fs.chdir(fs.Dir('subdir')) - target = DummyTarget(fs.getcwd()) + dir = fs.getcwd() fs.chdir(fs.Dir('..')) - deps2 = s.scan(fs.File('#fa.cpp'), env, target) + path = s.path(env, dir) + deps2 = s(fs.File('#fa.cpp'), env, path) headers1 = ['include/fa.h', 'include/fb.h'] headers2 = ['subdir/include/fa.h', 'subdir/include/fb.h'] deps_match(self, deps1, headers1) @@ -297,9 +304,10 @@ class CScannerTestCase9(unittest.TestCase): SCons.Warnings._warningOut = to test.write('fa.h','\n') fs = SCons.Node.FS.FS(test.workpath('')) - s = SCons.Scanner.C.CScan(fs=fs) env = DummyEnvironment([]) - deps = s.scan(fs.File('fa.cpp'), env, DummyTarget()) + s = SCons.Scanner.C.CScan(fs=fs) + path = s.path(env) + deps = s(fs.File('fa.cpp'), env, path) # Did we catch the warning associated with not finding fb.h? assert to.out @@ -311,10 +319,11 @@ class CScannerTestCase10(unittest.TestCase): def runTest(self): fs = SCons.Node.FS.FS(test.workpath('')) fs.chdir(fs.Dir('include')) - s = SCons.Scanner.C.CScan(fs=fs) env = DummyEnvironment([]) + s = SCons.Scanner.C.CScan(fs=fs) + path = s.path(env) test.write('include/fa.cpp', test.read('fa.cpp')) - deps = s.scan(fs.File('#include/fa.cpp'), env, DummyTarget()) + deps = s(fs.File('#include/fa.cpp'), env, path) deps_match(self, deps, [ 'include/fa.h', 'include/fb.h' ]) test.unlink('include/fa.cpp') @@ -328,9 +337,10 @@ class CScannerTestCase11(unittest.TestCase): # This was a bug at one time. f1=fs.File('include2/jjj.h') f1.builder=1 - s = SCons.Scanner.C.CScan(fs=fs) env = DummyEnvironment(['include', 'include2']) - deps = s.scan(fs.File('src/fff.c'), env, DummyTarget()) + s = SCons.Scanner.C.CScan(fs=fs) + path = s.path(env) + deps = s(fs.File('src/fff.c'), env, path) deps_match(self, deps, [ test.workpath('repository/include/iii.h'), 'include2/jjj.h' ]) os.chdir(test.workpath('')) @@ -343,13 +353,14 @@ class CScannerTestCase12(unittest.TestCase): fs.Repository(test.workpath('repository')) env = DummyEnvironment([]) s = SCons.Scanner.C.CScan(fs = fs) - deps1 = s.scan(fs.File('build1/aaa.c'), env, DummyTarget()) + path = s.path(env) + deps1 = s(fs.File('build1/aaa.c'), env, path) deps_match(self, deps1, [ 'build1/bbb.h' ]) - deps2 = s.scan(fs.File('build2/aaa.c'), env, DummyTarget()) + deps2 = s(fs.File('build2/aaa.c'), env, path) deps_match(self, deps2, [ 'src/bbb.h' ]) - deps3 = s.scan(fs.File('build1/ccc.c'), env, DummyTarget()) + deps3 = s(fs.File('build1/ccc.c'), env, path) deps_match(self, deps3, [ 'build1/ddd.h' ]) - deps4 = s.scan(fs.File('build2/ccc.c'), env, DummyTarget()) + deps4 = s(fs.File('build2/ccc.c'), env, path) deps_match(self, deps4, [ test.workpath('repository/src/ddd.h') ]) os.chdir(test.workpath('')) @@ -360,7 +371,8 @@ class CScannerTestCase13(unittest.TestCase): return test.workpath("d1") env = SubstEnvironment(["blah"]) s = SCons.Scanner.C.CScan() - deps = s.scan(make_node('f1.cpp'), env, DummyTarget()) + path = s.path(env) + deps = s(make_node('f1.cpp'), env, path) headers = ['d1/f2.h', 'f1.h'] deps_match(self, deps, map(test.workpath, headers)) diff --git a/src/engine/SCons/Scanner/Fortran.py b/src/engine/SCons/Scanner/Fortran.py index 5d908f7..e23c7a5 100644 --- a/src/engine/SCons/Scanner/Fortran.py +++ b/src/engine/SCons/Scanner/Fortran.py @@ -46,71 +46,53 @@ def FortranScan(fs = SCons.Node.FS.default_fs): """Return a prototype Scanner instance for scanning source files for Fortran INCLUDE statements""" scanner = SCons.Scanner.Recursive(scan, "FortranScan", fs, - [".f", ".F", ".for", ".FOR"]) + [".f", ".F", ".for", ".FOR"], + path_function = path) return scanner -def scan(node, env, target, fs = SCons.Node.FS.default_fs): +def path(env, dir, fs = SCons.Node.FS.default_fs): + try: + f77path = env['F77PATH'] + except KeyError: + return () + return tuple(fs.Rsearchall(SCons.Util.mapPaths(f77path, dir, env), + clazz = SCons.Node.FS.Dir, + must_exist = 0)) + +def scan(node, env, f77path = (), fs = SCons.Node.FS.default_fs): """ scan(node, Environment) -> [node] the Fortran dependency scanner function - - This function is intentionally simple. There are two rules it - follows: - - 1) #include <foo.h> - search for foo.h in F77PATH followed by the - directory 'filename' is in - 2) #include \"foo.h\" - search for foo.h in the directory 'filename' is - in followed by F77PATH - - These rules approximate the behaviour of most C/C++ compilers. - - This scanner also ignores #ifdef and other preprocessor conditionals, so - it may find more depencies than there really are, but it never misses - dependencies. """ - # This function caches various information in node and target: - # target.f77path - env['F77PATH'] converted to nodes - # node.found_includes - include files found by previous call to scan, - # keyed on f77path + node = node.rfile() + + # This function caches the following information: # node.includes - the result of include_re.findall() - if not hasattr(target, 'f77path'): - try: - target.f77path = tuple(fs.Rsearchall(SCons.Util.mapPaths(env['F77PATH'], target.cwd, env), clazz=SCons.Node.FS.Dir, must_exist=0)) - except KeyError: - target.f77path = () + if not node.exists(): + return [] - f77path = target.f77path + # cache the includes list in node so we only scan it once: + if node.includes != None: + includes = node.includes + else: + includes = include_re.findall(node.get_contents()) + node.includes = includes + source_dir = node.get_dir() + nodes = [] - - node = node.rfile() - try: - nodes = node.found_includes[f77path] - except KeyError: - if node.rexists(): - - # cache the includes list in node so we only scan it once: - if node.includes != None: - includes = node.includes - else: - includes = include_re.findall(node.get_contents()) - node.includes = includes - - source_dir = node.get_dir() - - for include in includes: - n = SCons.Node.FS.find_file(include, - (source_dir,) + f77path, - fs.File) - if not n is None: - nodes.append(n) - else: - SCons.Warnings.warn(SCons.Warnings.DependencyWarning, - "No dependency generated for file: %s (included from: %s) -- file not found" % (include, node)) - node.found_includes[f77path] = nodes + for include in includes: + n = SCons.Node.FS.find_file(include, + (source_dir,) + f77path, + fs.File) + if not n is None: + nodes.append(n) + else: + SCons.Warnings.warn(SCons.Warnings.DependencyWarning, + "No dependency generated for file: %s (included from: %s) -- file not found" % (include, node)) # Schwartzian transform from the Python FAQ Wizard def st(List, Metric): diff --git a/src/engine/SCons/Scanner/FortranTests.py b/src/engine/SCons/Scanner/FortranTests.py index e2cfcd6..f721d89 100644 --- a/src/engine/SCons/Scanner/FortranTests.py +++ b/src/engine/SCons/Scanner/FortranTests.py @@ -127,10 +127,6 @@ test.write([ 'repository', 'src', 'ddd.f'], "\n") # define some helpers: -class DummyTarget: - def __init__(self, cwd=None): - self.cwd = cwd - class DummyEnvironment: def __init__(self, listCppPath): self.path = listCppPath @@ -143,6 +139,9 @@ class DummyEnvironment: else: raise KeyError, "Dummy environment only has F77PATH attribute." + def has_key(self, key): + return self.Dictionary().has_key(key) + def __getitem__(self,key): return self.Dictionary()[key] @@ -171,12 +170,13 @@ class FortranScannerTestCase1(unittest.TestCase): test.write('f2.f', " INCLUDE 'fi.f'\n") env = DummyEnvironment([]) s = SCons.Scanner.Fortran.FortranScan() - fs = SCons.Node.FS.FS(original) - deps = s.scan(make_node('fff1.f', fs), env, DummyTarget()) + path = s.path(env) + fs = SCons.Node.FS.FS(original) + deps = s(make_node('fff1.f', fs), env, path) headers = ['f1.f', 'f2.f', 'fi.f'] deps_match(self, deps, map(test.workpath, headers)) - test.unlink('f1.f') - test.unlink('f2.f') + test.unlink('f1.f') + test.unlink('f2.f') class FortranScannerTestCase2(unittest.TestCase): def runTest(self): @@ -184,19 +184,21 @@ class FortranScannerTestCase2(unittest.TestCase): test.write('f2.f', " INCLUDE 'fi.f'\n") env = DummyEnvironment([test.workpath("d1")]) s = SCons.Scanner.Fortran.FortranScan() - fs = SCons.Node.FS.FS(original) - deps = s.scan(make_node('fff1.f', fs), env, DummyTarget()) + path = s.path(env) + fs = SCons.Node.FS.FS(original) + deps = s(make_node('fff1.f', fs), env, path) headers = ['f1.f', 'f2.f', 'fi.f'] deps_match(self, deps, map(test.workpath, headers)) - test.unlink('f1.f') - test.unlink('f2.f') + test.unlink('f1.f') + test.unlink('f2.f') class FortranScannerTestCase3(unittest.TestCase): def runTest(self): env = DummyEnvironment([test.workpath("d1")]) s = SCons.Scanner.Fortran.FortranScan() - fs = SCons.Node.FS.FS(original) - deps = s.scan(make_node('fff1.f', fs), env, DummyTarget()) + path = s.path(env) + fs = SCons.Node.FS.FS(original) + deps = s(make_node('fff1.f', fs), env, path) headers = ['d1/f1.f', 'd1/f2.f'] deps_match(self, deps, map(test.workpath, headers)) @@ -205,8 +207,9 @@ class FortranScannerTestCase4(unittest.TestCase): test.write(['d1', 'f2.f'], " INCLUDE 'fi.f'\n") env = DummyEnvironment([test.workpath("d1")]) s = SCons.Scanner.Fortran.FortranScan() - fs = SCons.Node.FS.FS(original) - deps = s.scan(make_node('fff1.f', fs), env, DummyTarget()) + path = s.path(env) + fs = SCons.Node.FS.FS(original) + deps = s(make_node('fff1.f', fs), env, path) headers = ['d1/f1.f', 'd1/f2.f'] deps_match(self, deps, map(test.workpath, headers)) test.write(['d1', 'f2.f'], "\n") @@ -215,8 +218,9 @@ class FortranScannerTestCase5(unittest.TestCase): def runTest(self): env = DummyEnvironment([test.workpath("d1")]) s = SCons.Scanner.Fortran.FortranScan() - fs = SCons.Node.FS.FS(original) - deps = s.scan(make_node('fff2.f', fs), env, DummyTarget()) + path = s.path(env) + fs = SCons.Node.FS.FS(original) + deps = s(make_node('fff2.f', fs), env, path) headers = ['d1/d2/f2.f', 'd1/f2.f', 'd1/f2.f'] deps_match(self, deps, map(test.workpath, headers)) @@ -225,8 +229,9 @@ class FortranScannerTestCase6(unittest.TestCase): test.write('f2.f', "\n") env = DummyEnvironment([test.workpath("d1")]) s = SCons.Scanner.Fortran.FortranScan() - fs = SCons.Node.FS.FS(original) - deps = s.scan(make_node('fff2.f', fs), env, DummyTarget()) + path = s.path(env) + fs = SCons.Node.FS.FS(original) + deps = s(make_node('fff2.f', fs), env, path) headers = ['d1/d2/f2.f', 'd1/f2.f', 'f2.f'] deps_match(self, deps, map(test.workpath, headers)) test.unlink('f2.f') @@ -235,8 +240,9 @@ class FortranScannerTestCase7(unittest.TestCase): def runTest(self): env = DummyEnvironment([test.workpath("d1/d2"), test.workpath("d1")]) s = SCons.Scanner.Fortran.FortranScan() - fs = SCons.Node.FS.FS(original) - deps = s.scan(make_node('fff2.f', fs), env, DummyTarget()) + path = s.path(env) + fs = SCons.Node.FS.FS(original) + deps = s(make_node('fff2.f', fs), env, path) headers = ['d1/d2/f2.f', 'd1/d2/f2.f', 'd1/f2.f'] deps_match(self, deps, map(test.workpath, headers)) @@ -245,8 +251,9 @@ class FortranScannerTestCase8(unittest.TestCase): test.write('f2.f', "\n") env = DummyEnvironment([test.workpath("d1/d2"), test.workpath("d1")]) s = SCons.Scanner.Fortran.FortranScan() - fs = SCons.Node.FS.FS(original) - deps = s.scan(make_node('fff2.f', fs), env, DummyTarget()) + path = s.path(env) + fs = SCons.Node.FS.FS(original) + deps = s(make_node('fff2.f', fs), env, path) headers = ['d1/d2/f2.f', 'd1/f2.f', 'f2.f'] deps_match(self, deps, map(test.workpath, headers)) test.unlink('f2.f') @@ -256,6 +263,7 @@ class FortranScannerTestCase9(unittest.TestCase): test.write('f3.f', "\n") env = DummyEnvironment([]) s = SCons.Scanner.Fortran.FortranScan() + path = s.path(env) n = make_node('fff3.f') def my_rexists(s=n): @@ -264,7 +272,7 @@ class FortranScannerTestCase9(unittest.TestCase): setattr(n, 'old_rexists', n.rexists) setattr(n, 'rexists', my_rexists) - deps = s.scan(n, env, DummyTarget()) + deps = s(n, env, path) # Make sure rexists() got called on the file node being # scanned, essential for cooperation with BuildDir functionality. @@ -279,11 +287,13 @@ class FortranScannerTestCase10(unittest.TestCase): fs = SCons.Node.FS.FS(test.workpath('')) env = DummyEnvironment(["include"]) s = SCons.Scanner.Fortran.FortranScan(fs = fs) - deps1 = s.scan(fs.File('fff4.f'), env, DummyTarget()) + path = s.path(env) + deps1 = s(fs.File('fff4.f'), env, path) fs.chdir(fs.Dir('subdir')) - target = DummyTarget(fs.getcwd()) + dir = fs.getcwd() fs.chdir(fs.Dir('..')) - deps2 = s.scan(fs.File('#fff4.f'), env, target) + path = s.path(env, dir) + deps2 = s(fs.File('#fff4.f'), env, path) headers1 = ['include/f4.f'] headers2 = ['subdir/include/f4.f'] deps_match(self, deps1, headers1) @@ -301,9 +311,10 @@ class FortranScannerTestCase11(unittest.TestCase): SCons.Warnings._warningOut = to test.write('f4.f'," INCLUDE 'not_there.f'\n") fs = SCons.Node.FS.FS(test.workpath('')) - s = SCons.Scanner.Fortran.FortranScan(fs=fs) env = DummyEnvironment([]) - deps = s.scan(fs.File('fff4.f'), env, DummyTarget()) + s = SCons.Scanner.Fortran.FortranScan(fs=fs) + path = s.path(env) + deps = s(fs.File('fff4.f'), env, path) # Did we catch the warning from not finding not_there.f? assert to.out @@ -315,10 +326,11 @@ class FortranScannerTestCase12(unittest.TestCase): def runTest(self): fs = SCons.Node.FS.FS(test.workpath('')) fs.chdir(fs.Dir('include')) - s = SCons.Scanner.Fortran.FortranScan(fs=fs) env = DummyEnvironment([]) + s = SCons.Scanner.Fortran.FortranScan(fs=fs) + path = s.path(env) test.write('include/fff4.f', test.read('fff4.f')) - deps = s.scan(fs.File('#include/fff4.f'), env, DummyTarget()) + deps = s(fs.File('#include/fff4.f'), env, path) deps_match(self, deps, ['include/f4.f']) test.unlink('include/fff4.f') @@ -332,9 +344,10 @@ class FortranScannerTestCase13(unittest.TestCase): # This was a bug at one time. f1=fs.File('include2/jjj.f') f1.builder=1 - s = SCons.Scanner.Fortran.FortranScan(fs=fs) env = DummyEnvironment(['include','include2']) - deps = s.scan(fs.File('src/fff.f'), env, DummyTarget()) + s = SCons.Scanner.Fortran.FortranScan(fs=fs) + path = s.path(env) + deps = s(fs.File('src/fff.f'), env, path) deps_match(self, deps, [test.workpath('repository/include/iii.f'), 'include2/jjj.f']) os.chdir(test.workpath('')) @@ -347,13 +360,14 @@ class FortranScannerTestCase14(unittest.TestCase): fs.Repository(test.workpath('repository')) env = DummyEnvironment([]) s = SCons.Scanner.Fortran.FortranScan(fs = fs) - deps1 = s.scan(fs.File('build1/aaa.f'), env, DummyTarget()) + path = s.path(env) + deps1 = s(fs.File('build1/aaa.f'), env, path) deps_match(self, deps1, [ 'build1/bbb.f' ]) - deps2 = s.scan(fs.File('build2/aaa.f'), env, DummyTarget()) + deps2 = s(fs.File('build2/aaa.f'), env, path) deps_match(self, deps2, [ 'src/bbb.f' ]) - deps3 = s.scan(fs.File('build1/ccc.f'), env, DummyTarget()) + deps3 = s(fs.File('build1/ccc.f'), env, path) deps_match(self, deps3, [ 'build1/ddd.f' ]) - deps4 = s.scan(fs.File('build2/ccc.f'), env, DummyTarget()) + deps4 = s(fs.File('build2/ccc.f'), env, path) deps_match(self, deps4, [ test.workpath('repository/src/ddd.f') ]) os.chdir(test.workpath('')) @@ -365,8 +379,9 @@ class FortranScannerTestCase15(unittest.TestCase): test.write(['d1', 'f2.f'], " INCLUDE 'fi.f'\n") env = SubstEnvironment(["junk"]) s = SCons.Scanner.Fortran.FortranScan() - fs = SCons.Node.FS.FS(original) - deps = s.scan(make_node('fff1.f', fs), env, DummyTarget()) + path = s.path(env) + fs = SCons.Node.FS.FS(original) + deps = s(make_node('fff1.f', fs), env, path) headers = ['d1/f1.f', 'd1/f2.f'] deps_match(self, deps, map(test.workpath, headers)) test.write(['d1', 'f2.f'], "\n") diff --git a/src/engine/SCons/Scanner/Prog.py b/src/engine/SCons/Scanner/Prog.py index 7cfd9f4..081fd5c 100644 --- a/src/engine/SCons/Scanner/Prog.py +++ b/src/engine/SCons/Scanner/Prog.py @@ -34,10 +34,19 @@ import SCons.Util def ProgScan(fs = SCons.Node.FS.default_fs): """Return a prototype Scanner instance for scanning executable files for static-lib dependencies""" - ps = SCons.Scanner.Base(scan, "ProgScan", fs) + ps = SCons.Scanner.Base(scan, "ProgScan", fs, path_function = path) return ps -def scan(node, env, target, fs): +def path(env, dir, fs = SCons.Node.FS.default_fs): + try: + libpath = env['LIBPATH'] + except KeyError: + return () + return tuple(fs.Rsearchall(SCons.Util.mapPaths(libpath, dir, env), + clazz = SCons.Node.FS.Dir, + must_exist = 0)) + +def scan(node, env, libpath = (), fs = SCons.Node.FS.default_fs): """ This scanner scans program files for static-library dependencies. It will search the LIBPATH environment variable @@ -45,17 +54,6 @@ def scan(node, env, target, fs): files it finds as dependencies. """ - # This function caches information in target: - # target.libpath - env['LIBPATH'] converted to nodes - - if not hasattr(target, 'libpath'): - try: - target.libpath = tuple(fs.Rsearchall(SCons.Util.mapPaths(env['LIBPATH'], target.cwd, env), clazz=SCons.Node.FS.Dir, must_exist=0)) - except KeyError: - target.libpath = () - - libpath = target.libpath - try: libs = env.Dictionary('LIBS') except KeyError: diff --git a/src/engine/SCons/Scanner/ProgTests.py b/src/engine/SCons/Scanner/ProgTests.py index 1302e46..0d0e5ad 100644 --- a/src/engine/SCons/Scanner/ProgTests.py +++ b/src/engine/SCons/Scanner/ProgTests.py @@ -43,10 +43,6 @@ for h in libs: # define some helpers: -class DummyTarget: - def __init__(self, cwd=None): - self.cwd = cwd - class DummyEnvironment: def __init__(self, **kw): self._dict = kw @@ -59,6 +55,10 @@ class DummyEnvironment: return self._dict[args[0]] else: return map(lambda x, s=self: s._dict[x], args) + + def has_key(self, key): + return self.Dictionary().has_key(key) + def __getitem__(self,key): return self.Dictionary()[key] @@ -86,7 +86,8 @@ class ProgScanTestCase1(unittest.TestCase): env = DummyEnvironment(LIBPATH=[ test.workpath("") ], LIBS=[ 'l1', 'l2', 'l3' ]) s = SCons.Scanner.Prog.ProgScan() - deps = s.scan('dummy', env, DummyTarget()) + path = s.path(env) + deps = s('dummy', env, path) assert deps_match(deps, ['l1.lib']), map(str, deps) class ProgScanTestCase2(unittest.TestCase): @@ -95,7 +96,8 @@ class ProgScanTestCase2(unittest.TestCase): ["", "d1", "d1/d2" ]), LIBS=[ 'l1', 'l2', 'l3' ]) s = SCons.Scanner.Prog.ProgScan() - deps = s.scan('dummy', env, DummyTarget()) + path = s.path(env) + deps = s('dummy', env, path) assert deps_match(deps, ['l1.lib', 'd1/l2.lib', 'd1/d2/l3.lib' ]), map(str, deps) class ProgScanTestCase3(unittest.TestCase): @@ -104,7 +106,8 @@ class ProgScanTestCase3(unittest.TestCase): test.workpath("d1")], LIBS=string.split('l2 l3')) s = SCons.Scanner.Prog.ProgScan() - deps = s.scan('dummy', env, DummyTarget()) + path = s.path(env) + deps = s('dummy', env, path) assert deps_match(deps, ['d1/l2.lib', 'd1/d2/l3.lib']), map(str, deps) class ProgScanTestCase5(unittest.TestCase): @@ -118,7 +121,8 @@ class ProgScanTestCase5(unittest.TestCase): env = SubstEnvironment(LIBPATH=[ "blah" ], LIBS=string.split('l2 l3')) s = SCons.Scanner.Prog.ProgScan() - deps = s.scan('dummy', env, DummyTarget()) + path = s.path(env) + deps = s('dummy', env, path) assert deps_match(deps, [ 'd1/l2.lib' ]), map(str, deps) def suite(): @@ -135,7 +139,8 @@ def suite(): test.workpath("d1")], LIBS=string.split(u'l2 l3')) s = SCons.Scanner.Prog.ProgScan() - deps = s.scan('dummy', env, DummyTarget()) + path = s.path(env) + deps = s('dummy', env, path) assert deps_match(deps, ['d1/l2.lib', 'd1/d2/l3.lib']), map(str, deps) suite.addTest(ProgScanTestCase4()) \n""" diff --git a/src/engine/SCons/Scanner/ScannerTests.py b/src/engine/SCons/Scanner/ScannerTests.py index 2d0e47a..7280c2f 100644 --- a/src/engine/SCons/Scanner/ScannerTests.py +++ b/src/engine/SCons/Scanner/ScannerTests.py @@ -27,10 +27,6 @@ import unittest import SCons.Scanner import sys -class DummyTarget: - cwd = None - - class ScannerTestBase: def func(self, filename, env, target, *args): @@ -46,7 +42,8 @@ class ScannerTestBase: def test(self, scanner, env, filename, deps, *args): self.deps = deps - scanned = scanner.scan(filename, env, DummyTarget()) + path = scanner.path(env) + scanned = scanner(filename, env, path) scanned_strs = map(lambda x: str(x), scanned) self.failUnless(self.filename == filename, "the filename was passed incorrectly") @@ -121,7 +118,7 @@ class ScannerHashTestCase(ScannerTestBase, unittest.TestCase): s = SCons.Scanner.Base(self.func, "Hash") dict = {} dict[s] = 777 - self.failUnless(hash(dict.keys()[0]) == hash(None), + self.failUnless(hash(dict.keys()[0]) == hash(repr(s)), "did not hash Scanner base class as expected") class ScannerCheckTestCase(unittest.TestCase): @@ -134,8 +131,10 @@ class ScannerCheckTestCase(unittest.TestCase): def check(node, s=self): s.checked[node] = 1 return 1 + env = DummyEnvironment() s = SCons.Scanner.Base(my_scan, "Check", scan_check = check) - scanned = s.scan('x', DummyEnvironment(), DummyTarget()) + path = s.path(env) + scanned = s('x', env, path) self.failUnless(self.checked['x'] == 1, "did not call check function") diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py index aef5d72..c27c762 100644 --- a/src/engine/SCons/Scanner/__init__.py +++ b/src/engine/SCons/Scanner/__init__.py @@ -52,23 +52,42 @@ class Base: name = "NONE", argument = _null, skeys = [], + path_function = None, + node_class = SCons.Node.FS.Entry, node_factory = SCons.Node.FS.default_fs.File, scan_check = None): """ Construct a new scanner object given a scanner function. - 'function' - a scanner function taking two or three arguments and - returning a list of strings. + 'function' - a scanner function taking two or three + arguments and returning a list of strings. 'name' - a name for identifying this scanner object. - 'argument' - an optional argument that will be passed to the - scanner function if it is given. + 'argument' - an optional argument that, if specified, will be + passed to both the scanner function and the path_function. - 'skeys; - an optional list argument that can be used to determine + 'skeys' - an optional list argument that can be used to determine which scanner should be used for a given Node. In the case of File nodes, for example, the 'skeys' would be file suffixes. + 'path_function' - a function that takes one to three arguments + (a construction environment, optional directory, and optional + argument for this instance) and returns a tuple of the + directories that can be searched for implicit dependency files. + + 'node_class' - the class of Nodes which this scan will return. + If node_class is None, then this scanner will not enforce any + Node conversion and will return the raw results from the + underlying scanner function. + + 'node_factory' - the factory function to be called to translate + the raw results returned by the scanner function into the + expected node_class objects. + + 'scan_check' - a function to be called to first check whether + this node really needs to be scanned. + The scanner function's first argument will be the name of a file that should be scanned for dependencies, the second argument will be an Environment object, the third argument will be the value @@ -91,13 +110,23 @@ class Base: # would need to be changed is the documentation. self.function = function + self.path_function = path_function self.name = name self.argument = argument self.skeys = skeys + self.node_class = node_class self.node_factory = node_factory self.scan_check = scan_check - def scan(self, node, env, target): + def path(self, env, dir = None): + if not self.path_function: + return () + if not self.argument is _null: + return self.path_function(env, dir, self.argument) + else: + return self.path_function(env, dir) + + def __call__(self, node, env, path = ()): """ This method scans a single object. 'node' is the node that will be passed to the scanner function, and 'env' is the @@ -108,15 +137,15 @@ class Base: return [] if not self.argument is _null: - list = self.function(node, env, target, self.argument) + list = self.function(node, env, path, self.argument) else: - list = self.function(node, env, target) + list = self.function(node, env, path) kw = {} if hasattr(node, 'dir'): kw['directory'] = node.dir nodes = [] for l in list: - if not isinstance(l, SCons.Node.FS.Entry): + if self.node_class and not isinstance(l, self.node_class): l = apply(self.node_factory, (l,), kw) nodes.append(l) return nodes @@ -125,7 +154,7 @@ class Base: return cmp(self.__dict__, other.__dict__) def __hash__(self): - return hash(None) + return hash(repr(self)) def add_skey(self, skey): """Add a skey to the list of skeys""" @@ -149,7 +178,7 @@ class Recursive(RExists): list of all dependencies. """ - def scan(self, node, env, target): + def __call__(self, node, env, path = ()): """ This method does the actual scanning. 'node' is the node that will be passed to the scanner function, and 'env' is the @@ -164,7 +193,7 @@ class Recursive(RExists): while nodes: n = nodes.pop(0) d = filter(lambda x, seen=seen: not seen.has_key(x), - Base.scan(self, n, env, target)) + Base.__call__(self, n, env, path)) if d: deps.extend(d) nodes.extend(d) |