From 3df74590f61e6de1271fc57df1a478f99ab28819 Mon Sep 17 00:00:00 2001 From: Steven Knight Date: Sat, 4 Dec 2004 18:50:33 +0000 Subject: Fix how scanners sort dependencies so it doesn't matter if the dependency file is found in a Repository or locally. --- src/CHANGES.txt | 4 + src/engine/SCons/Scanner/CTests.py | 16 ++-- src/engine/SCons/Scanner/Fortran.py | 40 +++------- src/engine/SCons/Scanner/FortranTests.py | 8 +- src/engine/SCons/Scanner/IDLTests.py | 19 ++--- src/engine/SCons/Scanner/__init__.py | 54 ++++++------- test/Repository/signature-order.py | 121 +++++++++++++++++++++++++++++ test/signature-order.py | 128 +++++++++++++++++++++++++++++++ 8 files changed, 311 insertions(+), 79 deletions(-) create mode 100644 test/Repository/signature-order.py create mode 100644 test/signature-order.py diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 507245a..fe0f879 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -155,6 +155,10 @@ RELEASE 0.97 - XXX - Make --debug={tree,dtree,stree} print something even when there's a build failure. + - Fix how Scanners sort the found dependencies so that it doesn't + matter whether the dependency file is in a Repository or not. + This may cause recompilations upon upgrade to this version. + From Wayne Lee: - Avoid "maximum recursion limit" errors when removing $(-$) pairs diff --git a/src/engine/SCons/Scanner/CTests.py b/src/engine/SCons/Scanner/CTests.py index 693ec18..e5830a0 100644 --- a/src/engine/SCons/Scanner/CTests.py +++ b/src/engine/SCons/Scanner/CTests.py @@ -227,7 +227,7 @@ class CScannerTestCase2(unittest.TestCase): s = SCons.Scanner.C.CScan() path = s.path(env) deps = s(make_node('f1.cpp'), env, path) - headers = ['d1/f2.h', 'f1.h'] + headers = ['f1.h', 'd1/f2.h'] deps_match(self, deps, map(test.workpath, headers)) class CScannerTestCase3(unittest.TestCase): @@ -236,7 +236,7 @@ class CScannerTestCase3(unittest.TestCase): s = SCons.Scanner.C.CScan() path = s.path(env) deps = s(make_node('f2.cpp'), env, path) - headers = ['d1/d2/f1.h', 'd1/f1.h', 'f1.h'] + headers = ['d1/f1.h', 'f1.h', 'd1/d2/f1.h'] deps_match(self, deps, map(test.workpath, headers)) class CScannerTestCase4(unittest.TestCase): @@ -245,7 +245,7 @@ class CScannerTestCase4(unittest.TestCase): s = SCons.Scanner.C.CScan() path = s.path(env) deps = s(make_node('f2.cpp'), env, path) - headers = ['d1/d2/f1.h', 'd1/d2/f4.h', 'd1/f1.h', 'f1.h'] + headers = ['d1/f1.h', 'f1.h', 'd1/d2/f1.h', 'd1/d2/f4.h'] deps_match(self, deps, map(test.workpath, headers)) class CScannerTestCase5(unittest.TestCase): @@ -267,8 +267,8 @@ class CScannerTestCase5(unittest.TestCase): # scanned, essential for cooperation with BuildDir functionality. assert n.rexists_called - headers = ['d1/f1.h', 'd1/f2.h', 'd1/f3-test.h', - 'f1.h', 'f2.h', 'f3-test.h'] + headers = ['f1.h', 'f2.h', 'f3-test.h', + 'd1/f1.h', 'd1/f2.h', 'd1/f3-test.h'] deps_match(self, deps, map(test.workpath, headers)) class CScannerTestCase6(unittest.TestCase): @@ -280,8 +280,8 @@ class CScannerTestCase6(unittest.TestCase): path2 = s.path(env2) deps1 = s(make_node('f1.cpp'), env1, path1) deps2 = s(make_node('f1.cpp'), env2, path2) - headers1 = ['d1/f2.h', 'f1.h'] - headers2 = ['d1/d2/f2.h', 'f1.h'] + headers1 = ['f1.h', 'd1/f2.h'] + headers2 = ['f1.h', 'd1/d2/f2.h'] deps_match(self, deps1, map(test.workpath, headers1)) deps_match(self, deps2, map(test.workpath, headers2)) @@ -384,7 +384,7 @@ class CScannerTestCase13(unittest.TestCase): s = SCons.Scanner.C.CScan() path = s.path(env) deps = s(make_node('f1.cpp'), env, path) - headers = ['d1/f2.h', 'f1.h'] + headers = ['f1.h', 'd1/f2.h'] deps_match(self, deps, map(test.workpath, headers)) class CScannerTestCase14(unittest.TestCase): diff --git a/src/engine/SCons/Scanner/Fortran.py b/src/engine/SCons/Scanner/Fortran.py index 786d4ab..6ab878f 100644 --- a/src/engine/SCons/Scanner/Fortran.py +++ b/src/engine/SCons/Scanner/Fortran.py @@ -94,42 +94,26 @@ class F90Scanner(SCons.Scanner.Classic): mods_and_includes = SCons.Util.unique(includes+modules) node.includes = mods_and_includes + # This is a hand-coded DSU (decorate-sort-undecorate, or + # Schwartzian transform) pattern. The sort key is the raw name + # of the file as specifed on the USE or INCLUDE line, which lets + # us keep the sort order constant regardless of whether the file + # is actually found in a Repository or locally. nodes = [] source_dir = node.get_dir() for dep in mods_and_includes: n, i = self.find_include(dep, source_dir, path) - if not n is None: - nodes.append(n) - else: + if n is None: SCons.Warnings.warn(SCons.Warnings.DependencyWarning, "No dependency generated for file: %s (referenced by: %s) -- file not found" % (i, node)) + else: + sortkey = self.sort_key(dep) + nodes.append((sortkey, n)) - # Sort the list of dependencies - - # Schwartzian transform from the Python FAQ Wizard - def st(List, Metric): - def pairing(element, M = Metric): - return (M(element), element) - def stripit(pair): - return pair[1] - paired = map(pairing, List) - paired.sort() - return map(stripit, paired) - - def normalize(node): - # We don't want the order of includes to be - # modified by case changes on case insensitive OSes, so - # normalize the case of the filename here: - # (see test/win32pathmadness.py for a test of this) - return SCons.Node.FS._my_normcase(str(node)) - - # Apply a Schwartzian transform to return the list of - # dependencies, sorted according to their normalized names - transformed = st(nodes, normalize) -# print "ClassicF90: " + str(node) + " => " + str(map(lambda x: str(x),list(transformed))) - return transformed - + nodes.sort() + nodes = map(lambda pair: pair[1], nodes) + return nodes def FortranScan(path_variable="FORTRANPATH", fs=SCons.Node.FS.default_fs): """Return a prototype Scanner instance for scanning source files diff --git a/src/engine/SCons/Scanner/FortranTests.py b/src/engine/SCons/Scanner/FortranTests.py index b7e527c..ebf7b56 100644 --- a/src/engine/SCons/Scanner/FortranTests.py +++ b/src/engine/SCons/Scanner/FortranTests.py @@ -311,7 +311,7 @@ class FortranScannerTestCase5(unittest.TestCase): path = s.path(env) fs = SCons.Node.FS.FS(original) deps = s(make_node('fff2.f', fs), env, path) - headers = ['d1/d2/f2.f', 'd1/f2.f', 'd1/f2.f'] + headers = ['d1/f2.f', 'd1/d2/f2.f', 'd1/f2.f'] deps_match(self, deps, map(test.workpath, headers)) class FortranScannerTestCase6(unittest.TestCase): @@ -322,7 +322,7 @@ class FortranScannerTestCase6(unittest.TestCase): path = s.path(env) fs = SCons.Node.FS.FS(original) deps = s(make_node('fff2.f', fs), env, path) - headers = ['d1/d2/f2.f', 'd1/f2.f', 'f2.f'] + headers = ['d1/f2.f', 'd1/d2/f2.f', 'f2.f'] deps_match(self, deps, map(test.workpath, headers)) test.unlink('f2.f') @@ -333,7 +333,7 @@ class FortranScannerTestCase7(unittest.TestCase): path = s.path(env) fs = SCons.Node.FS.FS(original) deps = s(make_node('fff2.f', fs), env, path) - headers = ['d1/d2/f2.f', 'd1/d2/f2.f', 'd1/f2.f'] + headers = ['d1/f2.f', 'd1/d2/f2.f', 'd1/d2/f2.f'] deps_match(self, deps, map(test.workpath, headers)) class FortranScannerTestCase8(unittest.TestCase): @@ -344,7 +344,7 @@ class FortranScannerTestCase8(unittest.TestCase): path = s.path(env) fs = SCons.Node.FS.FS(original) deps = s(make_node('fff2.f', fs), env, path) - headers = ['d1/d2/f2.f', 'd1/f2.f', 'f2.f'] + headers = ['d1/f2.f', 'd1/d2/f2.f', 'f2.f'] deps_match(self, deps, map(test.workpath, headers)) test.unlink('f2.f') diff --git a/src/engine/SCons/Scanner/IDLTests.py b/src/engine/SCons/Scanner/IDLTests.py index 876f492..f8683b0 100644 --- a/src/engine/SCons/Scanner/IDLTests.py +++ b/src/engine/SCons/Scanner/IDLTests.py @@ -243,7 +243,7 @@ class IDLScannerTestCase1(unittest.TestCase): s = SCons.Scanner.IDL.IDLScan() path = s.path(env) deps = s(make_node('t1.idl'), env, path) - headers = ['f1.idl', 'f2.idl', 'f3.idl'] + headers = ['f1.idl', 'f3.idl', 'f2.idl'] deps_match(self, deps, map(test.workpath, headers)) class IDLScannerTestCase2(unittest.TestCase): @@ -252,7 +252,7 @@ class IDLScannerTestCase2(unittest.TestCase): s = SCons.Scanner.IDL.IDLScan() path = s.path(env) deps = s(make_node('t1.idl'), env, path) - headers = ['d1/f2.idl', 'f1.idl', 'f3.idl'] + headers = ['f1.idl', 'f3.idl', 'd1/f2.idl'] deps_match(self, deps, map(test.workpath, headers)) class IDLScannerTestCase3(unittest.TestCase): @@ -261,7 +261,7 @@ class IDLScannerTestCase3(unittest.TestCase): s = SCons.Scanner.IDL.IDLScan() path = s.path(env) deps = s(make_node('t2.idl'), env, path) - headers = ['d1/d2/f1.idl', 'd1/f1.idl', 'f1.idl', 'f3.idl'] + headers = ['d1/f1.idl', 'f1.idl', 'd1/d2/f1.idl', 'f3.idl'] deps_match(self, deps, map(test.workpath, headers)) class IDLScannerTestCase4(unittest.TestCase): @@ -270,7 +270,7 @@ class IDLScannerTestCase4(unittest.TestCase): s = SCons.Scanner.IDL.IDLScan() path = s.path(env) deps = s(make_node('t2.idl'), env, path) - headers = ['d1/d2/f1.idl', 'd1/f1.idl', 'f1.idl', 'f3.idl'] + headers = ['d1/f1.idl', 'f1.idl', 'd1/d2/f1.idl', 'f3.idl'] deps_match(self, deps, map(test.workpath, headers)) class IDLScannerTestCase5(unittest.TestCase): @@ -292,8 +292,9 @@ class IDLScannerTestCase5(unittest.TestCase): # scanned, essential for cooperation with BuildDir functionality. assert n.rexists_called - headers = ['d1/f1.idl', 'd1/f1.idl', 'd1/f2.idl', 'd1/f2.idl', 'd1/f3-test.idl', - 'f1.idl', 'f2.idl', 'f3-test.idl'] + headers = ['d1/f1.idl', 'd1/f2.idl', + 'f1.idl', 'f2.idl', 'f3-test.idl', + 'd1/f1.idl', 'd1/f2.idl', 'd1/f3-test.idl'] deps_match(self, deps, map(test.workpath, headers)) class IDLScannerTestCase6(unittest.TestCase): @@ -305,8 +306,8 @@ class IDLScannerTestCase6(unittest.TestCase): path2 = s.path(env2) deps1 = s(make_node('t1.idl'), env1, path1) deps2 = s(make_node('t1.idl'), env2, path2) - headers1 = ['d1/f2.idl', 'f1.idl', 'f3.idl'] - headers2 = ['d1/d2/f2.idl', 'f1.idl', 'f3.idl'] + headers1 = ['f1.idl', 'f3.idl', 'd1/f2.idl'] + headers2 = ['f1.idl', 'f3.idl', 'd1/d2/f2.idl'] deps_match(self, deps1, map(test.workpath, headers1)) deps_match(self, deps2, map(test.workpath, headers2)) @@ -409,7 +410,7 @@ class IDLScannerTestCase12(unittest.TestCase): s = SCons.Scanner.IDL.IDLScan() path = s.path(env) deps = s(make_node('t1.idl'), env, path) - headers = ['d1/f2.idl', 'f1.idl', 'f3.idl'] + headers = ['f1.idl', 'f3.idl', 'd1/f2.idl'] deps_match(self, deps, map(test.workpath, headers)) diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py index 68b56bb..e5ac2c6 100644 --- a/src/engine/SCons/Scanner/__init__.py +++ b/src/engine/SCons/Scanner/__init__.py @@ -30,6 +30,7 @@ The Scanner package for the SCons software construction utility. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import re +import string import SCons.Node.FS import SCons.Sig @@ -285,9 +286,9 @@ class Classic(Current): regular expressions to find the includes. Note that in order for this to work "out of the box" (without - overriding the find_include() method), the regular expression passed - to the constructor must return the name of the include file in group - 0. + overriding the find_include() and sort_key() methods), the regular + expression passed to the constructor must return the name of the + include file in group 0. """ def __init__(self, name, suffixes, path_variable, regex, @@ -296,10 +297,7 @@ class Classic(Current): self.cre = re.compile(regex, re.M) self.fs = fs - def _scan(node, env, path, self=self, fs=fs): - return self.scan(node, env, path) - - kw['function'] = _scan + kw['function'] = self.scan kw['path_function'] = FindPathDirs(path_variable, fs) kw['recursive'] = 1 kw['skeys'] = suffixes @@ -313,6 +311,9 @@ class Classic(Current): self.fs.File) return n, include + def sort_key(self, include): + return SCons.Node.FS._my_normcase(include) + def scan(self, node, env, path=()): node = node.rfile() @@ -326,37 +327,27 @@ class Classic(Current): includes = self.cre.findall(node.get_contents()) node.includes = includes + # This is a hand-coded DSU (decorate-sort-undecorate, or + # Schwartzian transform) pattern. The sort key is the raw name + # of the file as specifed on the #include line (including the + # " or <, since that may affect what file is found), which lets + # us keep the sort order constant regardless of whether the file + # is actually found in a Repository or locally. nodes = [] source_dir = node.get_dir() for include in includes: n, i = self.find_include(include, source_dir, path) - if not n is None: - nodes.append(n) - else: + if n is None: SCons.Warnings.warn(SCons.Warnings.DependencyWarning, "No dependency generated for file: %s (included from: %s) -- file not found" % (i, node)) + else: + sortkey = self.sort_key(include) + nodes.append((sortkey, n)) - # Schwartzian transform from the Python FAQ Wizard - def st(List, Metric): - def pairing(element, M = Metric): - return (M(element), element) - def stripit(pair): - return pair[1] - paired = map(pairing, List) - paired.sort() - return map(stripit, paired) - - def normalize(node): - # We don't want the order of includes to be - # modified by case changes on case insensitive OSes, so - # normalize the case of the filename here: - # (see test/win32pathmadness.py for a test of this) - return SCons.Node.FS._my_normcase(str(node)) - - transformed = st(nodes, normalize) - # print "Classic: " + str(node) + " => " + str(map(lambda x: str(x),list(transformed))) - return transformed + nodes.sort() + nodes = map(lambda pair: pair[1], nodes) + return nodes class ClassicCPP(Classic): """ @@ -378,3 +369,6 @@ class ClassicCPP(Classic): tuple(path) + (source_dir,), self.fs.File) return n, include[1] + + def sort_key(self, include): + return SCons.Node.FS._my_normcase(string.join(include)) diff --git a/test/Repository/signature-order.py b/test/Repository/signature-order.py new file mode 100644 index 0000000..2329569 --- /dev/null +++ b/test/Repository/signature-order.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify that we don't rebuild things unnecessarily when the order of +signatures changes because an included file shifts from the local sandbox +to a Repository and vice versa. +""" + +import sys +import TestSCons + +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.subdir('repository', 'work') + +repository = test.workpath('repository') +work_foo = test.workpath('work', 'foo' + _exe) +work_foo_h = test.workpath('work', 'foo.h') + +test.write(['work', 'SConstruct'], """ +Repository(r'%s') +env = Environment(CPPPATH = ['.']) +env.Program(target = 'foo', source = 'foo.c') +""" % repository) + +foo_h_contents = """\ +#define STRING1 "foo.h" +""" + +bar_h_contents = """\ +#define STRING2 "bar.h" +""" + +test.write(['repository', 'foo.h'], foo_h_contents) + +test.write(['repository', 'bar.h'], bar_h_contents) + +test.write(['repository', 'foo.c'], r""" +#include +#include +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("%s\n", STRING1); + printf("%s\n", STRING2); + printf("repository/foo.c\n"); + exit (0); +} +""") + +# Make the entire repository non-writable, so we'll detect +# if we try to write into it accidentally. +test.writable('repository', 0) + + + +test.run(chdir = 'work', arguments = '.') + +test.run(program = work_foo, stdout = +"""foo.h +bar.h +repository/foo.c +""") + +test.up_to_date(chdir = 'work', arguments = '.') + + + +# Now create the bar.h file locally, and make sure it's still up-to-date. +# This will change the order in which the include files are listed when +# calculating the signature; the old order was something like: +# /var/tmp/.../bar.h +# /var/tmp/.../far.h +# and the new order will be: +# /var/tmp/.../far.h +# bar.h +# Because we write the same bar.h file contents, this should not cause +# a rebuild (because we sort the resulting signatures). + +test.write(['work', 'bar.h'], bar_h_contents) + +test.up_to_date(chdir = 'work', arguments = '.') + + + +# And for good measure, write the same foo.h file locally. + +test.write(['work', 'foo.h'], foo_h_contents) + +test.up_to_date(chdir = 'work', arguments = '.') + + + +test.pass_test() diff --git a/test/signature-order.py b/test/signature-order.py new file mode 100644 index 0000000..3fc9f5f --- /dev/null +++ b/test/signature-order.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" + +Verify that we do rebuild things when the contents of +two .h files are swapped, changing the order in which +dependency signatures show up in the calculated list, +but + +""" + +import sys +import TestSCons + +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.subdir('work1', 'work2') + +work1_foo = test.workpath('work1', 'foo' + _exe) +work2_foo = test.workpath('work2', 'foo' + _exe) + +content1 = """\ +#ifndef STRING +#define STRING "content1" +#endif +""" + +content2 = """\ +#ifndef STRING +#define STRING "content2" +#endif +""" + + + +test.write(['work1', 'SConstruct'], """\ +env = Environment(CPPPATH = ['.']) +env.Program(target = 'foo', source = 'foo.c') +""") + +test.write(['work1', 'foo.c'], r""" +#include +#include +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("%s\n", STRING); + exit (0); +} +""") + +test.write(['work1', 'aaa.h'], content1) +test.write(['work1', 'bbb.h'], content2) + +test.run(chdir = 'work1') + +test.run(program = work1_foo, stdout = "content1\n") + +test.write(['work1', 'aaa.h'], content2) +test.write(['work1', 'bbb.h'], content1) + +test.run(chdir = 'work1') + +test.run(program = work1_foo, stdout = "content2\n") + + + +test.write(['work2', 'SConstruct'], """\ +env = Environment(CPPPATH = ['.']) +env.Program(target = 'foo', source = 'foo.c') +""") + +test.write(['work2', 'foo.c'], r""" +#include "aaa.h" +#include "bbb.h" +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("%s\n", STRING); + exit (0); +} +""") + +test.write(['work2', 'aaa.h'], content1) +test.write(['work2', 'bbb.h'], content2) + +test.run(chdir = 'work2') + +test.run(program = work2_foo, stdout = "content1\n") + +test.write(['work2', 'aaa.h'], content2) +test.write(['work2', 'bbb.h'], content1) + +test.run(chdir = 'work2') + +test.run(program = work2_foo, stdout = "content2\n") + + + +test.pass_test() -- cgit v0.12