From 82a50acec0e8ac5d2c02d01bfad31df542563636 Mon Sep 17 00:00:00 2001 From: Steven Knight Date: Sun, 20 Mar 2005 00:23:12 +0000 Subject: Fix re-scanning of built files for implicit dependencies when the -j option is used. --- src/CHANGES.txt | 3 ++ src/engine/SCons/Debug.py | 23 ++++++++++ src/engine/SCons/EnvironmentTests.py | 6 --- src/engine/SCons/JobTests.py | 2 +- src/engine/SCons/Node/NodeTests.py | 10 +--- src/engine/SCons/Node/__init__.py | 17 +++++-- src/engine/SCons/SConf.py | 2 +- src/engine/SCons/Taskmaster.py | 88 ++++++++++++++++++++++-------------- src/engine/SCons/TaskmasterTests.py | 9 ++-- test/parallel-rescan.py | 72 +++++++++++++++++++++++++++++ 10 files changed, 172 insertions(+), 60 deletions(-) create mode 100644 test/parallel-rescan.py diff --git a/src/CHANGES.txt b/src/CHANGES.txt index ae16481..2e26545 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -237,6 +237,9 @@ RELEASE 0.97 - XXX - Add a global name for the Entry class (which had already been documented). + - Fix re-scanning of generated source files for implicit dependencies + when the -j option is used. + From Wayne Lee: - Avoid "maximum recursion limit" errors when removing $(-$) pairs diff --git a/src/engine/SCons/Debug.py b/src/engine/SCons/Debug.py index bf7ed43..0dbb116 100644 --- a/src/engine/SCons/Debug.py +++ b/src/engine/SCons/Debug.py @@ -152,3 +152,26 @@ def func_shorten(func_tuple): f = f[i:] break return (f,)+func_tuple[1:] + + + +TraceFP = {} +TraceDefault = '/dev/tty' + +def Trace(msg, file=None, mode='a'): + """Write a trace a message to a file. Whenever a file is specified, + it becomes the default for the next call to Trace().""" + global TraceDefault + if file is None: + file = TraceDefault + else: + TraceDefault = file + try: + fp = TraceFP[file] + except KeyError: + try: + fp = TraceFP[file] = open(file, mode) + except TypeError: + # Assume we were passed an open file pointer. + fp = file + fp.write(msg) diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py index 37a2dca..0e8fd56 100644 --- a/src/engine/SCons/EnvironmentTests.py +++ b/src/engine/SCons/EnvironmentTests.py @@ -2589,8 +2589,6 @@ f5: \ assert s.side_effect assert foo.side_effects == [s] assert bar.side_effects == [s] - assert s.depends_on([bar]) - assert s.depends_on([foo]) fff = env.Object('fff.obj', 'fff.cpp')[0] bbb = env.Object('bbb.obj', 'bbb.cpp')[0] @@ -2600,8 +2598,6 @@ f5: \ assert s.side_effect assert fff.side_effects == [s], fff.side_effects assert bbb.side_effects == [s], bbb.side_effects - assert s.depends_on([bbb]) - assert s.depends_on([fff]) ggg = env.Object('ggg.obj', 'ggg.cpp')[0] ccc = env.Object('ccc.obj', 'ccc.cpp')[0] @@ -2611,8 +2607,6 @@ f5: \ assert s.side_effect assert ggg.side_effects == [s], ggg.side_effects assert ccc.side_effects == [s], ccc.side_effects - assert s.depends_on([ccc]) - assert s.depends_on([ggg]) def test_SourceCode(self): """Test the SourceCode() method.""" diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py index 7dc8658..3326f33 100644 --- a/src/engine/SCons/JobTests.py +++ b/src/engine/SCons/JobTests.py @@ -425,7 +425,7 @@ class _SConsTaskTest(unittest.TestCase): # mislabelling of results). for N in testnodes: - self.failUnless(N.get_state() in [None, N.expect_to_be], + self.failUnless(N.get_state() in [SCons.Node.no_state, N.expect_to_be], "node ran but got unexpected result") self.failUnless(filter(lambda N: N.get_state(), testnodes), diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index 90bb332..3660b11 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -402,14 +402,6 @@ class NodeTestCase(unittest.TestCase): n = SCons.Node.Node() n.visited() - def test_depends_on(self): - """Test the depends_on() method - """ - parent = SCons.Node.Node() - child = SCons.Node.Node() - parent.add_dependency([child]) - assert parent.depends_on([child]) - def test_builder_set(self): """Test setting a Node's Builder """ @@ -1011,7 +1003,7 @@ class NodeTestCase(unittest.TestCase): """Test setting and getting the state of a node """ node = SCons.Node.Node() - assert node.get_state() == None + assert node.get_state() == SCons.Node.no_state node.set_state(SCons.Node.executing) assert node.get_state() == SCons.Node.executing assert SCons.Node.pending < SCons.Node.executing diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 9c15659..e1f872f 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -62,6 +62,7 @@ import SCons.Util # it has no builder of its own. The canonical example is a file # system directory, which is only up to date if all of its children # were up to date. +no_state = 0 pending = 1 executing = 2 up_to_date = 3 @@ -69,6 +70,16 @@ executed = 4 failed = 5 stack = 6 # nodes that are in the current Taskmaster execution stack +StateString = { + 0 : "0", + 1 : "pending", + 2 : "executing", + 3 : "up_to_date", + 4 : "executed", + 5 : "failed", + 6 : "stack", +} + # controls whether implicit dependencies are cached: implicit_cache = 0 @@ -124,7 +135,7 @@ class Node: self.wkids = None # Kids yet to walk, when it's an array self.env = None - self.state = None + self.state = no_state self.precious = None self.always_build = None self.found_includes = {} @@ -291,10 +302,6 @@ class Node: without requiring a build..""" pass - def depends_on(self, nodes): - """Does this node depend on any of 'nodes'? __cacheable__""" - return reduce(lambda D,N,C=self.children(): D or (N in C), nodes, 0) - def builder_set(self, builder): "__cache_reset__" self.builder = builder diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py index 5572a01..7bd9c1a 100644 --- a/src/engine/SCons/SConf.py +++ b/src/engine/SCons/SConf.py @@ -248,7 +248,7 @@ class SConfBuildTask(SCons.Taskmaster.Task): c_bi = isinstance(bi, SConfBuildInfo) if c_bi: if cache_mode == CACHE: - t.state = SCons.Node.up_to_date + t.set_state(SCons.Node.up_to_date) else: new_bsig = t.calc_signature(sconf_global.calc) if t.env.use_build_signature(): diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 1784798..b90bfe1 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -134,7 +134,7 @@ class Task: if self.targets[0].get_state() == SCons.Node.executing: for t in self.targets: for side_effect in t.side_effects: - side_effect.set_state(None) + side_effect.set_state(SCons.Node.no_state) t.set_state(SCons.Node.executed) t.built() else: @@ -275,14 +275,13 @@ class Taskmaster: return self.ready_exc = None - + while self.candidates: - node = self.candidates[-1] + node = self.candidates.pop() state = node.get_state() - # Skip this node if it has already been executed: - if state != None and state != SCons.Node.stack: - self.candidates.pop() + # Skip this node if it has already been handled: + if not state in [ SCons.Node.no_state, SCons.Node.stack ]: continue # Mark this node as being on the execution stack: @@ -296,7 +295,6 @@ class Taskmaster: exc_value = sys.exc_info()[1] e = SCons.Errors.ExplicitExit(node, exc_value.code) self.ready_exc = (SCons.Errors.ExplicitExit, e) - self.candidates.pop() self.ready = node break except KeyboardInterrupt: @@ -307,23 +305,24 @@ class Taskmaster: # BuildDir, or a Scanner threw something). Arrange to # raise the exception when the Task is "executed." self.ready_exc = sys.exc_info() - self.candidates.pop() self.ready = node break - # Skip this node if any of its children have failed. This # catches the case where we're descending a top-level target # and one of our children failed while trying to be built # by a *previous* descent of an earlier top-level target. if filter(lambda I: I[0] == SCons.Node.failed, childinfo): node.set_state(SCons.Node.failed) - self.candidates.pop() continue # Detect dependency cycles: cycle = filter(lambda I: I[0] == SCons.Node.stack, childinfo) if cycle: + # The node we popped from the candidate stack is part of + # the cycle we detected, so put it back before generating + # the message to report. + self.candidates.append(node) nodes = filter(lambda N: N.get_state() == SCons.Node.stack, self.candidates) + \ map(lambda I: I[2], cycle) @@ -331,19 +330,52 @@ class Taskmaster: desc = "Dependency cycle: " + string.join(map(str, nodes), " -> ") raise SCons.Errors.UserError, desc - # Find all of the derived dependencies (that is, - # children who have builders or are side effects): - # Add derived files that have not been built - # to the candidates list: - not_built = filter(lambda I: I[1] and not I[0], childinfo) + # Select all of the dependencies that are derived targets + # (that is, children who have builders or are side effects). + derived_children = filter(lambda I: I[1], childinfo) + + not_started = filter(lambda I: not I[0], derived_children) + if not_started: + not_started = map(lambda I: I[2], not_started) + + # We're waiting on one more derived targets that have + # not yet started building. Add this node to the + # waiting_parents lists of those derived files so that + # when they've finished building, our implicit dependency + # list will get cleared and we'll re-scan the newly-built + # file(s) for updated implicit dependencies. + map(lambda n, P=node: n.add_to_waiting_parents(P), not_started) + + # Now we add these derived targets to the candidates + # list so they can be examined and built. We have to + # add ourselves back to the list first, though, so we get + # a chance to re-scan and build after the dependencies. + # + # We reverse the order in which the children are added + # to the candidates stack so the order in which they're + # popped matches the order in which they show up in our + # children's list. This is more logical / efficient for + # builders with multiple targets, since the "primary" + # target will be examined first. + self.candidates.append(node) + not_started.reverse() + self.candidates.extend(self.order(not_started)) + continue + + not_built = filter(lambda I: I[0] <= SCons.Node.executing, derived_children) if not_built: - # We're waiting on one more derived files that have not - # yet been built. Add this node to the waiting_parents - # list of each of those derived files. + # We're waiting on one or more derived targets that have + # started building but not yet finished. Add this node + # to the waiting parents lists of those derived files + # so that when they've finished building, our implicit + # dependency list will get cleared and we'll re-scan the + # newly-built file(s) for updated implicit dependencies. map(lambda I, P=node: I[2].add_to_waiting_parents(P), not_built) - not_built.reverse() - self.candidates.extend(self.order(map(lambda I: I[2], - not_built))) + + # And add this node to the "pending" list, so it can get + # put back on the candidates list when appropriate. + self.pending.append(node) + node.set_state(SCons.Node.pending) continue # Skip this node if it has side-effects that are @@ -354,20 +386,10 @@ class Taskmaster: 0): self.pending.append(node) node.set_state(SCons.Node.pending) - self.candidates.pop() - continue - - # Skip this node if it is pending on a currently - # executing node: - if node.depends_on(self.executing) or node.depends_on(self.pending): - self.pending.append(node) - node.set_state(SCons.Node.pending) - self.candidates.pop() continue # The default when we've gotten through all of the checks above: # this node is ready to be built. - self.candidates.pop() self.ready = node break @@ -387,7 +409,7 @@ class Taskmaster: tlist = [node] self.executing.extend(tlist) self.executing.extend(node.side_effects) - + task = self.tasker(self, tlist, node in self.targets, node) try: task.make_ready() @@ -451,7 +473,7 @@ class Taskmaster: # (they may not all be ready to build, but _find_next_ready_node() # will figure out which ones are really ready) for node in self.pending: - node.set_state(None) + node.set_state(SCons.Node.no_state) self.pending.reverse() self.candidates.extend(self.pending) self.pending = [] diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index a3e4219..e30ca19 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -51,7 +51,7 @@ class Node: self.builder = Builder() self.bsig = None self.csig = None - self.state = None + self.state = SCons.Node.no_state self.prepared = None self.waiting_parents = [] self.side_effect = 0 @@ -157,7 +157,6 @@ class Node: def postprocess(self): self.postprocessed = 1 - class OtherError(Exception): pass @@ -220,11 +219,11 @@ class TaskmasterTestCase(unittest.TestCase): else: self.targets[0].build() - n1.set_state(None) + n1.set_state(SCons.Node.no_state) n1._current_val = 1 - n2.set_state(None) + n2.set_state(SCons.Node.no_state) n2._current_val = 1 - n3.set_state(None) + n3.set_state(SCons.Node.no_state) n3._current_val = 1 tm = SCons.Taskmaster.Taskmaster(targets = [n3], tasker = MyTask) diff --git a/test/parallel-rescan.py b/test/parallel-rescan.py new file mode 100644 index 0000000..4a1f27e --- /dev/null +++ b/test/parallel-rescan.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify that when a source file is generated and the -j option is used, +the source file correctly gets re-scanned for implicit dependencies +after it's built. +""" + +import TestSCons + +test = TestSCons.TestSCons() + +test.write('SConstruct', """\ +env = Environment() +env['BUILDERS']['COPY'] = Builder(action = Copy("$TARGET", "$SOURCE")) + +env.COPY('a.c', 'a.in') +env.COPY('b.c', 'b.in') + +env.StaticLibrary('lib', ['a.c', 'b.c']) +""") + +test.write("a.in", """\ +#include "a.h" +""") + +test.write("b.in", """\ +#include "b.h" +""") + +test.write("a.h", """\ +char *A_FILE = "b.in"; +""") + +test.write("b.h", """\ +char *B_FILE = "b.in"; +""") + +test.run(arguments = '-j4 .', + stderr=TestSCons.noisy_ar, + match=TestSCons.match_re_dotall) + +# If the dependencies weren't re-scanned properly, the .h files won't +# show up in the previous run's dependency lists, and the .o files and +# library will get rebuilt here. +test.up_to_date(arguments = '.') + +test.pass_test() -- cgit v0.12