Fix re-scanning of built files for implicit dependencies when the -j option is used.

9 files changed, 100 insertions, 60 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index ae16481..2e26545 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -237,6 +237,9 @@ RELEASE 0.97 - XXX
   - Add a global name for the Entry class (which had already been
     documented).
 
+  - Fix re-scanning of generated source files for implicit dependencies
+    when the -j option is used.
+
   From Wayne Lee:
 
   - Avoid "maximum recursion limit" errors when removing $(-$) pairs
diff --git a/src/engine/SCons/Debug.py b/src/engine/SCons/Debug.py
index bf7ed43..0dbb116 100644
--- a/src/engine/SCons/Debug.py
+++ b/src/engine/SCons/Debug.py
@@ -152,3 +152,26 @@ def func_shorten(func_tuple):
             f = f[i:]
             break
     return (f,)+func_tuple[1:]
+
+
+
+TraceFP = {}
+TraceDefault = '/dev/tty'
+
+def Trace(msg, file=None, mode='a'):
+    """Write a trace a message to a file.  Whenever a file is specified,
+    it becomes the default for the next call to Trace()."""
+    global TraceDefault
+    if file is None:
+        file = TraceDefault
+    else:
+        TraceDefault = file
+    try:
+        fp = TraceFP[file]
+    except KeyError:
+        try:
+            fp = TraceFP[file] = open(file, mode)
+        except TypeError:
+            # Assume we were passed an open file pointer.
+            fp = file
+    fp.write(msg)
diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py
index 37a2dca..0e8fd56 100644
--- a/src/engine/SCons/EnvironmentTests.py
+++ b/src/engine/SCons/EnvironmentTests.py
@@ -2589,8 +2589,6 @@ f5: \
         assert s.side_effect
         assert foo.side_effects == [s]
         assert bar.side_effects == [s]
-        assert s.depends_on([bar])
-        assert s.depends_on([foo])
 
         fff = env.Object('fff.obj', 'fff.cpp')[0]
         bbb = env.Object('bbb.obj', 'bbb.cpp')[0]
@@ -2600,8 +2598,6 @@ f5: \
         assert s.side_effect
         assert fff.side_effects == [s], fff.side_effects
         assert bbb.side_effects == [s], bbb.side_effects
-        assert s.depends_on([bbb])
-        assert s.depends_on([fff])
 
         ggg = env.Object('ggg.obj', 'ggg.cpp')[0]
         ccc = env.Object('ccc.obj', 'ccc.cpp')[0]
@@ -2611,8 +2607,6 @@ f5: \
         assert s.side_effect
         assert ggg.side_effects == [s], ggg.side_effects
         assert ccc.side_effects == [s], ccc.side_effects
-        assert s.depends_on([ccc])
-        assert s.depends_on([ggg])
 
     def test_SourceCode(self):
         """Test the SourceCode() method."""
diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py
index 7dc8658..3326f33 100644
--- a/src/engine/SCons/JobTests.py
+++ b/src/engine/SCons/JobTests.py
@@ -425,7 +425,7 @@ class _SConsTaskTest(unittest.TestCase):
         # mislabelling of results).
 
         for N in testnodes:
-            self.failUnless(N.get_state() in [None, N.expect_to_be],
+            self.failUnless(N.get_state() in [SCons.Node.no_state, N.expect_to_be],
                             "node ran but got unexpected result")
 
         self.failUnless(filter(lambda N: N.get_state(), testnodes),
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index 90bb332..3660b11 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -402,14 +402,6 @@ class NodeTestCase(unittest.TestCase):
         n = SCons.Node.Node()
         n.visited()
 
-    def test_depends_on(self):
-        """Test the depends_on() method
-        """
-        parent = SCons.Node.Node()
-        child = SCons.Node.Node()
-        parent.add_dependency([child])
-        assert parent.depends_on([child])
-
     def test_builder_set(self):
         """Test setting a Node's Builder
         """
@@ -1011,7 +1003,7 @@ class NodeTestCase(unittest.TestCase):
         """Test setting and getting the state of a node
         """
         node = SCons.Node.Node()
-        assert node.get_state() == None
+        assert node.get_state() == SCons.Node.no_state
         node.set_state(SCons.Node.executing)
         assert node.get_state() == SCons.Node.executing
         assert SCons.Node.pending < SCons.Node.executing
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index 9c15659..e1f872f 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -62,6 +62,7 @@ import SCons.Util
 # it has no builder of its own.  The canonical example is a file
 # system directory, which is only up to date if all of its children
 # were up to date.
+no_state = 0
 pending = 1
 executing = 2
 up_to_date = 3
@@ -69,6 +70,16 @@ executed = 4
 failed = 5
 stack = 6 # nodes that are in the current Taskmaster execution stack
 
+StateString = {
+    0 : "0",
+    1 : "pending",
+    2 : "executing",
+    3 : "up_to_date",
+    4 : "executed",
+    5 : "failed",
+    6 : "stack",
+}
+
 # controls whether implicit dependencies are cached:
 implicit_cache = 0
 
@@ -124,7 +135,7 @@ class Node:
         self.wkids = None       # Kids yet to walk, when it's an array
 
         self.env = None
-        self.state = None
+        self.state = no_state
         self.precious = None
         self.always_build = None
         self.found_includes = {}
@@ -291,10 +302,6 @@ class Node:
         without requiring a build.."""
         pass
 
-    def depends_on(self, nodes):
-        """Does this node depend on any of 'nodes'? __cacheable__"""
-        return reduce(lambda D,N,C=self.children(): D or (N in C), nodes, 0)
-
     def builder_set(self, builder):
         "__cache_reset__"
         self.builder = builder
diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py
index 5572a01..7bd9c1a 100644
--- a/src/engine/SCons/SConf.py
+++ b/src/engine/SCons/SConf.py
@@ -248,7 +248,7 @@ class SConfBuildTask(SCons.Taskmaster.Task):
             c_bi = isinstance(bi, SConfBuildInfo)
             if c_bi:
                 if cache_mode == CACHE:
-                    t.state = SCons.Node.up_to_date
+                    t.set_state(SCons.Node.up_to_date)
                 else:
                     new_bsig = t.calc_signature(sconf_global.calc)
                     if t.env.use_build_signature():
diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py
index 1784798..b90bfe1 100644
--- a/src/engine/SCons/Taskmaster.py
+++ b/src/engine/SCons/Taskmaster.py
@@ -134,7 +134,7 @@ class Task:
         if self.targets[0].get_state() == SCons.Node.executing:
             for t in self.targets:
                 for side_effect in t.side_effects:
-                    side_effect.set_state(None)
+                    side_effect.set_state(SCons.Node.no_state)
                 t.set_state(SCons.Node.executed)
                 t.built()
         else:
@@ -275,14 +275,13 @@ class Taskmaster:
             return
 
         self.ready_exc = None
-        
+
         while self.candidates:
-            node = self.candidates[-1]
+            node = self.candidates.pop()
             state = node.get_state()
 
-            # Skip this node if it has already been executed:
-            if state != None and state != SCons.Node.stack:
-                self.candidates.pop()
+            # Skip this node if it has already been handled:
+            if not state in [ SCons.Node.no_state, SCons.Node.stack ]:
                 continue
 
             # Mark this node as being on the execution stack:
@@ -296,7 +295,6 @@ class Taskmaster:
                 exc_value = sys.exc_info()[1]
                 e = SCons.Errors.ExplicitExit(node, exc_value.code)
                 self.ready_exc = (SCons.Errors.ExplicitExit, e)
-                self.candidates.pop()
                 self.ready = node
                 break
             except KeyboardInterrupt:
@@ -307,23 +305,24 @@ class Taskmaster:
                 # BuildDir, or a Scanner threw something).  Arrange to
                 # raise the exception when the Task is "executed."
                 self.ready_exc = sys.exc_info()
-                self.candidates.pop()
                 self.ready = node
                 break
 
-            
             # Skip this node if any of its children have failed.  This
             # catches the case where we're descending a top-level target
             # and one of our children failed while trying to be built
             # by a *previous* descent of an earlier top-level target.
             if filter(lambda I: I[0] == SCons.Node.failed, childinfo):
                 node.set_state(SCons.Node.failed)
-                self.candidates.pop()
                 continue
 
             # Detect dependency cycles:
             cycle = filter(lambda I: I[0] == SCons.Node.stack, childinfo)
             if cycle:
+                # The node we popped from the candidate stack is part of
+                # the cycle we detected, so put it back before generating
+                # the message to report.
+                self.candidates.append(node)
                 nodes = filter(lambda N: N.get_state() == SCons.Node.stack,
                                self.candidates) + \
                                map(lambda I: I[2], cycle)
@@ -331,19 +330,52 @@ class Taskmaster:
                 desc = "Dependency cycle: " + string.join(map(str, nodes), " -> ")
                 raise SCons.Errors.UserError, desc
 
-            # Find all of the derived dependencies (that is,
-            # children who have builders or are side effects):
-            # Add derived files that have not been built
-            # to the candidates list:
-            not_built = filter(lambda I: I[1] and not I[0], childinfo)
+            # Select all of the dependencies that are derived targets
+            # (that is, children who have builders or are side effects).
+            derived_children = filter(lambda I: I[1], childinfo)
+
+            not_started = filter(lambda I: not I[0], derived_children)
+            if not_started:
+                not_started = map(lambda I: I[2], not_started)
+
+                # We're waiting on one more derived targets that have
+                # not yet started building.  Add this node to the
+                # waiting_parents lists of those derived files so that
+                # when they've finished building, our implicit dependency
+                # list will get cleared and we'll re-scan the newly-built
+                # file(s) for updated implicit dependencies.
+                map(lambda n, P=node: n.add_to_waiting_parents(P), not_started)
+
+                # Now we add these derived targets to the candidates
+                # list so they can be examined and built.  We have to
+                # add ourselves back to the list first, though, so we get
+                # a chance to re-scan and build after the dependencies.
+                #
+                # We reverse the order in which the children are added
+                # to the candidates stack so the order in which they're
+                # popped matches the order in which they show up in our
+                # children's list.  This is more logical / efficient for
+                # builders with multiple targets, since the "primary"
+                # target will be examined first.
+                self.candidates.append(node)
+                not_started.reverse()
+                self.candidates.extend(self.order(not_started))
+                continue
+
+            not_built = filter(lambda I: I[0] <= SCons.Node.executing, derived_children)
             if not_built:
-                # We're waiting on one more derived files that have not
-                # yet been built.  Add this node to the waiting_parents
-                # list of each of those derived files.
+                # We're waiting on one or more derived targets that have
+                # started building but not yet finished.  Add this node
+                # to the waiting parents lists of those derived files
+                # so that when they've finished building, our implicit
+                # dependency list will get cleared and we'll re-scan the
+                # newly-built file(s) for updated implicit dependencies.
                 map(lambda I, P=node: I[2].add_to_waiting_parents(P), not_built)
-                not_built.reverse()
-                self.candidates.extend(self.order(map(lambda I: I[2],
-                                                      not_built)))
+
+                # And add this node to the "pending" list, so it can get
+                # put back on the candidates list when appropriate.
+                self.pending.append(node)
+                node.set_state(SCons.Node.pending)
                 continue
 
             # Skip this node if it has side-effects that are
@@ -354,20 +386,10 @@ class Taskmaster:
                       0):
                 self.pending.append(node)
                 node.set_state(SCons.Node.pending)
-                self.candidates.pop()
-                continue
-
-            # Skip this node if it is pending on a currently
-            # executing node:
-            if node.depends_on(self.executing) or node.depends_on(self.pending):
-                self.pending.append(node)
-                node.set_state(SCons.Node.pending)
-                self.candidates.pop()
                 continue
 
             # The default when we've gotten through all of the checks above:
             # this node is ready to be built.
-            self.candidates.pop()
             self.ready = node
             break
 
@@ -387,7 +409,7 @@ class Taskmaster:
             tlist = [node]
         self.executing.extend(tlist)
         self.executing.extend(node.side_effects)
-        
+
         task = self.tasker(self, tlist, node in self.targets, node)
         try:
             task.make_ready()
@@ -451,7 +473,7 @@ class Taskmaster:
         # (they may not all be ready to build, but _find_next_ready_node()
         #  will figure out which ones are really ready)
         for node in self.pending:
-            node.set_state(None)
+            node.set_state(SCons.Node.no_state)
         self.pending.reverse()
         self.candidates.extend(self.pending)
         self.pending = []
diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py
index a3e4219..e30ca19 100644
--- a/src/engine/SCons/TaskmasterTests.py
+++ b/src/engine/SCons/TaskmasterTests.py
@@ -51,7 +51,7 @@ class Node:
         self.builder = Builder()
         self.bsig = None
         self.csig = None
-        self.state = None
+        self.state = SCons.Node.no_state
         self.prepared = None
         self.waiting_parents = []
         self.side_effect = 0
@@ -157,7 +157,6 @@ class Node:
     def postprocess(self):
         self.postprocessed = 1
 
-
 class OtherError(Exception):
     pass
 
@@ -220,11 +219,11 @@ class TaskmasterTestCase(unittest.TestCase):
                 else:
                     self.targets[0].build()
 
-        n1.set_state(None)
+        n1.set_state(SCons.Node.no_state)
         n1._current_val = 1
-        n2.set_state(None)
+        n2.set_state(SCons.Node.no_state)
         n2._current_val = 1
-        n3.set_state(None)
+        n3.set_state(SCons.Node.no_state)
         n3._current_val = 1
         tm = SCons.Taskmaster.Taskmaster(targets = [n3], tasker = MyTask)