Fix a problem with the new Parallel job support when a command fails. (J.T. Conklin)

3 files changed, 43 insertions, 21 deletions

diff --git a/src/engine/SCons/Job.py b/src/engine/SCons/Job.py
index 202f86f..0c01b85 100644
--- a/src/engine/SCons/Job.py
+++ b/src/engine/SCons/Job.py
@@ -147,12 +147,7 @@ class ThreadPool:
 
     def __init__(self, num):
         """Create the request and reply queues, and 'num' worker threads."""
-        # Ideally we wouldn't have to artificially limit the number of
-        # tasks that can be posted to the request queue.  But this can
-        # result in a large number of pending tasks, which at the time
-        # of this writing causes the taskmaster's next_task method to
-        # take a very long time.
-        self.requestQueue = Queue.Queue(num)
+        self.requestQueue = Queue.Queue()
         self.resultsQueue = Queue.Queue()
 
         # Create worker threads
@@ -200,6 +195,9 @@ class Parallel:
         self.taskmaster = taskmaster
         self.tp = ThreadPool(num)
 
+        self.jobs = 0
+        self.maxjobs = num
+
     def start(self):
         """Start the job. This will begin pulling tasks from the
         taskmaster and executing them, and return when there are no
@@ -207,31 +205,34 @@ class Parallel:
         an exception), then the job will stop."""
 
         while 1:
-            task = self.taskmaster.next_task()
-            if task is None:
-                break
+            if self.jobs < self.maxjobs:
+                task = self.taskmaster.next_task()
+                if task is None:
+                    break
 
-            # prepare task for execution
-            try:
-                task.prepare()
-            except KeyboardInterrupt:
-                raise
-            except:
-                # Let the failed() callback function arrange for the
-                # build to stop if that's appropriate.
-                task.failed()
+                # prepare task for execution
+                try:
+                    task.prepare()
+                except KeyboardInterrupt:
+                    raise
+                except:
+                    # Let the failed() callback function arrange for the
+                    # build to stop if that's appropriate.
+                    task.failed()
 
-            # dispatch task
-            self.tp.put(task)
+                # dispatch task
+                self.tp.put(task)
+                self.jobs = self.jobs + 1
 
             while 1:
                 try:
                     task, ok = self.tp.get_nowait()
                 except Queue.Empty:
-                    if not self.taskmaster.is_blocked():
+                    if not (self.jobs is self.maxjobs or self.taskmaster.is_blocked()):
                         break
                     task, ok = self.tp.get()
 
+                self.jobs = self.jobs - 1
                 if ok:
                     task.executed()
                 else:
diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py
index 7760cfe..6eda8c1 100644
--- a/src/engine/SCons/Taskmaster.py
+++ b/src/engine/SCons/Taskmaster.py
@@ -140,6 +140,7 @@ class Task:
         """Explicit stop-the-build failure."""
         for t in self.targets:
             t.set_state(SCons.Node.failed)
+        self.tm.failed(self.node)
         self.tm.stop()
 
     def fail_continue(self):
@@ -366,6 +367,16 @@ class Taskmaster:
         self.ready = None
         self.pending = []
 
+    def failed(self, node):
+        try:
+            tlist = node.builder.targets(node)
+        except AttributeError:
+            tlist = [node]
+        for t in tlist:
+            self.executing.remove(t)
+        for side_effect in node.side_effects:
+            self.executing.remove(side_effect)
+
     def executed(self, node):
         try:
             tlist = node.builder.targets(node)
diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py
index a394151..fd21891 100644
--- a/src/engine/SCons/TaskmasterTests.py
+++ b/src/engine/SCons/TaskmasterTests.py
@@ -535,6 +535,16 @@ class TaskmasterTestCase(unittest.TestCase):
         assert built_text == "MyTM.stop()"
         assert tm.next_task() is None
 
+    def test_failed(self):
+        """Test when a task has failed
+        """
+        n1 = Node("n1")
+        tm = SCons.Taskmaster.Taskmaster([n1])
+        t = tm.next_task()
+        assert tm.executing == [n1], tm.executing
+        tm.failed(n1)
+        assert tm.executing == [], tm.executing
+
     def test_executed(self):
         """Test when a task has been executed
         """