Save memory by allowing Nodes to clean up their Executor's build environments after they've been built.

13 files changed, 161 insertions, 20 deletions

diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py
index 6714a3d..871a803 100644
--- a/src/engine/SCons/Builder.py
+++ b/src/engine/SCons/Builder.py
@@ -203,7 +203,7 @@ def _init_nodes(builder, env, overrides, tlist, slist):
             executor.add_sources(slist)
     if executor is None:
         executor = SCons.Executor.Executor(builder,
-                                           tlist[0].generate_build_env(env),
+                                           env,
                                            overrides,
                                            tlist,
                                            slist)
diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py
index 2a0882e..8151f87 100644
--- a/src/engine/SCons/Executor.py
+++ b/src/engine/SCons/Executor.py
@@ -32,6 +32,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 
 from SCons.Debug import logInstanceCreation
+import SCons.Util
 
 
 class Executor:
@@ -71,6 +72,13 @@ class Executor:
             overrides = {}
             overrides.update(self.builder.overrides)
             overrides.update(self.overrides)
+            try:
+                generate_build_dict = self.targets[0].generate_build_dict
+            except AttributeError:
+                pass
+            else:
+                overrides.update(generate_build_dict())
+            overrides.update(SCons.Util.subst_dict(self.targets, self.sources))
             self.build_env = env.Override(overrides)
             return self.build_env
 
@@ -101,6 +109,12 @@ class Executor:
         for action in action_list:
             func(action, self.targets, self.sources, env)
 
+    def cleanup(self):
+        try:
+            del self.build_env
+        except AttributeError:
+            pass
+
     def add_sources(self, sources):
         """Add source files to this Executor's list.  This is necessary
         for "multi" Builders that can be called repeatedly to build up
diff --git a/src/engine/SCons/ExecutorTests.py b/src/engine/SCons/ExecutorTests.py
index 2a79e2d..2cdc0e2 100644
--- a/src/engine/SCons/ExecutorTests.py
+++ b/src/engine/SCons/ExecutorTests.py
@@ -90,7 +90,8 @@ class ExecutorTestCase(unittest.TestCase):
                                     't',
                                     ['s1', 's2'])
         be = x.get_build_env()
-        assert be == {'O':'o2', 'X':'xxx'}, be
+        assert be['O'] == 'o2', be['O']
+        assert be['X'] == 'xxx', be['X']
 
         env = MyEnvironment(Y='yyy')
         x = SCons.Executor.Executor(MyBuilder(env, {'O':'ob3'}),
@@ -99,14 +100,16 @@ class ExecutorTestCase(unittest.TestCase):
                                     't',
                                     's')
         be = x.get_build_env()
-        assert be == {'O':'oo3', 'Y':'yyy'}, be
+        assert be['O'] == 'oo3', be['O']
+        assert be['Y'] == 'yyy', be['Y']
         x = SCons.Executor.Executor(MyBuilder(env, {'O':'ob3'}),
                                     None,
                                     {},
                                     't',
                                     's')
         be = x.get_build_env()
-        assert be == {'O':'ob3', 'Y':'yyy'}, be
+        assert be['O'] == 'ob3', be['O']
+        assert be['Y'] == 'yyy', be['Y']
 
     def test_get_action_list(self):
         """Test fetching and generating an action list"""
@@ -131,10 +134,24 @@ class ExecutorTestCase(unittest.TestCase):
             a.append(action)
             assert target == ['t1', 't2'], target
             assert source == ['s1', 's2'], source
-            assert env == {'CALL':'call'}, env
+            assert env['CALL'] == 'call', env['CALL']
         x(MyNode(['pre'], ['post']), func)
         assert actions == ['pre', 'action1', 'action2', 'post'], actions
 
+    def test_cleanup(self):
+        """Test cleaning up an Executor"""
+        x = SCons.Executor.Executor('b', 'e', 'o', 't', ['s1', 's2'])
+
+        x.cleanup()
+
+        x.build_env = 'eee'
+        be = x.get_build_env()
+        assert be == 'eee', be
+
+        x.cleanup()
+
+        assert not hasattr(x, 'build_env')
+
     def test_add_sources(self):
         """Test adding sources to an Executor"""
         x = SCons.Executor.Executor('b', 'e', 'o', 't', ['s1', 's2'])
diff --git a/src/engine/SCons/Job.py b/src/engine/SCons/Job.py
index ee84e34..1ba7d62 100644
--- a/src/engine/SCons/Job.py
+++ b/src/engine/SCons/Job.py
@@ -118,6 +118,8 @@ class Serial:
             else:
                 task.executed()
 
+            task.postprocess()
+
 
 # Trap import failure so that everything in the Job module but the
 # Parallel class (and its dependent classes) will work if the interpreter
@@ -251,3 +253,5 @@ else:
                         task.executed()
                     else:
                         task.failed()
+
+                    task.postprocess()
diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py
index 1b0128b..48caa16 100644
--- a/src/engine/SCons/JobTests.py
+++ b/src/engine/SCons/JobTests.py
@@ -100,6 +100,9 @@ class Task:
         self.taskmaster.test_case.failUnless(self.was_prepared,
                                   "the task wasn't prepared")
 
+    def postprocess(self):
+        self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1
+
 
 class ExceptionTask:
     """A dummy task class for testing purposes."""
@@ -130,6 +133,9 @@ class ExceptionTask:
         self.taskmaster.test_case.failUnless(self.was_prepared,
                                   "the task wasn't prepared")
 
+    def postprocess(self):
+        self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1
+
 class Taskmaster:
     """A dummy taskmaster class for testing the job classes."""
 
@@ -142,6 +148,7 @@ class Taskmaster:
         self.num_iterated = 0
         self.num_executed = 0
         self.num_failed = 0
+        self.num_postprocessed = 0
         self.Task = Task
         # 'guard' guards 'task_begin_list' and 'task_end_list'
         try:
@@ -170,6 +177,9 @@ class Taskmaster:
     def all_tasks_are_iterated(self):
         return self.num_iterated == self.num_tasks
 
+    def all_tasks_are_postprocessed(self):
+        return self.num_postprocessed == self.num_tasks
+
     def is_blocked(self):
         if self.stop or self.all_tasks_are_executed():
             return 0
@@ -206,6 +216,8 @@ class ParallelTestCase(unittest.TestCase):
                         "all the tests were not executed")
         self.failUnless(taskmaster.all_tasks_are_iterated(),
                         "all the tests were not iterated over")
+        self.failUnless(taskmaster.all_tasks_are_postprocessed(),
+                        "all the tests were not postprocessed")
         self.failIf(taskmaster.num_failed,
                     "some task(s) failed to execute") 
 
@@ -223,6 +235,8 @@ class SerialTestCase(unittest.TestCase):
                         "all the tests were not executed")
         self.failUnless(taskmaster.all_tasks_are_iterated(),
                         "all the tests were not iterated over")
+        self.failUnless(taskmaster.all_tasks_are_postprocessed(),
+                        "all the tests were not postprocessed")
         self.failIf(taskmaster.num_failed,
                     "some task(s) failed to execute") 
 
@@ -245,6 +259,8 @@ class NoParallelTestCase(unittest.TestCase):
                             "all the tests were not executed")
             self.failUnless(taskmaster.all_tasks_are_iterated(),
                             "all the tests were not iterated over")
+            self.failUnless(taskmaster.all_tasks_are_postprocessed(),
+                            "all the tests were not postprocessed")
             self.failIf(taskmaster.num_failed,
                         "some task(s) failed to execute") 
         finally:
@@ -265,6 +281,8 @@ class SerialExceptionTestCase(unittest.TestCase):
                     "exactly one task should have been iterated")
         self.failUnless(taskmaster.num_failed == 1,
                     "exactly one task should have failed")
+        self.failUnless(taskmaster.num_postprocessed == 1,
+                    "exactly one task should have been postprocessed")
 
 class ParallelExceptionTestCase(unittest.TestCase):
     def runTest(self):
@@ -280,6 +298,8 @@ class ParallelExceptionTestCase(unittest.TestCase):
                     "one or more task should have been iterated")
         self.failUnless(taskmaster.num_failed >= 1,
                     "one or more tasks should have failed") 
+        self.failUnless(taskmaster.num_postprocessed >= 1,
+                    "one or more tasks should have been postprocessed")
 
 
 def suite():
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 438cd7f..98c7dec 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -938,6 +938,8 @@ class DummyExecutor:
     """Dummy executor class returned by Dir nodes to bamboozle SCons
     into thinking we are an actual derived node, where our sources are
     our directory entries."""
+    def cleanup(self):
+        pass
     def get_raw_contents(self):
         return ''
     def get_contents(self):
@@ -1214,11 +1216,12 @@ class File(Base):
         return self.fs.Rsearchall(pathlist, clazz=Dir, must_exist=0,
                                   cwd=self.cwd)
 
-    def generate_build_env(self, env):
-        """Generate an appropriate Environment to build this File."""
-        return env.Override({'Dir' : self.Dir,
-                             'File' : self.File,
-                             'RDirs' : self.RDirs})
+    def generate_build_dict(self):
+        """Return an appropriate dictionary of values for building
+        this File."""
+        return {'Dir' : self.Dir,
+                'File' : self.File,
+                'RDirs' : self.RDirs}
 
     def _morph(self):
         """Turn a file system node into a File object."""
@@ -1343,9 +1346,9 @@ class File(Base):
         if b and self.fs.CachePath:
             if self.fs.cache_show:
                 if CacheRetrieveSilent(self, None, None) == 0:
-                    def do_print(action, targets, sources, env, self=self):
+                    def do_print(action, targets, sources, env, s=self):
                         if action.strfunction:
-                            al = action.strfunction(targets, self.sources, env)
+                            al = action.strfunction(targets, s.sources, env)
                             if not SCons.Util.is_List(al):
                                 al = [al]
                             for a in al:
diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py
index a84a993..35ccead 100644
--- a/src/engine/SCons/Node/FSTests.py
+++ b/src/engine/SCons/Node/FSTests.py
@@ -1694,6 +1694,7 @@ class CacheDirTestCase(unittest.TestCase):
 
 class clearTestCase(unittest.TestCase):
     def runTest(self):
+        """Test clearing FS nodes of cached data."""
         fs = SCons.Node.FS.FS()
 
         e = fs.Entry('e')
@@ -1717,6 +1718,20 @@ class clearTestCase(unittest.TestCase):
         assert not hasattr(f, '_exists')
         assert not hasattr(f, '_rexists')
 
+class postprocessTestCase(unittest.TestCase):
+    def runTest(self):
+        """Test calling the postprocess() method."""
+        fs = SCons.Node.FS.FS()
+
+        e = fs.Entry('e')
+        e.postprocess()
+
+        d = fs.Dir('d')
+        d.postprocess()
+
+        f = fs.File('f')
+        f.postprocess()
+
 class SpecialAttrTestCase(unittest.TestCase):
     def runTest(self):
         """Test special attributes of file nodes."""
@@ -1869,6 +1884,7 @@ if __name__ == "__main__":
     suite.addTest(SConstruct_dirTestCase())
     suite.addTest(CacheDirTestCase())
     suite.addTest(clearTestCase())
+    suite.addTest(postprocessTestCase())
     suite.addTest(SpecialAttrTestCase())
     if not unittest.TextTestRunner().run(suite).wasSuccessful():
         sys.exit(1)
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index 6b8c8ae..2e053be 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -925,6 +925,17 @@ class NodeTestCase(unittest.TestCase):
         siginfo = n.get_prevsiginfo()
         assert siginfo == (None, None, None), siginfo
 
+    def test_generate_build_dict(self):
+        """Test the base Node generate_build_dict() method"""
+        n = SCons.Node.Node()
+        dict = n.generate_build_dict()
+        assert dict == {}, dict
+
+    def test_postprocess(self):
+        """Test calling the base Node postprocess() method"""
+        n = SCons.Node.Node()
+        n.postprocess()
+
 
 
 if __name__ == "__main__":
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index 4d51370..88aa120 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -135,9 +135,10 @@ class Node:
         # what line in what file created the node, for example).
         Annotate(self)
 
-    def generate_build_env(self, env):
-        """Generate the appropriate Environment to build this node."""
-        return env
+    def generate_build_dict(self):
+        """Return an appropriate dictionary of values for building
+        this Node."""
+        return {}
 
     def get_build_env(self):
         """Fetch the appropriate Environment to build this node."""
@@ -157,10 +158,9 @@ class Node:
             if not create:
                 raise
             import SCons.Executor
-            env = self.generate_build_env(self.builder.env)
             executor = SCons.Executor.Executor(self.builder,
-                                               env,
-                                               self.builder.overrides,
+                                               self.builder.env,
+                                               {},
                                                [self],
                                                self.sources)
             self.executor = executor
@@ -196,10 +196,10 @@ class Node:
         so only do thread safe stuff here. Do thread unsafe stuff in
         built().
         """
-        def do_action(action, targets, sources, env, self=self):
+        def do_action(action, targets, sources, env, s=self):
             stat = action(targets, sources, env)
             if stat:
-                raise SCons.Errors.BuildError(node = self,
+                raise SCons.Errors.BuildError(node = s,
                                               errstr = "Error %d" % stat)
         self._for_each_action(do_action)
 
@@ -225,6 +225,16 @@ class Node:
         # node were presumably just changed:
         self.del_csig()
 
+    def postprocess(self):
+        """Clean up anything we don't need to hang onto after we've
+        been built."""
+        try:
+            executor = self.get_executor(create=None)
+        except AttributeError:
+            pass
+        else:
+            executor.cleanup()
+
     def clear(self):
         """Completely clear a Node of all its cached state (so that it
         can be re-evaluated by interfaces that do continuous integration
diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py
index 2028c3b..918f4b9 100644
--- a/src/engine/SCons/SConfTests.py
+++ b/src/engine/SCons/SConfTests.py
@@ -176,6 +176,8 @@ class SConfTestCase(unittest.TestCase):
                         return [], None
                     def depends_on(self, nodes):
                         return None
+                    def postprocess(self):
+                        pass
                 return [MyNode('n1'), MyNode('n2')]
         self.scons_env.Append(BUILDERS = {'SConfActionBuilder' : MyBuilder()})
         sconf.TryBuild(self.scons_env.SConfActionBuilder)
diff --git a/src/engine/SCons/Script/SConscript.py b/src/engine/SCons/Script/SConscript.py
index 90e1fe1..2fa1510 100644
--- a/src/engine/SCons/Script/SConscript.py
+++ b/src/engine/SCons/Script/SConscript.py
@@ -270,12 +270,18 @@ def SConscript_exception(file=sys.stderr):
     up to where we exec the SConscript."""
     stack = traceback.extract_tb(sys.exc_traceback)
     last_text = ""
+    found = 0
     i = 0
     for frame in stack:
         if is_our_exec_statement(last_text):
+            found = 1
             break
         i = i + 1
         last_text = frame[3]
+    if not found:
+        # We did not find our exec statement, so this was actually a bug
+        # in SCons itself.  Show the whole stack.
+        i = 0
     type = str(sys.exc_type)
     if type[:11] == "exceptions.":
         type = type[11:]
diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py
index 66492de..8afd847 100644
--- a/src/engine/SCons/Taskmaster.py
+++ b/src/engine/SCons/Taskmaster.py
@@ -181,6 +181,13 @@ class Task:
                     side_effect.set_state(state)
             t.set_state(state)
 
+    def postprocess(self):
+        """Post process a task after it's been executed."""
+        for t in self.targets:
+            t.postprocess()
+
+
+
 def order(dependencies):
     """Re-order a list of dependencies (if we need to)."""
     return dependencies
diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py
index 63190a5..d20b805 100644
--- a/src/engine/SCons/TaskmasterTests.py
+++ b/src/engine/SCons/TaskmasterTests.py
@@ -53,6 +53,7 @@ class Node:
         self.side_effect = 0
         self.side_effects = []
         self.alttargets = []
+        self.postprocessed = None
 
         for kid in kids:
             kid.parents.append(self)
@@ -137,6 +138,9 @@ class Node:
     def __str__(self):
         return self.name
 
+    def postprocess(self):
+        self.postprocessed = 1
+
 
 class OtherError(Exception):
     pass
@@ -791,6 +795,33 @@ class TaskmasterTestCase(unittest.TestCase):
         else:
             assert 0, "did not catch expected exception"
 
+    def test_postprocess(self):
+        """Test postprocessing targets to give them a chance to clean up
+        
+        """
+        n1 = Node("n1")
+        tm = SCons.Taskmaster.Taskmaster([n1])
+
+        t = tm.next_task()
+        assert not n1.postprocessed
+        t.postprocess()
+        assert n1.postprocessed
+
+        n2 = Node("n2")
+        n3 = Node("n3")
+        tm = SCons.Taskmaster.Taskmaster([n2, n3])
+
+        assert not n2.postprocessed
+        assert not n3.postprocessed
+        t = tm.next_task()
+        t.postprocess()
+        assert n2.postprocessed
+        assert not n3.postprocessed
+        t = tm.next_task()
+        t.postprocess()
+        assert n2.postprocessed
+        assert n3.postprocessed
+
 
 
 if __name__ == "__main__":