diff options
author | Steven Knight <knight@baldmt.com> | 2004-02-09 06:59:46 (GMT) |
---|---|---|
committer | Steven Knight <knight@baldmt.com> | 2004-02-09 06:59:46 (GMT) |
commit | e83632fa336430dcfb76881d9130169bf27420f7 (patch) | |
tree | 2179fb29cac6cb7112e1b9b423e6a42ebb6fdd8f /src | |
parent | 6cd1a4e5447a832a89d738b9cd545a2a9bc57f33 (diff) | |
download | SCons-e83632fa336430dcfb76881d9130169bf27420f7.zip SCons-e83632fa336430dcfb76881d9130169bf27420f7.tar.gz SCons-e83632fa336430dcfb76881d9130169bf27420f7.tar.bz2 |
Save memory by allowing Nodes to clean up their Executor's build environments after they've been built.
Diffstat (limited to 'src')
-rw-r--r-- | src/CHANGES.txt | 3 | ||||
-rw-r--r-- | src/engine/SCons/Builder.py | 2 | ||||
-rw-r--r-- | src/engine/SCons/Executor.py | 14 | ||||
-rw-r--r-- | src/engine/SCons/ExecutorTests.py | 25 | ||||
-rw-r--r-- | src/engine/SCons/Job.py | 4 | ||||
-rw-r--r-- | src/engine/SCons/JobTests.py | 20 | ||||
-rw-r--r-- | src/engine/SCons/Node/FS.py | 17 | ||||
-rw-r--r-- | src/engine/SCons/Node/FSTests.py | 16 | ||||
-rw-r--r-- | src/engine/SCons/Node/NodeTests.py | 11 | ||||
-rw-r--r-- | src/engine/SCons/Node/__init__.py | 26 | ||||
-rw-r--r-- | src/engine/SCons/SConfTests.py | 2 | ||||
-rw-r--r-- | src/engine/SCons/Script/SConscript.py | 6 | ||||
-rw-r--r-- | src/engine/SCons/Taskmaster.py | 7 | ||||
-rw-r--r-- | src/engine/SCons/TaskmasterTests.py | 31 |
14 files changed, 164 insertions, 20 deletions
diff --git a/src/CHANGES.txt b/src/CHANGES.txt index f93a78a..e6c3b53 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -170,6 +170,9 @@ RELEASE 0.95 - XXX - Fix the M4 Builder so that it chdirs to the Repository directory when the input file is in the source directory of a BuildDir. + - Save memory at build time by allowing Nodes to delete their build + environments after they've been built. + From Vincent Risi: - Add support for the bcc32, ilink32 and tlib Borland tools. diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py index 6714a3d..871a803 100644 --- a/src/engine/SCons/Builder.py +++ b/src/engine/SCons/Builder.py @@ -203,7 +203,7 @@ def _init_nodes(builder, env, overrides, tlist, slist): executor.add_sources(slist) if executor is None: executor = SCons.Executor.Executor(builder, - tlist[0].generate_build_env(env), + env, overrides, tlist, slist) diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index 2a0882e..8151f87 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -32,6 +32,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" from SCons.Debug import logInstanceCreation +import SCons.Util class Executor: @@ -71,6 +72,13 @@ class Executor: overrides = {} overrides.update(self.builder.overrides) overrides.update(self.overrides) + try: + generate_build_dict = self.targets[0].generate_build_dict + except AttributeError: + pass + else: + overrides.update(generate_build_dict()) + overrides.update(SCons.Util.subst_dict(self.targets, self.sources)) self.build_env = env.Override(overrides) return self.build_env @@ -101,6 +109,12 @@ class Executor: for action in action_list: func(action, self.targets, self.sources, env) + def cleanup(self): + try: + del self.build_env + except AttributeError: + pass + def add_sources(self, sources): """Add source files to this Executor's list. This is necessary for "multi" Builders that can be called repeatedly to build up diff --git a/src/engine/SCons/ExecutorTests.py b/src/engine/SCons/ExecutorTests.py index 2a79e2d..2cdc0e2 100644 --- a/src/engine/SCons/ExecutorTests.py +++ b/src/engine/SCons/ExecutorTests.py @@ -90,7 +90,8 @@ class ExecutorTestCase(unittest.TestCase): 't', ['s1', 's2']) be = x.get_build_env() - assert be == {'O':'o2', 'X':'xxx'}, be + assert be['O'] == 'o2', be['O'] + assert be['X'] == 'xxx', be['X'] env = MyEnvironment(Y='yyy') x = SCons.Executor.Executor(MyBuilder(env, {'O':'ob3'}), @@ -99,14 +100,16 @@ class ExecutorTestCase(unittest.TestCase): 't', 's') be = x.get_build_env() - assert be == {'O':'oo3', 'Y':'yyy'}, be + assert be['O'] == 'oo3', be['O'] + assert be['Y'] == 'yyy', be['Y'] x = SCons.Executor.Executor(MyBuilder(env, {'O':'ob3'}), None, {}, 't', 's') be = x.get_build_env() - assert be == {'O':'ob3', 'Y':'yyy'}, be + assert be['O'] == 'ob3', be['O'] + assert be['Y'] == 'yyy', be['Y'] def test_get_action_list(self): """Test fetching and generating an action list""" @@ -131,10 +134,24 @@ class ExecutorTestCase(unittest.TestCase): a.append(action) assert target == ['t1', 't2'], target assert source == ['s1', 's2'], source - assert env == {'CALL':'call'}, env + assert env['CALL'] == 'call', env['CALL'] x(MyNode(['pre'], ['post']), func) assert actions == ['pre', 'action1', 'action2', 'post'], actions + def test_cleanup(self): + """Test cleaning up an Executor""" + x = SCons.Executor.Executor('b', 'e', 'o', 't', ['s1', 's2']) + + x.cleanup() + + x.build_env = 'eee' + be = x.get_build_env() + assert be == 'eee', be + + x.cleanup() + + assert not hasattr(x, 'build_env') + def test_add_sources(self): """Test adding sources to an Executor""" x = SCons.Executor.Executor('b', 'e', 'o', 't', ['s1', 's2']) diff --git a/src/engine/SCons/Job.py b/src/engine/SCons/Job.py index ee84e34..1ba7d62 100644 --- a/src/engine/SCons/Job.py +++ b/src/engine/SCons/Job.py @@ -118,6 +118,8 @@ class Serial: else: task.executed() + task.postprocess() + # Trap import failure so that everything in the Job module but the # Parallel class (and its dependent classes) will work if the interpreter @@ -251,3 +253,5 @@ else: task.executed() else: task.failed() + + task.postprocess() diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py index 1b0128b..48caa16 100644 --- a/src/engine/SCons/JobTests.py +++ b/src/engine/SCons/JobTests.py @@ -100,6 +100,9 @@ class Task: self.taskmaster.test_case.failUnless(self.was_prepared, "the task wasn't prepared") + def postprocess(self): + self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1 + class ExceptionTask: """A dummy task class for testing purposes.""" @@ -130,6 +133,9 @@ class ExceptionTask: self.taskmaster.test_case.failUnless(self.was_prepared, "the task wasn't prepared") + def postprocess(self): + self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1 + class Taskmaster: """A dummy taskmaster class for testing the job classes.""" @@ -142,6 +148,7 @@ class Taskmaster: self.num_iterated = 0 self.num_executed = 0 self.num_failed = 0 + self.num_postprocessed = 0 self.Task = Task # 'guard' guards 'task_begin_list' and 'task_end_list' try: @@ -170,6 +177,9 @@ class Taskmaster: def all_tasks_are_iterated(self): return self.num_iterated == self.num_tasks + def all_tasks_are_postprocessed(self): + return self.num_postprocessed == self.num_tasks + def is_blocked(self): if self.stop or self.all_tasks_are_executed(): return 0 @@ -206,6 +216,8 @@ class ParallelTestCase(unittest.TestCase): "all the tests were not executed") self.failUnless(taskmaster.all_tasks_are_iterated(), "all the tests were not iterated over") + self.failUnless(taskmaster.all_tasks_are_postprocessed(), + "all the tests were not postprocessed") self.failIf(taskmaster.num_failed, "some task(s) failed to execute") @@ -223,6 +235,8 @@ class SerialTestCase(unittest.TestCase): "all the tests were not executed") self.failUnless(taskmaster.all_tasks_are_iterated(), "all the tests were not iterated over") + self.failUnless(taskmaster.all_tasks_are_postprocessed(), + "all the tests were not postprocessed") self.failIf(taskmaster.num_failed, "some task(s) failed to execute") @@ -245,6 +259,8 @@ class NoParallelTestCase(unittest.TestCase): "all the tests were not executed") self.failUnless(taskmaster.all_tasks_are_iterated(), "all the tests were not iterated over") + self.failUnless(taskmaster.all_tasks_are_postprocessed(), + "all the tests were not postprocessed") self.failIf(taskmaster.num_failed, "some task(s) failed to execute") finally: @@ -265,6 +281,8 @@ class SerialExceptionTestCase(unittest.TestCase): "exactly one task should have been iterated") self.failUnless(taskmaster.num_failed == 1, "exactly one task should have failed") + self.failUnless(taskmaster.num_postprocessed == 1, + "exactly one task should have been postprocessed") class ParallelExceptionTestCase(unittest.TestCase): def runTest(self): @@ -280,6 +298,8 @@ class ParallelExceptionTestCase(unittest.TestCase): "one or more task should have been iterated") self.failUnless(taskmaster.num_failed >= 1, "one or more tasks should have failed") + self.failUnless(taskmaster.num_postprocessed >= 1, + "one or more tasks should have been postprocessed") def suite(): diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 438cd7f..98c7dec 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -938,6 +938,8 @@ class DummyExecutor: """Dummy executor class returned by Dir nodes to bamboozle SCons into thinking we are an actual derived node, where our sources are our directory entries.""" + def cleanup(self): + pass def get_raw_contents(self): return '' def get_contents(self): @@ -1214,11 +1216,12 @@ class File(Base): return self.fs.Rsearchall(pathlist, clazz=Dir, must_exist=0, cwd=self.cwd) - def generate_build_env(self, env): - """Generate an appropriate Environment to build this File.""" - return env.Override({'Dir' : self.Dir, - 'File' : self.File, - 'RDirs' : self.RDirs}) + def generate_build_dict(self): + """Return an appropriate dictionary of values for building + this File.""" + return {'Dir' : self.Dir, + 'File' : self.File, + 'RDirs' : self.RDirs} def _morph(self): """Turn a file system node into a File object.""" @@ -1343,9 +1346,9 @@ class File(Base): if b and self.fs.CachePath: if self.fs.cache_show: if CacheRetrieveSilent(self, None, None) == 0: - def do_print(action, targets, sources, env, self=self): + def do_print(action, targets, sources, env, s=self): if action.strfunction: - al = action.strfunction(targets, self.sources, env) + al = action.strfunction(targets, s.sources, env) if not SCons.Util.is_List(al): al = [al] for a in al: diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index a84a993..35ccead 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -1694,6 +1694,7 @@ class CacheDirTestCase(unittest.TestCase): class clearTestCase(unittest.TestCase): def runTest(self): + """Test clearing FS nodes of cached data.""" fs = SCons.Node.FS.FS() e = fs.Entry('e') @@ -1717,6 +1718,20 @@ class clearTestCase(unittest.TestCase): assert not hasattr(f, '_exists') assert not hasattr(f, '_rexists') +class postprocessTestCase(unittest.TestCase): + def runTest(self): + """Test calling the postprocess() method.""" + fs = SCons.Node.FS.FS() + + e = fs.Entry('e') + e.postprocess() + + d = fs.Dir('d') + d.postprocess() + + f = fs.File('f') + f.postprocess() + class SpecialAttrTestCase(unittest.TestCase): def runTest(self): """Test special attributes of file nodes.""" @@ -1869,6 +1884,7 @@ if __name__ == "__main__": suite.addTest(SConstruct_dirTestCase()) suite.addTest(CacheDirTestCase()) suite.addTest(clearTestCase()) + suite.addTest(postprocessTestCase()) suite.addTest(SpecialAttrTestCase()) if not unittest.TextTestRunner().run(suite).wasSuccessful(): sys.exit(1) diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index 6b8c8ae..2e053be 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -925,6 +925,17 @@ class NodeTestCase(unittest.TestCase): siginfo = n.get_prevsiginfo() assert siginfo == (None, None, None), siginfo + def test_generate_build_dict(self): + """Test the base Node generate_build_dict() method""" + n = SCons.Node.Node() + dict = n.generate_build_dict() + assert dict == {}, dict + + def test_postprocess(self): + """Test calling the base Node postprocess() method""" + n = SCons.Node.Node() + n.postprocess() + if __name__ == "__main__": diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 4d51370..88aa120 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -135,9 +135,10 @@ class Node: # what line in what file created the node, for example). Annotate(self) - def generate_build_env(self, env): - """Generate the appropriate Environment to build this node.""" - return env + def generate_build_dict(self): + """Return an appropriate dictionary of values for building + this Node.""" + return {} def get_build_env(self): """Fetch the appropriate Environment to build this node.""" @@ -157,10 +158,9 @@ class Node: if not create: raise import SCons.Executor - env = self.generate_build_env(self.builder.env) executor = SCons.Executor.Executor(self.builder, - env, - self.builder.overrides, + self.builder.env, + {}, [self], self.sources) self.executor = executor @@ -196,10 +196,10 @@ class Node: so only do thread safe stuff here. Do thread unsafe stuff in built(). """ - def do_action(action, targets, sources, env, self=self): + def do_action(action, targets, sources, env, s=self): stat = action(targets, sources, env) if stat: - raise SCons.Errors.BuildError(node = self, + raise SCons.Errors.BuildError(node = s, errstr = "Error %d" % stat) self._for_each_action(do_action) @@ -225,6 +225,16 @@ class Node: # node were presumably just changed: self.del_csig() + def postprocess(self): + """Clean up anything we don't need to hang onto after we've + been built.""" + try: + executor = self.get_executor(create=None) + except AttributeError: + pass + else: + executor.cleanup() + def clear(self): """Completely clear a Node of all its cached state (so that it can be re-evaluated by interfaces that do continuous integration diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py index 2028c3b..918f4b9 100644 --- a/src/engine/SCons/SConfTests.py +++ b/src/engine/SCons/SConfTests.py @@ -176,6 +176,8 @@ class SConfTestCase(unittest.TestCase): return [], None def depends_on(self, nodes): return None + def postprocess(self): + pass return [MyNode('n1'), MyNode('n2')] self.scons_env.Append(BUILDERS = {'SConfActionBuilder' : MyBuilder()}) sconf.TryBuild(self.scons_env.SConfActionBuilder) diff --git a/src/engine/SCons/Script/SConscript.py b/src/engine/SCons/Script/SConscript.py index 90e1fe1..2fa1510 100644 --- a/src/engine/SCons/Script/SConscript.py +++ b/src/engine/SCons/Script/SConscript.py @@ -270,12 +270,18 @@ def SConscript_exception(file=sys.stderr): up to where we exec the SConscript.""" stack = traceback.extract_tb(sys.exc_traceback) last_text = "" + found = 0 i = 0 for frame in stack: if is_our_exec_statement(last_text): + found = 1 break i = i + 1 last_text = frame[3] + if not found: + # We did not find our exec statement, so this was actually a bug + # in SCons itself. Show the whole stack. + i = 0 type = str(sys.exc_type) if type[:11] == "exceptions.": type = type[11:] diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 66492de..8afd847 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -181,6 +181,13 @@ class Task: side_effect.set_state(state) t.set_state(state) + def postprocess(self): + """Post process a task after it's been executed.""" + for t in self.targets: + t.postprocess() + + + def order(dependencies): """Re-order a list of dependencies (if we need to).""" return dependencies diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index 63190a5..d20b805 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -53,6 +53,7 @@ class Node: self.side_effect = 0 self.side_effects = [] self.alttargets = [] + self.postprocessed = None for kid in kids: kid.parents.append(self) @@ -137,6 +138,9 @@ class Node: def __str__(self): return self.name + def postprocess(self): + self.postprocessed = 1 + class OtherError(Exception): pass @@ -791,6 +795,33 @@ class TaskmasterTestCase(unittest.TestCase): else: assert 0, "did not catch expected exception" + def test_postprocess(self): + """Test postprocessing targets to give them a chance to clean up + + """ + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster([n1]) + + t = tm.next_task() + assert not n1.postprocessed + t.postprocess() + assert n1.postprocessed + + n2 = Node("n2") + n3 = Node("n3") + tm = SCons.Taskmaster.Taskmaster([n2, n3]) + + assert not n2.postprocessed + assert not n3.postprocessed + t = tm.next_task() + t.postprocess() + assert n2.postprocessed + assert not n3.postprocessed + t = tm.next_task() + t.postprocess() + assert n2.postprocessed + assert n3.postprocessed + if __name__ == "__main__": |