summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDirk Baechle <dl9obn@darc.de>2013-12-19 20:27:00 (GMT)
committerDirk Baechle <dl9obn@darc.de>2013-12-19 20:27:00 (GMT)
commit4bbec153e3cb22fccb8c67212af1894f245bd8f8 (patch)
treedf9bc10f1461930597339c0a26d290d46e81c5dd
parent49f23291e62a741a5f53a3e37f606588a2e0d2f0 (diff)
downloadSCons-4bbec153e3cb22fccb8c67212af1894f245bd8f8.zip
SCons-4bbec153e3cb22fccb8c67212af1894f245bd8f8.tar.gz
SCons-4bbec153e3cb22fccb8c67212af1894f245bd8f8.tar.bz2
Added release_target_info() to File nodes, reduces memory consumption.
-rw-r--r--src/CHANGES.txt3
-rw-r--r--src/engine/SCons/Executor.py8
-rw-r--r--src/engine/SCons/Node/FS.py139
-rw-r--r--src/engine/SCons/Node/__init__.py76
-rw-r--r--src/engine/SCons/SConf.py3
-rw-r--r--src/engine/SCons/SConfTests.py2
-rw-r--r--src/engine/SCons/Script/Main.py1
-rw-r--r--src/engine/SCons/Taskmaster.py37
-rw-r--r--src/engine/SCons/TaskmasterTests.py5
9 files changed, 221 insertions, 53 deletions
diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 8b24696..5e3a05b 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -43,6 +43,9 @@ RELEASE 2.3.1.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE
SCons from a source (non-installed) dir.
- Count statistics of instances are now collected only when
the --debug=count command-line option is used (#2922).
+ - Added release_target_info() to File nodes, which helps to
+ reduce memory consumption in clean builds and update runs
+ of large projects.
From Gary Oberbrunner:
- Test harness: fail_test() can now print a message to help debugging.
diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py
index 7875537..051d275 100644
--- a/src/engine/SCons/Executor.py
+++ b/src/engine/SCons/Executor.py
@@ -230,6 +230,8 @@ class Executor(object):
self.action_list = action
def get_action_list(self):
+ if self.action_list is None:
+ return []
return self.pre_actions + self.action_list + self.post_actions
def get_all_targets(self):
@@ -268,7 +270,8 @@ class Executor(object):
"""
result = SCons.Util.UniqueList([])
for target in self.get_all_targets():
- result.extend(target.prerequisites)
+ if target.prerequisites is not None:
+ result.extend(target.prerequisites)
return result
def get_action_side_effects(self):
@@ -571,7 +574,7 @@ class Null(object):
"""A null Executor, with a null build Environment, that does
nothing when the rest of the methods call it.
- This might be able to disapper when we refactor things to
+ This might be able to disappear when we refactor things to
disassociate Builders from Nodes entirely, so we're not
going to worry about unit tests for this--at least for now.
"""
@@ -626,7 +629,6 @@ class Null(object):
self._morph()
self.set_action_list(action)
-
# Local Variables:
# tab-width:4
# indent-tabs-mode:nil
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 18400e8..aaa5b47 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -2398,6 +2398,8 @@ class File(Base):
self.scanner_paths = {}
if not hasattr(self, '_local'):
self._local = 0
+ if not hasattr(self, 'released_target_info'):
+ self.released_target_info = False
# If there was already a Builder set on this entry, then
# we need to make sure we call the target-decider function,
@@ -2725,7 +2727,7 @@ class File(Base):
return self.get_build_env().get_CacheDir().retrieve(self)
def visited(self):
- if self.exists():
+ if self.exists() and self.executor is not None:
self.get_build_env().get_CacheDir().push_if_forced(self)
ninfo = self.get_ninfo()
@@ -2747,6 +2749,58 @@ class File(Base):
self.store_info()
+ def release_target_info(self):
+ """Called just after this node has been marked
+ up-to-date or was built completely.
+
+ This is where we try to release as many target node infos
+ as possible for clean builds and update runs, in order
+ to minimize the overall memory consumption.
+
+ We'd like to remove a lot more attributes like self.sources
+ and self.sources_set, but they might get used
+ in a next build step. For example, during configuration
+ the source files for a built *.o file are used to figure out
+ which linker to use for the resulting Program (gcc vs. g++)!
+ That's why we check for the 'keep_targetinfo' attribute,
+ config Nodes and the Interactive mode just don't allow
+ an early release of most variables.
+
+ In the same manner, we can't simply remove the self.attributes
+ here. The smart linking relies on the shared flag, and some
+ parts of the java Tool use it to transport information
+ about nodes...
+
+ @see: built() and Node.release_target_info()
+ """
+ if (self.released_target_info or SCons.Node.interactive):
+ return
+
+ if not hasattr(self.attributes, 'keep_targetinfo'):
+ # Cache some required values, before releasing
+ # stuff like env, executor and builder...
+ self.changed()
+ self.get_contents_sig()
+ self.get_build_env()
+ # Now purge unneeded stuff to free memory...
+ self.executor = None
+ self._memo.pop('rfile', None)
+ self.prerequisites = None
+ # Cleanup lists, but only if they're empty
+ if not len(self.ignore_set):
+ self.ignore_set = None
+ if not len(self.implicit_set):
+ self.implicit_set = None
+ if not len(self.depends_set):
+ self.depends_set = None
+ if not len(self.ignore):
+ self.ignore = None
+ if not len(self.depends):
+ self.depends = None
+ # Mark this node as done, we only have to release
+ # the memory once...
+ self.released_target_info = True
+
def find_src_builder(self):
if self.rexists():
return None
@@ -2957,6 +3011,48 @@ class File(Base):
SCons.Node.Node.builder_set(self, builder)
self.changed_since_last_build = self.decide_target
+ def built(self):
+ """Called just after this File node is successfully built.
+
+ Just like for 'release_target_info' we try to release
+ some more target node attributes in order to minimize the
+ overall memory consumption.
+
+ @see: release_target_info
+ """
+
+ SCons.Node.Node.built(self)
+
+ if not hasattr(self.attributes, 'keep_targetinfo'):
+ # Ensure that the build infos get computed and cached...
+ self.store_info()
+ # ... then release some more variables.
+ self._specific_sources = False
+ self.labspath = None
+ self._save_str()
+ self.cwd = None
+
+ self.scanner_paths = None
+
+ def changed(self, node=None):
+ """
+ Returns if the node is up-to-date with respect to the BuildInfo
+ stored last time it was built.
+
+ For File nodes this is basically a wrapper around Node.changed(),
+ but we allow the return value to get cached after the reference
+ to the Executor got released in release_target_info().
+ """
+ if node is None:
+ try:
+ return self._memo['changed']
+ except KeyError:
+ pass
+
+ has_changed = SCons.Node.Node.changed(self, node)
+ self._memo['changed'] = has_changed
+ return has_changed
+
def changed_content(self, target, prev_ni):
cur_csig = self.get_csig()
try:
@@ -3090,25 +3186,50 @@ class File(Base):
self.cachedir_csig = self.get_csig()
return self.cachedir_csig
+ def get_contents_sig(self):
+ """
+ A helper method for get_cachedir_bsig.
+
+ It computes and returns the signature for this
+ node's contents.
+ """
+
+ try:
+ return self.contentsig
+ except AttributeError:
+ pass
+
+ executor = self.get_executor()
+
+ result = self.contentsig = SCons.Util.MD5signature(executor.get_contents())
+ return result
+
def get_cachedir_bsig(self):
+ """
+ Return the signature for a cached file, including
+ its children.
+
+ It adds the path of the cached file to the cache signature,
+ because multiple targets built by the same action will all
+ have the same build signature, and we have to differentiate
+ them somehow.
+ """
try:
return self.cachesig
except AttributeError:
pass
-
- # Add the path to the cache signature, because multiple
- # targets built by the same action will all have the same
- # build signature, and we have to differentiate them somehow.
+
+ # Collect signatures for all children
children = self.children()
- executor = self.get_executor()
- # sigs = [n.get_cachedir_csig() for n in children]
sigs = [n.get_cachedir_csig() for n in children]
- sigs.append(SCons.Util.MD5signature(executor.get_contents()))
+ # Append this node's signature...
+ sigs.append(self.get_contents_sig())
+ # ...and it's path
sigs.append(self.path)
+ # Merge this all into a single signature
result = self.cachesig = SCons.Util.MD5collect(sigs)
return result
-
default_fs = None
def get_default_fs():
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index d353245..d6dbf2e 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -100,6 +100,11 @@ def do_nothing(node): pass
Annotate = do_nothing
+# Gets set to 'True' if we're running in interactive mode. Is
+# currently used to release parts of a target's info during
+# clean builds and update runs (see release_target_info).
+interactive = False
+
# Classes for signature info for Nodes.
class NodeInfoBase(object):
@@ -209,7 +214,7 @@ class Node(object):
self.depends_set = set()
self.ignore = [] # dependencies to ignore
self.ignore_set = set()
- self.prerequisites = SCons.Util.UniqueList()
+ self.prerequisites = None
self.implicit = None # implicit (scanned) dependencies (None means not scanned yet)
self.waiting_parents = set()
self.waiting_s_e = set()
@@ -292,7 +297,8 @@ class Node(object):
except AttributeError:
pass
else:
- executor.cleanup()
+ if executor is not None:
+ executor.cleanup()
def reset_executor(self):
"Remove cached executor; forces recompute when needed."
@@ -352,10 +358,11 @@ class Node(object):
methods should call this base class method to get the child
check and the BuildInfo structure.
"""
- for d in self.depends:
- if d.missing():
- msg = "Explicit dependency `%s' not found, needed by target `%s'."
- raise SCons.Errors.StopError(msg % (d, self))
+ if self.depends is not None:
+ for d in self.depends:
+ if d.missing():
+ msg = "Explicit dependency `%s' not found, needed by target `%s'."
+ raise SCons.Errors.StopError(msg % (d, self))
if self.implicit is not None:
for i in self.implicit:
if i.missing():
@@ -413,6 +420,23 @@ class Node(object):
self.ninfo.update(self)
self.store_info()
+ def release_target_info(self):
+ """Called just after this node has been marked
+ up-to-date or was built completely.
+
+ This is where we try to release as many target node infos
+ as possible for clean builds and update runs, in order
+ to minimize the overall memory consumption.
+
+ By purging attributes that aren't needed any longer after
+ a Node (=File) got built, we don't have to care that much how
+ many KBytes a Node actually requires...as long as we free
+ the memory shortly afterwards.
+
+ @see: built() and File.release_target_info()
+ """
+ pass
+
#
#
#
@@ -514,7 +538,7 @@ class Node(object):
def is_derived(self):
"""
- Returns true iff this node is derived (i.e. built).
+ Returns true if this node is derived (i.e. built).
This should return true only for nodes whose path should be in
the variant directory when duplicate=0 and should contribute their build
@@ -854,6 +878,8 @@ class Node(object):
def add_prerequisite(self, prerequisite):
"""Adds prerequisites"""
+ if self.prerequisites is None:
+ self.prerequisites = SCons.Util.UniqueList()
self.prerequisites.extend(prerequisite)
self._children_reset()
@@ -941,20 +967,14 @@ class Node(object):
# dictionary patterns I found all ended up using "not in"
# internally anyway...)
if self.ignore_set:
- if self.implicit is None:
- iter = chain(self.sources,self.depends)
- else:
- iter = chain(self.sources, self.depends, self.implicit)
+ iter = chain.from_iterable(filter(None, [self.sources, self.depends, self.implicit]))
children = []
for i in iter:
if i not in self.ignore_set:
children.append(i)
else:
- if self.implicit is None:
- children = self.sources + self.depends
- else:
- children = self.sources + self.depends + self.implicit
+ children = self.all_children(scan=0)
self._memo['children_get'] = children
return children
@@ -981,10 +1001,7 @@ class Node(object):
# using dictionary keys, lose the order, and the only ordered
# dictionary patterns I found all ended up using "not in"
# internally anyway...)
- if self.implicit is None:
- return self.sources + self.depends
- else:
- return self.sources + self.depends + self.implicit
+ return list(chain.from_iterable(filter(None, [self.sources, self.depends, self.implicit])))
def children(self, scan=1):
"""Return a list of the node's direct children, minus those
@@ -1120,17 +1137,18 @@ class Node(object):
Return a text representation, suitable for displaying to the
user, of the include tree for the sources of this node.
"""
- if self.is_derived() and self.env:
+ if self.is_derived():
env = self.get_build_env()
- for s in self.sources:
- scanner = self.get_source_scanner(s)
- if scanner:
- path = self.get_build_scanner_path(scanner)
- else:
- path = None
- def f(node, env=env, scanner=scanner, path=path):
- return node.get_found_includes(env, scanner, path)
- return SCons.Util.render_tree(s, f, 1)
+ if env:
+ for s in self.sources:
+ scanner = self.get_source_scanner(s)
+ if scanner:
+ path = self.get_build_scanner_path(scanner)
+ else:
+ path = None
+ def f(node, env=env, scanner=scanner, path=path):
+ return node.get_found_includes(env, scanner, path)
+ return SCons.Util.render_tree(s, f, 1)
else:
return None
diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py
index f3a3545..7a8a0c2 100644
--- a/src/engine/SCons/SConf.py
+++ b/src/engine/SCons/SConf.py
@@ -483,6 +483,9 @@ class SConfBase(object):
# so we really control how it gets written.
for n in nodes:
n.store_info = n.do_not_store_info
+ if not hasattr(n, 'attributes'):
+ n.attributes = SCons.Node.Node.Attrs()
+ n.attributes.keep_targetinfo = 1
ret = 1
diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py
index e604886..1cfb05b 100644
--- a/src/engine/SCons/SConfTests.py
+++ b/src/engine/SCons/SConfTests.py
@@ -182,7 +182,7 @@ class SConfTestCase(unittest.TestCase):
self.waiting_parents = set()
self.side_effects = []
self.builder = None
- self.prerequisites = []
+ self.prerequisites = None
def disambiguate(self):
return self
def has_builder(self):
diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py
index 7c21519..93380fb 100644
--- a/src/engine/SCons/Script/Main.py
+++ b/src/engine/SCons/Script/Main.py
@@ -1074,6 +1074,7 @@ def _main(parser):
platform = SCons.Platform.platform_module()
if options.interactive:
+ SCons.Node.interactive = True
SCons.Script.Interactive.interact(fs, OptionsParser, options,
targets, target_top)
diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py
index 64ab84d..5de1cda 100644
--- a/src/engine/SCons/Taskmaster.py
+++ b/src/engine/SCons/Taskmaster.py
@@ -186,6 +186,8 @@ class Task(object):
# or implicit dependencies exists, and also initialize the
# .sconsign info.
executor = self.targets[0].get_executor()
+ if executor is None:
+ return
executor.prepare()
for t in executor.get_action_targets():
if print_prepare:
@@ -289,6 +291,7 @@ class Task(object):
post-visit actions that must take place regardless of whether
or not the target was an actual built target or a source Node.
"""
+ global print_prepare
T = self.tm.trace
if T: T.write(self.trace_message('Task.executed_with_callbacks()',
self.node))
@@ -301,7 +304,12 @@ class Task(object):
if not t.cached:
t.push_to_cache()
t.built()
- t.visited()
+ t.visited()
+ if (not print_prepare and
+ (not hasattr(self, 'options') or not self.options.debug_includes)):
+ t.release_target_info()
+ else:
+ t.visited()
executed = executed_with_callbacks
@@ -382,6 +390,7 @@ class Task(object):
This is the default behavior for building only what's necessary.
"""
+ global print_prepare
T = self.tm.trace
if T: T.write(self.trace_message(u'Task.make_ready_current()',
self.node))
@@ -414,6 +423,9 @@ class Task(object):
# parallel build...)
t.visited()
t.set_state(NODE_UP_TO_DATE)
+ if (not print_prepare and
+ (not hasattr(self, 'options') or not self.options.debug_includes)):
+ t.release_target_info()
make_ready = make_ready_current
@@ -453,14 +465,15 @@ class Task(object):
parents[p] = parents.get(p, 0) + 1
for t in targets:
- for s in t.side_effects:
- if s.get_state() == NODE_EXECUTING:
- s.set_state(NODE_NO_STATE)
- for p in s.waiting_parents:
- parents[p] = parents.get(p, 0) + 1
- for p in s.waiting_s_e:
- if p.ref_count == 0:
- self.tm.candidates.append(p)
+ if t.side_effects is not None:
+ for s in t.side_effects:
+ if s.get_state() == NODE_EXECUTING:
+ s.set_state(NODE_NO_STATE)
+ for p in s.waiting_parents:
+ parents[p] = parents.get(p, 0) + 1
+ for p in s.waiting_s_e:
+ if p.ref_count == 0:
+ self.tm.candidates.append(p)
for p, subtract in parents.items():
p.ref_count = p.ref_count - subtract
@@ -927,7 +940,11 @@ class Taskmaster(object):
if node is None:
return None
- tlist = node.get_executor().get_all_targets()
+ executor = node.get_executor()
+ if executor is None:
+ return None
+
+ tlist = executor.get_all_targets()
task = self.tasker(self, tlist, node in self.original_top, node)
try:
diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py
index 85ade8d..e875158 100644
--- a/src/engine/SCons/TaskmasterTests.py
+++ b/src/engine/SCons/TaskmasterTests.py
@@ -49,7 +49,7 @@ class Node(object):
self.scanned = 0
self.scanner = None
self.targets = [self]
- self.prerequisites = []
+ self.prerequisites = None
class Builder(object):
def targets(self, node):
return node.targets
@@ -141,6 +141,9 @@ class Node(object):
self.clear()
+ def release_target_info(self):
+ pass
+
def has_builder(self):
return not self.builder is None