diff options
| -rw-r--r-- | src/CHANGES.txt | 3 | ||||
| -rw-r--r-- | src/engine/SCons/Executor.py | 8 | ||||
| -rw-r--r-- | src/engine/SCons/Node/FS.py | 139 | ||||
| -rw-r--r-- | src/engine/SCons/Node/__init__.py | 76 | ||||
| -rw-r--r-- | src/engine/SCons/SConf.py | 3 | ||||
| -rw-r--r-- | src/engine/SCons/SConfTests.py | 2 | ||||
| -rw-r--r-- | src/engine/SCons/Script/Main.py | 1 | ||||
| -rw-r--r-- | src/engine/SCons/Taskmaster.py | 37 | ||||
| -rw-r--r-- | src/engine/SCons/TaskmasterTests.py | 5 |
9 files changed, 221 insertions, 53 deletions
diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 10f940a..978c82c 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -46,6 +46,9 @@ RELEASE 2.3.1.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE SCons from a source (non-installed) dir. - Count statistics of instances are now collected only when the --debug=count command-line option is used (#2922). + - Added release_target_info() to File nodes, which helps to + reduce memory consumption in clean builds and update runs + of large projects. From Gary Oberbrunner: - Test harness: fail_test() can now print a message to help debugging. diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index 7875537..051d275 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -230,6 +230,8 @@ class Executor(object): self.action_list = action def get_action_list(self): + if self.action_list is None: + return [] return self.pre_actions + self.action_list + self.post_actions def get_all_targets(self): @@ -268,7 +270,8 @@ class Executor(object): """ result = SCons.Util.UniqueList([]) for target in self.get_all_targets(): - result.extend(target.prerequisites) + if target.prerequisites is not None: + result.extend(target.prerequisites) return result def get_action_side_effects(self): @@ -571,7 +574,7 @@ class Null(object): """A null Executor, with a null build Environment, that does nothing when the rest of the methods call it. - This might be able to disapper when we refactor things to + This might be able to disappear when we refactor things to disassociate Builders from Nodes entirely, so we're not going to worry about unit tests for this--at least for now. """ @@ -626,7 +629,6 @@ class Null(object): self._morph() self.set_action_list(action) - # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 18400e8..aaa5b47 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -2398,6 +2398,8 @@ class File(Base): self.scanner_paths = {} if not hasattr(self, '_local'): self._local = 0 + if not hasattr(self, 'released_target_info'): + self.released_target_info = False # If there was already a Builder set on this entry, then # we need to make sure we call the target-decider function, @@ -2725,7 +2727,7 @@ class File(Base): return self.get_build_env().get_CacheDir().retrieve(self) def visited(self): - if self.exists(): + if self.exists() and self.executor is not None: self.get_build_env().get_CacheDir().push_if_forced(self) ninfo = self.get_ninfo() @@ -2747,6 +2749,58 @@ class File(Base): self.store_info() + def release_target_info(self): + """Called just after this node has been marked + up-to-date or was built completely. + + This is where we try to release as many target node infos + as possible for clean builds and update runs, in order + to minimize the overall memory consumption. + + We'd like to remove a lot more attributes like self.sources + and self.sources_set, but they might get used + in a next build step. For example, during configuration + the source files for a built *.o file are used to figure out + which linker to use for the resulting Program (gcc vs. g++)! + That's why we check for the 'keep_targetinfo' attribute, + config Nodes and the Interactive mode just don't allow + an early release of most variables. + + In the same manner, we can't simply remove the self.attributes + here. The smart linking relies on the shared flag, and some + parts of the java Tool use it to transport information + about nodes... + + @see: built() and Node.release_target_info() + """ + if (self.released_target_info or SCons.Node.interactive): + return + + if not hasattr(self.attributes, 'keep_targetinfo'): + # Cache some required values, before releasing + # stuff like env, executor and builder... + self.changed() + self.get_contents_sig() + self.get_build_env() + # Now purge unneeded stuff to free memory... + self.executor = None + self._memo.pop('rfile', None) + self.prerequisites = None + # Cleanup lists, but only if they're empty + if not len(self.ignore_set): + self.ignore_set = None + if not len(self.implicit_set): + self.implicit_set = None + if not len(self.depends_set): + self.depends_set = None + if not len(self.ignore): + self.ignore = None + if not len(self.depends): + self.depends = None + # Mark this node as done, we only have to release + # the memory once... + self.released_target_info = True + def find_src_builder(self): if self.rexists(): return None @@ -2957,6 +3011,48 @@ class File(Base): SCons.Node.Node.builder_set(self, builder) self.changed_since_last_build = self.decide_target + def built(self): + """Called just after this File node is successfully built. + + Just like for 'release_target_info' we try to release + some more target node attributes in order to minimize the + overall memory consumption. + + @see: release_target_info + """ + + SCons.Node.Node.built(self) + + if not hasattr(self.attributes, 'keep_targetinfo'): + # Ensure that the build infos get computed and cached... + self.store_info() + # ... then release some more variables. + self._specific_sources = False + self.labspath = None + self._save_str() + self.cwd = None + + self.scanner_paths = None + + def changed(self, node=None): + """ + Returns if the node is up-to-date with respect to the BuildInfo + stored last time it was built. + + For File nodes this is basically a wrapper around Node.changed(), + but we allow the return value to get cached after the reference + to the Executor got released in release_target_info(). + """ + if node is None: + try: + return self._memo['changed'] + except KeyError: + pass + + has_changed = SCons.Node.Node.changed(self, node) + self._memo['changed'] = has_changed + return has_changed + def changed_content(self, target, prev_ni): cur_csig = self.get_csig() try: @@ -3090,25 +3186,50 @@ class File(Base): self.cachedir_csig = self.get_csig() return self.cachedir_csig + def get_contents_sig(self): + """ + A helper method for get_cachedir_bsig. + + It computes and returns the signature for this + node's contents. + """ + + try: + return self.contentsig + except AttributeError: + pass + + executor = self.get_executor() + + result = self.contentsig = SCons.Util.MD5signature(executor.get_contents()) + return result + def get_cachedir_bsig(self): + """ + Return the signature for a cached file, including + its children. + + It adds the path of the cached file to the cache signature, + because multiple targets built by the same action will all + have the same build signature, and we have to differentiate + them somehow. + """ try: return self.cachesig except AttributeError: pass - - # Add the path to the cache signature, because multiple - # targets built by the same action will all have the same - # build signature, and we have to differentiate them somehow. + + # Collect signatures for all children children = self.children() - executor = self.get_executor() - # sigs = [n.get_cachedir_csig() for n in children] sigs = [n.get_cachedir_csig() for n in children] - sigs.append(SCons.Util.MD5signature(executor.get_contents())) + # Append this node's signature... + sigs.append(self.get_contents_sig()) + # ...and it's path sigs.append(self.path) + # Merge this all into a single signature result = self.cachesig = SCons.Util.MD5collect(sigs) return result - default_fs = None def get_default_fs(): diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index d353245..d6dbf2e 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -100,6 +100,11 @@ def do_nothing(node): pass Annotate = do_nothing +# Gets set to 'True' if we're running in interactive mode. Is +# currently used to release parts of a target's info during +# clean builds and update runs (see release_target_info). +interactive = False + # Classes for signature info for Nodes. class NodeInfoBase(object): @@ -209,7 +214,7 @@ class Node(object): self.depends_set = set() self.ignore = [] # dependencies to ignore self.ignore_set = set() - self.prerequisites = SCons.Util.UniqueList() + self.prerequisites = None self.implicit = None # implicit (scanned) dependencies (None means not scanned yet) self.waiting_parents = set() self.waiting_s_e = set() @@ -292,7 +297,8 @@ class Node(object): except AttributeError: pass else: - executor.cleanup() + if executor is not None: + executor.cleanup() def reset_executor(self): "Remove cached executor; forces recompute when needed." @@ -352,10 +358,11 @@ class Node(object): methods should call this base class method to get the child check and the BuildInfo structure. """ - for d in self.depends: - if d.missing(): - msg = "Explicit dependency `%s' not found, needed by target `%s'." - raise SCons.Errors.StopError(msg % (d, self)) + if self.depends is not None: + for d in self.depends: + if d.missing(): + msg = "Explicit dependency `%s' not found, needed by target `%s'." + raise SCons.Errors.StopError(msg % (d, self)) if self.implicit is not None: for i in self.implicit: if i.missing(): @@ -413,6 +420,23 @@ class Node(object): self.ninfo.update(self) self.store_info() + def release_target_info(self): + """Called just after this node has been marked + up-to-date or was built completely. + + This is where we try to release as many target node infos + as possible for clean builds and update runs, in order + to minimize the overall memory consumption. + + By purging attributes that aren't needed any longer after + a Node (=File) got built, we don't have to care that much how + many KBytes a Node actually requires...as long as we free + the memory shortly afterwards. + + @see: built() and File.release_target_info() + """ + pass + # # # @@ -514,7 +538,7 @@ class Node(object): def is_derived(self): """ - Returns true iff this node is derived (i.e. built). + Returns true if this node is derived (i.e. built). This should return true only for nodes whose path should be in the variant directory when duplicate=0 and should contribute their build @@ -854,6 +878,8 @@ class Node(object): def add_prerequisite(self, prerequisite): """Adds prerequisites""" + if self.prerequisites is None: + self.prerequisites = SCons.Util.UniqueList() self.prerequisites.extend(prerequisite) self._children_reset() @@ -941,20 +967,14 @@ class Node(object): # dictionary patterns I found all ended up using "not in" # internally anyway...) if self.ignore_set: - if self.implicit is None: - iter = chain(self.sources,self.depends) - else: - iter = chain(self.sources, self.depends, self.implicit) + iter = chain.from_iterable(filter(None, [self.sources, self.depends, self.implicit])) children = [] for i in iter: if i not in self.ignore_set: children.append(i) else: - if self.implicit is None: - children = self.sources + self.depends - else: - children = self.sources + self.depends + self.implicit + children = self.all_children(scan=0) self._memo['children_get'] = children return children @@ -981,10 +1001,7 @@ class Node(object): # using dictionary keys, lose the order, and the only ordered # dictionary patterns I found all ended up using "not in" # internally anyway...) - if self.implicit is None: - return self.sources + self.depends - else: - return self.sources + self.depends + self.implicit + return list(chain.from_iterable(filter(None, [self.sources, self.depends, self.implicit]))) def children(self, scan=1): """Return a list of the node's direct children, minus those @@ -1120,17 +1137,18 @@ class Node(object): Return a text representation, suitable for displaying to the user, of the include tree for the sources of this node. """ - if self.is_derived() and self.env: + if self.is_derived(): env = self.get_build_env() - for s in self.sources: - scanner = self.get_source_scanner(s) - if scanner: - path = self.get_build_scanner_path(scanner) - else: - path = None - def f(node, env=env, scanner=scanner, path=path): - return node.get_found_includes(env, scanner, path) - return SCons.Util.render_tree(s, f, 1) + if env: + for s in self.sources: + scanner = self.get_source_scanner(s) + if scanner: + path = self.get_build_scanner_path(scanner) + else: + path = None + def f(node, env=env, scanner=scanner, path=path): + return node.get_found_includes(env, scanner, path) + return SCons.Util.render_tree(s, f, 1) else: return None diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py index f3a3545..7a8a0c2 100644 --- a/src/engine/SCons/SConf.py +++ b/src/engine/SCons/SConf.py @@ -483,6 +483,9 @@ class SConfBase(object): # so we really control how it gets written. for n in nodes: n.store_info = n.do_not_store_info + if not hasattr(n, 'attributes'): + n.attributes = SCons.Node.Node.Attrs() + n.attributes.keep_targetinfo = 1 ret = 1 diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py index e604886..1cfb05b 100644 --- a/src/engine/SCons/SConfTests.py +++ b/src/engine/SCons/SConfTests.py @@ -182,7 +182,7 @@ class SConfTestCase(unittest.TestCase): self.waiting_parents = set() self.side_effects = [] self.builder = None - self.prerequisites = [] + self.prerequisites = None def disambiguate(self): return self def has_builder(self): diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index 7c21519..93380fb 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -1074,6 +1074,7 @@ def _main(parser): platform = SCons.Platform.platform_module() if options.interactive: + SCons.Node.interactive = True SCons.Script.Interactive.interact(fs, OptionsParser, options, targets, target_top) diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 64ab84d..5de1cda 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -186,6 +186,8 @@ class Task(object): # or implicit dependencies exists, and also initialize the # .sconsign info. executor = self.targets[0].get_executor() + if executor is None: + return executor.prepare() for t in executor.get_action_targets(): if print_prepare: @@ -289,6 +291,7 @@ class Task(object): post-visit actions that must take place regardless of whether or not the target was an actual built target or a source Node. """ + global print_prepare T = self.tm.trace if T: T.write(self.trace_message('Task.executed_with_callbacks()', self.node)) @@ -301,7 +304,12 @@ class Task(object): if not t.cached: t.push_to_cache() t.built() - t.visited() + t.visited() + if (not print_prepare and + (not hasattr(self, 'options') or not self.options.debug_includes)): + t.release_target_info() + else: + t.visited() executed = executed_with_callbacks @@ -382,6 +390,7 @@ class Task(object): This is the default behavior for building only what's necessary. """ + global print_prepare T = self.tm.trace if T: T.write(self.trace_message(u'Task.make_ready_current()', self.node)) @@ -414,6 +423,9 @@ class Task(object): # parallel build...) t.visited() t.set_state(NODE_UP_TO_DATE) + if (not print_prepare and + (not hasattr(self, 'options') or not self.options.debug_includes)): + t.release_target_info() make_ready = make_ready_current @@ -453,14 +465,15 @@ class Task(object): parents[p] = parents.get(p, 0) + 1 for t in targets: - for s in t.side_effects: - if s.get_state() == NODE_EXECUTING: - s.set_state(NODE_NO_STATE) - for p in s.waiting_parents: - parents[p] = parents.get(p, 0) + 1 - for p in s.waiting_s_e: - if p.ref_count == 0: - self.tm.candidates.append(p) + if t.side_effects is not None: + for s in t.side_effects: + if s.get_state() == NODE_EXECUTING: + s.set_state(NODE_NO_STATE) + for p in s.waiting_parents: + parents[p] = parents.get(p, 0) + 1 + for p in s.waiting_s_e: + if p.ref_count == 0: + self.tm.candidates.append(p) for p, subtract in parents.items(): p.ref_count = p.ref_count - subtract @@ -927,7 +940,11 @@ class Taskmaster(object): if node is None: return None - tlist = node.get_executor().get_all_targets() + executor = node.get_executor() + if executor is None: + return None + + tlist = executor.get_all_targets() task = self.tasker(self, tlist, node in self.original_top, node) try: diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index 85ade8d..e875158 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -49,7 +49,7 @@ class Node(object): self.scanned = 0 self.scanner = None self.targets = [self] - self.prerequisites = [] + self.prerequisites = None class Builder(object): def targets(self, node): return node.targets @@ -141,6 +141,9 @@ class Node(object): self.clear() + def release_target_info(self): + pass + def has_builder(self): return not self.builder is None |