Merge pull request #97, reduce memory consumption

9 files changed, 221 insertions, 53 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 10f940a..978c82c 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -46,6 +46,9 @@ RELEASE 2.3.1.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE
       SCons from a source (non-installed) dir.
     - Count statistics of instances are now collected only when
       the --debug=count command-line option is used (#2922).
+    - Added release_target_info() to File nodes, which helps to
+      reduce memory consumption in clean builds and update runs
+      of large projects.
 
   From Gary Oberbrunner:
     - Test harness: fail_test() can now print a message to help debugging.
diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py
index 7875537..051d275 100644
--- a/src/engine/SCons/Executor.py
+++ b/src/engine/SCons/Executor.py
@@ -230,6 +230,8 @@ class Executor(object):
         self.action_list = action
 
     def get_action_list(self):
+        if self.action_list is None:
+            return []
         return self.pre_actions + self.action_list + self.post_actions
 
     def get_all_targets(self):
@@ -268,7 +270,8 @@ class Executor(object):
         """
         result = SCons.Util.UniqueList([])
         for target in self.get_all_targets():
-            result.extend(target.prerequisites)
+            if target.prerequisites is not None:
+                result.extend(target.prerequisites)
         return result
 
     def get_action_side_effects(self):
@@ -571,7 +574,7 @@ class Null(object):
     """A null Executor, with a null build Environment, that does
     nothing when the rest of the methods call it.
 
-    This might be able to disapper when we refactor things to
+    This might be able to disappear when we refactor things to
     disassociate Builders from Nodes entirely, so we're not
     going to worry about unit tests for this--at least for now.
     """
@@ -626,7 +629,6 @@ class Null(object):
         self._morph()
         self.set_action_list(action)
 
-
 # Local Variables:
 # tab-width:4
 # indent-tabs-mode:nil
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 18400e8..aaa5b47 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -2398,6 +2398,8 @@ class File(Base):
         self.scanner_paths = {}
         if not hasattr(self, '_local'):
             self._local = 0
+        if not hasattr(self, 'released_target_info'):
+            self.released_target_info = False
 
         # If there was already a Builder set on this entry, then
         # we need to make sure we call the target-decider function,
@@ -2725,7 +2727,7 @@ class File(Base):
         return self.get_build_env().get_CacheDir().retrieve(self)
 
     def visited(self):
-        if self.exists():
+        if self.exists() and self.executor is not None:
             self.get_build_env().get_CacheDir().push_if_forced(self)
 
         ninfo = self.get_ninfo()
@@ -2747,6 +2749,58 @@ class File(Base):
 
         self.store_info()
 
+    def release_target_info(self):
+        """Called just after this node has been marked
+         up-to-date or was built completely.
+         
+         This is where we try to release as many target node infos
+         as possible for clean builds and update runs, in order
+         to minimize the overall memory consumption.
+         
+         We'd like to remove a lot more attributes like self.sources
+         and self.sources_set, but they might get used
+         in a next build step. For example, during configuration
+         the source files for a built *.o file are used to figure out
+         which linker to use for the resulting Program (gcc vs. g++)!
+         That's why we check for the 'keep_targetinfo' attribute,
+         config Nodes and the Interactive mode just don't allow
+         an early release of most variables. 
+
+         In the same manner, we can't simply remove the self.attributes
+         here. The smart linking relies on the shared flag, and some
+         parts of the java Tool use it to transport information
+         about nodes...
+         
+         @see: built() and Node.release_target_info()
+         """
+        if (self.released_target_info or SCons.Node.interactive):
+            return
+        
+        if not hasattr(self.attributes, 'keep_targetinfo'):
+            # Cache some required values, before releasing
+            # stuff like env, executor and builder...
+            self.changed()
+            self.get_contents_sig()
+            self.get_build_env()
+            # Now purge unneeded stuff to free memory...
+            self.executor = None
+            self._memo.pop('rfile', None)
+            self.prerequisites = None
+            # Cleanup lists, but only if they're empty
+            if not len(self.ignore_set):
+                self.ignore_set = None
+            if not len(self.implicit_set):
+                self.implicit_set = None
+            if not len(self.depends_set):
+                self.depends_set = None
+            if not len(self.ignore):
+                self.ignore = None
+            if not len(self.depends):
+                self.depends = None
+            # Mark this node as done, we only have to release
+            # the memory once...
+            self.released_target_info = True
+
     def find_src_builder(self):
         if self.rexists():
             return None
@@ -2957,6 +3011,48 @@ class File(Base):
         SCons.Node.Node.builder_set(self, builder)
         self.changed_since_last_build = self.decide_target
 
+    def built(self):
+        """Called just after this File node is successfully built.
+        
+         Just like for 'release_target_info' we try to release
+         some more target node attributes in order to minimize the
+         overall memory consumption.
+         
+         @see: release_target_info
+        """
+
+        SCons.Node.Node.built(self)
+
+        if not hasattr(self.attributes, 'keep_targetinfo'):
+            # Ensure that the build infos get computed and cached...        
+            self.store_info()
+            # ... then release some more variables.
+            self._specific_sources = False
+            self.labspath = None
+            self._save_str()
+            self.cwd = None
+             
+            self.scanner_paths = None
+
+    def changed(self, node=None):
+        """
+        Returns if the node is up-to-date with respect to the BuildInfo
+        stored last time it was built. 
+        
+        For File nodes this is basically a wrapper around Node.changed(),
+        but we allow the return value to get cached after the reference
+        to the Executor got released in release_target_info().
+        """
+        if node is None:
+            try:
+                return self._memo['changed']
+            except KeyError:
+                pass
+        
+        has_changed = SCons.Node.Node.changed(self, node)
+        self._memo['changed'] = has_changed
+        return has_changed
+
     def changed_content(self, target, prev_ni):
         cur_csig = self.get_csig()
         try:
@@ -3090,25 +3186,50 @@ class File(Base):
             self.cachedir_csig = self.get_csig()
         return self.cachedir_csig
 
+    def get_contents_sig(self):
+        """
+        A helper method for get_cachedir_bsig.
+
+        It computes and returns the signature for this
+        node's contents.
+        """
+        
+        try:
+            return self.contentsig
+        except AttributeError:
+            pass
+        
+        executor = self.get_executor()
+
+        result = self.contentsig = SCons.Util.MD5signature(executor.get_contents())
+        return result
+
     def get_cachedir_bsig(self):
+        """
+        Return the signature for a cached file, including
+        its children.
+
+        It adds the path of the cached file to the cache signature,
+        because multiple targets built by the same action will all
+        have the same build signature, and we have to differentiate
+        them somehow.
+        """
         try:
             return self.cachesig
         except AttributeError:
             pass
-
-        # Add the path to the cache signature, because multiple
-        # targets built by the same action will all have the same
-        # build signature, and we have to differentiate them somehow.
+        
+        # Collect signatures for all children
         children = self.children()
-        executor = self.get_executor()
-        # sigs = [n.get_cachedir_csig() for n in children]
         sigs = [n.get_cachedir_csig() for n in children]
-        sigs.append(SCons.Util.MD5signature(executor.get_contents()))
+        # Append this node's signature...
+        sigs.append(self.get_contents_sig())
+        # ...and it's path
         sigs.append(self.path)
+        # Merge this all into a single signature
         result = self.cachesig = SCons.Util.MD5collect(sigs)
         return result
 
-
 default_fs = None
 
 def get_default_fs():
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index d353245..d6dbf2e 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -100,6 +100,11 @@ def do_nothing(node): pass
 
 Annotate = do_nothing
 
+# Gets set to 'True' if we're running in interactive mode. Is
+# currently used to release parts of a target's info during
+# clean builds and update runs (see release_target_info).
+interactive = False
+
 # Classes for signature info for Nodes.
 
 class NodeInfoBase(object):
@@ -209,7 +214,7 @@ class Node(object):
         self.depends_set = set()
         self.ignore = []        # dependencies to ignore
         self.ignore_set = set()
-        self.prerequisites = SCons.Util.UniqueList()
+        self.prerequisites = None
         self.implicit = None    # implicit (scanned) dependencies (None means not scanned yet)
         self.waiting_parents = set()
         self.waiting_s_e = set()
@@ -292,7 +297,8 @@ class Node(object):
         except AttributeError:
             pass
         else:
-            executor.cleanup()
+            if executor is not None:
+                executor.cleanup()
 
     def reset_executor(self):
         "Remove cached executor; forces recompute when needed."
@@ -352,10 +358,11 @@ class Node(object):
         methods should call this base class method to get the child
         check and the BuildInfo structure.
         """
-        for d in self.depends:
-            if d.missing():
-                msg = "Explicit dependency `%s' not found, needed by target `%s'."
-                raise SCons.Errors.StopError(msg % (d, self))
+        if self.depends is not None:
+            for d in self.depends:
+                if d.missing():
+                    msg = "Explicit dependency `%s' not found, needed by target `%s'."
+                    raise SCons.Errors.StopError(msg % (d, self))
         if self.implicit is not None:
             for i in self.implicit:
                 if i.missing():
@@ -413,6 +420,23 @@ class Node(object):
             self.ninfo.update(self)
             self.store_info()
 
+    def release_target_info(self):
+        """Called just after this node has been marked
+         up-to-date or was built completely.
+         
+         This is where we try to release as many target node infos
+         as possible for clean builds and update runs, in order
+         to minimize the overall memory consumption.
+         
+         By purging attributes that aren't needed any longer after
+         a Node (=File) got built, we don't have to care that much how
+         many KBytes a Node actually requires...as long as we free
+         the memory shortly afterwards.
+         
+         @see: built() and File.release_target_info()
+         """
+        pass
+
     #
     #
     #
@@ -514,7 +538,7 @@ class Node(object):
 
     def is_derived(self):
         """
-        Returns true iff this node is derived (i.e. built).
+        Returns true if this node is derived (i.e. built).
 
         This should return true only for nodes whose path should be in
         the variant directory when duplicate=0 and should contribute their build
@@ -854,6 +878,8 @@ class Node(object):
 
     def add_prerequisite(self, prerequisite):
         """Adds prerequisites"""
+        if self.prerequisites is None:
+            self.prerequisites = SCons.Util.UniqueList()
         self.prerequisites.extend(prerequisite)
         self._children_reset()
 
@@ -941,20 +967,14 @@ class Node(object):
         # dictionary patterns I found all ended up using "not in"
         # internally anyway...)
         if self.ignore_set:
-            if self.implicit is None:
-                iter = chain(self.sources,self.depends)
-            else:
-                iter = chain(self.sources, self.depends, self.implicit)
+            iter = chain.from_iterable(filter(None, [self.sources, self.depends, self.implicit]))
 
             children = []
             for i in iter:
                 if i not in self.ignore_set:
                     children.append(i)
         else:
-            if self.implicit is None:
-                children = self.sources + self.depends
-            else:
-                children = self.sources + self.depends + self.implicit
+            children = self.all_children(scan=0)
 
         self._memo['children_get'] = children
         return children
@@ -981,10 +1001,7 @@ class Node(object):
         # using dictionary keys, lose the order, and the only ordered
         # dictionary patterns I found all ended up using "not in"
         # internally anyway...)
-        if self.implicit is None:
-            return self.sources + self.depends
-        else:
-            return self.sources + self.depends + self.implicit
+        return list(chain.from_iterable(filter(None, [self.sources, self.depends, self.implicit])))
 
     def children(self, scan=1):
         """Return a list of the node's direct children, minus those
@@ -1120,17 +1137,18 @@ class Node(object):
         Return a text representation, suitable for displaying to the
         user, of the include tree for the sources of this node.
         """
-        if self.is_derived() and self.env:
+        if self.is_derived():
             env = self.get_build_env()
-            for s in self.sources:
-                scanner = self.get_source_scanner(s)
-                if scanner:
-                    path = self.get_build_scanner_path(scanner)
-                else:
-                    path = None
-                def f(node, env=env, scanner=scanner, path=path):
-                    return node.get_found_includes(env, scanner, path)
-                return SCons.Util.render_tree(s, f, 1)
+            if env:
+                for s in self.sources:
+                    scanner = self.get_source_scanner(s)
+                    if scanner:
+                        path = self.get_build_scanner_path(scanner)
+                    else:
+                        path = None
+                    def f(node, env=env, scanner=scanner, path=path):
+                        return node.get_found_includes(env, scanner, path)
+                    return SCons.Util.render_tree(s, f, 1)
         else:
             return None
 
diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py
index f3a3545..7a8a0c2 100644
--- a/src/engine/SCons/SConf.py
+++ b/src/engine/SCons/SConf.py
@@ -483,6 +483,9 @@ class SConfBase(object):
         # so we really control how it gets written.
         for n in nodes:
             n.store_info = n.do_not_store_info
+            if not hasattr(n, 'attributes'):
+                n.attributes = SCons.Node.Node.Attrs()
+            n.attributes.keep_targetinfo = 1
 
         ret = 1
 
diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py
index e604886..1cfb05b 100644
--- a/src/engine/SCons/SConfTests.py
+++ b/src/engine/SCons/SConfTests.py
@@ -182,7 +182,7 @@ class SConfTestCase(unittest.TestCase):
                         self.waiting_parents = set()
                         self.side_effects = []
                         self.builder = None
-                        self.prerequisites = []
+                        self.prerequisites = None
                     def disambiguate(self):
                         return self
                     def has_builder(self):
diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py
index 7c21519..93380fb 100644
--- a/src/engine/SCons/Script/Main.py
+++ b/src/engine/SCons/Script/Main.py
@@ -1074,6 +1074,7 @@ def _main(parser):
     platform = SCons.Platform.platform_module()
 
     if options.interactive:
+        SCons.Node.interactive = True
         SCons.Script.Interactive.interact(fs, OptionsParser, options,
                                           targets, target_top)
 
diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py
index 64ab84d..5de1cda 100644
--- a/src/engine/SCons/Taskmaster.py
+++ b/src/engine/SCons/Taskmaster.py
@@ -186,6 +186,8 @@ class Task(object):
         # or implicit dependencies exists, and also initialize the
         # .sconsign info.
         executor = self.targets[0].get_executor()
+        if executor is None:
+            return
         executor.prepare()
         for t in executor.get_action_targets():
             if print_prepare:
@@ -289,6 +291,7 @@ class Task(object):
         post-visit actions that must take place regardless of whether
         or not the target was an actual built target or a source Node.
         """
+        global print_prepare
         T = self.tm.trace
         if T: T.write(self.trace_message('Task.executed_with_callbacks()',
                                          self.node))
@@ -301,7 +304,12 @@ class Task(object):
                 if not t.cached:
                     t.push_to_cache()
                 t.built()
-            t.visited()
+                t.visited()
+                if (not print_prepare and 
+                    (not hasattr(self, 'options') or not self.options.debug_includes)):
+                    t.release_target_info()
+            else:
+                t.visited()
 
     executed = executed_with_callbacks
 
@@ -382,6 +390,7 @@ class Task(object):
 
         This is the default behavior for building only what's necessary.
         """
+        global print_prepare
         T = self.tm.trace
         if T: T.write(self.trace_message(u'Task.make_ready_current()',
                                          self.node))
@@ -414,6 +423,9 @@ class Task(object):
                 # parallel build...)
                 t.visited()
                 t.set_state(NODE_UP_TO_DATE)
+                if (not print_prepare and 
+                    (not hasattr(self, 'options') or not self.options.debug_includes)):
+                    t.release_target_info()
 
     make_ready = make_ready_current
 
@@ -453,14 +465,15 @@ class Task(object):
                 parents[p] = parents.get(p, 0) + 1
 
         for t in targets:
-            for s in t.side_effects:
-                if s.get_state() == NODE_EXECUTING:
-                    s.set_state(NODE_NO_STATE)
-                    for p in s.waiting_parents:
-                        parents[p] = parents.get(p, 0) + 1
-                for p in s.waiting_s_e:
-                    if p.ref_count == 0:
-                        self.tm.candidates.append(p)
+            if t.side_effects is not None:
+                for s in t.side_effects:
+                    if s.get_state() == NODE_EXECUTING:
+                        s.set_state(NODE_NO_STATE)
+                        for p in s.waiting_parents:
+                            parents[p] = parents.get(p, 0) + 1
+                    for p in s.waiting_s_e:
+                        if p.ref_count == 0:
+                            self.tm.candidates.append(p)
 
         for p, subtract in parents.items():
             p.ref_count = p.ref_count - subtract
@@ -927,7 +940,11 @@ class Taskmaster(object):
         if node is None:
             return None
 
-        tlist = node.get_executor().get_all_targets()
+        executor = node.get_executor()
+        if executor is None:
+            return None
+        
+        tlist = executor.get_all_targets()
 
         task = self.tasker(self, tlist, node in self.original_top, node)
         try:
diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py
index 85ade8d..e875158 100644
--- a/src/engine/SCons/TaskmasterTests.py
+++ b/src/engine/SCons/TaskmasterTests.py
@@ -49,7 +49,7 @@ class Node(object):
         self.scanned = 0
         self.scanner = None
         self.targets = [self]
-        self.prerequisites = []
+        self.prerequisites = None
         class Builder(object):
             def targets(self, node):
                 return node.targets
@@ -141,6 +141,9 @@ class Node(object):
 
         self.clear()
         
+    def release_target_info(self):
+        pass
+
     def has_builder(self):
         return not self.builder is None