Performance improvement: memo-ize Node.FS string values when appropriate.

6 files changed, 138 insertions, 10 deletions

diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py
index 5971056..4bca1c8 100644
--- a/src/engine/SCons/Builder.py
+++ b/src/engine/SCons/Builder.py
@@ -667,9 +667,18 @@ class MultiStepBuilder(BuilderBase):
         src_suffixes = self.src_suffixes(env)
 
         for snode in slist:
-            base, ext = self.splitext(str(snode))
-            if sdict.has_key(ext):
-                tgt = sdict[ext]._execute(env, None, snode, overwarn)
+            try:
+                get_suffix = snode.get_suffix
+            except AttributeError:
+                ext = self.splitext(str(snode))
+            else:
+                ext = get_suffix()
+            try:
+                subsidiary_builder = sdict[ext]
+            except KeyError:
+                final_sources.append(snode)
+            else:
+                tgt = subsidiary_builder._execute(env, None, snode, overwarn)
                 # Only supply the builder with sources it is capable
                 # of building.
                 if SCons.Util.is_List(tgt):
@@ -680,8 +689,6 @@ class MultiStepBuilder(BuilderBase):
                     final_sources.append(tgt)
                 else:
                     final_sources.extend(tgt)
-            else:
-                final_sources.append(snode)
 
         return BuilderBase._execute(self, env, target, final_sources, overwarn)
 
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index a349f77..a30994d 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -53,6 +53,30 @@ import SCons.Util
 import SCons.Warnings
 
 #
+# We stringify these file system Nodes a lot.  Turning a file system Node
+# into a string is non-trivial, because the final string representation
+# can depend on a lot of factors:  whether it's a derived target or not,
+# whether it's linked to a repository or source directory, and whether
+# there's duplication going on.  The normal technique for optimizing
+# calculations like this is to memoize (cache) the string value, so you
+# only have to do the calculation once.
+#
+# A number of the above factors, however, can be set after we've already
+# been asked to return a string for a Node, because a Repository() or
+# BuildDir() call or the like may not occur until later in SConscript
+# files.  So this variable controls whether we bother trying to save
+# string values for Nodes.  The wrapper interface can set this whenever
+# they're done mucking with Repository and BuildDir and the other stuff,
+# to let this module know it can start returning saved string values
+# for Nodes.
+#
+Save_Strings = None
+
+def save_strings(val):
+    global Save_Strings
+    Save_Strings = val
+
+#
 # SCons.Action objects for interacting with the outside world.
 #
 # The Node.FS methods in this module should use these actions to
@@ -417,16 +441,32 @@ class Base(SCons.Node.Node):
             delattr(self, '_rexists')
         except AttributeError:
             pass
+        try:
+            delattr(self, '_str_val')
+        except AttributeError:
+            pass
+        self.relpath = {}
 
     def get_dir(self):
         return self.dir
 
+    def get_suffix(self):
+        return SCons.Util.splitext(self.name)[1]
+
     def __str__(self):
         """A Node.FS.Base object's string representation is its path
         name."""
-        if self.duplicate or self.is_derived():
-            return self.get_path()
-        return self.srcnode().get_path()
+        try:
+            return self._str_val
+        except AttributeError:
+            global Save_Strings
+            if self.duplicate or self.is_derived():
+                str_val = self.get_path()
+            else:
+                str_val = self.srcnode().get_path()
+            if Save_Strings:
+                self._str_val = str_val
+            return str_val
 
     def exists(self):
         try:
@@ -573,7 +613,7 @@ class Entry(Base):
         return node.get_found_includes(env, scanner, target)
 
     def scanner_key(self):
-        return SCons.Util.splitext(self.name)[1]
+        return self.get_suffix()
 
     def get_contents(self):
         """Fetch the contents of the entry.
@@ -1124,6 +1164,10 @@ class Dir(Base):
                         del node._srcnode
                     except AttributeError:
                         pass
+                    try:
+                        del node._str_val
+                    except AttributeError:
+                        pass
                     if duplicate != None:
                         node.duplicate=duplicate
     
@@ -1347,7 +1391,7 @@ class File(Base):
         return self.dir.root()
 
     def scanner_key(self):
-        return SCons.Util.splitext(self.name)[1]
+        return self.get_suffix()
 
     def get_contents(self):
         if not self.rexists():
diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py
index 6fbcfdc..d5e04e1 100644
--- a/src/engine/SCons/Node/FSTests.py
+++ b/src/engine/SCons/Node/FSTests.py
@@ -1735,23 +1735,29 @@ class clearTestCase(unittest.TestCase):
         e = fs.Entry('e')
         e._exists = 1
         e._rexists = 1
+        e._str_val = 'e'
         e.clear()
         assert not hasattr(e, '_exists')
         assert not hasattr(e, '_rexists')
+        assert not hasattr(e, '_str_val')
 
         d = fs.Dir('d')
         d._exists = 1
         d._rexists = 1
+        d._str_val = 'd'
         d.clear()
         assert not hasattr(d, '_exists')
         assert not hasattr(d, '_rexists')
+        assert not hasattr(d, '_str_val')
 
         f = fs.File('f')
         f._exists = 1
         f._rexists = 1
+        f._str_val = 'f'
         f.clear()
         assert not hasattr(f, '_exists')
         assert not hasattr(f, '_rexists')
+        assert not hasattr(f, '_str_val')
 
 class postprocessTestCase(unittest.TestCase):
     def runTest(self):
@@ -1903,6 +1909,61 @@ class SpecialAttrTestCase(unittest.TestCase):
             caught = 1
         assert caught, "did not catch expected AttributeError"
 
+class SaveStringsTestCase(unittest.TestCase):
+    def runTest(self):
+        """Test caching string values of nodes."""
+        test=TestCmd(workdir='')
+
+        def setup(fs):
+            fs.Dir('src')
+            fs.Dir('d0')
+            fs.Dir('d1')
+
+            d0_f = fs.File('d0/f')
+            d1_f = fs.File('d1/f')
+            d0_b = fs.File('d0/b')
+            d1_b = fs.File('d1/b')
+            d1_f.duplicate = 1
+            d1_b.duplicate = 1
+            d0_b.builder = 1
+            d1_b.builder = 1
+
+            return [d0_f, d1_f, d0_b, d1_b]
+
+        def modify(nodes):
+            d0_f, d1_f, d0_b, d1_b = nodes
+            d1_f.duplicate = 0
+            d1_b.duplicate = 0
+            d0_b.builder = 0
+            d1_b.builder = 0
+
+        fs1 = SCons.Node.FS.FS(test.workpath('fs1'))
+        nodes = setup(fs1)
+        fs1.BuildDir('d0', 'src', duplicate=0)
+        fs1.BuildDir('d1', 'src', duplicate=1)
+
+        s = map(str, nodes)
+        assert s == ['src/f', 'd1/f', 'd0/b', 'd1/b'], s
+
+        modify(nodes)
+
+        s = map(str, nodes)
+        assert s == ['src/f', 'src/f', 'd0/b', 'd1/b'], s
+
+        SCons.Node.FS.save_strings(1)
+        fs2 = SCons.Node.FS.FS(test.workpath('fs2'))
+        nodes = setup(fs2)
+        fs2.BuildDir('d0', 'src', duplicate=0)
+        fs2.BuildDir('d1', 'src', duplicate=1)
+
+        s = map(str, nodes)
+        assert s == ['src/f', 'd1/f', 'd0/b', 'd1/b'], s
+
+        modify(nodes)
+
+        s = map(str, nodes)
+        assert s == ['src/f', 'd1/f', 'd0/b', 'd1/b'], s
+
 
 
 if __name__ == "__main__":
@@ -1921,5 +1982,6 @@ if __name__ == "__main__":
     suite.addTest(clearTestCase())
     suite.addTest(postprocessTestCase())
     suite.addTest(SpecialAttrTestCase())
+    suite.addTest(SaveStringsTestCase())
     if not unittest.TextTestRunner().run(suite).wasSuccessful():
         sys.exit(1)
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index e9d779c..4ea73ab 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -954,6 +954,12 @@ class NodeTestCase(unittest.TestCase):
         siginfo = n.get_prevsiginfo()
         assert siginfo == (None, None, None), siginfo
 
+    def test_get_suffix(self):
+        """Test the base Node get_suffix() method"""
+        n = SCons.Node.Node()
+        s = n.get_suffix()
+        assert s == '', s
+
     def test_generate_build_dict(self):
         """Test the base Node generate_build_dict() method"""
         n = SCons.Node.Node()
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index a9581e4..ceac5ca 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -135,6 +135,9 @@ class Node:
         # what line in what file created the node, for example).
         Annotate(self)
 
+    def get_suffix(self):
+        return ''
+
     def generate_build_dict(self):
         """Return an appropriate dictionary of values for building
         this Node."""
diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py
index 245fea9..ee1ec81 100644
--- a/src/engine/SCons/Script/__init__.py
+++ b/src/engine/SCons/Script/__init__.py
@@ -963,6 +963,12 @@ def _main(args, parser):
         sys.exit(0)
     progress_display("scons: done reading SConscript files.")
 
+    # Tell the Node.FS subsystem that we're all done reading the
+    # SConscript files and calling Repository() and BuildDir() and the
+    # like, so it can go ahead and start memoizing the string values of
+    # file system nodes.
+    SCons.Node.FS.save_strings(1)
+
     if not memory_stats is None: memory_stats.append(SCons.Debug.memory())
 
     fs.chdir(fs.Top)