Implement implicit dependency caching.

2 files changed, 176 insertions, 46 deletions

diff --git a/src/engine/SCons/Sig/SigTests.py b/src/engine/SCons/Sig/SigTests.py
index 2a2560f..cc39656 100644
--- a/src/engine/SCons/Sig/SigTests.py
+++ b/src/engine/SCons/Sig/SigTests.py
@@ -58,11 +58,11 @@ class DummyNode:
         self.oldtime = 0
         self.oldbsig = 0
         self.oldcsig = 0
-        
+
     def get_contents(self):
         # a file that doesn't exist has no contents:
         assert self.exists()
-        
+
         return self.file.contents
 
     def get_timestamp(self):
@@ -80,11 +80,11 @@ class DummyNode:
         except AttributeError:
             self.exists_cache = self.exists()
             return self.exists_cache
-        
+
     def children(self):
         return filter(lambda x, i=self.ignore: x not in i,
                       self.sources + self.depends)
-        
+
     def all_children(self):
         return self.sources + self.depends
 
@@ -113,7 +113,16 @@ class DummyNode:
 
     def get_prevsiginfo(self):
         return (self.oldtime, self.oldbsig, self.oldcsig)
-    
+
+    def get_stored_implicit(self):
+        return None
+
+    def store_csig(self):
+        pass
+
+    def store_bsig(self):
+        pass
+
     def builder_sig_adapter(self):
         class Adapter:
             def get_contents(self):
@@ -135,7 +144,7 @@ def create_files(test):
              (test.workpath('d1/test.c'), 'blah blah', 111, 0),#8
              (test.workpath('d1/test.o'), None, 0, 1),         #9
              (test.workpath('d1/test'), None, 0, 1)]           #10
-    
+
     files = map(lambda x: apply(DummyFile, x), args)
 
     return files
@@ -173,12 +182,12 @@ def clear(nodes):
         node.bsig = None
 
 class SigTestBase:
-    
+
     def runTest(self):
 
         test = TestCmd.TestCmd(workdir = '')
         test.subdir('d1')
-        
+
         self.files = create_files(test)
         self.test_initial()
         self.test_built()
@@ -186,9 +195,9 @@ class SigTestBase:
         self.test_modify_same_time()
         self.test_delete()
         self.test_cache()
-        
+
     def test_initial(self):
-        
+
         nodes = create_nodes(self.files)
         calc = SCons.Sig.Calculator(self.module)
 
@@ -209,7 +218,7 @@ class SigTestBase:
         calc = SCons.Sig.Calculator(self.module)
 
         write(calc, nodes)
-        
+
         for node in nodes:
             self.failUnless(current(calc, node),
                             "all of the nodes should be current")
@@ -242,7 +251,6 @@ class SigTestBase:
         self.failUnless(not current(calc, nodes[9]), "direct source modified")
         self.failUnless(not current(calc, nodes[10]), "indirect source modified")
 
-
     def test_modify_same_time(self):
 
         nodes = create_nodes(self.files)
@@ -264,9 +272,9 @@ class SigTestBase:
                             "all of the nodes should be current")
 
     def test_delete(self):
-        
+
         nodes = create_nodes(self.files)
-        
+
         calc = SCons.Sig.Calculator(self.module)
 
         write(calc, nodes)
@@ -292,13 +300,13 @@ class SigTestBase:
     def test_cache(self):
         """Test that signatures are cached properly."""
         nodes = create_nodes(self.files)
-        
+
         calc = SCons.Sig.Calculator(self.module)
         nodes[0].set_csig(1)
         nodes[1].set_bsig(1)
         assert calc.csig(nodes[0]) == 1, calc.csig(nodes[0])
         assert calc.bsig(nodes[1]) == 1, calc.bsig(nodes[1])
-        
+
 
 class MD5TestCase(unittest.TestCase, SigTestBase):
     """Test MD5 signatures"""
@@ -311,7 +319,7 @@ class TimeStampTestCase(unittest.TestCase, SigTestBase):
     module = SCons.Sig.TimeStamp
 
 class CalcTestCase(unittest.TestCase):
-    
+
     def runTest(self):
         class MySigModule:
             def collect(self, signatures):
@@ -348,6 +356,8 @@ class CalcTestCase(unittest.TestCase):
                 return self.csig
             def get_prevsiginfo(self):
                 return 0, self.bsig, self.csig
+            def get_stored_implicit(self):
+                return None
             def get_timestamp(self):
                 return 1
             def builder_sig_adapter(self):
@@ -443,11 +453,52 @@ class CalcTestCase(unittest.TestCase):
         assert not self.calc.current(nn, 30)
         assert self.calc.current(nn, 33)
 
+class SConsignEntryTestCase(unittest.TestCase):
+
+    def runTest(self):
+        class DummyModule:
+            def to_string(self, sig):
+                return str(sig)
+
+            def from_string(self, sig):
+                return int(sig)
+
+        m = DummyModule()
+        e = SCons.Sig.SConsignEntry(m)
+        assert e.timestamp == None
+        assert e.csig == None
+        assert e.bsig == None
+        assert e.get_implicit() == None
+        assert e.render(m) == "- - - -"
+
+        e = SCons.Sig.SConsignEntry(m, "- - - -")
+        assert e.timestamp == None
+        assert e.csig == None
+        assert e.bsig == None
+        assert e.get_implicit() == None
+        assert e.render(m) == "- - - -"
+
+        e = SCons.Sig.SConsignEntry(m, "- - - foo\0bar")
+        assert e.timestamp == None
+        assert e.csig == None
+        assert e.bsig == None
+        assert e.get_implicit() == ['foo', 'bar']
+        assert e.render(m) == "- - - foo\0bar"
+
+        e = SCons.Sig.SConsignEntry(m, "123 456 789 foo bletch\0bar")
+        assert e.timestamp == 123
+        assert e.bsig == 456
+        assert e.csig == 789
+        assert e.get_implicit() == ['foo bletch', 'bar']
+        assert e.render(m) == "123 456 789 foo bletch\0bar"
+
+
 def suite():
     suite = unittest.TestSuite()
     suite.addTest(MD5TestCase())
     suite.addTest(TimeStampTestCase())
     suite.addTest(CalcTestCase())
+    suite.addTest(SConsignEntryTestCase())
     return suite
 
 if __name__ == "__main__":
@@ -455,4 +506,4 @@ if __name__ == "__main__":
     result = runner.run(suite())
     if not result.wasSuccessful():
         sys.exit(1)
-    
+
diff --git a/src/engine/SCons/Sig/__init__.py b/src/engine/SCons/Sig/__init__.py
index d077114..386c9e5 100644
--- a/src/engine/SCons/Sig/__init__.py
+++ b/src/engine/SCons/Sig/__init__.py
@@ -43,19 +43,74 @@ def write():
     for sig_file in sig_files:
         sig_file.write()
 
+class SConsignEntry:
+    def __init__(self, module, entry=None):
+
+        self.timestamp = self.csig = self.bsig = self.implicit = None
+
+        if not entry is None:
+            arr = map(string.strip, string.split(entry, " ", 3))
+
+            try:
+                if arr[0] == '-': self.timestamp = None
+                else:             self.timestamp = int(arr[0])
+
+                if arr[1] == '-': self.bsig = None
+                else:             self.bsig = module.from_string(arr[1])
+
+                if arr[2] == '-': self.csig = None
+                else:             self.csig = module.from_string(arr[2])
+
+                if arr[3] == '-': self.implicit = None
+                else:             self.implicit = arr[3]
+            except IndexError:
+                pass
+
+    def render(self, module):
+        if self.timestamp is None: timestamp = '-'
+        else:                      timestamp = "%d"%self.timestamp
+
+        if self.bsig is None: bsig = '-'
+        else:                 bsig = module.to_string(self.bsig)
+
+        if self.csig is None: csig = '-'
+        else:                 csig = module.to_string(self.csig)
+
+        if self.implicit is None: implicit = '-'
+        else:                     implicit = self.implicit
+
+        return '%s %s %s %s' % (timestamp, bsig, csig, implicit)
+
+    def get_implicit(self):
+        if self.implicit is None:
+            return None
+        else:
+            return string.split(self.implicit, '\0')
+
+    def set_implicit(self, implicit):
+        if implicit is None:
+            self.implicit = None
+        else:
+            self.implicit = string.join(map(str, implicit), '\0')
+
+
 class SConsignFile:
     """
     Encapsulates reading and writing a .sconsign file.
     """
 
-    def __init__(self, dir, module):
+    def __init__(self, dir, module=None):
         """
         dir - the directory for the file
         module - the signature module being used
         """
 
         self.dir = dir
-        self.module = module
+
+        if module is None:
+            self.module = default_calc.module
+        else:
+            self.module = module
         self.sconsign = os.path.join(dir.path, '.sconsign')
         self.entries = {}
         self.dirty = None
@@ -66,8 +121,8 @@ class SConsignFile:
             pass
         else:
             for line in file.readlines():
-                filename, rest = map(string.strip, string.split(line, ":"))
-                self.entries[filename] = rest
+                filename, rest = map(string.strip, string.split(line, ":", 1))
+                self.entries[filename] = SConsignEntry(self.module, rest)
 
         global sig_files
         sig_files.append(self)
@@ -77,24 +132,13 @@ class SConsignFile:
         Get the .sconsign entry for a file
 
         filename - the filename whose signature will be returned
-        returns - (timestamp, bsig, csig)
+        returns - (timestamp, bsig, csig, implicit)
         """
-
         try:
-            arr = map(string.strip, string.split(self.entries[filename], " "))
+            entry = self.entries[filename]
+            return (entry.timestamp, entry.bsig, entry.csig)
         except KeyError:
             return (None, None, None)
-        try:
-            if arr[1] == '-': bsig = None
-            else:             bsig = self.module.from_string(arr[1])
-        except IndexError:
-            bsig = None
-        try:
-            if arr[2] == '-': csig = None
-            else:             csig = self.module.from_string(arr[2])
-        except IndexError:
-            csig = None
-        return (int(arr[0]), bsig, csig)
 
     def set(self, filename, timestamp, bsig = None, csig = None):
         """
@@ -106,13 +150,37 @@ class SConsignFile:
         bsig - the file's build signature
         csig - the file's content signature
         """
-        if bsig is None: bsig = '-'
-        else:            bsig = self.module.to_string(bsig)
-        if csig is None: csig = ''
-        else:            csig = ' ' + self.module.to_string(csig)
-        self.entries[filename] = "%d %s%s" % (timestamp, bsig, csig)
+
+        try:
+            entry = self.entries[filename]
+        except KeyError:
+            entry = SConsignEntry(self.module)
+            self.entries[filename] = entry
+
+        entry.timestamp = timestamp
+        entry.bsig = bsig
+        entry.csig = csig
+
         self.dirty = 1
 
+    def get_implicit(self, filename):
+        """Fetch the cached implicit dependencies for 'filename'"""
+        try:
+            entry = self.entries[filename]
+            return entry.get_implicit()
+        except KeyError:
+            return None
+
+    def set_implicit(self, filename, implicit):
+        """Cache the implicit dependencies for 'filename'."""
+        try:
+            entry = self.entries[filename]
+        except KeyError:
+            entry = SConsignEntry(self.module)
+            self.entries[filename] = entry
+
+        entry.set_implicit(implicit)
+
     def write(self):
         """
         Write the .sconsign file to disk.
@@ -132,12 +200,15 @@ class SConsignFile:
                 file = open(temp, 'wt')
                 fname = temp
             except:
-                file = open(self.sconsign, 'wt')
-                fname = self.sconsign
+                try:
+                    file = open(self.sconsign, 'wt')
+                    fname = self.sconsign
+                except:
+                    return
             keys = self.entries.keys()
             keys.sort()
             for name in keys:
-                file.write("%s: %s\n" % (name, self.entries[name]))
+                file.write("%s: %s\n" % (name, self.entries[name].render(self.module)))
             file.close()
             if fname != self.sconsign:
                 try:
@@ -163,7 +234,7 @@ class Calculator:
     for the build engine.
     """
 
-    def __init__(self, module, max_drift=2*24*60*60):
+    def __init__(self, module=None, max_drift=2*24*60*60):
         """
         Initialize the calculator.
 
@@ -172,7 +243,11 @@ class Calculator:
           cache content signatures. A negative value means to never cache
           content signatures. (defaults to 2 days)
         """
-        self.module = module
+        if module is None:
+            import MD5
+            self.module = MD5
+        else:
+            self.module = module
         self.max_drift = max_drift
 
     def bsig(self, node):
@@ -198,6 +273,7 @@ class Calculator:
         sigs = map(self.get_signature, node.children())
         if node.builder:
             sigs.append(self.module.signature(node.builder_sig_adapter()))
+
         bsig = self.module.collect(filter(lambda x: not x is None, sigs))
 
         node.set_bsig(bsig)
@@ -294,3 +370,6 @@ class Calculator:
             return 1
 
         return self.module.current(newsig, oldbsig)
+
+
+default_calc = Calculator()