Implement content signature caching and --max-drift (Anthony Roach)

8 files changed, 148 insertions, 37 deletions

diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 0f8425f..c49361d 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -520,10 +520,19 @@ class File(Entry):
         else:
             return 0
 
-    def store_sigs(self):
-        """Update a file's .sconsign entry with its current info."""
-        self.dir.sconsign().set(self.name, self.get_timestamp(),
-                                self.get_bsig(), self.get_csig())
+    def store_csig(self):
+        old = self.get_prevsiginfo()
+        self.dir.sconsign().set(self.name,
+                                self.get_timestamp(),
+                                old[1],
+                                self.get_csig())
+
+    def store_bsig(self):
+        old = self.get_prevsiginfo()
+        self.dir.sconsign().set(self.name,
+                                self.get_timestamp(),
+                                self.get_bsig(),
+                                old[2])
 
     def get_prevsiginfo(self):
         """Fetch the previous signature information from the
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index 4caebb5..447ef93 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -266,11 +266,17 @@ class NodeTestCase(unittest.TestCase):
         node.set_csig('zzz')
         assert node.get_csig() == 'zzz'
 
+    def test_store_bsig(self):
+        """Test calling the method to store a build signature
+        """
+        node = SCons.Node.Node()
+        node.store_bsig()
+
     def test_store_sigs(self):
-        """Test calling the method to store signatures
+        """Test calling the method to store a content signature
         """
         node = SCons.Node.Node()
-        node.store_sigs()
+        node.store_csig()
 
     def test_set_precious(self):
         """Test setting a Node's precious value
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index 9c39d25..a9ca790 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -120,9 +120,14 @@ class Node:
             def get_parents(node, parent): return node.get_parents()
             def clear_cache(node, parent): 
                 node.implicit = None
+                node.bsig = None
             w = Walker(self, get_parents, ignore_cycle, clear_cache)
             while w.next(): pass
 
+        # clear out the content signature, since the contents of this
+        # node were presumably just changed:
+        self.csig = None
+
     def depends_on(self, nodes):
         """Does this node depend on any of 'nodes'?"""
         for node in nodes:
@@ -194,6 +199,11 @@ class Node:
         of its dependency files and build information)."""
         self.bsig = bsig
 
+    def store_bsig(self):
+        """Make the build signature permanent (that is, store it in the
+        .sconsign file or equivalent)."""
+        pass
+
     def get_csig(self):
         """Get the signature of the node's content."""
         return self.csig
@@ -202,11 +212,19 @@ class Node:
         """Set the signature of the node's content."""
         self.csig = csig
 
-    def store_sigs(self):
-        """Make the signatures permanent (that is, store them in the
+    def store_csig(self):
+        """Make the content signature permanent (that is, store it in the
         .sconsign file or equivalent)."""
         pass
 
+    def get_prevsiginfo(self):
+        """Fetch the previous signature information from the
+        .sconsign entry."""
+        return None
+
+    def get_timestamp(self):
+        return 0
+
     def set_precious(self, precious = 1):
         """Set the Node's precious value."""
         self.precious = precious
diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py
index 126eb6b..d9406ec 100644
--- a/src/engine/SCons/Script/__init__.py
+++ b/src/engine/SCons/Script/__init__.py
@@ -157,6 +157,7 @@ climb_up = 0
 target_top = None
 exit_status = 0 # exit status, assume success by default
 profiling = 0
+max_drift = None
 
 # utility functions
 
@@ -510,6 +511,18 @@ def options_init():
 	long = ['list-where'],
 	help = "Don't build; list files and where defined.")
 
+    def opt_max_drift(opt, arg):
+        global max_drift
+        try:
+            max_drift = int(arg)
+        except ValueError: 
+            raise UserError, "The argument for --max-drift must be an integer."
+
+    Option(func = opt_max_drift,
+        long = ['max-drift'],
+        arg = 'SECONDS',
+        help = "Set the maximum system clock drift to be SECONDS.")
+
     def opt_n(opt, arg):
 	SCons.Action.execute_actions = None
 	CleanTask.execute = CleanTask.show
@@ -797,7 +810,10 @@ def _main():
     nodes = filter(lambda x: x is not None, map(Entry, targets))
 
     if not calc:
-        calc = SCons.Sig.Calculator(SCons.Sig.MD5)
+        if max_drift is None:
+            calc = SCons.Sig.Calculator(SCons.Sig.MD5)
+        else:
+            calc = SCons.Sig.Calculator(SCons.Sig.MD5, max_drift)
 
     taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, calc)
 
diff --git a/src/engine/SCons/Sig/SigTests.py b/src/engine/SCons/Sig/SigTests.py
index 90488e1..2a2560f 100644
--- a/src/engine/SCons/Sig/SigTests.py
+++ b/src/engine/SCons/Sig/SigTests.py
@@ -99,15 +99,21 @@ class DummyNode:
     def get_bsig(self):
         return self.bsig
 
+    def store_bsig(self):
+        pass
+
     def set_csig(self, csig):
         self.csig = csig
 
     def get_csig(self):
         return self.csig
 
+    def store_csig(self):
+        pass
+
     def get_prevsiginfo(self):
         return (self.oldtime, self.oldbsig, self.oldcsig)
-
+    
     def builder_sig_adapter(self):
         class Adapter:
             def get_contents(self):
@@ -160,7 +166,11 @@ def write(calc, nodes):
         node.oldtime = node.file.timestamp
         node.oldbsig = calc.bsig(node)
         node.oldcsig = calc.csig(node)
-        
+
+def clear(nodes):
+    for node in nodes:
+        node.csig = None
+        node.bsig = None
 
 class SigTestBase:
     
@@ -173,6 +183,7 @@ class SigTestBase:
         self.test_initial()
         self.test_built()
         self.test_modify()
+        self.test_modify_same_time()
         self.test_delete()
         self.test_cache()
         
@@ -217,6 +228,8 @@ class SigTestBase:
         self.files[6].modify('blah blah blah', 333)
         self.files[8].modify('blah blah blah', 333)
 
+        clear(nodes)
+
         self.failUnless(not current(calc, nodes[0]), "modified directly")
         self.failUnless(not current(calc, nodes[1]), "direct source modified")
         self.failUnless(current(calc, nodes[2]))
@@ -229,6 +242,27 @@ class SigTestBase:
         self.failUnless(not current(calc, nodes[9]), "direct source modified")
         self.failUnless(not current(calc, nodes[10]), "indirect source modified")
 
+
+    def test_modify_same_time(self):
+
+        nodes = create_nodes(self.files)
+
+        calc = SCons.Sig.Calculator(self.module, 0)
+
+        write(calc, nodes)
+
+        #simulate a modification of some files without changing the timestamp:
+        self.files[0].modify('blah blah blah blah', 333)
+        self.files[3].modify('blah blah blah blah', 333)
+        self.files[6].modify('blah blah blah blah', 333)
+        self.files[8].modify('blah blah blah blah', 333)
+
+        clear(nodes)
+
+        for node in nodes:
+            self.failUnless(current(calc, node),
+                            "all of the nodes should be current")
+
     def test_delete(self):
         
         nodes = create_nodes(self.files)
@@ -306,6 +340,10 @@ class CalcTestCase(unittest.TestCase):
                 return 1
             def get_bsig(self):
                 return self.bsig
+            def set_bsig(self, bsig):
+                self.bsig = bsig
+            def store_sigs(self):
+                pass
             def get_csig(self):
                 return self.csig
             def get_prevsiginfo(self):
diff --git a/src/engine/SCons/Sig/__init__.py b/src/engine/SCons/Sig/__init__.py
index cb03630..d077114 100644
--- a/src/engine/SCons/Sig/__init__.py
+++ b/src/engine/SCons/Sig/__init__.py
@@ -33,6 +33,7 @@ import os
 import os.path
 import string
 import SCons.Node
+import time
 
 #XXX Get rid of the global array so this becomes re-entrant.
 sig_files = []
@@ -52,13 +53,13 @@ class SConsignFile:
         dir - the directory for the file
         module - the signature module being used
         """
-        
+
         self.dir = dir
         self.module = module
         self.sconsign = os.path.join(dir.path, '.sconsign')
         self.entries = {}
         self.dirty = None
-                    
+
         try:
             file = open(self.sconsign, 'rt')
         except:
@@ -78,7 +79,7 @@ class SConsignFile:
         filename - the filename whose signature will be returned
         returns - (timestamp, bsig, csig)
         """
-        
+
         try:
             arr = map(string.strip, string.split(self.entries[filename], " "))
         except KeyError:
@@ -162,13 +163,17 @@ class Calculator:
     for the build engine.
     """
 
-    def __init__(self, module):
+    def __init__(self, module, max_drift=2*24*60*60):
         """
         Initialize the calculator.
 
         module - the signature module to use for signature calculations
+        max_drift - the maximum system clock drift used to determine when to
+          cache content signatures. A negative value means to never cache
+          content signatures. (defaults to 2 days)
         """
         self.module = module
+        self.max_drift = max_drift
 
     def bsig(self, node):
         """
@@ -185,21 +190,23 @@ class Calculator:
         """
         if not node.use_signature:
             return None
-        #XXX If configured, use the content signatures from the
-        #XXX .sconsign file if the timestamps match.
 
         bsig = node.get_bsig()
         if not bsig is None:
             return bsig
 
-        sigs = []
-        for child in node.children():
-            sigs.append(self.get_signature(child))
+        sigs = map(self.get_signature, node.children())
         if node.builder:
             sigs.append(self.module.signature(node.builder_sig_adapter()))
+        bsig = self.module.collect(filter(lambda x: not x is None, sigs))
+
+        node.set_bsig(bsig)
+
+        # don't store the bsig here, because it isn't accurate until
+        # the node is actually built.
+
+        return bsig
 
-        return self.module.collect(filter(lambda x: not x is None, sigs))
-        
     def csig(self, node):
         """
         Generate a node's content signature, the digested signature
@@ -210,14 +217,35 @@ class Calculator:
         """
         if not node.use_signature:
             return None
-        #XXX If configured, use the content signatures from the
-        #XXX .sconsign file if the timestamps match.
+
         csig = node.get_csig()
         if not csig is None:
             return csig
-        
-        return self.module.signature(node)
-        
+
+        if self.max_drift >= 0:
+            info = node.get_prevsiginfo()
+        else:
+            info = None
+
+        mtime = node.get_timestamp()
+
+        if (info and info[0] and info[2] and info[0] == mtime):
+            # use the signature stored in the .sconsign file
+            csig = info[2]
+            # Set the csig here so it doesn't get recalculated unnecessarily
+            # and so it's set when the .sconsign file gets written
+            node.set_csig(csig)
+        else:
+            csig = self.module.signature(node)
+            # Set the csig here so it doesn't get recalculated unnecessarily
+            # and so it's set when the .sconsign file gets written
+            node.set_csig(csig)
+
+            if self.max_drift >= 0 and (time.time() - mtime) > self.max_drift:
+                node.store_csig()
+
+        return csig
+
     def get_signature(self, node):
         """
         Get the appropriate signature for a node.
@@ -264,5 +292,5 @@ class Calculator:
 
         if not node.builder and node.get_timestamp() == oldtime:
             return 1
-        
+
         return self.module.current(newsig, oldbsig)
diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py
index 47707b0..22c22cb 100644
--- a/src/engine/SCons/Taskmaster.py
+++ b/src/engine/SCons/Taskmaster.py
@@ -90,7 +90,7 @@ class Task:
         if self.targets[0].get_state() == SCons.Node.executing:
             self.set_tstates(SCons.Node.executed)
             for t in self.targets:
-                t.store_sigs()
+                t.store_bsig()
                 t.built()
 
         self.tm.executed(self.node)
@@ -190,13 +190,6 @@ class Taskmaster:
                 desc = "Dependency cycle: " + string.join(map(str, nodes), " -> ")
                 raise SCons.Errors.UserError, desc
 
-            for child in children:
-                if not child.builder:
-                    # set the signature for non-derived files
-                    # here so they don't get recalculated over
-                    # and over again:
-                    child.set_csig(self.calc.csig(child))
-
             # Add non-derived files that have not been built
             # to the candidates list:
             def derived(node):
diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py
index 44b1e40..3a67aa2 100644
--- a/src/engine/SCons/TaskmasterTests.py
+++ b/src/engine/SCons/TaskmasterTests.py
@@ -93,7 +93,10 @@ class Node:
     def set_csig(self, csig):
         self.csig = csig
 
-    def store_sigs(self):
+    def store_csig(self):
+        pass
+
+    def store_bsig(self):
         pass