Add CacheDir support.

12 files changed, 312 insertions, 34 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index b29889e..b258199 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -54,6 +54,10 @@ RELEASE 0.11 - XXX
     objects have strfunction() methods, and the functions for building
     and returning a string both take the same arguments.
 
+  - Add support for new CacheDir() functionality to share derived files
+    between builds, with related options --cache-disable, --cache-force,
+    and --cache-show.
+
   From Steve Leblanc:
 
   - Fix the output of -c -n when directories are involved, so it
diff --git a/src/RELEASE.txt b/src/RELEASE.txt
index d87808b..ccb301c 100644
--- a/src/RELEASE.txt
+++ b/src/RELEASE.txt
@@ -187,21 +187,17 @@ RELEASE 0.10 - Thu, 16 Jan 2003 04:11:46 -0600
 
         command < $SOURCE > $TARGET
 
-      If you don't put space (for example, "<$SOURCE"), SCons will not
-      recognize the redirection.
+      If you don't supply a space (for example, "<$SOURCE"), SCons will
+      not recognize the redirection.
 
     - Executing the -u or -U option from a source directory that has an
       associated BuildDir() does not build the targets in the BuildDir().
 
-    - No support yet for the following future features:
+    - No support yet for the following planned command-line options:
 
-        - No support for caching built files.
-
-        - No support yet for the following command-line options:
-
-             -d -e -l --list-actions --list-derived --list-where
-             -o --override -p -r -R --random -w --write-filenames
-             -W --warn-undefined-variables
+         -d -e -l --list-actions --list-derived --list-where
+         -o --override -p -r -R --random -w --write-filenames
+         -W --warn-undefined-variables
 
 
 
diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py
index 3ab910b..960ffce 100644
--- a/src/engine/SCons/Action.py
+++ b/src/engine/SCons/Action.py
@@ -155,7 +155,8 @@ class ActionBase:
         return cmp(self.__dict__, other.__dict__)
 
     def show(self, string):
-        print string
+        if print_actions:
+            print string
 
     def get_actions(self):
         return [self]
diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py
index 2d3540c..61cea53 100644
--- a/src/engine/SCons/ActionTests.py
+++ b/src/engine/SCons/ActionTests.py
@@ -187,6 +187,30 @@ class ActionBaseTestCase(unittest.TestCase):
         assert a1 != a3
         assert a2 != a3
 
+    def test_show(self):
+        """Test the show() method
+        """
+        save = SCons.Action.print_actions
+        SCons.Action.print_actions = 0
+
+        sio = StringIO.StringIO()
+        sys.stdout = sio
+        a = SCons.Action.Action("x")
+        a.show("xyzzy")
+        s = sio.getvalue()
+        assert s == "", s
+
+        SCons.Action.print_actions = 1
+
+        sio = StringIO.StringIO()
+        sys.stdout = sio
+        a.show("foobar")
+        s = sio.getvalue()
+        assert s == "foobar\n", s
+
+        SCons.Action.print_actions = save
+        sys.stdout = StringIO.StringIO()
+
     def test_get_actions(self):
         """Test the get_actions() method
         """
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 923615f..08a47a7 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -38,6 +38,8 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 import os
 import os.path
+import shutil
+import stat
 import string
 from UserDict import UserDict
 
@@ -80,8 +82,6 @@ def LinkFunc(target, source, env):
         try :
             os.symlink(src, dest)
         except (AttributeError, OSError):
-            import shutil
-            import stat
             shutil.copy2(src, dest)
             st=os.stat(src)
             os.chmod(dest, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
@@ -106,6 +106,38 @@ def MkdirFunc(target, source, env):
 
 Mkdir = SCons.Action.Action(MkdirFunc, None)
 
+def CacheRetrieveFunc(target, source, env):
+    t = target[0]
+    cachedir, cachefile = t.cachepath()
+    if os.path.exists(cachefile):
+        shutil.copy2(cachefile, t.path)
+        st = os.stat(cachefile)
+        os.chmod(t.path, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
+        return 0
+    return 1
+
+def CacheRetrieveString(target, source, env):
+    t = target[0]
+    cachedir, cachefile = t.cachepath()
+    if os.path.exists(cachefile):
+        return "Retrieved `%s' from cache" % t.path
+    return None
+
+CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
+
+CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
+
+def CachePushFunc(target, source, env):
+    t = target[0]
+    cachedir, cachefile = t.cachepath()
+    if not os.path.isdir(cachedir):
+        os.mkdir(cachedir)
+    shutil.copy2(t.path, cachefile)
+    st = os.stat(t.path)
+    os.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
+
+CachePush = SCons.Action.Action(CachePushFunc, None)
+
 #
 class ParentOfRoot:
     """
@@ -283,6 +315,9 @@ class FS:
             self.pathTop = path
         self.Root = {}
         self.Top = None
+        self.CachePath = None
+        self.cache_force = None
+        self.cache_show = None
 
     def set_toplevel_dir(self, path):
         assert not self.Top, "You can only set the top-level path on an FS object that has not had its File, Dir, or Entry methods called yet."
@@ -572,6 +607,9 @@ class FS:
                     d = d.get_dir()
         return ret
 
+    def CacheDir(self, path):
+        self.CachePath = path
+
 # XXX TODO?
 # Annotate with the creator
 # rel_path
@@ -924,14 +962,48 @@ class File(Entry):
             except OSError:
                 pass
 
+    def build(self):
+        """Actually build the file.
+
+        This overrides the base class build() method to check for the
+        existence of derived files in a CacheDir before going ahead and
+        building them.
+
+        This method is called from multiple threads in a parallel build,
+        so only do thread safe stuff here. Do thread unsafe stuff in
+        built().
+        """
+        if not self.has_builder():
+            return
+        if self.fs.CachePath:
+            if self.fs.cache_show:
+                if CacheRetrieveSilent(self, None, None) == 0:
+                    def do_print(action, targets, sources, env, self=self):
+                        al = action.strfunction(targets, self.sources, env)
+                        if not SCons.Util.is_List(al):
+                            al = [al]
+                        for a in al:
+                            action.show(a)
+                    self._for_each_action(do_print)
+                    return
+            elif CacheRetrieve(self, None, None) == 0:
+                return
+        SCons.Node.Node.build(self)
+
     def built(self):
         SCons.Node.Node.built(self)
+        if self.fs.CachePath and os.path.exists(self.path):
+            CachePush(self, None, None)
         self.found_includes = {}
         if hasattr(self, '_exists'):
             delattr(self, '_exists')
         if hasattr(self, '_rexists'):
             delattr(self, '_rexists')
 
+    def visited(self):
+        if self.fs.CachePath and self.fs.cache_force and os.path.exists(self.path):
+            CachePush(self, None, None)
+
     def prepare(self):
         """Prepare for this file to be created."""
 
@@ -1020,6 +1092,13 @@ class File(Entry):
     def rstr(self):
         return str(self.rfile())
 
+    def cachepath(self):
+        if self.fs.CachePath:
+            bsig = str(self.get_bsig())
+            subdir = string.upper(bsig[0])
+            dir = os.path.join(self.fs.CachePath, subdir)
+            return dir, os.path.join(dir, bsig)
+        return None, None
 
 default_fs = FS()
 
diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py
index 83bdccf..fa32c25 100644
--- a/src/engine/SCons/Node/FSTests.py
+++ b/src/engine/SCons/Node/FSTests.py
@@ -54,6 +54,10 @@ class Builder:
                 global built_it
                 built_it = 1
                 return 0
+            def show(self, string):
+                pass
+            def strfunction(self, targets, sources, env):
+                return ""
         return [Action()]
 
     def targets(self, t):
@@ -1206,6 +1210,116 @@ class get_actionsTestCase(unittest.TestCase):
         a = dir.get_actions()
         assert a == [], a
 
+class CacheDirTestCase(unittest.TestCase):
+    def runTest(self):
+        """Test CacheDir functionality"""
+        global built_it
+
+        fs = SCons.Node.FS.FS()
+        assert fs.CachePath is None, fs.CachePath
+        assert fs.cache_force is None, fs.cache_force
+        assert fs.cache_show is None, fs.cache_show
+
+        fs.CacheDir('cache')
+        assert fs.CachePath == 'cache', fs.CachePath
+
+        save_CacheRetrieve = SCons.Node.FS.CacheRetrieve
+        self.retrieved = []
+        def retrieve_succeed(target, source, env, self=self):
+            self.retrieved.append(target)
+            return 0
+        def retrieve_fail(target, source, env, self=self):
+            self.retrieved.append(target)
+            return 1
+
+        f1 = fs.File("cd.f1")
+        f1.builder_set(Builder(fs.File))
+        f1.env_set(Environment())
+        try:
+            SCons.Node.FS.CacheRetrieve = retrieve_succeed
+            self.retrieved = []
+            built_it = None
+
+            f1.build()
+            assert self.retrieved == [f1], self.retrieved
+            assert built_it is None, built_it
+
+            SCons.Node.FS.CacheRetrieve = retrieve_fail
+            self.retrieved = []
+            built_it = None
+
+            f1.build()
+            assert self.retrieved == [f1], self.retrieved
+            assert built_it, built_it
+        finally:
+            SCons.Node.FS.CacheRetrieve = save_CacheRetrieve
+
+        save_CacheRetrieveSilent = SCons.Node.FS.CacheRetrieveSilent
+
+        fs.cache_show = 1
+
+        f2 = fs.File("cd.f2")
+        f2.builder_set(Builder(fs.File))
+        f2.env_set(Environment())
+        try:
+            SCons.Node.FS.CacheRetrieveSilent = retrieve_succeed
+            self.retrieved = []
+            built_it = None
+
+            f2.build()
+            assert self.retrieved == [f2], self.retrieved
+            assert built_it is None, built_it
+
+            SCons.Node.FS.CacheRetrieveSilent = retrieve_fail
+            self.retrieved = []
+            built_it = None
+
+            f2.build()
+            assert self.retrieved == [f2], self.retrieved
+            assert built_it, built_it
+        finally:
+            SCons.Node.FS.CacheRetrieveSilent = save_CacheRetrieveSilent
+
+        save_CachePush = SCons.Node.FS.CachePush
+        self.pushed = []
+        def push(target, source, env, self=self):
+            self.pushed.append(target)
+            return 0
+        SCons.Node.FS.CachePush = push
+
+        try:
+            test = TestCmd(workdir='')
+
+            cd_f3 = test.workpath("cd.f3")
+            f3 = fs.File(cd_f3)
+            f3.built()
+            assert self.pushed == [], self.pushed
+            test.write(cd_f3, "cd.f3\n")
+            f3.built()
+            assert self.pushed == [f3], self.pushed
+
+            self.pushed = []
+
+            cd_f4 = test.workpath("cd.f4")
+            f4 = fs.File(cd_f4)
+            f4.visited()
+            assert self.pushed == [], self.pushed
+            test.write(cd_f4, "cd.f4\n")
+            f4.visited()
+            assert self.pushed == [], self.pushed
+            fs.cache_force = 1
+            f4.visited()
+            assert self.pushed == [f4], self.pushed
+        finally:
+            SCons.Node.FS.CachePush = save_CachePush
+
+        f5 = fs.File("cd.f5")
+        f5.set_bsig('a_fake_bsig')
+        cp = f5.cachepath()
+        dirname = os.path.join('cache', 'A')
+        filename = os.path.join(dirname, 'a_fake_bsig')
+        assert cp == (dirname, filename), cp
+
 
 if __name__ == "__main__":
     suite = unittest.TestSuite()
@@ -1216,5 +1330,6 @@ if __name__ == "__main__":
     suite.addTest(StringDirTestCase())
     suite.addTest(prepareTestCase())
     suite.addTest(get_actionsTestCase())
+    suite.addTest(CacheDirTestCase())
     if not unittest.TextTestRunner().run(suite).wasSuccessful():
         sys.exit(1)
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index 3b66f43..48da843 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -215,6 +215,14 @@ class NodeTestCase(unittest.TestCase):
             assert type(act.built_target[0]) == type(MyNode("bar")), type(act.built_target[0])
             assert str(act.built_target[0]) == "xxx", str(act.built_target[0])
             assert act.built_source == ["yyy", "zzz"], act.built_source
+
+    def test_visited(self):
+        """Test the base visited() method
+
+        Just make sure it's there and we can call it.
+        """
+        n = SCons.Node.Node()
+        n.visited()
             
     def test_depends_on(self):
         """Test the depends_on() method
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index 9641ea7..8b9a542 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -108,13 +108,7 @@ class Node:
     def generate_build_env(self):
         return self.env.Override(self.overrides)
 
-    def build(self):
-        """Actually build the node.
-
-        This method is called from multiple threads in a parallel build,
-        so only do thread safe stuff here. Do thread unsafe stuff in
-        built().
-        """
+    def _for_each_action(self, func):
         if not self.has_builder():
             return None
         action_list = self.pre_actions + self.builder.get_actions() + \
@@ -124,10 +118,21 @@ class Node:
         targets = self.builder.targets(self)
         env = self.generate_build_env()
         for action in action_list:
-            stat = action(targets, self.sources, env)
+            func(action, targets, self.sources, env)
+
+    def build(self):
+        """Actually build the node.
+
+        This method is called from multiple threads in a parallel build,
+        so only do thread safe stuff here. Do thread unsafe stuff in
+        built().
+        """
+        def do_action(action, targets, sources, env, self=self):
+            stat = action(targets, sources, env)
             if stat:
                 raise SCons.Errors.BuildError(node = self,
                                               errstr = "Error %d" % stat)
+        self._for_each_action(do_action)
 
     def built(self):
         """Called just after this node is sucessfully built."""
@@ -151,6 +156,11 @@ class Node:
         # node were presumably just changed:
         self.del_csig()
 
+    def visited(self):
+        """Called just after this node has been visited
+        without requiring a build.."""
+        pass
+
     def depends_on(self, nodes):
         """Does this node depend on any of 'nodes'?"""
         for node in nodes:
diff --git a/src/engine/SCons/Script/SConscript.py b/src/engine/SCons/Script/SConscript.py
index c5cccb6..7cfb312 100644
--- a/src/engine/SCons/Script/SConscript.py
+++ b/src/engine/SCons/Script/SConscript.py
@@ -360,6 +360,7 @@ def BuildDefaultGlobals():
     globals['ARGUMENTS']         = arguments
     globals['BuildDir']          = BuildDir
     globals['Builder']           = SCons.Builder.Builder
+    globals['CacheDir']          = SCons.Node.FS.default_fs.CacheDir
     globals['Clean']             = Clean
     globals['CScan']             = SCons.Defaults.CScan
     globals['Default']           = Default
diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py
index a14c7ae..8bc3da4 100644
--- a/src/engine/SCons/Script/__init__.py
+++ b/src/engine/SCons/Script/__init__.py
@@ -427,6 +427,18 @@ class OptParser(OptionParser):
         self.add_option('-C', '--directory', type="string", action = "append",
                         help="Change to DIRECTORY before doing anything.")
 
+        self.add_option('--cache-disable', '--no-cache',
+                        action="store_true", dest='cache_disable', default=0,
+                        help="Do not retrieve built targets from CacheDir.")
+
+        self.add_option('--cache-force', '--cache-populate',
+                        action="store_true", dest='cache_force', default=0,
+                        help="Copy already-built targets into the CacheDir.")
+
+        self.add_option('--cache-show',
+                        action="store_true", dest='cache_show', default=0,
+                        help="Print build actions for files from CacheDir.")
+
         def opt_not_yet(option, opt, value, parser):
             sys.stderr.write("Warning:  the %s option is not yet implemented\n" % opt)
             sys.exit(0)
@@ -555,18 +567,6 @@ class OptParser(OptionParser):
         self.add_option('-Y', '--repository', nargs=1, action="append",
                         help="Search REPOSITORY for source and target files.")
 
-        self.add_option('--cache-disable', '--no-cache', action="callback",
-                        callback=opt_not_yet,
-                        # help = "Do not retrieve built targets from Cache."
-                        help=SUPPRESS_HELP)
-        self.add_option('--cache-force', '--cache-populate', action="callback",
-                        callback=opt_not_yet,
-                        # help = "Copy already-built targets into the Cache."
-                        help=SUPPRESS_HELP)
-        self.add_option('--cache-show', action="callback",
-                        callback=opt_not_yet,
-                        # help = "Print what would have built Cached targets.",
-                        help=SUPPRESS_HELP)
         self.add_option('-e', '--environment-overrides', action="callback",
                         callback=opt_not_yet,
                         # help="Environment variables override makefiles."
@@ -676,6 +676,13 @@ def _main():
         display.set_mode(0)
     if options.silent:
         SCons.Action.print_actions = None
+    if options.cache_disable:
+        def disable(self): pass
+        SCons.Node.FS.default_fs.CacheDir = disable
+    if options.cache_force:
+        SCons.Node.FS.default_fs.cache_force = 1
+    if options.cache_show:
+        SCons.Node.FS.default_fs.cache_show = 1
     if options.directory:
         cdir = _create_path(options.directory)
         try:
diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py
index 10f074f..e0d933e 100644
--- a/src/engine/SCons/Taskmaster.py
+++ b/src/engine/SCons/Taskmaster.py
@@ -115,6 +115,9 @@ class Task:
                     side_effect.set_state(None)
                 t.set_state(SCons.Node.executed)
                 t.built()
+        else:
+            for t in self.targets:
+                t.visited()
 
         self.tm.executed(self.node)
 
diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py
index cc0b437..2b8a362 100644
--- a/src/engine/SCons/TaskmasterTests.py
+++ b/src/engine/SCons/TaskmasterTests.py
@@ -31,6 +31,7 @@ import SCons.Errors
 
 
 built_text = None
+visited_nodes = []
 executed = None
 scan_called = 0
 
@@ -64,6 +65,10 @@ class Node:
         global built_text
         built_text = built_text + " really"
 
+    def visited(self):
+        global visited_nodes
+        visited_nodes.append(self.name)
+
     def prepare(self):
         self.prepared = 1
 
@@ -449,7 +454,32 @@ class TaskmasterTestCase(unittest.TestCase):
     def test_executed(self):
         """Test when a task has been executed
         """
-        pass
+        global built_text
+        global visited_nodes
+
+        n1 = Node("n1")
+        tm = SCons.Taskmaster.Taskmaster([n1])
+        t = tm.next_task()
+        built_text = "xxx"
+        visited_nodes = []
+        n1.set_state(SCons.Node.executing)
+        t.executed()
+        s = n1.get_state()
+        assert s == SCons.Node.executed, s
+        assert built_text == "xxx really", built_text
+        assert visited_nodes == [], visited_nodes
+
+        n2 = Node("n2")
+        tm = SCons.Taskmaster.Taskmaster([n2])
+        t = tm.next_task()
+        built_text = "should_not_change"
+        visited_nodes = []
+        n2.set_state(None)
+        t.executed()
+        s = n1.get_state()
+        assert s == SCons.Node.executed, s
+        assert built_text == "should_not_change", built_text
+        assert visited_nodes == ["n2"], visited_nodes
 
     def test_prepare(self):
         """Test preparation of multiple Nodes for a task