Apply Memoizer to cache more return values from various methods. (Kevin Quick)

6 files changed, 159 insertions, 105 deletions

diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py
index 2e146e3..8c2e767 100644
--- a/src/engine/SCons/Environment.py
+++ b/src/engine/SCons/Environment.py
@@ -264,12 +264,14 @@ class SubstitutionEnvironment:
         return cmp(sdict, odict)
 
     def __delitem__(self, key):
+        "__cache_reset__"
         del self._dict[key]
 
     def __getitem__(self, key):
         return self._dict[key]
 
     def __setitem__(self, key, value):
+        "__cache_reset__"
         if key in reserved_construction_var_names:
             SCons.Warnings.warn(SCons.Warnings.ReservedVariableWarning,
                                 "Ignoring attempt to set reserved variable `%s'" % key)
@@ -599,6 +601,7 @@ class Base(SubstitutionEnvironment):
     def _update(self, dict):
         """Update an environment's values directly, bypassing the normal
         checks that occur when users try to set items.
+        __cache_reset__
         """
         self._dict.update(dict)
 
@@ -743,7 +746,7 @@ class Base(SubstitutionEnvironment):
         return clone
 
     def Detect(self, progs):
-        """Return the first available program in progs.
+        """Return the first available program in progs.  __cacheable__
         """
         if not SCons.Util.is_List(progs):
             progs = [ progs ]
@@ -1011,7 +1014,7 @@ class Base(SubstitutionEnvironment):
         except KeyError:
             pass
         kw = copy_non_reserved_keywords(kw)
-        self._dict.update(our_deepcopy(kw))
+        self._update(our_deepcopy(kw))
         self.scanner_map_delete(kw)
 
     def ReplaceIxes(self, path, old_prefix, old_suffix, new_prefix, new_suffix):
@@ -1052,7 +1055,7 @@ class Base(SubstitutionEnvironment):
         tool(self)
 
     def WhereIs(self, prog, path=None, pathext=None, reject=[]):
-        """Find prog in the path.  
+        """Find prog in the path.  __cacheable__
         """
         if path is None:
             try:
diff --git a/src/engine/SCons/Memoize.py b/src/engine/SCons/Memoize.py
index ce3d9e6..72adbcd 100644
--- a/src/engine/SCons/Memoize.py
+++ b/src/engine/SCons/Memoize.py
@@ -89,6 +89,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 #TBD: for pickling, should probably revert object to unclassed state...
 
 import copy
+import os
 import string
 import sys
 
@@ -448,6 +449,15 @@ def Analyze_Class(klass):
 # for traceback or profile output, which generate things like 'File
 # "<string>", line X'.  X will be the number of \n's plus 1.
 
+# Also use the following routine to specify the "filename" portion so
+# that it provides useful information.  In addition, make sure it
+# contains 'os.sep + "SCons" + os.sep' for the
+# SCons.Script.find_deepest_user_frame operation.
+
+def whoami(memoizer_funcname, real_funcname):
+    return '...'+os.sep+'SCons'+os.sep+'Memoizer-'+ \
+           memoizer_funcname+'-lambda<'+real_funcname+'>'
+
 def memoize_classdict(modelklass, new_klassdict, cacheable, resetting):
     new_klassdict.update(modelklass.__dict__)
     new_klassdict['_MeMoIZeR_converted'] = 1
@@ -457,35 +467,39 @@ def memoize_classdict(modelklass, new_klassdict, cacheable, resetting):
                not code.func_code.co_flags & 0xC:
             newmethod = eval(
                 compile("\n"*1 +
-                "lambda self: Memoizer_cache_get_self(methcode, methcached, self)",
-                        "Memoizer_cache_get_self_lambda",
+                        "lambda self: MCGS(methcode, methcached, self)",
+                        whoami('cache_get_self', name),
                         "eval"),
                 {'methcode':code, 'methcached':{},
-                 'Memoizer_cache_get_self':Memoizer_cache_get_self},
+                 'MCGS':Memoizer_cache_get_self},
                 {})
         elif code.func_code.co_argcount == 2 and \
                not code.func_code.co_flags & 0xC:
             newmethod = eval(
                 compile("\n"*2 +
-                "lambda self, arg: Memoizer_cache_get_one(methcode, methcached, self, arg)",
-                        "Memoizer_cache_get_one_lambda",
+                "lambda self, arg: MCGO(methcode, methcached, self, arg)",
+                        whoami('cache_get_one', name),
                         "eval"),
                 {'methcode':code, 'methcached':{},
-                 'Memoizer_cache_get_one':Memoizer_cache_get_one},
+                 'MCGO':Memoizer_cache_get_one},
                 {})
         else:
             newmethod = eval(
                 compile("\n"*3 +
-                "lambda *args, **kw: Memoizer_cache_get(methcode, methcached, args, kw)",
-                        "Memoizer_cache_get_lambda",
+                "lambda *args, **kw: MCG(methcode, methcached, args, kw)",
+                        whoami('cache_get', name),
                         "eval"),
                 {'methcode':code, 'methcached':{},
-                 'Memoizer_cache_get':Memoizer_cache_get}, {})
+                 'MCG':Memoizer_cache_get}, {})
         new_klassdict[name] = newmethod
 
     for name,code in resetting.items():
-        newmethod = eval("lambda obj_self, *args, **kw: (obj_self._MeMoIZeR_reset(), apply(rmethcode, (obj_self,)+args, kw))[1]",
-                         {'rmethcode':code}, {})
+        newmethod = eval(
+            compile(
+            "lambda obj_self, *args, **kw: (obj_self._MeMoIZeR_reset(), apply(rmethcode, (obj_self,)+args, kw))[1]",
+            whoami('cache_reset', name),
+            'eval'),
+            {'rmethcode':code}, {})
         new_klassdict[name] = newmethod
 
     return new_klassdict
@@ -667,7 +681,8 @@ else:
             newinitcode = compile(
                 "\n"*(init.func_code.co_firstlineno-1) +
                 "lambda self, args, kw: _MeMoIZeR_init(real_init, self, args, kw)",
-                init.func_code.co_filename, 'eval')
+                whoami('init', init.func_code.co_filename),
+                'eval')
             newinit = eval(newinitcode,
                            {'real_init':init,
                             '_MeMoIZeR_init':_MeMoIZeR_init},
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index a67fa76..1f2b0a8 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -630,11 +630,11 @@ class Entry(Base):
         if self.fs.isfile(self.abspath):
             self.__class__ = File
             self._morph()
-            return File.get_contents(self)
+            return self.get_contents()
         if self.fs.isdir(self.abspath):
             self.__class__ = Dir
             self._morph()
-            return Dir.get_contents(self)
+            return self.get_contents()
         if self.fs.islink(self.abspath):
             return ''             # avoid errors for dangling symlinks
         raise AttributeError
@@ -679,6 +679,9 @@ _classEntry = Entry
 
 
 class LocalFS:
+
+    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    
     # This class implements an abstraction layer for operations involving
     # a local file system.  Essentially, this wraps any function in
     # the os, os.path or shutil modules that we use to actually go do
@@ -736,6 +739,14 @@ class LocalFS:
             return 0                    # no symlinks
         exists_or_islink = exists
 
+if not SCons.Memoize.has_metaclass:
+    _FSBase = LocalFS
+    class LocalFS(SCons.Memoize.Memoizer, _FSBase):
+        def __init__(self, *args, **kw):
+            apply(_FSBase.__init__, (self,)+args, kw)
+            SCons.Memoize.Memoizer.__init__(self)
+
+
 #class RemoteFS:
 #    # Skeleton for the obvious methods we might need from the
 #    # abstraction layer for a remote filesystem.
@@ -746,6 +757,7 @@ class LocalFS:
 
 
 class FS(LocalFS):
+
     def __init__(self, path = None):
         """Initialize the Node.FS subsystem.
 
@@ -771,6 +783,10 @@ class FS(LocalFS):
         assert not self.Top, "You can only set the top-level path on an FS object that has not had its File, Dir, or Entry methods called yet."
         self.pathTop = path
 
+    def clear_cache(self):
+        "__cache_reset__"
+        pass
+    
     def set_SConstruct_dir(self, dir):
         self.SConstruct_dir = dir
         
@@ -800,7 +816,8 @@ class FS(LocalFS):
         In this method, if directory is None or not supplied, the supplied
         name is expected to be an absolute path.  If you try to look up a
         relative path with directory=None, then an AssertionError will be
-        raised."""
+        raised.
+        __cacheable__"""
 
         if not name:
             # This is a stupid hack to compensate for the fact
@@ -996,7 +1013,9 @@ class FS(LocalFS):
 
     def Rsearch(self, path, clazz=_classEntry, cwd=None):
         """Search for something in a Repository.  Returns the first
-        one found in the list, or None if there isn't one."""
+        one found in the list, or None if there isn't one.
+        __cacheable__
+        """
         if isinstance(path, SCons.Node.Node):
             return path
         else:
@@ -1036,7 +1055,9 @@ class FS(LocalFS):
         return None
 
     def Rsearchall(self, pathlist, must_exist=1, clazz=_classEntry, cwd=None):
-        """Search for a list of somethings in the Repository list."""
+        """Search for a list of somethings in the Repository list.
+        __cacheable__
+        """
         ret = []
         if SCons.Util.is_String(pathlist):
             pathlist = string.split(pathlist, os.pathsep)
@@ -1091,6 +1112,7 @@ class FS(LocalFS):
 
         Climb the directory tree, and look up path names
         relative to any linked build directories we find.
+        __cacheable__
         """
         targets = []
         message = None
@@ -1109,6 +1131,7 @@ class FS(LocalFS):
             message = fmt % string.join(map(str, targets))
         return targets, message
 
+
 class Dir(Base):
     """A class for directories in a file system.
     """
@@ -1415,6 +1438,7 @@ class File(Base):
         self.dir.sconsign().set_entry(self.name, entry)
 
     def get_stored_info(self):
+        "__cacheable__"
         try:
             stored = self.dir.sconsign().get_entry(self.name)
         except (KeyError, OSError):
@@ -1543,6 +1567,7 @@ class File(Base):
         # will do if this file has a source scanner.
         if self.fs.CachePath and self.fs.exists(self.path):
             CachePush(self, [], None)
+        self.fs.clear_cache()
         SCons.Node.Node.built(self)
 
     def visited(self):
@@ -1594,6 +1619,7 @@ class File(Base):
         return self.fs.build_dir_target_climb(self, self.dir, [self.name])
 
     def is_pseudo_derived(self):
+        "__cacheable__"
         return self.has_src_builder()
 
     def _rmv_existing(self):
@@ -1712,6 +1738,9 @@ class File(Base):
 
     def current(self, calc=None):
         self.binfo = self.gen_binfo(calc)
+        return self._cur2()
+    def _cur2(self):
+        "__cacheable__"
         if self.always_build:
             return None
         if not self.exists():
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index e239e93..7b443ab 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -286,7 +286,7 @@ class Node:
         pass
 
     def depends_on(self, nodes):
-        """Does this node depend on any of 'nodes'?"""
+        """Does this node depend on any of 'nodes'? __cacheable__"""
         return reduce(lambda D,N,C=self.children(): D or (N in C), nodes, 0)
 
     def builder_set(self, builder):
@@ -342,6 +342,7 @@ class Node:
         signatures when they are used as source files to other derived files. For
         example: source with source builders are not derived in this sense,
         and hence should not return true.
+        __cacheable__
         """
         return self.has_builder() or self.side_effect
 
@@ -564,6 +565,7 @@ class Node:
         node's children's signatures.  We expect that they're
         already built and updated by someone else, if that's
         what's wanted.
+        __cacheable__
         """
 
         if calc is None:
diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py
index 404ded1..1784798 100644
--- a/src/engine/SCons/Taskmaster.py
+++ b/src/engine/SCons/Taskmaster.py
@@ -201,10 +201,7 @@ class Task:
 
         This is the default behavior for building only what's necessary.
         """
-        self.out_of_date = []
-        for t in self.targets:
-            if not t.current():
-                self.out_of_date.append(t)
+        self.out_of_date = filter(lambda T: not T.current(), self.targets)
         if self.out_of_date:
             self.mark_targets_and_side_effects(SCons.Node.executing)
         else:
diff --git a/src/engine/SCons/Util.py b/src/engine/SCons/Util.py
index 25713a8..08977cb 100644
--- a/src/engine/SCons/Util.py
+++ b/src/engine/SCons/Util.py
@@ -667,7 +667,7 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={
             """Substitute expansions in an argument or list of arguments.
 
             This serves as a wrapper for splitting up a string into
-            separate tokens.
+            separate tokens.  __cacheable__
             """
             if is_String(args) and not isinstance(args, CmdStringHolder):
                 try:
@@ -1274,95 +1274,103 @@ if can_read_reg:
 
 if sys.platform == 'win32':
 
-    def WhereIs(file, path=None, pathext=None, reject=[]):
-        if path is None:
-            try:
-                path = os.environ['PATH']
-            except KeyError:
-                return None
-        if is_String(path):
-            path = string.split(path, os.pathsep)
-        if pathext is None:
-            try:
-                pathext = os.environ['PATHEXT']
-            except KeyError:
-                pathext = '.COM;.EXE;.BAT;.CMD'
-        if is_String(pathext):
-            pathext = string.split(pathext, os.pathsep)
-        for ext in pathext:
-            if string.lower(ext) == string.lower(file[-len(ext):]):
-                pathext = ['']
-                break
-        if not is_List(reject):
-            reject = [reject]
-        for dir in path:
-            f = os.path.join(dir, file)
+    class _WhereIs:
+        def __call__(self, file, path=None, pathext=None, reject=[]):
+            "__cacheable__"
+            if path is None:
+                try:
+                    path = os.environ['PATH']
+                except KeyError:
+                    return None
+            if is_String(path):
+                path = string.split(path, os.pathsep)
+            if pathext is None:
+                try:
+                    pathext = os.environ['PATHEXT']
+                except KeyError:
+                    pathext = '.COM;.EXE;.BAT;.CMD'
+            if is_String(pathext):
+                pathext = string.split(pathext, os.pathsep)
             for ext in pathext:
-                fext = f + ext
-                if os.path.isfile(fext):
-                    try:
-                        reject.index(fext)
-                    except ValueError:
-                        return os.path.normpath(fext)
-                    continue
-        return None
+                if string.lower(ext) == string.lower(file[-len(ext):]):
+                    pathext = ['']
+                    break
+            if not is_List(reject):
+                reject = [reject]
+            for dir in path:
+                f = os.path.join(dir, file)
+                for ext in pathext:
+                    fext = f + ext
+                    if os.path.isfile(fext):
+                        try:
+                            reject.index(fext)
+                        except ValueError:
+                            return os.path.normpath(fext)
+                        continue
+            return None
 
 elif os.name == 'os2':
 
-    def WhereIs(file, path=None, pathext=None, reject=[]):
-        if path is None:
-            try:
-                path = os.environ['PATH']
-            except KeyError:
-                return None
-        if is_String(path):
-            path = string.split(path, os.pathsep)
-        if pathext is None:
-            pathext = ['.exe', '.cmd']
-        for ext in pathext:
-            if string.lower(ext) == string.lower(file[-len(ext):]):
-                pathext = ['']
-                break
-        if not is_List(reject):
-            reject = [reject]
-        for dir in path:
-            f = os.path.join(dir, file)
+    class _WhereIs:
+        def __call__(self, file, path=None, pathext=None, reject=[]):
+            "__cacheable__"
+            if path is None:
+                try:
+                    path = os.environ['PATH']
+                except KeyError:
+                    return None
+            if is_String(path):
+                path = string.split(path, os.pathsep)
+            if pathext is None:
+                pathext = ['.exe', '.cmd']
             for ext in pathext:
-                fext = f + ext
-                if os.path.isfile(fext):
-                    try:
-                        reject.index(fext)
-                    except ValueError:
-                        return os.path.normpath(fext)
-                    continue
-        return None
+                if string.lower(ext) == string.lower(file[-len(ext):]):
+                    pathext = ['']
+                    break
+            if not is_List(reject):
+                reject = [reject]
+            for dir in path:
+                f = os.path.join(dir, file)
+                for ext in pathext:
+                    fext = f + ext
+                    if os.path.isfile(fext):
+                        try:
+                            reject.index(fext)
+                        except ValueError:
+                            return os.path.normpath(fext)
+                        continue
+            return None
 
 else:
 
-    def WhereIs(file, path=None, pathext=None, reject=[]):
-        if path is None:
-            try:
-                path = os.environ['PATH']
-            except KeyError:
-                return None
-        if is_String(path):
-            path = string.split(path, os.pathsep)
-        if not is_List(reject):
-            reject = [reject]
-        for d in path:
-            f = os.path.join(d, file)
-            if os.path.isfile(f):
+    class _WhereIs:
+        def __call__(self, file, path=None, pathext=None, reject=[]):
+            "__cacheable__"
+            if path is None:
                 try:
-                    st = os.stat(f)
-                except OSError:
-                    continue
-                if stat.S_IMODE(st[stat.ST_MODE]) & 0111:
+                    path = os.environ['PATH']
+                except KeyError:
+                    return None
+            if is_String(path):
+                path = string.split(path, os.pathsep)
+            if not is_List(reject):
+                reject = [reject]
+            for d in path:
+                f = os.path.join(d, file)
+                if os.path.isfile(f):
                     try:
-                        reject.index(f)
-                    except ValueError:
-                        return os.path.normpath(f)
-                    continue
-        return None
+                        st = os.stat(f)
+                    except OSError:
+                        continue
+                    if stat.S_IMODE(st[stat.ST_MODE]) & 0111:
+                        try:
+                            reject.index(f)
+                        except ValueError:
+                            return os.path.normpath(f)
+                        continue
+            return None
+
+WhereIs = _WhereIs()
 
 def PrependPath(oldpath, newpath, sep = os.pathsep):
     """This prepends newpath elements to the given oldpath.  Will only