Add a Memoizer metaclass to collect the logic for caching values in one location. Convert by-hand caching to use of Memoizer. (Kevin Quick)

6 files changed, 73 insertions, 56 deletions

diff --git a/src/engine/SCons/Scanner/CTests.py b/src/engine/SCons/Scanner/CTests.py
index e5830a0..5296f93 100644
--- a/src/engine/SCons/Scanner/CTests.py
+++ b/src/engine/SCons/Scanner/CTests.py
@@ -333,8 +333,8 @@ class CScannerTestCase10(unittest.TestCase):
         s = SCons.Scanner.C.CScan(fs=fs)
         path = s.path(env)
         test.write('include/fa.cpp', test.read('fa.cpp'))
-        deps = s(fs.File('#include/fa.cpp'), env, path)
         fs.chdir(fs.Dir('..'))
+        deps = s(fs.File('#include/fa.cpp'), env, path)
         deps_match(self, deps, [ 'include/fa.h', 'include/fb.h' ])
         test.unlink('include/fa.cpp')
 
diff --git a/src/engine/SCons/Scanner/D.py b/src/engine/SCons/Scanner/D.py
index 27ba165..fba973f 100644
--- a/src/engine/SCons/Scanner/D.py
+++ b/src/engine/SCons/Scanner/D.py
@@ -47,6 +47,7 @@ def DScan(fs = SCons.Node.FS.default_fs):
 
 class DScanner(SCons.Scanner.Classic):
     def find_include(self, include, source_dir, path):
+        if callable(path): path=path()
         # translate dots (package separators) to slashes
         inc = string.replace(include, '.', '/')
 
diff --git a/src/engine/SCons/Scanner/Fortran.py b/src/engine/SCons/Scanner/Fortran.py
index 6ab878f..4da0914 100644
--- a/src/engine/SCons/Scanner/Fortran.py
+++ b/src/engine/SCons/Scanner/Fortran.py
@@ -62,6 +62,11 @@ class F90Scanner(SCons.Scanner.Classic):
         self.fs = fs
 
         def _scan(node, env, path, self=self, fs=fs):
+            node = node.rfile()
+
+            if not node.exists():
+                return []
+
             return self.scan(node, env, path)
 
         kw['function'] = _scan
@@ -73,11 +78,8 @@ class F90Scanner(SCons.Scanner.Classic):
         apply(SCons.Scanner.Current.__init__, (self,) + args, kw)
 
     def scan(self, node, env, path=()):
-        node = node.rfile()
-
-        if not node.exists():
-            return []
-
+        "__cacheable__"
+        
         # cache the includes list in node so we only scan it once:
         if node.includes != None:
             mods_and_includes = node.includes
diff --git a/src/engine/SCons/Scanner/Prog.py b/src/engine/SCons/Scanner/Prog.py
index 512e512..d9b57b9 100644
--- a/src/engine/SCons/Scanner/Prog.py
+++ b/src/engine/SCons/Scanner/Prog.py
@@ -80,6 +80,8 @@ def scan(node, env, libpath = (), fs = SCons.Node.FS.default_fs):
 
     result = []
 
+    if callable(libpath): libpath = libpath()
+    
     find_file = SCons.Node.FS.find_file
     adjustixes = SCons.Util.adjustixes
     for lib in libs:
diff --git a/src/engine/SCons/Scanner/ScannerTests.py b/src/engine/SCons/Scanner/ScannerTests.py
index 00ad7fb..e418c17 100644
--- a/src/engine/SCons/Scanner/ScannerTests.py
+++ b/src/engine/SCons/Scanner/ScannerTests.py
@@ -69,7 +69,7 @@ class FindPathDirsTestCase(unittest.TestCase):
 
         fpd = SCons.Scanner.FindPathDirs('LIBPATH', FS())
         result = fpd(env, dir)
-        assert result == ('xxx', 'foo'), result
+        assert str(result) == "('xxx', 'foo')", result
 
 class ScannerTestCase(unittest.TestCase):
 
@@ -434,7 +434,7 @@ class ClassicTestCase(unittest.TestCase):
         # Verify that overall scan results are cached even if individual
         # results are de-cached
         ret = s.function(n, env, ('foo2',))
-        assert ret == ['abc'], ret
+        assert ret == ['abc'], 'caching inactive; got: %s'%ret
 
         # Verify that it sorts what it finds.
         n.includes = ['xyz', 'uvw']
@@ -459,6 +459,8 @@ class ClassicCPPTestCase(unittest.TestCase):
         s = SCons.Scanner.ClassicCPP("Test", [], None, "")
 
         def _find_file(filename, paths, factory):
+            if callable(paths):
+                paths = paths()
             return paths[0]+'/'+filename
 
         save = SCons.Node.FS.find_file
@@ -474,7 +476,7 @@ class ClassicCPPTestCase(unittest.TestCase):
             assert i == 'bbb', i
 
         finally:
-            SCons.Node.FS.find_file = _find_file
+            SCons.Node.FS.find_file = save
 
 def suite():
     suite = unittest.TestSuite()
diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py
index cbab50c..1968a9e 100644
--- a/src/engine/SCons/Scanner/__init__.py
+++ b/src/engine/SCons/Scanner/__init__.py
@@ -34,7 +34,6 @@ import string
 
 import SCons.Node.FS
 import SCons.Sig
-import SCons.UserTuple
 import SCons.Util
 
 
@@ -54,51 +53,51 @@ def Scanner(function, *args, **kw):
     else:
         return apply(Base, (function,) + args, kw)
 
-# Important, important, important performance optimization:
-#
-# The paths of Nodes returned from a FindPathDirs will be used to index
-# a dictionary that caches the values, so we don't have to look up the
-# same path over and over and over.  If FindPathDir returns just a tuple,
-# though, then the time it takes to compute the hash of the tuple grows
-# proportionally to the length of the tuple itself--and some people can
-# have very, very long strings of include directories...
-#
-# The solution is to wrap the tuple in an object, a UserTuple class
-# whose *id()* our caller can use to cache the appropriate value.
-# This means we have to guarantee that these ids genuinely represent
-# unique values, which we do by maintaining our own cache that maps the
-# expensive-to-hash tuple values to the easy-to-hash UniqueUserTuple
-# values that our caller uses.
-#
-# *Major* kudos to Eric Frias and his colleagues for finding this.
-class UniqueUserTuple(SCons.UserTuple.UserTuple):
-    def __hash__(self):
-        return id(self)
-
-PathCache = {}
 
+class Binder:
+    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    def __init__(self, bindval):
+        self._val = bindval
+    def __call__(self):
+        return self._val
+    def __str__(self):
+        return str(self._val)
+        #debug: return 'B<%s>'%str(self._val)
+    
 class FindPathDirs:
     """A class to bind a specific *PATH variable name and the fs object
     to a function that will return all of the *path directories."""
+    __metaclass__ = SCons.Memoize.Memoized_Metaclass
     def __init__(self, variable, fs):
         self.variable = variable
         self.fs = fs
     def __call__(self, env, dir, argument=None):
+        "__cacheable__"
         try:
             path = env[self.variable]
         except KeyError:
             return ()
 
         path_tuple = tuple(self.fs.Rsearchall(env.subst_path(path),
-                                              must_exist = 0,
+                                              must_exist = 0, #kwq!
                                               clazz = SCons.Node.FS.Dir,
                                               cwd = dir))
-        try:
-            return PathCache[path_tuple]
-        except KeyError:
-            path_UserTuple = UniqueUserTuple(path_tuple)
-            PathCache[path_tuple] = path_UserTuple
-            return path_UserTuple
+        return Binder(path_tuple)
+
+if not SCons.Memoize.has_metaclass:
+    _FPD_Base = FindPathDirs
+    class FindPathDirs(SCons.Memoize.Memoizer, _FPD_Base):
+        "Cache-backed version of FindPathDirs"
+        def __init__(self, *args, **kw):
+            apply(_FPD_Base.__init__, (self,)+args, kw)
+            SCons.Memoize.Memoizer.__init__(self)
+    _BinderBase = Binder
+    class Binder(SCons.Memoize.Memoizer, _BinderBase):
+        "Cache-backed version of Binder"
+        def __init__(self, *args, **kw):
+            apply(_BinderBase.__init__, (self,)+args, kw)
+            SCons.Memoize.Memoizer.__init__(self)
+
 
 class Base:
     """
@@ -106,6 +105,8 @@ class Base:
     straightforward, single-pass scanning of a single file.
     """
 
+    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+
     def __init__(self,
                  function,
                  name = "NONE",
@@ -135,6 +136,8 @@ class Base:
         (a construction environment, optional directory, and optional
         argument for this instance) and returns a tuple of the
         directories that can be searched for implicit dependency files.
+        May also return a callable() which is called with no args and
+        returns the tuple (supporting Bindable class).
 
         'node_class' - the class of Nodes which this scan will return.
         If node_class is None, then this scanner will not enforce any
@@ -186,6 +189,7 @@ class Base:
         self.recursive = recursive
 
     def path(self, env, dir = None):
+        "__cacheable__"
         if not self.path_function:
             return ()
         if not self.argument is _null:
@@ -242,6 +246,14 @@ class Base:
     def select(self, node):
         return self
 
+if not SCons.Memoize.has_metaclass:
+    _Base = Base
+    class Base(SCons.Memoize.Memoizer, _Base):
+        "Cache-backed version of Scanner Base"
+        def __init__(self, *args, **kw):
+            apply(_Base.__init__, (self,)+args, kw)
+            SCons.Memoize.Memoizer.__init__(self)
+
 
 class Selector(Base):
     """
@@ -296,22 +308,12 @@ class Classic(Current):
 
         self.cre = re.compile(regex, re.M)
         self.fs = fs
-        self._cached = {}
 
         def _scan(node, env, path=(), self=self):
             node = node.rfile()
-
             if not node.exists():
                 return []
-
-            key = str(id(node)) + '|' + string.join(map(str, path), ':')
-            try:
-                return self._cached[key]
-            except KeyError:
-                pass
-
-            self._cached[key] = scan_result = self.scan(node, path)
-            return scan_result
+            return self.scan(node, path)
 
         kw['function'] = _scan
         kw['path_function'] = FindPathDirs(path_variable, fs)
@@ -322,6 +324,8 @@ class Classic(Current):
         apply(Current.__init__, (self,) + args, kw)
 
     def find_include(self, include, source_dir, path):
+        "__cacheable__"
+        if callable(path): path = path()
         n = SCons.Node.FS.find_file(include,
                                     (source_dir,) + tuple(path),
                                     self.fs.File)
@@ -331,6 +335,7 @@ class Classic(Current):
         return SCons.Node.FS._my_normcase(include)
 
     def scan(self, node, path=()):
+        "__cacheable__"
 
         # cache the includes list in node so we only scan it once:
         if node.includes != None:
@@ -372,14 +377,19 @@ class ClassicCPP(Classic):
     the contained filename in group 1.
     """
     def find_include(self, include, source_dir, path):
+        "__cacheable__"
+        if callable(path):
+            path = path()   #kwq: extend callable to find_file...
+
         if include[0] == '"':
-            n = SCons.Node.FS.find_file(include[1],
-                                        (source_dir,) + tuple(path),
-                                        self.fs.File)
+            paths = Binder( (source_dir,) + tuple(path) )
         else:
-            n = SCons.Node.FS.find_file(include[1],
-                                        tuple(path) + (source_dir,),
-                                        self.fs.File)
+            paths = Binder( tuple(path) + (source_dir,) )
+
+        n = SCons.Node.FS.find_file(include[1],
+                                    paths,
+                                    self.fs.File)
+
         return n, include[1]
 
     def sort_key(self, include):