More performance improvements?

3 files changed, 188 insertions, 47 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 04444f2..0b47dd9 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -174,6 +174,10 @@ RELEASE 0.97 - XXX
     all of the normally-available global functions and variables by saying
     "from SCons.Script import *".
 
+  - Add a --debug=memoizer option to print Memoizer hit/mass statistics.
+
+  - Allow more than one --debug= option to be set at a time.
+
   From Wayne Lee:
 
   - Avoid "maximum recursion limit" errors when removing $(-$) pairs
diff --git a/src/engine/SCons/Memoize.py b/src/engine/SCons/Memoize.py
index 72adbcd..1dc95cb 100644
--- a/src/engine/SCons/Memoize.py
+++ b/src/engine/SCons/Memoize.py
@@ -352,6 +352,128 @@ def Memoizer_cache_get_one(func, cdict, self, arg):
 
     return rval
 
+#
+# Caching stuff is tricky, because the tradeoffs involved are often so
+# non-obvious, so we're going to support an alternate set of functions
+# that also count the hits and misses, to try to get a concrete idea of
+# which Memoizations seem to pay off.
+#
+# Because different configurations can have such radically different
+# performance tradeoffs, interpreting the hit/miss results will likely be
+# more of an art than a science.  In other words, don't assume that just
+# because you see no hits in one configuration that it's not worthwhile
+# Memoizing that method.
+#
+# Note that these are essentially cut-and-paste copies of the above
+# Memozer_cache_get*() implementations, with the addition of the
+# counting logic.  If the above implementations change, the
+# corresponding change should probably be made down below as well,
+# just to try to keep things in sync.
+#
+
+class CounterEntry:
+    def __init__(self):
+        self.hit = 0
+        self.miss = 0
+
+import UserDict
+class Counter(UserDict.UserDict):
+    def __call__(self, klass, code):
+        k = (klass.__name__, id(code))
+        try:
+            return self[k]
+        except KeyError:
+            c = self[k] = CounterEntry()
+            return c
+
+CacheCount = Counter()
+CacheCountSelf = Counter()
+CacheCountOne = Counter()
+
+Code_to_Name = {}
+
+def Dump():
+    items = CacheCount.items() + CacheCountSelf.items() + CacheCountOne.items()
+    def keyify(t):
+        return Code_to_Name[(t[0], t[1])]
+    items = map(lambda t, k=keyify: (k(t[0]), t[1]), items)
+    items.sort()
+    for k, v in items:
+        print "    %7d hits %7d misses   %s()" % (v.hit, v.miss, k)
+
+def Count_cache_get(func, cdict, args, kw):
+    """Called instead of name to see if this method call's return
+    value has been cached.  If it has, just return the cached
+    value; if not, call the actual method and cache the return."""
+
+    obj = args[0]
+
+    ckey = obj._MeMoIZeR_Key + ':' + _MeMoIZeR_gen_key(args, kw)
+
+    c = CacheCount(obj.__class__, func)
+    rval = cdict.get(ckey, "_MeMoIZeR")
+    if rval is "_MeMoIZeR":
+        rval = cdict[ckey] = apply(func, args, kw)
+        c.miss = c.miss + 1
+    else:
+        c.hit = c.hit + 1
+
+    return rval
+
+def Count_cache_get_self(func, cdict, self):
+    """Called instead of func(self) to see if this method call's
+    return value has been cached.  If it has, just return the cached
+    value; if not, call the actual method and cache the return.
+    Optimized version of Memoizer_cache_get for methods that take the
+    object instance as the only argument."""
+
+    ckey = self._MeMoIZeR_Key
+
+    c = CacheCountSelf(self.__class__, func)
+    rval = cdict.get(ckey, "_MeMoIZeR")
+    if rval is "_MeMoIZeR":
+        rval = cdict[ckey] = func(self)
+        c.miss = c.miss + 1
+    else:
+        c.hit = c.hit + 1
+
+    return rval
+
+def Count_cache_get_one(func, cdict, self, arg):
+    """Called instead of func(self, arg) to see if this method call's
+    return value has been cached.  If it has, just return the cached
+    value; if not, call the actual method and cache the return.
+    Optimized version of Memoizer_cache_get for methods that take the
+    object instance and one other argument only."""
+
+    ckey = self._MeMoIZeR_Key + ':' + \
+           (getattr(arg, "_MeMoIZeR_Key", None) or repr(arg))
+
+    c = CacheCountOne(self.__class__, func)
+    rval = cdict.get(ckey, "_MeMoIZeR")
+    if rval is "_MeMoIZeR":
+        rval = cdict[ckey] = func(self, arg)
+        c.miss = c.miss + 1
+    else:
+        c.hit = c.hit + 1
+
+    return rval
+
+MCG_dict = {
+    'MCG'  : Memoizer_cache_get,
+    'MCGS' : Memoizer_cache_get_self,
+    'MCGO' : Memoizer_cache_get_one,
+}
+
+def EnableCounting():
+    global MCG_dict
+    MCG_dict = {
+        'MCG'  : Count_cache_get,
+        'MCGS' : Count_cache_get_self,
+        'MCGO' : Count_cache_get_one,
+    }
+
+
 
 class _Memoizer_Simple:
 
@@ -438,7 +560,7 @@ def Analyze_Class(klass):
         modelklass = _Memoizer_Simple
         lcldict = {}
 
-    klass.__dict__.update(memoize_classdict(modelklass, lcldict, D, R))
+    klass.__dict__.update(memoize_classdict(klass, modelklass, lcldict, D, R))
 
     return klass
 
@@ -458,39 +580,38 @@ def whoami(memoizer_funcname, real_funcname):
     return '...'+os.sep+'SCons'+os.sep+'Memoizer-'+ \
            memoizer_funcname+'-lambda<'+real_funcname+'>'
 
-def memoize_classdict(modelklass, new_klassdict, cacheable, resetting):
+def memoize_classdict(klass, modelklass, new_klassdict, cacheable, resetting):
     new_klassdict.update(modelklass.__dict__)
     new_klassdict['_MeMoIZeR_converted'] = 1
 
     for name,code in cacheable.items():
+        Code_to_Name[(klass.__name__, id(code))] = klass.__name__ + '.' + name
+        eval_dict = {
+            'methcode' : code,
+            'methcached' : {},
+        }
+        eval_dict.update(MCG_dict)
         if code.func_code.co_argcount == 1 and \
                not code.func_code.co_flags & 0xC:
-            newmethod = eval(
+            compiled = \
                 compile("\n"*1 +
                         "lambda self: MCGS(methcode, methcached, self)",
                         whoami('cache_get_self', name),
-                        "eval"),
-                {'methcode':code, 'methcached':{},
-                 'MCGS':Memoizer_cache_get_self},
-                {})
+                        "eval")
         elif code.func_code.co_argcount == 2 and \
                not code.func_code.co_flags & 0xC:
-            newmethod = eval(
+            compiled = \
                 compile("\n"*2 +
                 "lambda self, arg: MCGO(methcode, methcached, self, arg)",
                         whoami('cache_get_one', name),
-                        "eval"),
-                {'methcode':code, 'methcached':{},
-                 'MCGO':Memoizer_cache_get_one},
-                {})
+                        "eval")
         else:
-            newmethod = eval(
+            compiled = \
                 compile("\n"*3 +
                 "lambda *args, **kw: MCG(methcode, methcached, args, kw)",
                         whoami('cache_get', name),
-                        "eval"),
-                {'methcode':code, 'methcached':{},
-                 'MCG':Memoizer_cache_get}, {})
+                        "eval")
+        newmethod = eval(compiled, eval_dict, {})
         new_klassdict[name] = newmethod
 
     for name,code in resetting.items():
@@ -662,7 +783,7 @@ else:
                 cls_dict['_MeMoIZeR_cmp'] = C
             else:
                 modelklass = _Memoizer_Simple
-            klassdict = memoize_classdict(modelklass, cls_dict, D, R)
+            klassdict = memoize_classdict(cls, modelklass, cls_dict, D, R)
 
             init = klassdict.get('__init__', None)
             if not init:
diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py
index 1bd6939..1bfd248 100644
--- a/src/engine/SCons/Script/Main.py
+++ b/src/engine/SCons/Script/Main.py
@@ -246,6 +246,7 @@ print_dtree = 0
 print_explanations = 0
 print_includes = 0
 print_objects = 0
+print_memoizer = 0
 print_stacktrace = 0
 print_stree = 0
 print_time = 0
@@ -407,7 +408,7 @@ def _SConstruct_exists(dirname=''):
 def _set_globals(options):
     global repositories, keep_going_on_error, ignore_errors
     global print_count, print_dtree
-    global print_explanations, print_includes
+    global print_explanations, print_includes, print_memoizer
     global print_objects, print_stacktrace, print_stree
     global print_time, print_tree
     global memory_outf, memory_stats
@@ -416,34 +417,40 @@ def _set_globals(options):
         repositories.extend(options.repository)
     keep_going_on_error = options.keep_going
     try:
-        if options.debug:
-            if options.debug == "count":
-                print_count = 1
-            elif options.debug == "dtree":
-                print_dtree = 1
-            elif options.debug == "explain":
-                print_explanations = 1
-            elif options.debug == "findlibs":
-                SCons.Scanner.Prog.print_find_libs = "findlibs"
-            elif options.debug == "includes":
-                print_includes = 1
-            elif options.debug == "memory":
-                memory_stats = []
-                memory_outf = sys.stdout
-            elif options.debug == "objects":
-                print_objects = 1
-            elif options.debug == "presub":
-                SCons.Action.print_actions_presub = 1
-            elif options.debug == "stacktrace":
-                print_stacktrace = 1
-            elif options.debug == "stree":
-                print_stree = 1
-            elif options.debug == "time":
-                print_time = 1
-            elif options.debug == "tree":
-                print_tree = 1
+        debug_values = options.debug
+        if debug_values is None:
+            debug_values = []
     except AttributeError:
         pass
+    else:
+        if "count" in debug_values:
+            print_count = 1
+        if "dtree" in debug_values:
+            print_dtree = 1
+        if "explain" in debug_values:
+            print_explanations = 1
+        if "findlibs" in debug_values:
+            SCons.Scanner.Prog.print_find_libs = "findlibs"
+        if "includes" in debug_values:
+            print_includes = 1
+        if "memoizer" in debug_values:
+            SCons.Memoize.EnableCounting()
+            print_memoizer = 1
+        if "memory" in debug_values:
+            memory_stats = []
+            memory_outf = sys.stdout
+        if "objects" in debug_values:
+            print_objects = 1
+        if "presub" in debug_values:
+            SCons.Action.print_actions_presub = 1
+        if "stacktrace" in debug_values:
+            print_stacktrace = 1
+        if "stree" in debug_values:
+            print_stree = 1
+        if "time" in debug_values:
+            print_time = 1
+        if "tree" in debug_values:
+            print_tree = 1
     ignore_errors = options.ignore_errors
 
 def _create_path(plist):
@@ -534,13 +541,18 @@ class OptParser(OptionParser):
                              "build all Default() targets.")
 
         debug_options = ["count", "dtree", "explain", "findlibs",
-                         "includes", "memory", "objects",
+                         "includes", "memoizer", "memory", "objects",
                          "pdb", "presub", "stacktrace", "stree",
                          "time", "tree"]
 
         def opt_debug(option, opt, value, parser, debug_options=debug_options):
             if value in debug_options:
-                parser.values.debug = value
+                try:
+                    if parser.values.debug is None:
+                        parser.values.debug = []
+                except AttributeError:
+                    parser.values.debug = []
+                parser.values.debug.append(value)
             else:
                 raise OptionValueError("Warning:  %s is not a valid debug type" % value)
         self.add_option('--debug', action="callback", type="string",
@@ -1093,6 +1105,10 @@ def _main(args, parser):
         SCons.Debug.listLoggedInstances('*')
         #SCons.Debug.dumpLoggedInstances('*')
 
+    if print_memoizer:
+        print "Memoizer (memory cache) hits and misses:"
+        SCons.Memoize.Dump()
+
 def _exec_main():
     all_args = sys.argv[1:]
     try:
@@ -1103,7 +1119,7 @@ def _exec_main():
     parser = OptParser()
     global options
     options, args = parser.parse_args(all_args)
-    if options.debug == "pdb":
+    if type(options.debug) == type([]) and "pdb" in options.debug:
         import pdb
         pdb.Pdb().runcall(_main, args, parser)
     else: