Add a --debug=nomemoizer option to disable memoization.

13 files changed, 169 insertions, 48 deletions

diff --git a/doc/man/scons.1 b/doc/man/scons.1
index 5eae209..6e95311 100644
--- a/doc/man/scons.1
+++ b/doc/man/scons.1
@@ -552,6 +552,22 @@ before and after reading the SConscript files
 and before and after building targets.
 
 .TP
+--debug=nomemoizer
+Disables use of the Memoizer,
+the internal SCons subsystem for caching
+various values in memory instead of
+recomputing them each time they're needed.
+This provides more accurate counts of the
+underlying function calls in the 
+Python profiler output when using the
+.R --profile=
+option.
+(When the Memoizer is used,
+the profiler counts all
+memoized functions as being executed
+by the Memoizer's wrapper calls.)
+
+.TP
 --debug=objects
 Prints a list of the various objects
 of the various classes used internally by SCons.
diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 9a202a3..6765364 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -285,6 +285,10 @@ RELEASE 0.97 - XXX
   - Allow the source directory of a BuildDir / build_dir to be outside
     of the top-level SConstruct directory tree.
 
+  - Add a --debug=nomemoizer option that disables the Memoizer for clearer
+    looks at the counts and profiles of the underlying function calls,
+    not the Memoizer wrappers.
+
   From Wayne Lee:
 
   - Avoid "maximum recursion limit" errors when removing $(-$) pairs
diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py
index f606bb2..59ab261 100644
--- a/src/engine/SCons/Action.py
+++ b/src/engine/SCons/Action.py
@@ -234,7 +234,8 @@ class ActionBase:
     other objects (Builders, Executors, etc.)  This provides the
     common methods for manipulating and combining those actions."""
     
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
 
     def __cmp__(self, other):
         return cmp(self.__dict__, other)
@@ -265,7 +266,7 @@ class ActionBase:
         return SCons.Executor.Executor(self, env, overrides,
                                        tlist, slist, executor_kw)
 
-if not SCons.Memoize.has_metaclass:
+if SCons.Memoize.use_old_memoization():
     _Base = ActionBase
     class ActionBase(SCons.Memoize.Memoizer, _Base):
         "Cache-backed version of ActionBase"
@@ -549,7 +550,8 @@ class CommandGeneratorAction(ActionBase):
 
 class LazyAction(CommandGeneratorAction, CommandAction):
 
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
 
     def __init__(self, var, *args, **kw):
         if __debug__: logInstanceCreation(self, 'Action.LazyAction')
diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py
index edc3f71..57fbe3e 100644
--- a/src/engine/SCons/Builder.py
+++ b/src/engine/SCons/Builder.py
@@ -356,7 +356,8 @@ class BuilderBase:
     nodes (files) from input nodes (files).
     """
 
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
 
     def __init__(self,  action = None,
                         prefix = '',
@@ -656,7 +657,7 @@ class BuilderBase:
         """
         self.emitter[suffix] = emitter
 
-if not SCons.Memoize.has_metaclass:
+if SCons.Memoize.use_old_memoization():
     _Base = BuilderBase
     class BuilderBase(SCons.Memoize.Memoizer, _Base):
         "Cache-backed version of BuilderBase"
diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py
index 0e9b642..847b130 100644
--- a/src/engine/SCons/Environment.py
+++ b/src/engine/SCons/Environment.py
@@ -244,7 +244,8 @@ class SubstitutionEnvironment:
     class actually becomes useful.)
     """
 
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
 
     def __init__(self, **kw):
         """Initialization of an underlying SubstitutionEnvironment class.
@@ -450,7 +451,8 @@ class Base(SubstitutionEnvironment):
     Environment.
     """
 
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
 
     #######################################################################
     # This is THE class for interacting with the SCons build engine,
@@ -1466,7 +1468,8 @@ class OverrideEnvironment(Base):
     values from the overrides dictionary.
     """
 
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
 
     def __init__(self, subject, overrides={}):
         if __debug__: logInstanceCreation(self, 'Environment.OverrideEnvironment')
@@ -1590,7 +1593,7 @@ def NoSubstitutionProxy(subject):
             return apply(SCons.Util.scons_subst, nargs, nkw)
     return _NoSubstitutionProxy(subject)
 
-if not SCons.Memoize.has_metaclass:
+if SCons.Memoize.use_old_memoization():
     _Base = Base
     class Base(SCons.Memoize.Memoizer, _Base):
         def __init__(self, *args, **kw):
diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py
index d8bcafb..4f4e650 100644
--- a/src/engine/SCons/Executor.py
+++ b/src/engine/SCons/Executor.py
@@ -44,7 +44,8 @@ class Executor:
     and sources for later processing as needed.
     """
 
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
 
     def __init__(self, action, env=None, overridelist=[{}],
                  targets=[], sources=[], builder_kw={}):
@@ -257,7 +258,7 @@ class Null(_Executor):
 
 
 
-if not SCons.Memoize.has_metaclass:
+if SCons.Memoize.use_old_memoization():
     _Base = Executor
     class Executor(SCons.Memoize.Memoizer, _Base):
         def __init__(self, *args, **kw):
diff --git a/src/engine/SCons/Memoize.py b/src/engine/SCons/Memoize.py
index afa77ee..134206f 100644
--- a/src/engine/SCons/Memoize.py
+++ b/src/engine/SCons/Memoize.py
@@ -58,7 +58,7 @@ Some advantages:
       might affect results.
 
     * Caching can be globally disabled very easily (for testing, etc.)
-    
+
 """
 
 #
@@ -93,6 +93,9 @@ import os
 import string
 import sys
 
+# A flag controlling whether or not we actually use memoization.
+use_memoizer = 1
+
 #
 # Generate a key for an object that is to be used as the caching key
 # for that object.
@@ -247,7 +250,7 @@ def ALT9_MeMoIZeR_gen_key(argtuple, kwdict):
 #_MeMoIZeR_gen_key = ALT9_MeMoIZeR_gen_key    # 8.8, 0.20
 _MeMoIZeR_gen_key = ALT8_MeMoIZeR_gen_key    # 8.5, 0.18
 #_MeMoIZeR_gen_key = ALT7_MeMoIZeR_gen_key    # 8.7, 0.17
-#_MeMoIZeR_gen_key = ALT6_MeMoIZeR_gen_key    # 
+#_MeMoIZeR_gen_key = ALT6_MeMoIZeR_gen_key    #
 #_MeMoIZeR_gen_key = ALT5_MeMoIZeR_gen_key    # 9.7, 0.20
 #_MeMoIZeR_gen_key = ALT4_MeMoIZeR_gen_key    # 8.6, 0.19
 #_MeMoIZeR_gen_key = ALT3_MeMoIZeR_gen_key    # 8.5, 0.20
@@ -534,7 +537,7 @@ class _Memoizer_Comparable:
         # Same thing for the other, but we should try to convert it
         # here in case the _MeMoIZeR_cmp compares __dict__ objects
         # directly.
-        
+
         saved_other = None
         try:
             if other.__dict__.has_key('_MeMoIZeR_Key'):
@@ -547,7 +550,7 @@ class _Memoizer_Comparable:
 
         # Both self and other have been prepared: perform the test,
         # then restore the original dictionaries and exit
-        
+
         rval = self._MeMoIZeR_cmp(other)
 
         self.__dict__ = saved_d1
@@ -561,7 +564,7 @@ def Analyze_Class(klass):
     if klass.__dict__.has_key('_MeMoIZeR_converted'): return klass
 
     original_name = str(klass)
-    
+
     D,R,C = _analyze_classmethods(klass.__dict__, klass.__bases__)
 
     if C:
@@ -628,7 +631,6 @@ def memoize_classdict(klass, modelklass, new_klassdict, cacheable, resetting):
         new_klassdict[name] = newmethod
 
     return new_klassdict
-        
 
 def _analyze_classmethods(klassdict, klassbases):
     """Given a class, performs a scan of methods for that class and
@@ -640,7 +642,7 @@ def _analyze_classmethods(klassdict, klassbases):
     D = {}
     R = {}
     C = None
-    
+
     # Get cache/reset/cmp methods from subclasses
 
     for K in klassbases:
@@ -685,7 +687,7 @@ def _scan_classdict(klassdict):
 
     Each dict has the name of the method as a key and the corresponding
     value is the method body."""
-    
+
     cache_setters = {}
     cache_resetters = {}
     cmp_if_exists = None
@@ -711,7 +713,7 @@ def _scan_classdict(klassdict):
             continue
     if already_cache_modified: cmp_if_exists = 'already_cache_modified'
     return cache_setters, cache_resetters, cmp_if_exists
-        
+
 #
 # Primary Memoizer class.  This should be a base-class for any class
 # that wants method call results to be cached.  The sub-class should
@@ -725,9 +727,7 @@ class Memoizer:
     def __init__(self):
         self.__class__ = Analyze_Class(self.__class__)
         self._MeMoIZeR_Key =  Next_Memoize_Key()
-    
 
-has_metaclass = 1
 # Find out if we are pre-2.2
 
 try:
@@ -744,14 +744,18 @@ except AttributeError:
 need_version = (2, 2) # actual
 #need_version = (33, 0)  # always
 #need_version = (0, 0)  # never
-if vinfo[0] < need_version[0] or \
-   (vinfo[0] == need_version[0] and
-    vinfo[1] < need_version[1]):
-    has_metaclass = 0
+
+has_metaclass =  (vinfo[0] > need_version[0] or \
+                  (vinfo[0] == need_version[0] and
+                   vinfo[1] >= need_version[1]))
+
+if not has_metaclass:
+
     class Memoized_Metaclass:
         # Just a place-holder so pre-metaclass Python versions don't
         # have to have special code for the Memoized classes.
         pass
+
 else:
 
     # Initialization is a wee bit of a hassle.  We want to do some of
@@ -768,14 +772,14 @@ else:
     # to obtain the __init__ because we don't want to re-instrument
     # parent-class __init__ operations (and we want to avoid the
     # Object object's slot init if the class has no __init__).
-    
+
     def _MeMoIZeR_init(actual_init, self, args, kw):
         self.__dict__['_MeMoIZeR_Key'] =  Next_Memoize_Key()
         apply(actual_init, (self,)+args, kw)
 
     def _MeMoIZeR_superinit(self, cls, args, kw):
         apply(super(cls, self).__init__, args, kw)
-        
+
     class Memoized_Metaclass(type):
         def __init__(cls, name, bases, cls_dict):
             # Note that cls_dict apparently contains a *copy* of the
@@ -802,7 +806,7 @@ else:
                                  {'cls':cls,
                                   'MPI':_MeMoIZeR_superinit})
                 init = superinit
-                
+
             newinitcode = compile(
                 "\n"*(init.func_code.co_firstlineno-1) +
                 "lambda self, args, kw: _MeMoIZeR_init(real_init, self, args, kw)",
@@ -819,4 +823,10 @@ else:
             # definition itself, apply klassdict.
             for attr in klassdict.keys():
                 setattr(cls, attr, klassdict[attr])
-                
+
+def DisableMemoization():
+    global use_memoizer
+    use_memoizer = None
+
+def use_old_memoization():
+    return use_memoizer and not has_metaclass
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 527d1e3..07093d0 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -659,7 +659,8 @@ _classEntry = Entry
 
 class LocalFS:
 
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
     
     # This class implements an abstraction layer for operations involving
     # a local file system.  Essentially, this wraps any function in
@@ -717,7 +718,7 @@ class LocalFS:
         def islink(self, path):
             return 0                    # no symlinks
 
-if not SCons.Memoize.has_metaclass:
+if SCons.Memoize.use_old_memoization():
     _FSBase = LocalFS
     class LocalFS(SCons.Memoize.Memoizer, _FSBase):
         def __init__(self, *args, **kw):
diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py
index 4a1faaa..b9d5a75 100644
--- a/src/engine/SCons/Node/__init__.py
+++ b/src/engine/SCons/Node/__init__.py
@@ -104,7 +104,8 @@ class Node:
     build, or use to build other Nodes.
     """
 
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
 
     class Attrs:
         pass
@@ -995,7 +996,7 @@ else:
 del l
 del ul
 
-if not SCons.Memoize.has_metaclass:
+if SCons.Memoize.use_old_memoization():
     _Base = Node
     class Node(SCons.Memoize.Memoizer, _Base):
         def __init__(self, *args, **kw):
diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py
index 3010159..ce9ae18 100644
--- a/src/engine/SCons/Scanner/__init__.py
+++ b/src/engine/SCons/Scanner/__init__.py
@@ -97,7 +97,8 @@ class Base:
     straightforward, single-pass scanning of a single file.
     """
 
-    __metaclass__ = SCons.Memoize.Memoized_Metaclass
+    if SCons.Memoize.use_memoizer:
+        __metaclass__ = SCons.Memoize.Memoized_Metaclass
 
     def __init__(self,
                  function,
@@ -254,7 +255,7 @@ class Base:
 
     recurse_nodes = _recurse_no_nodes
 
-if not SCons.Memoize.has_metaclass:
+if SCons.Memoize.use_old_memoization():
     _Base = Base
     class Base(SCons.Memoize.Memoizer, _Base):
         "Cache-backed version of Scanner Base"
diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py
index 7dcb987..73f6a6c 100644
--- a/src/engine/SCons/Script/Main.py
+++ b/src/engine/SCons/Script/Main.py
@@ -592,7 +592,8 @@ class OptParser(OptionParser):
                              "build all Default() targets.")
 
         debug_options = ["count", "dtree", "explain", "findlibs",
-                         "includes", "memoizer", "memory", "objects",
+                         "includes", "memoizer", "memory",
+                         "nomemoizer", "objects",
                          "pdb", "presub", "stacktrace", "stree",
                          "time", "tree"]
 
diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py
index 4327ac2..734ce3f 100644
--- a/src/engine/SCons/Script/__init__.py
+++ b/src/engine/SCons/Script/__init__.py
@@ -44,22 +44,30 @@ import string
 import sys
 import UserList
 
-# Special chicken-and-egg handling of the "--debug=memoizer" flags:
+# Special chicken-and-egg handling of the "--debug=memoizer"
+# and "--debug=nomemoizer" flags:
+#
 # SCons.Memoize contains a metaclass implementation that affects how
 # the other classes are instantiated.  The Memoizer handles optional
 # counting of the hits and misses by using a different, parallel set of
 # functions, so we don't slow down normal operation any more than we
-# have to.  But if we wait to enable the counting until we've parsed
-# the command line options normally, it will be too late, because the
-# Memoizer will have already analyzed the classes that it's Memoizing
-# and bound the non-counting versions of the functions.  So we have to
-# use a special-case, up-front check for the "--debug=memoizer" flag
-# and turn on Memoizer counting, if desired, before we import any of
-# the other modules that use it.
-sconsflags = string.split(os.environ.get('SCONSFLAGS', ''))
-if "--debug=memoizer" in sys.argv + sconsflags:
+# have to.  We can also tell it disable memoization completely.
+#
+# If we wait to enable the counting or disable memoization completely
+# until we've parsed the command line options normally, it will be too
+# late, because the Memoizer will have already analyzed the classes
+# that it's Memoizing and bound the (non-counting) versions of the
+# functions.  So we have to use a special-case, up-front check for
+# the "--debug=memoizer" and "--debug=nomemoizer" flags and do what's
+# appropriate before we import any of the other modules that use it.
+_args = sys.argv + string.split(os.environ.get('SCONSFLAGS', ''))
+if "--debug=memoizer" in _args:
     import SCons.Memoize
     SCons.Memoize.EnableCounting()
+if "--debug=nomemoizer" in _args:
+    import SCons.Memoize
+    SCons.Memoize.DisableMemoization()
+del _args
 
 import SCons.Action
 import SCons.Builder
@@ -315,7 +323,7 @@ for name in GlobalDefaultEnvironmentFunctions + GlobalDefaultBuilders:
 # "SConscript" in this namespace is no longer a module, it's a global
 # function call--or more precisely, an object that implements a global
 # function call through the default Environment.  Nevertheless, we can
-# aintain backwards compatibility for SConscripts that were reaching in
+# maintain backwards compatibility for SConscripts that were reaching in
 # this way by hanging some attributes off the "SConscript" object here.
 SConscript = _SConscript.DefaultEnvironmentCall('SConscript')
 
diff --git a/test/option/debug-nomemoizer.py b/test/option/debug-nomemoizer.py
new file mode 100644
index 0000000..633a46d
--- /dev/null
+++ b/test/option/debug-nomemoizer.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+"""
+Test calling the --debug=nomemoizer option.
+"""
+
+import pstats
+import string
+import StringIO
+import sys
+
+import TestSCons
+
+test = TestSCons.TestSCons()
+
+scons_prof = test.workpath('scons.prof')
+
+test.write('SConstruct', """
+def cat(target, source, env):
+    open(str(target[0]), 'wb').write(open(str(source[0]), 'rb').read())
+env = Environment(BUILDERS={'Cat':Builder(action=Action(cat))})
+env.Cat('file.out', 'file.in')
+""")
+
+test.write('file.in', "file.in\n")
+
+test.run(arguments = "--profile=%s --debug=nomemoizer " % scons_prof)
+
+stats = pstats.Stats(scons_prof)
+stats.sort_stats('time')
+
+try:
+    save_stdout = sys.stdout
+    sys.stdout = StringIO.StringIO()
+
+    stats.strip_dirs().print_stats()
+
+    s = sys.stdout.getvalue()
+finally:
+    sys.stdout = save_stdout
+
+test.fail_test(string.find(s, '_MeMoIZeR_init') != -1)
+test.fail_test(string.find(s, '_MeMoIZeR_reset') != -1)
+test.fail_test(string.find(s, 'Count_cache_get') != -1)
+test.fail_test(string.find(s, 'Count_cache_get_self') != -1)
+test.fail_test(string.find(s, 'Count_cache_get_one') != -1)
+
+test.pass_test()