From 1d3c36ee983d5f26bbd9cf9733a8084eced1b2c4 Mon Sep 17 00:00:00 2001 From: Steven Knight Date: Wed, 1 Jun 2005 12:13:56 +0000 Subject: Add a --debug=nomemoizer option to disable memoization. --- doc/man/scons.1 | 16 ++++++++ src/CHANGES.txt | 4 ++ src/engine/SCons/Action.py | 8 ++-- src/engine/SCons/Builder.py | 5 ++- src/engine/SCons/Environment.py | 11 ++++-- src/engine/SCons/Executor.py | 5 ++- src/engine/SCons/Memoize.py | 48 ++++++++++++++---------- src/engine/SCons/Node/FS.py | 5 ++- src/engine/SCons/Node/__init__.py | 5 ++- src/engine/SCons/Scanner/__init__.py | 5 ++- src/engine/SCons/Script/Main.py | 3 +- src/engine/SCons/Script/__init__.py | 30 +++++++++------ test/option/debug-nomemoizer.py | 72 ++++++++++++++++++++++++++++++++++++ 13 files changed, 169 insertions(+), 48 deletions(-) create mode 100644 test/option/debug-nomemoizer.py diff --git a/doc/man/scons.1 b/doc/man/scons.1 index 5eae209..6e95311 100644 --- a/doc/man/scons.1 +++ b/doc/man/scons.1 @@ -552,6 +552,22 @@ before and after reading the SConscript files and before and after building targets. .TP +--debug=nomemoizer +Disables use of the Memoizer, +the internal SCons subsystem for caching +various values in memory instead of +recomputing them each time they're needed. +This provides more accurate counts of the +underlying function calls in the +Python profiler output when using the +.R --profile= +option. +(When the Memoizer is used, +the profiler counts all +memoized functions as being executed +by the Memoizer's wrapper calls.) + +.TP --debug=objects Prints a list of the various objects of the various classes used internally by SCons. diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 9a202a3..6765364 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -285,6 +285,10 @@ RELEASE 0.97 - XXX - Allow the source directory of a BuildDir / build_dir to be outside of the top-level SConstruct directory tree. + - Add a --debug=nomemoizer option that disables the Memoizer for clearer + looks at the counts and profiles of the underlying function calls, + not the Memoizer wrappers. + From Wayne Lee: - Avoid "maximum recursion limit" errors when removing $(-$) pairs diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py index f606bb2..59ab261 100644 --- a/src/engine/SCons/Action.py +++ b/src/engine/SCons/Action.py @@ -234,7 +234,8 @@ class ActionBase: other objects (Builders, Executors, etc.) This provides the common methods for manipulating and combining those actions.""" - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass def __cmp__(self, other): return cmp(self.__dict__, other) @@ -265,7 +266,7 @@ class ActionBase: return SCons.Executor.Executor(self, env, overrides, tlist, slist, executor_kw) -if not SCons.Memoize.has_metaclass: +if SCons.Memoize.use_old_memoization(): _Base = ActionBase class ActionBase(SCons.Memoize.Memoizer, _Base): "Cache-backed version of ActionBase" @@ -549,7 +550,8 @@ class CommandGeneratorAction(ActionBase): class LazyAction(CommandGeneratorAction, CommandAction): - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass def __init__(self, var, *args, **kw): if __debug__: logInstanceCreation(self, 'Action.LazyAction') diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py index edc3f71..57fbe3e 100644 --- a/src/engine/SCons/Builder.py +++ b/src/engine/SCons/Builder.py @@ -356,7 +356,8 @@ class BuilderBase: nodes (files) from input nodes (files). """ - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass def __init__(self, action = None, prefix = '', @@ -656,7 +657,7 @@ class BuilderBase: """ self.emitter[suffix] = emitter -if not SCons.Memoize.has_metaclass: +if SCons.Memoize.use_old_memoization(): _Base = BuilderBase class BuilderBase(SCons.Memoize.Memoizer, _Base): "Cache-backed version of BuilderBase" diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py index 0e9b642..847b130 100644 --- a/src/engine/SCons/Environment.py +++ b/src/engine/SCons/Environment.py @@ -244,7 +244,8 @@ class SubstitutionEnvironment: class actually becomes useful.) """ - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass def __init__(self, **kw): """Initialization of an underlying SubstitutionEnvironment class. @@ -450,7 +451,8 @@ class Base(SubstitutionEnvironment): Environment. """ - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass ####################################################################### # This is THE class for interacting with the SCons build engine, @@ -1466,7 +1468,8 @@ class OverrideEnvironment(Base): values from the overrides dictionary. """ - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass def __init__(self, subject, overrides={}): if __debug__: logInstanceCreation(self, 'Environment.OverrideEnvironment') @@ -1590,7 +1593,7 @@ def NoSubstitutionProxy(subject): return apply(SCons.Util.scons_subst, nargs, nkw) return _NoSubstitutionProxy(subject) -if not SCons.Memoize.has_metaclass: +if SCons.Memoize.use_old_memoization(): _Base = Base class Base(SCons.Memoize.Memoizer, _Base): def __init__(self, *args, **kw): diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index d8bcafb..4f4e650 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -44,7 +44,8 @@ class Executor: and sources for later processing as needed. """ - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass def __init__(self, action, env=None, overridelist=[{}], targets=[], sources=[], builder_kw={}): @@ -257,7 +258,7 @@ class Null(_Executor): -if not SCons.Memoize.has_metaclass: +if SCons.Memoize.use_old_memoization(): _Base = Executor class Executor(SCons.Memoize.Memoizer, _Base): def __init__(self, *args, **kw): diff --git a/src/engine/SCons/Memoize.py b/src/engine/SCons/Memoize.py index afa77ee..134206f 100644 --- a/src/engine/SCons/Memoize.py +++ b/src/engine/SCons/Memoize.py @@ -58,7 +58,7 @@ Some advantages: might affect results. * Caching can be globally disabled very easily (for testing, etc.) - + """ # @@ -93,6 +93,9 @@ import os import string import sys +# A flag controlling whether or not we actually use memoization. +use_memoizer = 1 + # # Generate a key for an object that is to be used as the caching key # for that object. @@ -247,7 +250,7 @@ def ALT9_MeMoIZeR_gen_key(argtuple, kwdict): #_MeMoIZeR_gen_key = ALT9_MeMoIZeR_gen_key # 8.8, 0.20 _MeMoIZeR_gen_key = ALT8_MeMoIZeR_gen_key # 8.5, 0.18 #_MeMoIZeR_gen_key = ALT7_MeMoIZeR_gen_key # 8.7, 0.17 -#_MeMoIZeR_gen_key = ALT6_MeMoIZeR_gen_key # +#_MeMoIZeR_gen_key = ALT6_MeMoIZeR_gen_key # #_MeMoIZeR_gen_key = ALT5_MeMoIZeR_gen_key # 9.7, 0.20 #_MeMoIZeR_gen_key = ALT4_MeMoIZeR_gen_key # 8.6, 0.19 #_MeMoIZeR_gen_key = ALT3_MeMoIZeR_gen_key # 8.5, 0.20 @@ -534,7 +537,7 @@ class _Memoizer_Comparable: # Same thing for the other, but we should try to convert it # here in case the _MeMoIZeR_cmp compares __dict__ objects # directly. - + saved_other = None try: if other.__dict__.has_key('_MeMoIZeR_Key'): @@ -547,7 +550,7 @@ class _Memoizer_Comparable: # Both self and other have been prepared: perform the test, # then restore the original dictionaries and exit - + rval = self._MeMoIZeR_cmp(other) self.__dict__ = saved_d1 @@ -561,7 +564,7 @@ def Analyze_Class(klass): if klass.__dict__.has_key('_MeMoIZeR_converted'): return klass original_name = str(klass) - + D,R,C = _analyze_classmethods(klass.__dict__, klass.__bases__) if C: @@ -628,7 +631,6 @@ def memoize_classdict(klass, modelklass, new_klassdict, cacheable, resetting): new_klassdict[name] = newmethod return new_klassdict - def _analyze_classmethods(klassdict, klassbases): """Given a class, performs a scan of methods for that class and @@ -640,7 +642,7 @@ def _analyze_classmethods(klassdict, klassbases): D = {} R = {} C = None - + # Get cache/reset/cmp methods from subclasses for K in klassbases: @@ -685,7 +687,7 @@ def _scan_classdict(klassdict): Each dict has the name of the method as a key and the corresponding value is the method body.""" - + cache_setters = {} cache_resetters = {} cmp_if_exists = None @@ -711,7 +713,7 @@ def _scan_classdict(klassdict): continue if already_cache_modified: cmp_if_exists = 'already_cache_modified' return cache_setters, cache_resetters, cmp_if_exists - + # # Primary Memoizer class. This should be a base-class for any class # that wants method call results to be cached. The sub-class should @@ -725,9 +727,7 @@ class Memoizer: def __init__(self): self.__class__ = Analyze_Class(self.__class__) self._MeMoIZeR_Key = Next_Memoize_Key() - -has_metaclass = 1 # Find out if we are pre-2.2 try: @@ -744,14 +744,18 @@ except AttributeError: need_version = (2, 2) # actual #need_version = (33, 0) # always #need_version = (0, 0) # never -if vinfo[0] < need_version[0] or \ - (vinfo[0] == need_version[0] and - vinfo[1] < need_version[1]): - has_metaclass = 0 + +has_metaclass = (vinfo[0] > need_version[0] or \ + (vinfo[0] == need_version[0] and + vinfo[1] >= need_version[1])) + +if not has_metaclass: + class Memoized_Metaclass: # Just a place-holder so pre-metaclass Python versions don't # have to have special code for the Memoized classes. pass + else: # Initialization is a wee bit of a hassle. We want to do some of @@ -768,14 +772,14 @@ else: # to obtain the __init__ because we don't want to re-instrument # parent-class __init__ operations (and we want to avoid the # Object object's slot init if the class has no __init__). - + def _MeMoIZeR_init(actual_init, self, args, kw): self.__dict__['_MeMoIZeR_Key'] = Next_Memoize_Key() apply(actual_init, (self,)+args, kw) def _MeMoIZeR_superinit(self, cls, args, kw): apply(super(cls, self).__init__, args, kw) - + class Memoized_Metaclass(type): def __init__(cls, name, bases, cls_dict): # Note that cls_dict apparently contains a *copy* of the @@ -802,7 +806,7 @@ else: {'cls':cls, 'MPI':_MeMoIZeR_superinit}) init = superinit - + newinitcode = compile( "\n"*(init.func_code.co_firstlineno-1) + "lambda self, args, kw: _MeMoIZeR_init(real_init, self, args, kw)", @@ -819,4 +823,10 @@ else: # definition itself, apply klassdict. for attr in klassdict.keys(): setattr(cls, attr, klassdict[attr]) - + +def DisableMemoization(): + global use_memoizer + use_memoizer = None + +def use_old_memoization(): + return use_memoizer and not has_metaclass diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 527d1e3..07093d0 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -659,7 +659,8 @@ _classEntry = Entry class LocalFS: - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass # This class implements an abstraction layer for operations involving # a local file system. Essentially, this wraps any function in @@ -717,7 +718,7 @@ class LocalFS: def islink(self, path): return 0 # no symlinks -if not SCons.Memoize.has_metaclass: +if SCons.Memoize.use_old_memoization(): _FSBase = LocalFS class LocalFS(SCons.Memoize.Memoizer, _FSBase): def __init__(self, *args, **kw): diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 4a1faaa..b9d5a75 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -104,7 +104,8 @@ class Node: build, or use to build other Nodes. """ - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass class Attrs: pass @@ -995,7 +996,7 @@ else: del l del ul -if not SCons.Memoize.has_metaclass: +if SCons.Memoize.use_old_memoization(): _Base = Node class Node(SCons.Memoize.Memoizer, _Base): def __init__(self, *args, **kw): diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py index 3010159..ce9ae18 100644 --- a/src/engine/SCons/Scanner/__init__.py +++ b/src/engine/SCons/Scanner/__init__.py @@ -97,7 +97,8 @@ class Base: straightforward, single-pass scanning of a single file. """ - __metaclass__ = SCons.Memoize.Memoized_Metaclass + if SCons.Memoize.use_memoizer: + __metaclass__ = SCons.Memoize.Memoized_Metaclass def __init__(self, function, @@ -254,7 +255,7 @@ class Base: recurse_nodes = _recurse_no_nodes -if not SCons.Memoize.has_metaclass: +if SCons.Memoize.use_old_memoization(): _Base = Base class Base(SCons.Memoize.Memoizer, _Base): "Cache-backed version of Scanner Base" diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index 7dcb987..73f6a6c 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -592,7 +592,8 @@ class OptParser(OptionParser): "build all Default() targets.") debug_options = ["count", "dtree", "explain", "findlibs", - "includes", "memoizer", "memory", "objects", + "includes", "memoizer", "memory", + "nomemoizer", "objects", "pdb", "presub", "stacktrace", "stree", "time", "tree"] diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index 4327ac2..734ce3f 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -44,22 +44,30 @@ import string import sys import UserList -# Special chicken-and-egg handling of the "--debug=memoizer" flags: +# Special chicken-and-egg handling of the "--debug=memoizer" +# and "--debug=nomemoizer" flags: +# # SCons.Memoize contains a metaclass implementation that affects how # the other classes are instantiated. The Memoizer handles optional # counting of the hits and misses by using a different, parallel set of # functions, so we don't slow down normal operation any more than we -# have to. But if we wait to enable the counting until we've parsed -# the command line options normally, it will be too late, because the -# Memoizer will have already analyzed the classes that it's Memoizing -# and bound the non-counting versions of the functions. So we have to -# use a special-case, up-front check for the "--debug=memoizer" flag -# and turn on Memoizer counting, if desired, before we import any of -# the other modules that use it. -sconsflags = string.split(os.environ.get('SCONSFLAGS', '')) -if "--debug=memoizer" in sys.argv + sconsflags: +# have to. We can also tell it disable memoization completely. +# +# If we wait to enable the counting or disable memoization completely +# until we've parsed the command line options normally, it will be too +# late, because the Memoizer will have already analyzed the classes +# that it's Memoizing and bound the (non-counting) versions of the +# functions. So we have to use a special-case, up-front check for +# the "--debug=memoizer" and "--debug=nomemoizer" flags and do what's +# appropriate before we import any of the other modules that use it. +_args = sys.argv + string.split(os.environ.get('SCONSFLAGS', '')) +if "--debug=memoizer" in _args: import SCons.Memoize SCons.Memoize.EnableCounting() +if "--debug=nomemoizer" in _args: + import SCons.Memoize + SCons.Memoize.DisableMemoization() +del _args import SCons.Action import SCons.Builder @@ -315,7 +323,7 @@ for name in GlobalDefaultEnvironmentFunctions + GlobalDefaultBuilders: # "SConscript" in this namespace is no longer a module, it's a global # function call--or more precisely, an object that implements a global # function call through the default Environment. Nevertheless, we can -# aintain backwards compatibility for SConscripts that were reaching in +# maintain backwards compatibility for SConscripts that were reaching in # this way by hanging some attributes off the "SConscript" object here. SConscript = _SConscript.DefaultEnvironmentCall('SConscript') diff --git a/test/option/debug-nomemoizer.py b/test/option/debug-nomemoizer.py new file mode 100644 index 0000000..633a46d --- /dev/null +++ b/test/option/debug-nomemoizer.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test calling the --debug=nomemoizer option. +""" + +import pstats +import string +import StringIO +import sys + +import TestSCons + +test = TestSCons.TestSCons() + +scons_prof = test.workpath('scons.prof') + +test.write('SConstruct', """ +def cat(target, source, env): + open(str(target[0]), 'wb').write(open(str(source[0]), 'rb').read()) +env = Environment(BUILDERS={'Cat':Builder(action=Action(cat))}) +env.Cat('file.out', 'file.in') +""") + +test.write('file.in', "file.in\n") + +test.run(arguments = "--profile=%s --debug=nomemoizer " % scons_prof) + +stats = pstats.Stats(scons_prof) +stats.sort_stats('time') + +try: + save_stdout = sys.stdout + sys.stdout = StringIO.StringIO() + + stats.strip_dirs().print_stats() + + s = sys.stdout.getvalue() +finally: + sys.stdout = save_stdout + +test.fail_test(string.find(s, '_MeMoIZeR_init') != -1) +test.fail_test(string.find(s, '_MeMoIZeR_reset') != -1) +test.fail_test(string.find(s, 'Count_cache_get') != -1) +test.fail_test(string.find(s, 'Count_cache_get_self') != -1) +test.fail_test(string.find(s, 'Count_cache_get_one') != -1) + +test.pass_test() -- cgit v0.12