From 7db087e4c1efd7c5029befea03f904a6f0d21a44 Mon Sep 17 00:00:00 2001 From: Steven Knight Date: Sat, 7 Feb 2004 08:42:48 +0000 Subject: Add options to investigate object creation and memory consumption. --- bin/files | 1 + doc/man/scons.1 | 58 +++++++++++++------- src/CHANGES.txt | 4 ++ src/engine/MANIFEST.in | 1 + src/engine/SCons/Action.py | 6 +++ src/engine/SCons/Builder.py | 9 +++- src/engine/SCons/Debug.py | 102 ++++++++++++++++++++++++++++++++++++ src/engine/SCons/Environment.py | 14 ++--- src/engine/SCons/Executor.py | 4 ++ src/engine/SCons/Node/FS.py | 11 ++-- src/engine/SCons/Node/__init__.py | 4 +- src/engine/SCons/Script/__init__.py | 58 ++++++++++++++++---- test/option--debug.py | 41 ++++++++++++++- 13 files changed, 269 insertions(+), 44 deletions(-) create mode 100644 src/engine/SCons/Debug.py diff --git a/bin/files b/bin/files index 3365f4c..e15853f 100644 --- a/bin/files +++ b/bin/files @@ -1,6 +1,7 @@ ./SCons/Action.py ./SCons/Builder.py ./SCons/Conftest.py +./SCons/Debug.py ./SCons/Defaults.py ./SCons/Environment.py ./SCons/Errors.py diff --git a/doc/man/scons.1 b/doc/man/scons.1 index 7014cf3..c5a1c4a 100644 --- a/doc/man/scons.1 +++ b/doc/man/scons.1 @@ -457,6 +457,39 @@ Debug the build process. specifies what type of debugging: .TP +--debug=count +Print a count of how many objects are created +of the various classes used internally by SCons. +This only works when run under Python 2.1 or later. + +.TP +--debug=dtree +Print the dependency tree +after each top-level target is built. This prints out only derived files. + +.TP +--debug=includes +Print the include tree after each top-level target is built. +This is generally used to find out what files are included by the sources +of a given derived file: + +.ES +$ scons --debug=includes foo.o +.EE + +.TP +--debug=memory +Prints how much memory SCons uses +before and after reading the SConscript files +and before and after building. + +.TP +--debug=objects +Prints a list of the various objects +of the various classes used internally by SCons. +This only works when run under Python 2.1 or later. + +.TP --debug=pdb Re-run SCons under the control of the .RI pdb @@ -468,18 +501,6 @@ but all other arguments will be passed in-order to the SCons invocation run by the debugger. .TP ---debug=tree -Print the dependency tree -after each top-level target is built. This prints out the complete -dependency tree including implicit dependencies and ignored -dependencies. - -.TP ---debug=dtree -Print the dependency tree -after each top-level target is built. This prints out only derived files. - -.TP --debug=time Prints various time profiling information: the time spent executing each build command, the total build time, the total time spent @@ -487,14 +508,11 @@ executing build commands, the total time spent executing SConstruct and SConscript files, and the total time spent executing SCons itself. .TP ---debug=includes -Print the include tree after each top-level target is built. -This is generally used to find out what files are included by the sources -of a given derived file: - -.ES -$ scons --debug=includes foo.o -.EE +--debug=tree +Print the dependency tree +after each top-level target is built. This prints out the complete +dependency tree including implicit dependencies and ignored +dependencies. .\" .TP .\" -e, --environment-overrides diff --git a/src/CHANGES.txt b/src/CHANGES.txt index b66333d..2f1540b 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -150,6 +150,10 @@ RELEASE 0.95 - XXX - Fix transparent checkout of implicit dependency files from SCCS and RCS. + - Added new --debug=count, --debug=memory and --debug=objects options. + --debug=count and --debug=objects only print anything when run + under Python 2.1 or later. + From Vincent Risi: - Add support for the bcc32, ilink32 and tlib Borland tools. diff --git a/src/engine/MANIFEST.in b/src/engine/MANIFEST.in index e939790..4338e89 100644 --- a/src/engine/MANIFEST.in +++ b/src/engine/MANIFEST.in @@ -2,6 +2,7 @@ SCons/__init__.py SCons/Action.py SCons/Builder.py SCons/Conftest.py +SCons/Debug.py SCons/Defaults.py SCons/Environment.py SCons/Errors.py diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py index e2ceb2a..d9dbca1 100644 --- a/src/engine/SCons/Action.py +++ b/src/engine/SCons/Action.py @@ -35,6 +35,7 @@ import re import string import sys +from SCons.Debug import logInstanceCreation import SCons.Errors import SCons.Util @@ -173,6 +174,7 @@ class CommandAction(ActionBase): # Cmd list can actually be a list or a single item...basically # anything that we could pass in as the first arg to # Environment.subst_list(). + if __debug__: logInstanceCreation(self) self.cmd_list = cmd def strfunction(self, target, source, env): @@ -289,6 +291,7 @@ class CommandAction(ActionBase): class CommandGeneratorAction(ActionBase): """Class for command-generator actions.""" def __init__(self, generator): + if __debug__: logInstanceCreation(self) self.generator = generator def __generate(self, target, source, env, for_signature): @@ -331,6 +334,7 @@ class LazyCmdGenerator: until execution time to see what type it is, then tries to create an Action out of it.""" def __init__(self, var): + if __debug__: logInstanceCreation(self) self.var = SCons.Util.to_String(var) def strfunction(self, target, source, env): @@ -354,6 +358,7 @@ class FunctionAction(ActionBase): """Class for Python function actions.""" def __init__(self, execfunction, strfunction=_null, varlist=[]): + if __debug__: logInstanceCreation(self) self.execfunction = execfunction if strfunction is _null: def strfunction(target, source, env, execfunction=execfunction): @@ -407,6 +412,7 @@ class FunctionAction(ActionBase): class ListAction(ActionBase): """Class for lists of other actions.""" def __init__(self, list): + if __debug__: logInstanceCreation(self) self.list = map(lambda x: Action(x), list) def get_actions(self): diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py index 339a376..3010e98 100644 --- a/src/engine/SCons/Builder.py +++ b/src/engine/SCons/Builder.py @@ -47,6 +47,7 @@ import os.path import UserDict import SCons.Action +from SCons.Debug import logInstanceCreation from SCons.Errors import InternalError, UserError import SCons.Executor import SCons.Node.FS @@ -269,6 +270,7 @@ class BuilderBase: multi = 0, env = None, overrides = {}): + if __debug__: logInstanceCreation(self, 'BuilderBase') self.action = SCons.Action.Action(action) self.multi = multi if SCons.Util.is_Dict(prefix): @@ -357,7 +359,7 @@ class BuilderBase: if not t.is_derived(): t.builder = self new_targets.append(t) - + target, source = self.emitter(target=tlist, source=slist, env=env) # Now delete the temporary builders that we attached to any @@ -450,6 +452,7 @@ class ListBuilder(SCons.Util.Proxy): """ def __init__(self, builder, env, tlist): + if __debug__: logInstanceCreation(self) SCons.Util.Proxy.__init__(self, builder) self.builder = builder self.scanner = builder.scanner @@ -491,6 +494,7 @@ class MultiStepBuilder(BuilderBase): source_factory = None, scanner=None, emitter=None): + if __debug__: logInstanceCreation(self) BuilderBase.__init__(self, action, prefix, suffix, src_suffix, node_factory, target_factory, source_factory, scanner, emitter) @@ -581,6 +585,7 @@ class CompositeBuilder(SCons.Util.Proxy): """ def __init__(self, builder, cmdgen): + if __debug__: logInstanceCreation(self) SCons.Util.Proxy.__init__(self, builder) # cmdgen should always be an instance of DictCmdGenerator. @@ -590,6 +595,6 @@ class CompositeBuilder(SCons.Util.Proxy): def add_action(self, suffix, action): self.cmdgen.add_action(suffix, action) self.set_src_suffix(self.cmdgen.src_suffixes()) - + def __cmp__(self, other): return cmp(self.__dict__, other.__dict__) diff --git a/src/engine/SCons/Debug.py b/src/engine/SCons/Debug.py new file mode 100644 index 0000000..410f390 --- /dev/null +++ b/src/engine/SCons/Debug.py @@ -0,0 +1,102 @@ +"""SCons.Debug + +Code for debugging SCons internal things. Not everything here is +guaranteed to work all the way back to Python 1.5.2, and shouldn't be +needed by most users. + +""" + +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + + +# Recipe 14.10 from the Python Cookbook. +import string +import sys +try: + import weakref +except ImportError: + def logInstanceCreation(instance, name=None): + pass +else: + def logInstanceCreation(instance, name=None): + if name is None: + name = instance.__class__.__name__ + if not tracked_classes.has_key(name): + tracked_classes[name] = [] + tracked_classes[name].append(weakref.ref(instance)) + + + +tracked_classes = {} + +def string_to_classes(s): + if s == '*': + c = tracked_classes.keys() + c.sort() + return c + else: + return string.split(s) + +def countLoggedInstances(classes, file=sys.stdout): + for classname in string_to_classes(classes): + file.write("%s: %d\n" % (classname, len(tracked_classes[classname]))) + +def listLoggedInstances(classes, file=sys.stdout): + for classname in string_to_classes(classes): + file.write('\n%s:\n' % classname) + for ref in tracked_classes[classname]: + obj = ref() + if obj is not None: + file.write(' %s\n' % repr(obj)) + +def dumpLoggedInstances(classes, file=sys.stdout): + for classname in string_to_classes(classes): + file.write('\n%s:\n' % classname) + for ref in tracked_classes[classname]: + obj = ref() + if obj is not None: + file.write(' %s:\n' % obj) + for key, value in obj.__dict__.items(): + file.write(' %20s : %s\n' % (key, value)) + + + +if sys.platform[:5] == "linux": + # Linux doesn't actually support memory usage stats from getrusage(). + def memory(): + mstr = open('/proc/self/stat').read() + mstr = string.split(mstr)[22] + return int(mstr) +else: + try: + import resource + except ImportError: + def memory(): + return 0 + else: + def memory(): + res = resource.getrusage(resource.RUSAGE_SELF) + return res[4] diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py index 3819c27..43cb2c3 100644 --- a/src/engine/SCons/Environment.py +++ b/src/engine/SCons/Environment.py @@ -45,6 +45,7 @@ from UserDict import UserDict import SCons.Action import SCons.Builder +from SCons.Debug import logInstanceCreation import SCons.Defaults import SCons.Errors import SCons.Node @@ -211,6 +212,7 @@ class Base: toolpath=[], options=None, **kw): + if __debug__: logInstanceCreation(self) self.fs = SCons.Node.FS.default_fs self.ans = SCons.Node.Alias.default_ans self.lookup_list = SCons.Node.arg2nodes_lookups @@ -500,8 +502,8 @@ class Base: suffix - construction variable for the suffix. """ - suffix = self.subst('$%s'%suffix) - prefix = self.subst('$%s'%prefix) + suffix = self.subst('$'+suffix) + prefix = self.subst('$'+prefix) for path in paths: dir,name = os.path.split(str(path)) @@ -643,11 +645,11 @@ class Base: new_prefix - construction variable for the new prefix. new_suffix - construction variable for the new suffix. """ - old_prefix = self.subst('$%s'%old_prefix) - old_suffix = self.subst('$%s'%old_suffix) + old_prefix = self.subst('$'+old_prefix) + old_suffix = self.subst('$'+old_suffix) - new_prefix = self.subst('$%s'%new_prefix) - new_suffix = self.subst('$%s'%new_suffix) + new_prefix = self.subst('$'+new_prefix) + new_suffix = self.subst('$'+new_suffix) dir,name = os.path.split(str(path)) if name[:len(old_prefix)] == old_prefix: diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index 12dfcc7..797bdda 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -31,6 +31,9 @@ Nodes. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +from SCons.Debug import logInstanceCreation + + class Executor: """A class for controlling instances of executing an action. @@ -40,6 +43,7 @@ class Executor: """ def __init__(self, builder, env, overrides, targets, sources): + if __debug__: logInstanceCreation(self) self.builder = builder self.env = env self.overrides = overrides diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index cbfe396..e1aeda3 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -45,10 +45,11 @@ import sys import cStringIO import SCons.Action +from SCons.Debug import logInstanceCreation import SCons.Errors import SCons.Node -import SCons.Util import SCons.Sig.MD5 +import SCons.Util import SCons.Warnings # @@ -322,6 +323,7 @@ class Base(SCons.Node.Node): our relative and absolute paths, identify our parent directory, and indicate that this node should use signatures.""" + if __debug__: logInstanceCreation(self, 'Node.FS.Base') SCons.Node.Node.__init__(self) self.name = name @@ -571,6 +573,7 @@ class FS: The path argument must be a valid absolute path. """ + if __debug__: logInstanceCreation(self) if path == None: self.pathTop = os.getcwd() else: @@ -936,6 +939,7 @@ class Dir(Base): """ def __init__(self, name, directory, fs): + if __debug__: logInstanceCreation(self, 'Node.FS.Dir') Base.__init__(self, name, directory, fs) self._morph() @@ -1175,6 +1179,7 @@ class File(Base): """A class for files in a file system. """ def __init__(self, name, directory, fs): + if __debug__: logInstanceCreation(self, 'Node.FS.File') Base.__init__(self, name, directory, fs) self._morph() @@ -1197,13 +1202,13 @@ class File(Base): """Search for a list of directories in the Repository list.""" return self.fs.Rsearchall(pathlist, clazz=Dir, must_exist=0, cwd=self.cwd) - + def generate_build_env(self, env): """Generate an appropriate Environment to build this File.""" return env.Override({'Dir' : self.Dir, 'File' : self.File, 'RDirs' : self.RDirs}) - + def _morph(self): """Turn a file system node into a File object.""" self.scanner_paths = {} diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 0e453d2..4d51370 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -48,6 +48,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import copy +from SCons.Debug import logInstanceCreation import SCons.Sig import SCons.Util @@ -89,6 +90,7 @@ class Node: pass def __init__(self): + if __debug__: logInstanceCreation(self, 'Node') # Note that we no longer explicitly initialize a self.builder # attribute to None here. That's because the self.builder # attribute may be created on-the-fly later by a subclass (the @@ -154,7 +156,7 @@ class Node: except AttributeError: if not create: raise - import SCons.Builder + import SCons.Executor env = self.generate_build_env(self.builder.env) executor = SCons.Executor.Executor(self.builder, env, diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index ac05f3c..7067ab1 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -55,6 +55,7 @@ import traceback # 'lib', # 'scons-%d' % SCons.__version__)] + sys.path[1:] +import SCons.Debug import SCons.Defaults import SCons.Environment import SCons.Errors @@ -72,7 +73,6 @@ import SCons.Warnings display = SCons.Util.display progress_display = SCons.Util.DisplayEngine() -# # Task control. # class BuildTask(SCons.Taskmaster.Task): @@ -223,10 +223,13 @@ class QuestionTask(SCons.Taskmaster.Task): # Global variables keep_going_on_error = 0 -print_tree = 0 +print_count = 0 print_dtree = 0 -print_time = 0 print_includes = 0 +print_objects = 0 +print_time = 0 +print_tree = 0 +memory_stats = None ignore_errors = 0 sconscript_time = 0 command_time = 0 @@ -385,22 +388,31 @@ def _SConstruct_exists(dirname=''): return None def _set_globals(options): - global repositories, keep_going_on_error, print_tree, print_dtree - global print_time, ignore_errors, print_includes + global repositories, keep_going_on_error, ignore_errors + global print_count, print_dtree, print_includes + global print_objects, print_time, print_tree + global memory_outf, memory_stats if options.repository: repositories.extend(options.repository) keep_going_on_error = options.keep_going try: if options.debug: - if options.debug == "tree": - print_tree = 1 + if options.debug == "count": + print_count = 1 elif options.debug == "dtree": print_dtree = 1 - elif options.debug == "time": - print_time = 1 elif options.debug == "includes": print_includes = 1 + elif options.debug == "memory": + memory_stats = [] + memory_outf = sys.stdout + elif options.debug == "objects": + print_objects = 1 + elif options.debug == "time": + print_time = 1 + elif options.debug == "tree": + print_tree = 1 except AttributeError: pass ignore_errors = options.ignore_errors @@ -480,7 +492,7 @@ class OptParser(OptionParser): "build all Default() targets.") def opt_debug(option, opt, value, parser): - if value in ["pdb","tree", "dtree", "time", "includes"]: + if value in ["count", "dtree", "includes", "memory", "objects", "pdb", "time", "tree"]: setattr(parser.values, 'debug', value) else: raise OptionValueError("Warning: %s is not a valid debug type" % value) @@ -488,7 +500,7 @@ class OptParser(OptionParser): callback=opt_debug, nargs=1, dest="debug", metavar="TYPE", help="Print various types of debugging information: " - "pdb, tree, dtree, time, or includes.") + "count, dtree, includes, memory, objects, pdb, time, tree.") self.add_option('-f', '--file', '--makefile', '--sconstruct', action="append", nargs=1, @@ -825,6 +837,8 @@ def _main(args, parser): for rep in repositories: fs.Repository(rep) + if not memory_stats is None: memory_stats.append(SCons.Debug.memory()) + progress_display("scons: Reading SConscript files ...") try: start_time = time.time() @@ -850,6 +864,8 @@ def _main(args, parser): sys.exit(0) progress_display("scons: done reading SConscript files.") + if not memory_stats is None: memory_stats.append(SCons.Debug.memory()) + fs.chdir(fs.Top) if options.help_msg: @@ -982,6 +998,8 @@ def _main(args, parser): "\tignoring -j or num_jobs option.\n" SCons.Warnings.warn(SCons.Warnings.NoParallelSupportWarning, msg) + if not memory_stats is None: memory_stats.append(SCons.Debug.memory()) + try: jobs.run() finally: @@ -992,6 +1010,24 @@ def _main(args, parser): if not options.noexec: SCons.Sig.write() + if not memory_stats is None: + memory_stats.append(SCons.Debug.memory()) + when = [ + 'before SConscript files', + 'after SConscript files', + 'before building', + 'after building', + ] + for i in xrange(len(when)): + memory_outf.write('Memory %s: %d\n' % (when[i], memory_stats[i])) + + if print_count: + SCons.Debug.countLoggedInstances('*') + + if print_objects: + SCons.Debug.listLoggedInstances('*') + #SCons.Debug.dumpLoggedInstances('*') + def _exec_main(): all_args = sys.argv[1:] try: diff --git a/test/option--debug.py b/test/option--debug.py index 0583912..dfa5503 100644 --- a/test/option--debug.py +++ b/test/option--debug.py @@ -191,5 +191,44 @@ assert check(expected_command_time, command_time, 0.01) assert check(total_time, sconscript_time+scons_time+command_time, 0.01) assert check(total_time, expected_total_time, 0.1) -test.pass_test() +try: + import resource +except ImportError: + print "Python version has no `resource' module;" + print "skipping test of --debug=memory." +else: + ############################ + # test --debug=memory + + test.run(arguments = "--debug=memory") + lines = string.split(test.stdout(), '\n') + test.fail_test(re.match(r'Memory before SConscript files: \d+', lines[-5]) is None) + test.fail_test(re.match(r'Memory after SConscript files: \d+', lines[-4]) is None) + test.fail_test(re.match(r'Memory before building: \d+', lines[-3]) is None) + test.fail_test(re.match(r'Memory after building: \d+', lines[-2]) is None) + +try: + import weakref +except ImportError: + print "Python version has no `weakref' module;" + print "skipping tests of --debug=count and --debug=objects." +else: + ############################ + # test --debug=count + # Just check that object counts for some representative classes + # show up in the output. + test.run(arguments = "--debug=count") + stdout = test.stdout() + test.fail_test(re.search('BuilderBase: \d+', stdout) is None) + test.fail_test(re.search('FS: \d+', stdout) is None) + test.fail_test(re.search('Node: \d+', stdout) is None) + test.fail_test(re.search('SConsEnvironment: \d+', stdout) is None) + + ############################ + # test --debug=objects + # Just check that it runs, we're not overly concerned about the actual + # output at this point. + test.run(arguments = "--debug=objects") + +test.pass_test() -- cgit v0.12