From b3744e8862927899e3d0ebcb41297f9b4c142c63 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 22 Apr 2023 16:49:38 -0700 Subject: added --debug=json, which will dump object counts and memory into scons_stats.json in invocation directory --- SCons/Script/Main.py | 106 +++++++++-------------------- SCons/Script/SConsOptions.py | 2 +- SCons/Util/stats.py | 156 +++++++++++++++++++++++++++++++++++++++++++ SCons/__init__.py | 10 +-- 4 files changed, 192 insertions(+), 82 deletions(-) create mode 100644 SCons/Util/stats.py diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index d4228fa..ec34c12 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -37,7 +37,6 @@ deprecated_python_version = (3, 6, 0) import SCons.compat -import atexit import importlib.util import os import re @@ -63,6 +62,7 @@ import SCons.Taskmaster import SCons.Util import SCons.Warnings import SCons.Script.Interactive +from SCons.Util.stats import COUNT_STATS, MEMORY_STATS, ENABLE_JSON, WriteJsonFile from SCons import __version__ as SConsVersion @@ -82,10 +82,18 @@ num_jobs = None delayed_warnings = [] +display = SCons.Util.display +progress_display = SCons.Util.DisplayEngine() +ProgressObject = SCons.Util.Null() +_BuildFailures = [] + + def revert_io(): - # This call is added to revert stderr and stdout to the original - # ones just in case some build rule or something else in the system - # has redirected them elsewhere. + """ + This call is added to revert stderr and stdout to the original + ones just in case some build rule or something else in the system + has redirected them elsewhere. + """ sys.stderr = sys.__stderr__ sys.stdout = sys.__stdout__ @@ -94,10 +102,6 @@ class SConsPrintHelpException(Exception): pass -display = SCons.Util.display -progress_display = SCons.Util.DisplayEngine() - - class Progressor: prev = '' count = 0 @@ -150,7 +154,6 @@ class Progressor: self.erase_previous() self.func(node) -ProgressObject = SCons.Util.Null() def Progress(*args, **kw): global ProgressObject @@ -159,9 +162,6 @@ def Progress(*args, **kw): # Task control. # -_BuildFailures = [] - - def GetBuildFailures(): return _BuildFailures @@ -515,59 +515,8 @@ def ValidateOptions(throw_exception=False) -> None: def PrintHelp(file=None): OptionsParser.print_help(file=file) -class Stats: - def __init__(self): - self.stats = [] - self.labels = [] - self.append = self.do_nothing - self.print_stats = self.do_nothing - def enable(self, outfp): - self.outfp = outfp - self.append = self.do_append - self.print_stats = self.do_print - def do_nothing(self, *args, **kw): - pass -class CountStats(Stats): - def do_append(self, label): - self.labels.append(label) - self.stats.append(SCons.Debug.fetchLoggedInstances()) - def do_print(self): - stats_table = {} - for s in self.stats: - for n in [t[0] for t in s]: - stats_table[n] = [0, 0, 0, 0] - i = 0 - for s in self.stats: - for n, c in s: - stats_table[n][i] = c - i = i + 1 - self.outfp.write("Object counts:\n") - pre = [" "] - post = [" %s\n"] - l = len(self.stats) - fmt1 = ''.join(pre + [' %7s']*l + post) - fmt2 = ''.join(pre + [' %7d']*l + post) - labels = self.labels[:l] - labels.append(("", "Class")) - self.outfp.write(fmt1 % tuple([x[0] for x in labels])) - self.outfp.write(fmt1 % tuple([x[1] for x in labels])) - for k in sorted(stats_table.keys()): - r = stats_table[k][:l] + [k] - self.outfp.write(fmt2 % tuple(r)) - -count_stats = CountStats() - -class MemStats(Stats): - def do_append(self, label): - self.labels.append(label) - self.stats.append(SCons.Debug.memory()) - def do_print(self): - fmt = 'Memory %-32s %12d\n' - for label, stats in zip(self.labels, self.stats): - self.outfp.write(fmt % (label, stats)) - -memory_stats = MemStats() + # utility functions @@ -671,7 +620,7 @@ def _set_debug_values(options): enable_count = False if __debug__: enable_count = True if enable_count: - count_stats.enable(sys.stdout) + COUNT_STATS.enable(sys.stdout) SCons.Debug.track_instances = True else: msg = "--debug=count is not supported when running SCons\n" + \ @@ -685,7 +634,7 @@ def _set_debug_values(options): options.debug_includes = ("includes" in debug_values) print_memoizer = ("memoizer" in debug_values) if "memory" in debug_values: - memory_stats.enable(sys.stdout) + MEMORY_STATS.enable(sys.stdout) print_objects = ("objects" in debug_values) if print_objects: SCons.Debug.track_instances = True @@ -706,6 +655,8 @@ def _set_debug_values(options): SCons.Taskmaster.print_prepare = True if "duplicate" in debug_values: SCons.Node.print_duplicate = True + if "json" in debug_values: + SCons.Util.stats.ENABLE_JSON = True def _create_path(plist): path = '.' @@ -1039,8 +990,8 @@ def _main(parser): if not hasattr(sys.stderr, 'isatty') or not sys.stderr.isatty(): sys.stderr = SCons.Util.Unbuffered(sys.stderr) - memory_stats.append('before reading SConscript files:') - count_stats.append(('pre-', 'read')) + MEMORY_STATS.append('before reading SConscript files:') + COUNT_STATS.append(('pre-', 'read')) # And here's where we (finally) read the SConscript files. @@ -1066,8 +1017,8 @@ def _main(parser): progress_display("scons: done reading SConscript files.") - memory_stats.append('after reading SConscript files:') - count_stats.append(('post-', 'read')) + MEMORY_STATS.append('after reading SConscript files:') + COUNT_STATS.append(('post-', 'read')) # Re-{enable,disable} warnings in case they disabled some in # the SConscript file. @@ -1319,8 +1270,8 @@ def _build_targets(fs, options, targets, target_top): if msg: SCons.Warnings.warn(SCons.Warnings.NoParallelSupportWarning, msg) - memory_stats.append('before building targets:') - count_stats.append(('pre-', 'build')) + MEMORY_STATS.append('before building targets:') + COUNT_STATS.append(('pre-', 'build')) def jobs_postfunc( jobs=jobs, @@ -1348,8 +1299,8 @@ def _build_targets(fs, options, targets, target_top): progress_display("scons: " + opening_message) jobs.run(postfunc = jobs_postfunc) - memory_stats.append('after building targets:') - count_stats.append(('post-', 'build')) + MEMORY_STATS.append('after building targets:') + COUNT_STATS.append(('post-', 'build')) return nodes @@ -1437,8 +1388,8 @@ def main(): SCons.Script._SConscript.SConscript_exception() sys.exit(2) - memory_stats.print_stats() - count_stats.print_stats() + MEMORY_STATS.print_stats() + COUNT_STATS.print_stats() if print_objects: SCons.Debug.listLoggedInstances('*') @@ -1469,6 +1420,9 @@ def main(): print("Total SCons execution time: %f seconds"%scons_time) print("Total command execution time: %f seconds"%ct) + if SCons.Util.stats.ENABLE_JSON: + WriteJsonFile() + sys.exit(exit_status) # Local Variables: diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 8391d62..25c6046 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -701,7 +701,7 @@ def Parser(version): debug_options = ["count", "duplicate", "explain", "findlibs", "includes", "memoizer", "memory", "objects", "pdb", "prepare", "presub", "stacktrace", - "time", "action-timestamps"] + "time", "action-timestamps", "json"] def opt_debug(option, opt, value__, parser, debug_options=debug_options, diff --git a/SCons/Util/stats.py b/SCons/Util/stats.py new file mode 100644 index 0000000..ccad165 --- /dev/null +++ b/SCons/Util/stats.py @@ -0,0 +1,156 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +This package provides a way to gather various statistics during a SCons run and dump that info in several formats + +Additionally, it probably makes sense to do stderr/stdout output of those statistics here as well + +There are basically two types of stats. +1. Timer (start/stop/time) for specific event. These events can be hierarchical. So you can record the children events of some parent. + Think program compile could contain the total Program builder time, which could include linking, and stripping the executable +2. Counter. Counting the number of events and/or objects created. This would likely only be reported at the end of a given SCons run, + though it might be useful to query during a run. + +""" +import json + +import SCons.Debug + +ALL_STATS = {} +ENABLE_JSON = False + + +def AddStatType(name, stat_object): + """ + Add a statistic type to the global collection + """ + if name in ALL_STATS: + raise UserWarning(f'Stat type {name} already exists') + else: + ALL_STATS[name] = stat_object + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: +class Stats: + def __init__(self): + self.stats = [] + self.labels = [] + self.append = self.do_nothing + self.print_stats = self.do_nothing + self.enabled = False + + def do_append(self, label): + raise NotImplementedError + + def do_print(self): + raise NotImplementedError + + def enable(self, outfp): + self.outfp = outfp + self.append = self.do_append + self.print_stats = self.do_print + self.enabled = True + + def do_nothing(self, *args, **kw): + pass + + +class CountStats(Stats): + + def __init__(self): + super().__init__() + self.stats_table = {} + + def do_append(self, label): + self.labels.append(label) + self.stats.append(SCons.Debug.fetchLoggedInstances()) + + def do_print(self): + self.stats_table = {} + for s in self.stats: + for n in [t[0] for t in s]: + self.stats_table[n] = [0, 0, 0, 0] + i = 0 + for s in self.stats: + for n, c in s: + self.stats_table[n][i] = c + i = i + 1 + self.outfp.write("Object counts:\n") + pre = [" "] + post = [" %s\n"] + l = len(self.stats) + fmt1 = ''.join(pre + [' %7s'] * l + post) + fmt2 = ''.join(pre + [' %7d'] * l + post) + labels = self.labels[:l] + labels.append(("", "Class")) + self.outfp.write(fmt1 % tuple([x[0] for x in labels])) + self.outfp.write(fmt1 % tuple([x[1] for x in labels])) + for k in sorted(self.stats_table.keys()): + r = self.stats_table[k][:l] + [k] + self.outfp.write(fmt2 % tuple(r)) + + +class MemStats(Stats): + def do_append(self, label): + self.labels.append(label) + self.stats.append(SCons.Debug.memory()) + + def do_print(self): + fmt = 'Memory %-32s %12d\n' + for label, stats in zip(self.labels, self.stats): + self.outfp.write(fmt % (label, stats)) + + +COUNT_STATS = CountStats() +MEMORY_STATS = MemStats() + + +def WriteJsonFile(): + """ + + """ + print("DUMPING JSON FILE") + json_structure = {} + if COUNT_STATS.enabled: + json_structure['Object counts'] = {} + + oc = json_structure['Object counts'] + for c in COUNT_STATS.stats_table: + oc[c] = {} + for l, v in zip(COUNT_STATS.labels, COUNT_STATS.stats_table[c]): + oc[c][''.join(l)] = v + + if MEMORY_STATS.enabled: + json_structure['Memory'] = {} + + m = json_structure['Memory'] + for label, stats in zip(MEMORY_STATS.labels, MEMORY_STATS.stats): + m[label] =stats + + with open("scons_stats.json", 'w') as sf: + sf.write(json.dumps(json_structure)) diff --git a/SCons/__init__.py b/SCons/__init__.py index f53583b..9804f4f 100644 --- a/SCons/__init__.py +++ b/SCons/__init__.py @@ -1,9 +1,9 @@ -__version__="4.5.1" +__version__="4.5.2" __copyright__="Copyright (c) 2001 - 2023 The SCons Foundation" __developer__="bdbaddog" -__date__="Mon, 06 Mar 2023 23:32:38 -0400" -__buildsys__="M1DOG2021" -__revision__="49578b34582d9e92dac7d713a8e58599ae35aa63" -__build__="49578b34582d9e92dac7d713a8e58599ae35aa63" +__date__="Sat, 22 Apr 2023 16:48:09 -0700" +__buildsys__="M1Dog2021" +__revision__="176711bd1fde814e9c441ae44132eb600063ec7a" +__build__="176711bd1fde814e9c441ae44132eb600063ec7a" # make sure compatibility is always in place import SCons.compat # noqa \ No newline at end of file -- cgit v0.12