From 17879c1de6f589009c781e7f36e8a6a95b45c33f Mon Sep 17 00:00:00 2001 From: Steven Knight Date: Thu, 10 Dec 2009 06:19:43 +0000 Subject: Add a script for calibrating settings for timing configurations. Update the timings scripts with calibrated settings that run a full build between 9.5 and 10.0 seconds on the buildbot slave. --- QMTest/TestSCons.py | 15 ++++---- bin/calibrate.py | 76 ++++++++++++++++++++++++++++++++++++++++ timings/CPPPATH/TimeSCons-run.py | 23 +++++++++--- timings/JTimer/TimeSCons-run.py | 29 ++++++++++++--- timings/hundred/TimeSCons-run.py | 21 ++++++++--- 5 files changed, 142 insertions(+), 22 deletions(-) create mode 100644 bin/calibrate.py diff --git a/QMTest/TestSCons.py b/QMTest/TestSCons.py index 002baaa..c172632 100644 --- a/QMTest/TestSCons.py +++ b/QMTest/TestSCons.py @@ -1011,7 +1011,9 @@ class TimeSCons(TestSCons): self.variables[variable] = value del kw['variables'] - if not kw.has_key('verbose'): + self.calibrate = os.environ.get('TIMESCONS_CALIBRATE', '0') != '0' + + if not kw.has_key('verbose') and not self.calibrate: kw['verbose'] = True # TODO(1.5) @@ -1048,8 +1050,11 @@ class TimeSCons(TestSCons): for variable, value in self.variables.items(): options.append('%s=%s' % (variable, value)) kw['options'] = ' '.join(options) - calibrate = os.environ.get('TIMESCONS_CALIBRATE') - if calibrate in (None, '0'): + if self.calibrate: + # TODO(1.5) + #self.calibration(*args, **kw) + apply(self.calibration, args, kw) + else: # TODO(1.5) #self.help(*args, **kw) #self.full(*args, **kw) @@ -1057,10 +1062,6 @@ class TimeSCons(TestSCons): apply(self.help, args, kw) apply(self.full, args, kw) apply(self.null, args, kw) - else: - # TODO(1.5) - #self.calibration(*args, **kw) - apply(self.calibration, args, kw) def trace(self, graph, name, value, units, sort=None): fmt = "TRACE: graph=%s name=%s value=%s units=%s" diff --git a/bin/calibrate.py b/bin/calibrate.py new file mode 100644 index 0000000..c1b4f11 --- /dev/null +++ b/bin/calibrate.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Copyright (c) 2009 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import optparse +import os +import re +import subprocess +import sys + +variable_re = re.compile('^VARIABLE: (.*)$', re.M) +elapsed_re = re.compile('^ELAPSED: (.*)$', re.M) + +def main(argv=None): + if argv is None: + argv = sys.argv + + parser = optparse.OptionParser(usage="calibrate.py [-h] [--min time] [--max time] timings/*/*-run.py") + parser.add_option('--min', type='float', default=9.5, + help="minimum acceptable execution time (default 9.5)") + parser.add_option('--max', type='float', default=10.00, + help="maximum acceptable execution time (default 10.00)") + opts, args = parser.parse_args(argv[1:]) + + os.environ['TIMESCONS_CALIBRATE'] = '1' + + for arg in args: + if len(args) > 1: + print arg + ':' + + command = [sys.executable, 'runtest.py', '--noqmtest', arg] + + run = 1 + good = 0 + while good < 3: + p = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = p.communicate()[0] + vm = variable_re.search(output) + em = elapsed_re.search(output) + elapsed = float(em.group(1)) + print "run %3d: %7.3f: %s" % (run, elapsed, ' '.join(vm.groups())) + if opts.min < elapsed and elapsed < opts.max: + good += 1 + else: + good = 0 + for v in vm.groups(): + var, value = v.split('=', 1) + value = int((int(value) * opts.max) / elapsed) + os.environ[var] = str(value) + run += 1 + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/timings/CPPPATH/TimeSCons-run.py b/timings/CPPPATH/TimeSCons-run.py index dcee7c5..97d9e05 100644 --- a/timings/CPPPATH/TimeSCons-run.py +++ b/timings/CPPPATH/TimeSCons-run.py @@ -24,15 +24,28 @@ """ This configuration times searching long lists of CPPPATH directories. -We create 5000 on-disk directories. A single checked-in .h file exists -in the 'include' directory. The SConstruct sets CPPPATH to a list of Dir -Nodes for the created directories, followed by 'include'. A checked-in .c -file #includes the .h file to be found in the last directory in the list. +We create $DIR_COUNT on-disk directories. A single checked-in .h file +exists in the 'include' directory. The SConstruct sets CPPPATH to a +list of Dir Nodes for the created directories, followed by 'include'. +A checked-in .c file #includes the .h file to be found in the last +directory in the list. """ import TestSCons -test = TestSCons.TimeSCons(variables={'DIR_COUNT':5000}) +# Full-build time of just under 10 seconds on ubuntu-timings slave, +# as determined by bin/calibrate.py on 9 December 2009: +# +# run 1: 2.235: DIR_COUNT=50 +# run 2: 3.976: DIR_COUNT=223 +# run 3: 7.353: DIR_COUNT=560 +# run 4: 9.569: DIR_COUNT=761 +# run 5: 9.353: DIR_COUNT=761 +# run 6: 9.972: DIR_COUNT=813 +# run 7: 9.930: DIR_COUNT=813 +# run 8: 9.983: DIR_COUNT=813 + +test = TestSCons.TimeSCons(variables={'DIR_COUNT':813}) for d in xrange(test.variables['DIR_COUNT']): test.subdir('inc_%04d' % d) diff --git a/timings/JTimer/TimeSCons-run.py b/timings/JTimer/TimeSCons-run.py index 05ffbfb..7fe1bf4 100644 --- a/timings/JTimer/TimeSCons-run.py +++ b/timings/JTimer/TimeSCons-run.py @@ -25,10 +25,10 @@ This configuration is for timing how we evaluate long chains of dependencies, specifically when -j is used. -We set up a chain of 500 targets that get built from a Python function -action with no source files (equivalent to "echo junk > $TARGET"). -Each target explicitly depends on the next target in turn, so the -Taskmaster will do a deep walk of the dependency graph. +We set up a chain of $TARGET_COUNT targets that get built from a +Python function action with no source files (equivalent to "echo junk > +$TARGET"). Each target explicitly depends on the next target in turn, +so the Taskmaster will do a deep walk of the dependency graph. This test case was contributed by Kevin Massey. Prior to revision 1468, we had a serious O(N^2) problem in the Taskmaster when handling long @@ -38,7 +38,26 @@ to the Taskmaster so it could be smarter about not re-evaluating Nodes. import TestSCons -test = TestSCons.TimeSCons(variables={'TARGET_COUNT':500}) +# Full-build time of just under 10 seconds on ubuntu-timings slave, +# as determined by bin/calibrate.py on 9 December 2009: +# +# run 1: 3.211: TARGET_COUNT=50 +# run 2: 11.920: TARGET_COUNT=155 +# run 3: 9.182: TARGET_COUNT=130 +# run 4: 10.185: TARGET_COUNT=141 +# run 5: 9.945: TARGET_COUNT=138 +# run 6: 10.035: TARGET_COUNT=138 +# run 7: 9.898: TARGET_COUNT=137 +# run 8: 9.840: TARGET_COUNT=137 +# run 9: 10.054: TARGET_COUNT=137 +# run 10: 9.747: TARGET_COUNT=136 +# run 11: 9.778: TARGET_COUNT=136 +# run 12: 9.743: TARGET_COUNT=136 +# +# The fact that this varies so much suggests that it's pretty +# non-deterministic, which makes sense for a test involving -j. + +test = TestSCons.TimeSCons(variables={'TARGET_COUNT':136}) test.main() diff --git a/timings/hundred/TimeSCons-run.py b/timings/hundred/TimeSCons-run.py index 915c131..c90c26b 100644 --- a/timings/hundred/TimeSCons-run.py +++ b/timings/hundred/TimeSCons-run.py @@ -25,15 +25,26 @@ This configuration is for timing how we handle the NxM interaction when we build a lot of targets from a lot of source files. -We create a list of 500 target files that will each be built by copying -a file from a corresponding list of 500 source files. The source -files themselves are each built by a Python function action that's the -equivalent of "echo contents > $TARGET". +We create a list of $TARGET_COUNT target files that will each be built by +copying a file from a corresponding list of $TARGET_COUNT source files. +The source files themselves are each built by a Python function action +that's the equivalent of "echo contents > $TARGET". """ import TestSCons -test = TestSCons.TimeSCons(variables={'TARGET_COUNT':500}) +# Full-build time of just under 10 seconds on ubuntu-timings slave, +# as determined by bin/calibrate.py on 9 December 2009: +# +# run 1: 3.124: TARGET_COUNT=50 +# run 2: 11.936: TARGET_COUNT=160 +# run 3: 9.175: TARGET_COUNT=134 +# run 4: 10.489: TARGET_COUNT=146 +# run 5: 9.798: TARGET_COUNT=139 +# run 6: 9.695: TARGET_COUNT=139 +# run 7: 9.670: TARGET_COUNT=139 + +test = TestSCons.TimeSCons(variables={'TARGET_COUNT':139}) for t in xrange(test.variables['TARGET_COUNT']): open('source_%04d' % t, 'wb' ).write('contents\n') -- cgit v0.12