diff options
Diffstat (limited to 'Tools/pybench/pybench.py')
| -rwxr-xr-x | Tools/pybench/pybench.py | 170 |
1 files changed, 110 insertions, 60 deletions
diff --git a/Tools/pybench/pybench.py b/Tools/pybench/pybench.py index b20c3f3..f190d77 100755 --- a/Tools/pybench/pybench.py +++ b/Tools/pybench/pybench.py @@ -35,7 +35,7 @@ WITH THE USE OR PERFORMANCE OF THIS SOFTWARE ! """ # Version number -__version__ = '1.3' +__version__ = '1.4' # # NOTE: Use xrange for all test loops unless you want to face @@ -98,7 +98,7 @@ class Test: # Number of rounds to execute per test run. This should be # adjusted to a figure that results in a test run-time of between # 20-50 seconds. - rounds = 100000 + rounds = 10000 ### Internal variables @@ -115,6 +115,8 @@ class Test: if warp > 1: self.rounds = self.rounds / warp + if self.rounds == 0: + self.rounds = 1 self.warp = warp self.times = [] self.overhead = [] @@ -137,12 +139,13 @@ class Test: cruns = self.cruns # first calibrate offset = 0.0 - for i in range(cruns): - t = clock() - calibrate() - t = clock() - t - offset = offset + t - offset = offset / cruns + if cruns: + for i in range(cruns): + t = clock() + calibrate() + t = clock() - t + offset = offset + t + offset = offset / cruns # now the real thing t = clock() test() @@ -175,13 +178,18 @@ class Test: def stat(self): - """ Returns two value: average time per run and average per - operation. + """ Returns four values: + minimum round time + average time per round + average time per operation + average overhead time + XXX Should this take warp factors into account? """ runs = len(self.times) if runs == 0: return 0,0 + mintime = min(self.times) totaltime = reduce(operator.add,self.times,0.0) avg = totaltime / float(runs) op_avg = totaltime / float(runs * self.rounds * self.operations) @@ -191,7 +199,7 @@ class Test: else: # use self.last_timing - not too accurate ov_avg = self.last_timing[2] - return avg,op_avg,ov_avg + return mintime, avg, op_avg, ov_avg ### Load Setup @@ -210,105 +218,132 @@ class Benchmark: roundtime = 0 # Average round time version = None # Benchmark version number (see __init__) # as float x.yy - starttime = None # Benchmark start time def __init__(self): self.tests = {} self.version = 0.31 - def load_tests(self,setupmod,warp=1): + def load_tests(self, setupmod, warp=1, limitnames="", verbose=0): self.warp = warp + if limitnames: + limitnames = re.compile(limitnames, re.I) + else: + limitnames = None tests = self.tests - print 'Searching for tests...' + if verbose: + print 'Searching for tests ...', setupmod.__dict__.values() for c in setupmod.__dict__.values(): - if hasattr(c,'is_a_test') and c.__name__ != 'Test': - tests[c.__name__] = c(warp) + if not hasattr(c,'is_a_test'): + continue + name = c.__name__ + if name == 'Test': + continue + if limitnames is not None and limitnames.search(name) is None: + continue + tests[name] = c(warp) l = tests.keys() l.sort() - for t in l: - print ' ',t + if verbose: + print + for t in l: + print ' ', t + print len(l), "tests found" print - def run(self): + def run(self, verbose): tests = self.tests.items() tests.sort() clock = time.clock - print 'Running %i round(s) of the suite: ' % self.rounds + print 'Running %i round(s) of the suite at warp factor %i:' % (self.rounds, self.warp) print - self.starttime = time.time() roundtime = clock() for i in range(self.rounds): - print ' Round %-25i real abs overhead' % (i+1) + roundstarttime = clock() + if verbose: + print ' Round %-25i real abs overhead' % (i+1) for j in range(len(tests)): name,t = tests[j] - print '%30s:' % name, + if verbose: + print '%30s:' % name, t.run() - print ' %.3fr %.3fa %.3fo' % t.last_timing - print ' ----------------------' - print ' Average round time: %.3f seconds' % \ - ((clock() - roundtime)/(i+1)) - print + if verbose: + print ' %.3fr %.3fa %.3fo' % t.last_timing + if verbose: + print ' ----------------------' + print ' Average round time: %.3f seconds' % \ + ((clock() - roundtime)/(i+1)) + print + else: + print '%d done in %.3f seconds' % (i+1, (clock() - roundstarttime)) self.roundtime = (clock() - roundtime) / self.rounds print def print_stat(self, compare_to=None, hidenoise=0): if not compare_to: - print '%-30s per run per oper. overhead' % 'Tests:' - print '-'*72 + print '%-30s min run avg run per oprn overhead' % 'Tests:' + print '-'*77 tests = self.tests.items() tests.sort() + totalmintime = 0 for name,t in tests: - avg,op_avg,ov_avg = t.stat() - print '%30s: %10.2f ms %7.2f us %7.2f ms' % \ - (name,avg*1000.0,op_avg*1000000.0,ov_avg*1000.0) - print '-'*72 - print '%30s: %10.2f ms' % \ - ('Average round time',self.roundtime * 1000.0) + mintime,avg,op_avg,ov_avg = t.stat() + totalmintime += mintime + print '%30s: %9.2f ms %9.2f ms %6.2f us %6.2f' % \ + (name,mintime*1000.0,avg*1000.0,op_avg*1000000.0,ov_avg*1000.0) + print '-'*77 + print '%30s: %9.2f ms' % \ + ('Notional minimum round time', totalmintime * 1000.0) else: - print '%-30s per run per oper. diff *)' % \ + print 'Comparing with: %s (rounds=%i, warp=%i)' % \ + (compare_to.name,compare_to.rounds,compare_to.warp) + print '%-30s min run cmp run avg run diff' % \ 'Tests:' - print '-'*72 + print '-'*77 tests = self.tests.items() tests.sort() compatible = 1 - for name,t in tests: - avg,op_avg,ov_avg = t.stat() + totalmintime = other_totalmintime = 0 + for name, t in tests: + mintime, avg, op_avg, ov_avg = t.stat() + totalmintime += mintime try: other = compare_to.tests[name] except KeyError: other = None if other and other.version == t.version and \ other.operations == t.operations: - avg1,op_avg1,ov_avg1 = other.stat() - qop_avg = (op_avg/op_avg1-1.0)*100.0 + mintime1, avg1, op_avg1, ov_avg1 = other.stat() + other_totalmintime += mintime1 + diff = ((mintime*self.warp)/(mintime1*other.warp) - 1.0)*100.0 if hidenoise and abs(qop_avg) < 10: - qop_avg = '' + diff = '' else: - qop_avg = '%+7.2f%%' % qop_avg + diff = '%+7.2f%%' % diff else: - qavg,qop_avg = 'n/a', 'n/a' + qavg, diff = 'n/a', 'n/a' compatible = 0 - print '%30s: %10.2f ms %7.2f us %8s' % \ - (name,avg*1000.0,op_avg*1000000.0,qop_avg) - print '-'*72 + print '%30s: %8.2f ms %8.2f ms %8.2f ms %8s' % \ + (name,mintime*1000.0,mintime1*1000.0 * compare_to.warp/self.warp, avg*1000.0,diff) + print '-'*77 + # + # Summarise test results + # if compatible and compare_to.roundtime > 0 and \ compare_to.version == self.version: - print '%30s: %10.2f ms %+7.2f%%' % \ - ('Average round time',self.roundtime * 1000.0, - ((self.roundtime*self.warp)/ - (compare_to.roundtime*compare_to.warp)-1.0)*100.0) + print '%30s: %8.2f ms %8.2f ms %+7.2f%%' % \ + ('Notional minimum round time', totalmintime * 1000.0, + other_totalmintime * 1000.0 * compare_to.warp/self.warp, + ((totalmintime*self.warp)/ + (other_totalmintime*compare_to.warp)-1.0)*100.0) else: - print '%30s: %10.2f ms n/a' % \ - ('Average round time',self.roundtime * 1000.0) - print - print '*) measured against: %s (rounds=%i, warp=%i)' % \ - (compare_to.name,compare_to.rounds,compare_to.warp) + print '%30s: %9.2f ms n/a' % \ + ('Notional minimum round time', totalmintime * 1000.0) print def print_machine(): @@ -339,7 +374,12 @@ class PyBenchCmdline(Application): SwitchOption('-S','show statistics of benchmarks',0), ArgumentOption('-w','set warp factor to arg',Setup.Warp_factor), SwitchOption('-d','hide noise in compares', 0), + SwitchOption('-v','verbose output (not recommended)', 0), SwitchOption('--no-gc','disable garbage collection', 0), + SwitchOption('--no-syscheck', + '"disable" sys check interval (set to sys.maxint)', 0), + ArgumentOption('-t', 'tests containing substring', ''), + ArgumentOption('-C', 'number of calibration runs (default 0)', '') ] about = """\ @@ -380,6 +420,11 @@ python pybench.py -s p15 -c p14 hidenoise = self.values['-d'] warp = self.values['-w'] nogc = self.values['--no-gc'] + limitnames = self.values['-t'] + verbose = self.verbose + nosyscheck = self.values['--no-syscheck'] + + print 'PYBENCH',__version__ # Switch off GC if nogc: @@ -390,8 +435,13 @@ python pybench.py -s p15 -c p14 else: if self.values['--no-gc']: gc.disable() + print 'NO GC' + + # maximise sys check interval + if nosyscheck: + sys.setcheckinterval(sys.maxint) + print 'CHECKINTERVAL =', sys.maxint - print 'PYBENCH',__version__ print if not compare_to: @@ -436,9 +486,9 @@ python pybench.py -s p15 -c p14 # Create benchmark object bench = Benchmark() bench.rounds = rounds - bench.load_tests(Setup,warp) + bench.load_tests(Setup, warp, limitnames, verbose) try: - bench.run() + bench.run(verbose) except KeyboardInterrupt: print print '*** KeyboardInterrupt -- Aborting' |