summaryrefslogtreecommitdiffstats
path: root/Tools/pybench/pybench.py
blob: 942f56d2926de9dda37b14199e2e799c78f7c2a4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
#!/usr/local/bin/python -O

""" A Python Benchmark Suite

"""
# Note: Please keep this module compatible to Python 2.6.
#
# Tests may include features in later Python versions, but these
# should then be embedded in try-except clauses in the configuration
# module Setup.py.
#

from __future__ import print_function

# pybench Copyright
__copyright__ = """\
Copyright (c), 1997-2006, Marc-Andre Lemburg (mal@lemburg.com)
Copyright (c), 2000-2006, eGenix.com Software GmbH (info@egenix.com)

                   All Rights Reserved.

Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee or royalty is hereby
granted, provided that the above copyright notice appear in all copies
and that both that copyright notice and this permission notice appear
in supporting documentation or portions thereof, including
modifications, that you make.

THE AUTHOR MARC-ANDRE LEMBURG DISCLAIMS ALL WARRANTIES WITH REGARD TO
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
WITH THE USE OR PERFORMANCE OF THIS SOFTWARE !
"""

import sys
import time
import platform
from CommandLine import *

try:
    import cPickle
    pickle = cPickle
except ImportError:
    import pickle

# Version number; version history: see README file !
__version__ = '2.1'

### Constants

# Second fractions
MILLI_SECONDS = 1e3
MICRO_SECONDS = 1e6

# Percent unit
PERCENT = 100

# Horizontal line length
LINE = 79

# Minimum test run-time
MIN_TEST_RUNTIME = 1e-3

# Number of calibration runs to use for calibrating the tests
CALIBRATION_RUNS = 20

# Number of calibration loops to run for each calibration run
CALIBRATION_LOOPS = 20

# Allow skipping calibration ?
ALLOW_SKIPPING_CALIBRATION = 1

# Timer types
TIMER_TIME_TIME = 'time.time'
TIMER_TIME_PROCESS_TIME = 'time.process_time'
TIMER_TIME_PERF_COUNTER = 'time.perf_counter'
TIMER_TIME_CLOCK = 'time.clock'
TIMER_SYSTIMES_PROCESSTIME = 'systimes.processtime'

# Choose platform default timer
if hasattr(time, 'perf_counter'):
    TIMER_PLATFORM_DEFAULT = TIMER_TIME_PERF_COUNTER
elif sys.platform[:3] == 'win':
    # On WinXP this has 2.5ms resolution
    TIMER_PLATFORM_DEFAULT = TIMER_TIME_CLOCK
else:
    # On Linux this has 1ms resolution
    TIMER_PLATFORM_DEFAULT = TIMER_TIME_TIME

# Print debug information ?
_debug = 0

### Helpers

def get_timer(timertype):

    if timertype == TIMER_TIME_TIME:
        return time.time
    elif timertype == TIMER_TIME_PROCESS_TIME:
        return time.process_time
    elif timertype == TIMER_TIME_PERF_COUNTER:
        return time.perf_counter
    elif timertype == TIMER_TIME_CLOCK:
        return time.clock
    elif timertype == TIMER_SYSTIMES_PROCESSTIME:
        import systimes
        return systimes.processtime
    else:
        raise TypeError('unknown timer type: %s' % timertype)

def get_machine_details():

    if _debug:
        print('Getting machine details...')
    buildno, builddate = platform.python_build()
    python = platform.python_version()
    # XXX this is now always UCS4, maybe replace it with 'PEP393' in 3.3+?
    if sys.maxunicode == 65535:
        # UCS2 build (standard)
        unitype = 'UCS2'
    else:
        # UCS4 build (most recent Linux distros)
        unitype = 'UCS4'
    bits, linkage = platform.architecture()
    return {
        'platform': platform.platform(),
        'processor': platform.processor(),
        'executable': sys.executable,
        'implementation': getattr(platform, 'python_implementation',
                                  lambda:'n/a')(),
        'python': platform.python_version(),
        'compiler': platform.python_compiler(),
        'buildno': buildno,
        'builddate': builddate,
        'unicode': unitype,
        'bits': bits,
        }

def print_machine_details(d, indent=''):

    l = ['Machine Details:',
         '   Platform ID:    %s' % d.get('platform', 'n/a'),
         '   Processor:      %s' % d.get('processor', 'n/a'),
         '',
         'Python:',
         '   Implementation: %s' % d.get('implementation', 'n/a'),
         '   Executable:     %s' % d.get('executable', 'n/a'),
         '   Version:        %s' % d.get('python', 'n/a'),
         '   Compiler:       %s' % d.get('compiler', 'n/a'),
         '   Bits:           %s' % d.get('bits', 'n/a'),
         '   Build:          %s (#%s)' % (d.get('builddate', 'n/a'),
                                          d.get('buildno', 'n/a')),
         '   Unicode:        %s' % d.get('unicode', 'n/a'),
         ]
    joiner = '\n' + indent
    print(indent + joiner.join(l) + '\n')

### Test baseclass

class Test:

    """ All test must have this class as baseclass. It provides
        the necessary interface to the benchmark machinery.

        The tests must set .rounds to a value high enough to let the
        test run between 20-50 seconds. This is needed because
        clock()-timing only gives rather inaccurate values (on Linux,
        for example, it is accurate to a few hundreths of a
        second). If you don't want to wait that long, use a warp
        factor larger than 1.

        It is also important to set the .operations variable to a
        value representing the number of "virtual operations" done per
        call of .run().

        If you change a test in some way, don't forget to increase
        its version number.

    """

    ### Instance variables that each test should override

    # Version number of the test as float (x.yy); this is important
    # for comparisons of benchmark runs - tests with unequal version
    # number will not get compared.
    version = 2.1

    # The number of abstract operations done in each round of the
    # test. An operation is the basic unit of what you want to
    # measure. The benchmark will output the amount of run-time per
    # operation. Note that in order to raise the measured timings
    # significantly above noise level, it is often required to repeat
    # sets of operations more than once per test round. The measured
    # overhead per test round should be less than 1 second.
    operations = 1

    # Number of rounds to execute per test run. This should be
    # adjusted to a figure that results in a test run-time of between
    # 1-2 seconds.
    rounds = 100000

    ### Internal variables

    # Mark this class as implementing a test
    is_a_test = 1

    # Last timing: (real, run, overhead)
    last_timing = (0.0, 0.0, 0.0)

    # Warp factor to use for this test
    warp = 1

    # Number of calibration runs to use
    calibration_runs = CALIBRATION_RUNS

    # List of calibration timings
    overhead_times = None

    # List of test run timings
    times = []

    # Timer used for the benchmark
    timer = TIMER_PLATFORM_DEFAULT

    def __init__(self, warp=None, calibration_runs=None, timer=None):

        # Set parameters
        if warp is not None:
            self.rounds = int(self.rounds / warp)
            if self.rounds == 0:
                raise ValueError('warp factor set too high')
            self.warp = warp
        if calibration_runs is not None:
            if (not ALLOW_SKIPPING_CALIBRATION and
                calibration_runs < 1):
                raise ValueError('at least one calibration run is required')
            self.calibration_runs = calibration_runs
        if timer is not None:
            self.timer = timer

        # Init variables
        self.times = []
        self.overhead_times = []

        # We want these to be in the instance dict, so that pickle
        # saves them
        self.version = self.version
        self.operations = self.operations
        self.rounds = self.rounds

    def get_timer(self):

        """ Return the timer function to use for the test.

        """
        return get_timer(self.timer)

    def compatible(self, other):

        """ Return 1/0 depending on whether the test is compatible
            with the other Test instance or not.

        """
        if self.version != other.version:
            return 0
        if self.rounds != other.rounds:
            return 0
        return 1

    def calibrate_test(self):

        if self.calibration_runs == 0:
            self.overhead_times = [0.0]
            return

        calibrate = self.calibrate
        timer = self.get_timer()
        calibration_loops = range(CALIBRATION_LOOPS)

        # Time the calibration loop overhead
        prep_times = []
        for i in range(self.calibration_runs):
            t = timer()
            for i in calibration_loops:
                pass
            t = timer() - t
            prep_times.append(t / CALIBRATION_LOOPS)
        min_prep_time = min(prep_times)
        if _debug:
            print()
            print('Calib. prep time     = %.6fms' % (
                min_prep_time * MILLI_SECONDS))

        # Time the calibration runs (doing CALIBRATION_LOOPS loops of
        # .calibrate() method calls each)
        for i in range(self.calibration_runs):
            t = timer()
            for i in calibration_loops:
                calibrate()
            t = timer() - t
            self.overhead_times.append(t / CALIBRATION_LOOPS
                                       - min_prep_time)

        # Check the measured times
        min_overhead = min(self.overhead_times)
        max_overhead = max(self.overhead_times)
        if _debug:
            print('Calib. overhead time = %.6fms' % (
                min_overhead * MILLI_SECONDS))
        if min_overhead < 0.0:
            raise ValueError('calibration setup did not work')
        if max_overhead - min_overhead > 0.1:
            raise ValueError(
                'overhead calibration timing range too inaccurate: '
                '%r - %r' % (min_overhead, max_overhead))

    def run(self):

        """ Run the test in two phases: first calibrate, then
            do the actual test. Be careful to keep the calibration
            timing low w/r to the test timing.

        """
        test = self.test
        timer = self.get_timer()

        # Get calibration
        min_overhead = min(self.overhead_times)

        # Test run
        t = timer()
        test()
        t = timer() - t
        if t < MIN_TEST_RUNTIME:
            raise ValueError('warp factor too high: '
                             'test times are < 10ms')
        eff_time = t - min_overhead
        if eff_time < 0:
            raise ValueError('wrong calibration')
        self.last_timing = (eff_time, t, min_overhead)
        self.times.append(eff_time)

    def calibrate(self):

        """ Calibrate the test.

            This method should execute everything that is needed to
            setup and run the test - except for the actual operations
            that you intend to measure. pybench uses this method to
            measure the test implementation overhead.

        """
        return

    def test(self):

        """ Run the test.

            The test needs to run self.rounds executing
            self.operations number of operations each.

        """
        return

    def stat(self):

        """ Return test run statistics as tuple:

            (minimum run time,
             average run time,
             total run time,
             average time per operation,
             minimum overhead time)

        """
        runs = len(self.times)
        if runs == 0:
            return 0.0, 0.0, 0.0, 0.0
        min_time = min(self.times)
        total_time = sum(self.times)
        avg_time = total_time / float(runs)
        operation_avg = total_time / float(runs
                                           * self.rounds
                                           * self.operations)
        if self.overhead_times:
            min_overhead = min(self.overhead_times)
        else:
            min_overhead = self.last_timing[2]
        return min_time, avg_time, total_time, operation_avg, min_overhead

### Load Setup

# This has to be done after the definition of the Test class, since
# the Setup module will import subclasses using this class.

import Setup

### Benchmark base class

class Benchmark:

    # Name of the benchmark
    name = ''

    # Number of benchmark rounds to run
    rounds = 1

    # Warp factor use to run the tests
    warp = 1                    # Warp factor

    # Average benchmark round time
    roundtime = 0

    # Benchmark version number as float x.yy
    version = 2.1

    # Produce verbose output ?
    verbose = 0

    # Dictionary with the machine details
    machine_details = None

    # Timer used for the benchmark
    timer = TIMER_PLATFORM_DEFAULT

    def __init__(self, name, verbose=None, timer=None, warp=None,
                 calibration_runs=None):

        if name:
            self.name = name
        else:
            self.name = '%04i-%02i-%02i %02i:%02i:%02i' % \
                        (time.localtime(time.time())[:6])
        if verbose is not None:
            self.verbose = verbose
        if timer is not None:
            self.timer = timer
        if warp is not None:
            self.warp = warp
        if calibration_runs is not None:
            self.calibration_runs = calibration_runs

        # Init vars
        self.tests = {}
        if _debug:
            print('Getting machine details...')
        self.machine_details = get_machine_details()

        # Make .version an instance attribute to have it saved in the
        # Benchmark pickle
        self.version = self.version

    def get_timer(self):

        """ Return the timer function to use for the test.

        """
        return get_timer(self.timer)

    def compatible(self, other):

        """ Return 1/0 depending on whether the benchmark is
            compatible with the other Benchmark instance or not.

        """
        if self.version != other.version:
            return 0
        if (self.machine_details == other.machine_details and
            self.timer != other.timer):
            return 0
        if (self.calibration_runs == 0 and
            other.calibration_runs != 0):
            return 0
        if (self.calibration_runs != 0 and
            other.calibration_runs == 0):
            return 0
        return 1

    def load_tests(self, setupmod, limitnames=None):

        # Add tests
        if self.verbose:
            print('Searching for tests ...')
            print('--------------------------------------')
        for testclass in setupmod.__dict__.values():
            if not hasattr(testclass, 'is_a_test'):
                continue
            name = testclass.__name__
            if  name == 'Test':
                continue
            if (limitnames is not None and
                limitnames.search(name) is None):
                continue
            self.tests[name] = testclass(
                warp=self.warp,
                calibration_runs=self.calibration_runs,
                timer=self.timer)
        l = sorted(self.tests)
        if self.verbose:
            for name in l:
                print('  %s' % name)
            print('--------------------------------------')
            print('  %i tests found' % len(l))
            print()

    def calibrate(self):

        print('Calibrating tests. Please wait...', end=' ')
        sys.stdout.flush()
        if self.verbose:
            print()
            print()
            print('Test                              min      max')
            print('-' * LINE)
        tests = sorted(self.tests.items())
        for i in range(len(tests)):
            name, test = tests[i]
            test.calibrate_test()
            if self.verbose:
                print('%30s:  %6.3fms  %6.3fms' % \
                      (name,
                       min(test.overhead_times) * MILLI_SECONDS,
                       max(test.overhead_times) * MILLI_SECONDS))
        if self.verbose:
            print()
            print('Done with the calibration.')
        else:
            print('done.')
        print()

    def run(self):

        tests = sorted(self.tests.items())
        timer = self.get_timer()
        print('Running %i round(s) of the suite at warp factor %i:' % \
              (self.rounds, self.warp))
        print()
        self.roundtimes = []
        for i in range(self.rounds):
            if self.verbose:
                print(' Round %-25i  effective   absolute  overhead' % (i+1))
            total_eff_time = 0.0
            for j in range(len(tests)):
                name, test = tests[j]
                if self.verbose:
                    print('%30s:' % name, end=' ')
                test.run()
                (eff_time, abs_time, min_overhead) = test.last_timing
                total_eff_time = total_eff_time + eff_time
                if self.verbose:
                    print('    %5.0fms    %5.0fms %7.3fms' % \
                          (eff_time * MILLI_SECONDS,
                           abs_time * MILLI_SECONDS,
                           min_overhead * MILLI_SECONDS))
            self.roundtimes.append(total_eff_time)
            if self.verbose:
                print('                   '
                       '               ------------------------------')
                print('                   '
                       '     Totals:    %6.0fms' %
                       (total_eff_time * MILLI_SECONDS))
                print()
            else:
                print('* Round %i done in %.3f seconds.' % (i+1,
                                                            total_eff_time))
        print()

    def stat(self):

        """ Return benchmark run statistics as tuple:

            (minimum round time,
             average round time,
             maximum round time)

            XXX Currently not used, since the benchmark does test
                statistics across all rounds.

        """
        runs = len(self.roundtimes)
        if runs == 0:
            return 0.0, 0.0
        min_time = min(self.roundtimes)
        total_time = sum(self.roundtimes)
        avg_time = total_time / float(runs)
        max_time = max(self.roundtimes)
        return (min_time, avg_time, max_time)

    def print_header(self, title='Benchmark'):

        print('-' * LINE)
        print('%s: %s' % (title, self.name))
        print('-' * LINE)
        print()
        print('    Rounds: %s' % self.rounds)
        print('    Warp:   %s' % self.warp)
        print('    Timer:  %s' % self.timer)
        print()
        if self.machine_details:
            print_machine_details(self.machine_details, indent='    ')
            print()

    def print_benchmark(self, hidenoise=0, limitnames=None):

        print('Test                          '
               '   minimum  average  operation  overhead')
        print('-' * LINE)
        tests = sorted(self.tests.items())
        total_min_time = 0.0
        total_avg_time = 0.0
        for name, test in tests:
            if (limitnames is not None and
                limitnames.search(name) is None):
                continue
            (min_time,
             avg_time,
             total_time,
             op_avg,
             min_overhead) = test.stat()
            total_min_time = total_min_time + min_time
            total_avg_time = total_avg_time + avg_time
            print('%30s:  %5.0fms  %5.0fms  %6.2fus  %7.3fms' % \
                  (name,
                   min_time * MILLI_SECONDS,
                   avg_time * MILLI_SECONDS,
                   op_avg * MICRO_SECONDS,
                   min_overhead *MILLI_SECONDS))
        print('-' * LINE)
        print('Totals:                        '
               ' %6.0fms %6.0fms' %
               (total_min_time * MILLI_SECONDS,
                total_avg_time * MILLI_SECONDS,
                ))
        print()

    def print_comparison(self, compare_to, hidenoise=0, limitnames=None):

        # Check benchmark versions
        if compare_to.version != self.version:
            print('* Benchmark versions differ: '
                   'cannot compare this benchmark to "%s" !' %
                   compare_to.name)
            print()
            self.print_benchmark(hidenoise=hidenoise,
                                 limitnames=limitnames)
            return

        # Print header
        compare_to.print_header('Comparing with')
        print('Test                          '
               '   minimum run-time        average  run-time')
        print('                              '
               '   this    other   diff    this    other   diff')
        print('-' * LINE)

        # Print test comparisons
        tests = sorted(self.tests.items())
        total_min_time = other_total_min_time = 0.0
        total_avg_time = other_total_avg_time = 0.0
        benchmarks_compatible = self.compatible(compare_to)
        tests_compatible = 1
        for name, test in tests:
            if (limitnames is not None and
                limitnames.search(name) is None):
                continue
            (min_time,
             avg_time,
             total_time,
             op_avg,
             min_overhead) = test.stat()
            total_min_time = total_min_time + min_time
            total_avg_time = total_avg_time + avg_time
            try:
                other = compare_to.tests[name]
            except KeyError:
                other = None
            if other is None:
                # Other benchmark doesn't include the given test
                min_diff, avg_diff = 'n/a', 'n/a'
                other_min_time = 0.0
                other_avg_time = 0.0
                tests_compatible = 0
            else:
                (other_min_time,
                 other_avg_time,
                 other_total_time,
                 other_op_avg,
                 other_min_overhead) = other.stat()
                other_total_min_time = other_total_min_time + other_min_time
                other_total_avg_time = other_total_avg_time + other_avg_time
                if (benchmarks_compatible and
                    test.compatible(other)):
                    # Both benchmark and tests are comparable
                    min_diff = ((min_time * self.warp) /
                                (other_min_time * other.warp) - 1.0)
                    avg_diff = ((avg_time * self.warp) /
                                (other_avg_time * other.warp) - 1.0)
                    if hidenoise and abs(min_diff) < 10.0:
                        min_diff = ''
                    else:
                        min_diff = '%+5.1f%%' % (min_diff * PERCENT)
                    if hidenoise and abs(avg_diff) < 10.0:
                        avg_diff = ''
                    else:
                        avg_diff = '%+5.1f%%' % (avg_diff * PERCENT)
                else:
                    # Benchmark or tests are not comparable
                    min_diff, avg_diff = 'n/a', 'n/a'
                    tests_compatible = 0
            print('%30s: %5.0fms %5.0fms %7s %5.0fms %5.0fms %7s' % \
                  (name,
                   min_time * MILLI_SECONDS,
                   other_min_time * MILLI_SECONDS * compare_to.warp / self.warp,
                   min_diff,
                   avg_time * MILLI_SECONDS,
                   other_avg_time * MILLI_SECONDS * compare_to.warp / self.warp,
                   avg_diff))
        print('-' * LINE)

        # Summarise test results
        if not benchmarks_compatible or not tests_compatible:
            min_diff, avg_diff = 'n/a', 'n/a'
        else:
            if other_total_min_time != 0.0:
                min_diff = '%+5.1f%%' % (
                    ((total_min_time * self.warp) /
                     (other_total_min_time * compare_to.warp) - 1.0) * PERCENT)
            else:
                min_diff = 'n/a'
            if other_total_avg_time != 0.0:
                avg_diff = '%+5.1f%%' % (
                    ((total_avg_time * self.warp) /
                     (other_total_avg_time * compare_to.warp) - 1.0) * PERCENT)
            else:
                avg_diff = 'n/a'
        print('Totals:                       '
               '  %5.0fms %5.0fms %7s %5.0fms %5.0fms %7s' %
               (total_min_time * MILLI_SECONDS,
                (other_total_min_time * compare_to.warp/self.warp
                 * MILLI_SECONDS),
                min_diff,
                total_avg_time * MILLI_SECONDS,
                (other_total_avg_time * compare_to.warp/self.warp
                 * MILLI_SECONDS),
                avg_diff
               ))
        print()
        print('(this=%s, other=%s)' % (self.name,
                                       compare_to.name))
        print()

class PyBenchCmdline(Application):

    header = ("PYBENCH - a benchmark test suite for Python "
              "interpreters/compilers.")

    version = __version__

    debug = _debug

    options = [ArgumentOption('-n',
                              'number of rounds',
                              Setup.Number_of_rounds),
               ArgumentOption('-f',
                              'save benchmark to file arg',
                              ''),
               ArgumentOption('-c',
                              'compare benchmark with the one in file arg',
                              ''),
               ArgumentOption('-s',
                              'show benchmark in file arg, then exit',
                              ''),
               ArgumentOption('-w',
                              'set warp factor to arg',
                              Setup.Warp_factor),
               ArgumentOption('-t',
                              'run only tests with names matching arg',
                              ''),
               ArgumentOption('-C',
                              'set the number of calibration runs to arg',
                              CALIBRATION_RUNS),
               SwitchOption('-d',
                            'hide noise in comparisons',
                            0),
               SwitchOption('-v',
                            'verbose output (not recommended)',
                            0),
               SwitchOption('--with-gc',
                            'enable garbage collection',
                            0),
               SwitchOption('--with-syscheck',
                            'use default sys check interval',
                            0),
               ArgumentOption('--timer',
                            'use given timer',
                            TIMER_PLATFORM_DEFAULT),
               ]

    about = """\
The normal operation is to run the suite and display the
results. Use -f to save them for later reuse or comparisons.

Available timers:

   time.time
   time.clock
   systimes.processtime

Examples:

python2.1 pybench.py -f p21.pybench
python2.5 pybench.py -f p25.pybench
python pybench.py -s p25.pybench -c p21.pybench
"""
    copyright = __copyright__

    def main(self):

        rounds = self.values['-n']
        reportfile = self.values['-f']
        show_bench = self.values['-s']
        compare_to = self.values['-c']
        hidenoise = self.values['-d']
        warp = int(self.values['-w'])
        withgc = self.values['--with-gc']
        limitnames = self.values['-t']
        if limitnames:
            if _debug:
                print('* limiting test names to one with substring "%s"' % \
                      limitnames)
            limitnames = re.compile(limitnames, re.I)
        else:
            limitnames = None
        verbose = self.verbose
        withsyscheck = self.values['--with-syscheck']
        calibration_runs = self.values['-C']
        timer = self.values['--timer']

        print('-' * LINE)
        print('PYBENCH %s' % __version__)
        print('-' * LINE)
        print('* using %s %s' % (
            getattr(platform, 'python_implementation', lambda:'Python')(),
            ' '.join(sys.version.split())))

        # Switch off garbage collection
        if not withgc:
            try:
                import gc
            except ImportError:
                print('* Python version doesn\'t support garbage collection')
            else:
                try:
                    gc.disable()
                except NotImplementedError:
                    print('* Python version doesn\'t support gc.disable')
                else:
                    print('* disabled garbage collection')

        # "Disable" sys check interval
        if not withsyscheck:
            # Too bad the check interval uses an int instead of a long...
            value = 2147483647
            try:
                sys.setcheckinterval(value)
            except (AttributeError, NotImplementedError):
                print('* Python version doesn\'t support sys.setcheckinterval')
            else:
                print('* system check interval set to maximum: %s' % value)

        if timer == TIMER_SYSTIMES_PROCESSTIME:
            import systimes
            print('* using timer: systimes.processtime (%s)' % \
                  systimes.SYSTIMES_IMPLEMENTATION)
        else:
            # Check that the clock function does exist
            try:
                get_timer(timer)
            except TypeError:
                print("* Error: Unknown timer: %s" % timer)
                return

            print('* using timer: %s' % timer)
            if hasattr(time, 'get_clock_info'):
                info = time.get_clock_info(timer[5:])
                print('* timer: resolution=%s, implementation=%s'
                      % (info.resolution, info.implementation))

        print()

        if compare_to:
            try:
                f = open(compare_to,'rb')
                bench = pickle.load(f)
                bench.name = compare_to
                f.close()
                compare_to = bench
            except IOError as reason:
                print('* Error opening/reading file %s: %s' % (
                    repr(compare_to),
                    reason))
                compare_to = None

        if show_bench:
            try:
                f = open(show_bench,'rb')
                bench = pickle.load(f)
                bench.name = show_bench
                f.close()
                bench.print_header()
                if compare_to:
                    bench.print_comparison(compare_to,
                                           hidenoise=hidenoise,
                                           limitnames=limitnames)
                else:
                    bench.print_benchmark(hidenoise=hidenoise,
                                          limitnames=limitnames)
            except IOError as reason:
                print('* Error opening/reading file %s: %s' % (
                    repr(show_bench),
                    reason))
                print()
            return

        if reportfile:
            print('Creating benchmark: %s (rounds=%i, warp=%i)' % \
                  (reportfile, rounds, warp))
            print()

        # Create benchmark object
        bench = Benchmark(reportfile,
                          verbose=verbose,
                          timer=timer,
                          warp=warp,
                          calibration_runs=calibration_runs)
        bench.rounds = rounds
        bench.load_tests(Setup, limitnames=limitnames)
        try:
            bench.calibrate()
            bench.run()
        except KeyboardInterrupt:
            print()
            print('*** KeyboardInterrupt -- Aborting')
            print()
            return
        bench.print_header()
        if compare_to:
            bench.print_comparison(compare_to,
                                   hidenoise=hidenoise,
                                   limitnames=limitnames)
        else:
            bench.print_benchmark(hidenoise=hidenoise,
                                  limitnames=limitnames)

        # Ring bell
        sys.stderr.write('\007')

        if reportfile:
            try:
                f = open(reportfile,'wb')
                bench.name = reportfile
                pickle.dump(bench,f)
                f.close()
            except IOError as reason:
                print('* Error opening/writing reportfile %s: %s' % (
                    reportfile,
                    reason))
                print()

if __name__ == '__main__':
    PyBenchCmdline()