diff options
Diffstat (limited to 'Tools')
| -rw-r--r-- | Tools/buildbot/Makefile | 6 | ||||
| -rw-r--r-- | Tools/buildbot/kill_python.c | 14 | ||||
| -rw-r--r-- | Tools/msi/msi.py | 18 | ||||
| -rw-r--r-- | Tools/msi/uuids.py | 1 | ||||
| -rw-r--r-- | Tools/pybench/Arithmetic.py | 16 | ||||
| -rw-r--r-- | Tools/pybench/Calls.py | 107 | ||||
| -rw-r--r-- | Tools/pybench/CommandLine.py | 2 | ||||
| -rw-r--r-- | Tools/pybench/Constructs.py | 10 | ||||
| -rw-r--r-- | Tools/pybench/Dict.py | 78 | ||||
| -rw-r--r-- | Tools/pybench/Exceptions.py | 30 | ||||
| -rw-r--r-- | Tools/pybench/Imports.py | 12 | ||||
| -rw-r--r-- | Tools/pybench/Instances.py | 4 | ||||
| -rw-r--r-- | Tools/pybench/Lists.py | 146 | ||||
| -rw-r--r-- | Tools/pybench/Lookups.py | 10 | ||||
| -rw-r--r--[-rwxr-xr-x] | Tools/pybench/NewInstances.py | 11 | ||||
| -rw-r--r-- | Tools/pybench/Numbers.py | 12 | ||||
| -rw-r--r-- | Tools/pybench/README | 371 | ||||
| -rw-r--r-- | Tools/pybench/Setup.py | 6 | ||||
| -rw-r--r-- | Tools/pybench/Strings.py | 24 | ||||
| -rw-r--r-- | Tools/pybench/Tuples.py | 16 | ||||
| -rw-r--r-- | Tools/pybench/Unicode.py | 18 | ||||
| -rw-r--r-- | Tools/pybench/clockres.py | 43 | ||||
| -rwxr-xr-x | Tools/pybench/pybench.py | 934 | ||||
| -rw-r--r-- | Tools/pybench/systimes.py | 44 | ||||
| -rw-r--r-- | Tools/scripts/README | 5 | ||||
| -rwxr-xr-x | Tools/webchecker/webchecker.py | 3 |
26 files changed, 1274 insertions, 667 deletions
diff --git a/Tools/buildbot/Makefile b/Tools/buildbot/Makefile new file mode 100644 index 0000000..1660231 --- /dev/null +++ b/Tools/buildbot/Makefile @@ -0,0 +1,6 @@ +all: kill_python.exe + ./kill_python.exe + +kill_python.exe: kill_python.c + gcc -o kill_python.exe kill_python.c -lpsapi + diff --git a/Tools/buildbot/kill_python.c b/Tools/buildbot/kill_python.c index ebc9aa4..023ff2d 100644 --- a/Tools/buildbot/kill_python.c +++ b/Tools/buildbot/kill_python.c @@ -42,7 +42,19 @@ int main() _strlwr(path); /* printf("%s\n", path); */ - if (strstr(path, "build\\pcbuild\\python_d.exe") != NULL) { + + /* Check if we are running a buildbot version of Python. + + On Windows, this will always be a debug build from the + PCbuild directory. build\\PCbuild\\python_d.exe + + On Cygwin, the pathname is similar to other Unixes. + Use \\build\\python.exe to ensure we don't match + PCbuild\\python.exe which could be a normal instance + of Python running on vanilla Windows. + */ + if ((strstr(path, "build\\pcbuild\\python_d.exe") != NULL) || + (strstr(path, "\\build\\python.exe") != NULL)) { printf("Terminating %s (pid %d)\n", path, pids[i]); if (!TerminateProcess(hProcess, 1)) { printf("Termination failed: %d\n", GetLastError()); diff --git a/Tools/msi/msi.py b/Tools/msi/msi.py index 2576380..aebab98 100644 --- a/Tools/msi/msi.py +++ b/Tools/msi/msi.py @@ -367,7 +367,7 @@ def add_ui(db): ("VerdanaRed9", "Verdana", 9, 255, 0), ]) - compileargs = r"-Wi [TARGETDIR]Lib\compileall.py -f -x bad_coding|badsyntax|site-packages [TARGETDIR]Lib" + compileargs = r'-Wi "[TARGETDIR]Lib\compileall.py" -f -x bad_coding|badsyntax|site-packages "[TARGETDIR]Lib"' # See "CustomAction Table" add_data(db, "CustomAction", [ # msidbCustomActionTypeFirstSequence + msidbCustomActionTypeTextData + msidbCustomActionTypeProperty @@ -962,6 +962,14 @@ def add_files(db): continue dlls.append(f) lib.add_file(f) + # Add sqlite + if msilib.msi_type=="Intel64;1033": + sqlite_arch = "/ia64" + elif msilib.msi_type=="x64;1033": + sqlite_arch = "/amd64" + else: + sqlite_arch = "" + lib.add_file(srcdir+"/"+sqlite_dir+sqlite_arch+"/sqlite3.dll") if have_tcl: if not os.path.exists(srcdir+"/PCBuild/_tkinter.pyd"): print "WARNING: Missing _tkinter.pyd" @@ -972,14 +980,6 @@ def add_files(db): tcldir = os.path.normpath(srcdir+"/../tcltk/bin") for f in glob.glob1(tcldir, "*.dll"): lib.add_file(f, src=os.path.join(tcldir, f)) - # Add sqlite - if msilib.msi_type=="Intel64;1033": - sqlite_arch = "/ia64" - elif msilib.msi_type=="x64;1033": - sqlite_arch = "/amd64" - else: - sqlite_arch = "" - lib.add_file(srcdir+"/"+sqlite_dir+sqlite_arch+"/sqlite3.dll") # check whether there are any unknown extensions for f in glob.glob1(srcdir+"/PCBuild", "*.pyd"): if f.endswith("_d.pyd"): continue # debug version diff --git a/Tools/msi/uuids.py b/Tools/msi/uuids.py index ce7e604..3064975 100644 --- a/Tools/msi/uuids.py +++ b/Tools/msi/uuids.py @@ -27,6 +27,7 @@ product_codes = { '2.5.103': '{73dcd966-ffec-415f-bb39-8342c1f47017}', # 2.5a3 '2.5.111': '{c797ecf8-a8e6-4fec-bb99-526b65f28626}', # 2.5b1 '2.5.112': '{32beb774-f625-439d-b587-7187487baf15}', # 2.5b2 + '2.5.113': '{89f23918-11cf-4f08-be13-b9b2e6463fd9}', # 2.5b3 '2.5.121': '{8e9321bc-6b24-48a3-8fd4-c95f8e531e5f}', # 2.5c1 '2.5.122': '{a6cd508d-9599-45da-a441-cbffa9f7e070}', # 2.5c2 '2.5.150': '{0a2c5854-557e-48c8-835a-3b9f074bdcaa}', # 2.5.0 diff --git a/Tools/pybench/Arithmetic.py b/Tools/pybench/Arithmetic.py index 4ed6219..6923b4b 100644 --- a/Tools/pybench/Arithmetic.py +++ b/Tools/pybench/Arithmetic.py @@ -2,7 +2,7 @@ from pybench import Test class SimpleIntegerArithmetic(Test): - version = 0.3 + version = 2.0 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3) rounds = 120000 @@ -157,9 +157,9 @@ class SimpleIntegerArithmetic(Test): class SimpleFloatArithmetic(Test): - version = 0.3 + version = 2.0 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3) - rounds = 100000 + rounds = 120000 def test(self): @@ -312,7 +312,7 @@ class SimpleFloatArithmetic(Test): class SimpleIntFloatArithmetic(Test): - version = 0.3 + version = 2.0 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3) rounds = 120000 @@ -468,9 +468,9 @@ class SimpleIntFloatArithmetic(Test): class SimpleLongArithmetic(Test): - version = 0.3 + version = 2.0 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3) - rounds = 30000 + rounds = 60000 def test(self): @@ -623,9 +623,9 @@ class SimpleLongArithmetic(Test): class SimpleComplexArithmetic(Test): - version = 0.3 + version = 2.0 operations = 5 * (3 + 5 + 5 + 3 + 3 + 3) - rounds = 40000 + rounds = 80000 def test(self): diff --git a/Tools/pybench/Calls.py b/Tools/pybench/Calls.py index e295243..72ccd0e 100644 --- a/Tools/pybench/Calls.py +++ b/Tools/pybench/Calls.py @@ -2,7 +2,7 @@ from pybench import Test class PythonFunctionCalls(Test): - version = 0.3 + version = 2.0 operations = 5*(1+4+4+2) rounds = 60000 @@ -111,9 +111,9 @@ class PythonFunctionCalls(Test): class BuiltinFunctionCalls(Test): - version = 0.4 + version = 2.0 operations = 5*(2+5+5+5) - rounds = 30000 + rounds = 60000 def test(self): @@ -232,9 +232,9 @@ class BuiltinFunctionCalls(Test): class PythonMethodCalls(Test): - version = 0.3 + version = 2.0 operations = 5*(6 + 5 + 4) - rounds = 20000 + rounds = 30000 def test(self): @@ -374,9 +374,9 @@ class PythonMethodCalls(Test): class Recursion(Test): - version = 0.3 + version = 2.0 operations = 5 - rounds = 50000 + rounds = 100000 def test(self): @@ -407,3 +407,96 @@ class Recursion(Test): for i in xrange(self.rounds): pass + + +### Test to make Fredrik happy... + +if __name__ == '__main__': + import timeit + if 0: + timeit.TestClass = PythonFunctionCalls + timeit.main(['-s', 'test = TestClass(); test.rounds = 1000', + 'test.test()']) + else: + setup = """\ +global f,f1,g,h + +# define functions +def f(): + pass + +def f1(x): + pass + +def g(a,b,c): + return a,b,c + +def h(a,b,c,d=1,e=2,f=3): + return d,e,f + +i = 1 +""" + test = """\ +f() +f1(i) +f1(i) +f1(i) +f1(i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +h(i,i,3,i,i) +h(i,i,i,2,i,3) + +f() +f1(i) +f1(i) +f1(i) +f1(i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +h(i,i,3,i,i) +h(i,i,i,2,i,3) + +f() +f1(i) +f1(i) +f1(i) +f1(i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +h(i,i,3,i,i) +h(i,i,i,2,i,3) + +f() +f1(i) +f1(i) +f1(i) +f1(i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +h(i,i,3,i,i) +h(i,i,i,2,i,3) + +f() +f1(i) +f1(i) +f1(i) +f1(i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +g(i,i,i) +h(i,i,3,i,i) +h(i,i,i,2,i,3) +""" + + timeit.main(['-s', setup, + test]) diff --git a/Tools/pybench/CommandLine.py b/Tools/pybench/CommandLine.py index 13e4f9b..6601be5 100644 --- a/Tools/pybench/CommandLine.py +++ b/Tools/pybench/CommandLine.py @@ -358,7 +358,7 @@ class Application: except self.InternalError: print - print '* Internal Error' + print '* Internal Error (use --debug to display the traceback)' if self.debug: print traceback.print_exc(20, sys.stdout) diff --git a/Tools/pybench/Constructs.py b/Tools/pybench/Constructs.py index 00045bd..5105461 100644 --- a/Tools/pybench/Constructs.py +++ b/Tools/pybench/Constructs.py @@ -2,7 +2,7 @@ from pybench import Test class IfThenElse(Test): - version = 0.31 + version = 2.0 operations = 30*3 # hard to say... rounds = 150000 @@ -469,9 +469,9 @@ class IfThenElse(Test): class NestedForLoops(Test): - version = 0.3 + version = 2.0 operations = 1000*10*5 - rounds = 150 + rounds = 300 def test(self): @@ -494,9 +494,9 @@ class NestedForLoops(Test): class ForLoops(Test): - version = 0.1 + version = 2.0 operations = 5 * 5 - rounds = 8000 + rounds = 10000 def test(self): diff --git a/Tools/pybench/Dict.py b/Tools/pybench/Dict.py index 54aeae7..9cdd682 100644 --- a/Tools/pybench/Dict.py +++ b/Tools/pybench/Dict.py @@ -2,9 +2,9 @@ from pybench import Test class DictCreation(Test): - version = 0.3 + version = 2.0 operations = 5*(5 + 5) - rounds = 60000 + rounds = 80000 def test(self): @@ -77,7 +77,7 @@ class DictCreation(Test): class DictWithStringKeys(Test): - version = 0.1 + version = 2.0 operations = 5*(6 + 6) rounds = 200000 @@ -166,9 +166,9 @@ class DictWithStringKeys(Test): class DictWithFloatKeys(Test): - version = 0.1 + version = 2.0 operations = 5*(6 + 6) - rounds = 200000 + rounds = 150000 def test(self): @@ -255,7 +255,7 @@ class DictWithFloatKeys(Test): class DictWithIntegerKeys(Test): - version = 0.1 + version = 2.0 operations = 5*(6 + 6) rounds = 200000 @@ -344,13 +344,14 @@ class DictWithIntegerKeys(Test): class SimpleDictManipulation(Test): - version = 0.3 + version = 2.0 operations = 5*(6 + 6 + 6 + 6) - rounds = 50000 + rounds = 100000 def test(self): d = {} + has_key = d.has_key for i in xrange(self.rounds): @@ -368,12 +369,12 @@ class SimpleDictManipulation(Test): x = d[4] x = d[5] - d.has_key(0) - d.has_key(2) - d.has_key(4) - d.has_key(6) - d.has_key(8) - d.has_key(10) + has_key(0) + has_key(2) + has_key(4) + has_key(6) + has_key(8) + has_key(10) del d[0] del d[1] @@ -396,12 +397,12 @@ class SimpleDictManipulation(Test): x = d[4] x = d[5] - d.has_key(0) - d.has_key(2) - d.has_key(4) - d.has_key(6) - d.has_key(8) - d.has_key(10) + has_key(0) + has_key(2) + has_key(4) + has_key(6) + has_key(8) + has_key(10) del d[0] del d[1] @@ -424,12 +425,12 @@ class SimpleDictManipulation(Test): x = d[4] x = d[5] - d.has_key(0) - d.has_key(2) - d.has_key(4) - d.has_key(6) - d.has_key(8) - d.has_key(10) + has_key(0) + has_key(2) + has_key(4) + has_key(6) + has_key(8) + has_key(10) del d[0] del d[1] @@ -452,12 +453,12 @@ class SimpleDictManipulation(Test): x = d[4] x = d[5] - d.has_key(0) - d.has_key(2) - d.has_key(4) - d.has_key(6) - d.has_key(8) - d.has_key(10) + has_key(0) + has_key(2) + has_key(4) + has_key(6) + has_key(8) + has_key(10) del d[0] del d[1] @@ -480,12 +481,12 @@ class SimpleDictManipulation(Test): x = d[4] x = d[5] - d.has_key(0) - d.has_key(2) - d.has_key(4) - d.has_key(6) - d.has_key(8) - d.has_key(10) + has_key(0) + has_key(2) + has_key(4) + has_key(6) + has_key(8) + has_key(10) del d[0] del d[1] @@ -497,6 +498,7 @@ class SimpleDictManipulation(Test): def calibrate(self): d = {} + has_key = d.has_key for i in xrange(self.rounds): pass diff --git a/Tools/pybench/Exceptions.py b/Tools/pybench/Exceptions.py index 7e55708..eff69c7 100644 --- a/Tools/pybench/Exceptions.py +++ b/Tools/pybench/Exceptions.py @@ -2,9 +2,9 @@ from pybench import Test class TryRaiseExcept(Test): - version = 0.1 - operations = 2 + 3 - rounds = 60000 + version = 2.0 + operations = 2 + 3 + 3 + rounds = 80000 def test(self): @@ -31,6 +31,18 @@ class TryRaiseExcept(Test): raise error,"something" except: pass + try: + raise error("something") + except: + pass + try: + raise error("something") + except: + pass + try: + raise error("something") + except: + pass def calibrate(self): @@ -42,9 +54,9 @@ class TryRaiseExcept(Test): class TryExcept(Test): - version = 0.1 + version = 2.0 operations = 15 * 10 - rounds = 200000 + rounds = 150000 def test(self): @@ -677,3 +689,11 @@ class TryExcept(Test): for i in xrange(self.rounds): pass + +### Test to make Fredrik happy... + +if __name__ == '__main__': + import timeit + timeit.TestClass = TryRaiseExcept + timeit.main(['-s', 'test = TestClass(); test.rounds = 1000', + 'test.test()']) diff --git a/Tools/pybench/Imports.py b/Tools/pybench/Imports.py index 85eb604..afc728b 100644 --- a/Tools/pybench/Imports.py +++ b/Tools/pybench/Imports.py @@ -6,9 +6,9 @@ import package.submodule class SecondImport(Test): - version = 0.1 + version = 2.0 operations = 5 * 5 - rounds = 20000 + rounds = 40000 def test(self): @@ -51,9 +51,9 @@ class SecondImport(Test): class SecondPackageImport(Test): - version = 0.1 + version = 2.0 operations = 5 * 5 - rounds = 20000 + rounds = 40000 def test(self): @@ -95,9 +95,9 @@ class SecondPackageImport(Test): class SecondSubmoduleImport(Test): - version = 0.1 + version = 2.0 operations = 5 * 5 - rounds = 20000 + rounds = 40000 def test(self): diff --git a/Tools/pybench/Instances.py b/Tools/pybench/Instances.py index 9b1929d..1dfc82f 100644 --- a/Tools/pybench/Instances.py +++ b/Tools/pybench/Instances.py @@ -2,9 +2,9 @@ from pybench import Test class CreateInstances(Test): - version = 0.2 + version = 2.0 operations = 3 + 7 + 4 - rounds = 60000 + rounds = 80000 def test(self): diff --git a/Tools/pybench/Lists.py b/Tools/pybench/Lists.py index 4c18e99..67760db 100644 --- a/Tools/pybench/Lists.py +++ b/Tools/pybench/Lists.py @@ -2,22 +2,23 @@ from pybench import Test class SimpleListManipulation(Test): - version = 0.3 + version = 2.0 operations = 5* (6 + 6 + 6) - rounds = 60000 + rounds = 130000 def test(self): l = [] + append = l.append for i in xrange(self.rounds): - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -33,12 +34,12 @@ class SimpleListManipulation(Test): x = l[4] x = l[5] - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -54,12 +55,12 @@ class SimpleListManipulation(Test): x = l[4] x = l[5] - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -75,12 +76,12 @@ class SimpleListManipulation(Test): x = l[4] x = l[5] - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -96,12 +97,12 @@ class SimpleListManipulation(Test): x = l[4] x = l[5] - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -124,15 +125,16 @@ class SimpleListManipulation(Test): def calibrate(self): l = [] + append = l.append for i in xrange(self.rounds): pass class ListSlicing(Test): - version = 0.4 + version = 2.0 operations = 25*(3+1+2+1) - rounds = 400 + rounds = 800 def test(self): @@ -141,7 +143,7 @@ class ListSlicing(Test): for i in xrange(self.rounds): - l = range(100) + l = n[:] for j in r: @@ -159,17 +161,14 @@ class ListSlicing(Test): r = range(25) for i in xrange(self.rounds): - - l = range(100) - for j in r: pass class SmallLists(Test): - version = 0.3 + version = 2.0 operations = 5*(1+ 6 + 6 + 3 + 1) - rounds = 60000 + rounds = 80000 def test(self): @@ -177,12 +176,13 @@ class SmallLists(Test): l = [] - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append = l.append + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -199,12 +199,13 @@ class SmallLists(Test): l = [] - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append = l.append + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -221,12 +222,13 @@ class SmallLists(Test): l = [] - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append = l.append + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -243,12 +245,13 @@ class SmallLists(Test): l = [] - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append = l.append + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -265,12 +268,13 @@ class SmallLists(Test): l = [] - l.append(2) - l.append(3) - l.append(4) - l.append(2) - l.append(3) - l.append(4) + append = l.append + append(2) + append(3) + append(4) + append(2) + append(3) + append(4) l[0] = 3 l[1] = 4 @@ -288,4 +292,4 @@ class SmallLists(Test): def calibrate(self): for i in xrange(self.rounds): - l = [] + pass diff --git a/Tools/pybench/Lookups.py b/Tools/pybench/Lookups.py index e5529cd..f20e7da 100644 --- a/Tools/pybench/Lookups.py +++ b/Tools/pybench/Lookups.py @@ -2,7 +2,7 @@ from pybench import Test class SpecialClassAttribute(Test): - version = 0.3 + version = 2.0 operations = 5*(12 + 12) rounds = 100000 @@ -183,7 +183,7 @@ class SpecialClassAttribute(Test): class NormalClassAttribute(Test): - version = 0.3 + version = 2.0 operations = 5*(12 + 12) rounds = 100000 @@ -369,7 +369,7 @@ class NormalClassAttribute(Test): class SpecialInstanceAttribute(Test): - version = 0.3 + version = 2.0 operations = 5*(12 + 12) rounds = 100000 @@ -557,7 +557,7 @@ class SpecialInstanceAttribute(Test): class NormalInstanceAttribute(Test): - version = 0.3 + version = 2.0 operations = 5*(12 + 12) rounds = 100000 @@ -745,7 +745,7 @@ class NormalInstanceAttribute(Test): class BuiltinMethodLookup(Test): - version = 0.3 + version = 2.0 operations = 5*(3*5 + 3*5) rounds = 70000 diff --git a/Tools/pybench/NewInstances.py b/Tools/pybench/NewInstances.py index a352638..258beba 100755..100644 --- a/Tools/pybench/NewInstances.py +++ b/Tools/pybench/NewInstances.py @@ -1,8 +1,17 @@ from pybench import Test +# Check for new-style class support: +try: + class c(object): + pass +except NameError: + raise ImportError + +### + class CreateNewInstances(Test): - version = 0.1 + version = 2.0 operations = 3 + 7 + 4 rounds = 60000 diff --git a/Tools/pybench/Numbers.py b/Tools/pybench/Numbers.py index a6aea33..10c8940 100644 --- a/Tools/pybench/Numbers.py +++ b/Tools/pybench/Numbers.py @@ -2,7 +2,7 @@ from pybench import Test class CompareIntegers(Test): - version = 0.1 + version = 2.0 operations = 30 * 5 rounds = 120000 @@ -198,9 +198,9 @@ class CompareIntegers(Test): class CompareFloats(Test): - version = 0.1 + version = 2.0 operations = 30 * 5 - rounds = 60000 + rounds = 80000 def test(self): @@ -394,7 +394,7 @@ class CompareFloats(Test): class CompareFloatsIntegers(Test): - version = 0.1 + version = 2.0 operations = 30 * 5 rounds = 60000 @@ -590,9 +590,9 @@ class CompareFloatsIntegers(Test): class CompareLongs(Test): - version = 0.1 + version = 2.0 operations = 30 * 5 - rounds = 60000 + rounds = 70000 def test(self): diff --git a/Tools/pybench/README b/Tools/pybench/README index 95ae392..022c8de 100644 --- a/Tools/pybench/README +++ b/Tools/pybench/README @@ -28,12 +28,37 @@ and then print out a report to stdout. Micro-Manual ------------ -Run 'pybench.py -h' to see the help screen. -Run 'pybench.py' to just let the benchmark suite do it's thing and -'pybench.py -f <file>' to have it store the results in a file too. +Run 'pybench.py -h' to see the help screen. Run 'pybench.py' to run +the benchmark suite using default settings and 'pybench.py -f <file>' +to have it store the results in a file too. + +It is usually a good idea to run pybench.py multiple times to see +whether the environment, timers and benchmark run-times are suitable +for doing benchmark tests. + +You can use the comparison feature of pybench.py ('pybench.py -c +<file>') to check how well the system behaves in comparison to a +reference run. + +If the differences are well below 10% for each test, then you have a +system that is good for doing benchmark testings. Of you get random +differences of more than 10% or significant differences between the +values for minimum and average time, then you likely have some +background processes running which cause the readings to become +inconsistent. Examples include: web-browsers, email clients, RSS +readers, music players, backup programs, etc. + +If you are only interested in a few tests of the whole suite, you can +use the filtering option, e.g. 'pybench.py -t string' will only +run/show the tests that have 'string' in their name. This is the current output of pybench.py --help: +""" +------------------------------------------------------------------------ +PYBENCH - a benchmark test suite for Python interpreters/compilers. +------------------------------------------------------------------------ + Synopsis: pybench.py [option] files... @@ -42,14 +67,14 @@ Options and default settings: -f arg save benchmark to file arg () -c arg compare benchmark with the one in file arg () -s arg show benchmark in file arg, then exit () - -S show statistics of benchmarks (0) - -w arg set warp factor to arg (20) - -d hide noise in compares (0) - --no-gc disable garbage collection (0) - --no-syscheck "disable" sys check interval (set to sys.maxint) (0) - -t arg tests containing substring () - -C arg number of calibration runs (20) - -v generate verbose output + -w arg set warp factor to arg (10) + -t arg run only tests with names matching arg () + -C arg set the number of calibration runs to arg (20) + -d hide noise in comparisons (0) + -v verbose output (not recommended) (0) + --with-gc enable garbage collection (0) + --with-syscheck use default sys check interval (0) + --timer arg use given timer (time.time) -h show this help text --help show this help text --debug enable debugging @@ -57,17 +82,23 @@ Options and default settings: --examples show examples of usage Version: - 1.3 + 2.0 The normal operation is to run the suite and display the -results. Use -f to save them for later reuse or comparisms. +results. Use -f to save them for later reuse or comparisons. -Examples: +Available timers: -python1.5 pybench.py -w 100 -f p15 -python1.4 pybench.py -w 100 -f p14 -python pybench.py -s p15 -c p14 + time.time + time.clock + systimes.processtime +Examples: + +python2.1 pybench.py -f p21.pybench +python2.5 pybench.py -f p25.pybench +python pybench.py -s p25.pybench -c p21.pybench +""" License ------- @@ -78,184 +109,103 @@ See LICENSE file. Sample output ------------- -PYBENCH 1.3 - -Machine Details: - Platform ID: Linux-2.6.8-24.19-default-x86_64-with-SuSE-9.2-x86-64 - Executable: /home/lemburg/projects/Python/Installation/bin/python - Python: 2.5a1.0 - Compiler: GCC 3.3.4 (pre 3.3.5 20040809) - Build: Apr 9 2006 01:50:57 (#trunk) - -Searching for tests... - BuiltinFunctionCalls - BuiltinMethodLookup - CompareFloats - CompareFloatsIntegers - CompareIntegers - CompareInternedStrings - CompareLongs - CompareStrings - CompareUnicode - ConcatStrings - ConcatUnicode - CreateInstances - CreateStringsWithConcat - CreateUnicodeWithConcat - DictCreation - DictWithFloatKeys - DictWithIntegerKeys - DictWithStringKeys - ForLoops - IfThenElse - ListSlicing - NestedForLoops - NormalClassAttribute - NormalInstanceAttribute - PythonFunctionCalls - PythonMethodCalls - Recursion - SecondImport - SecondPackageImport - SecondSubmoduleImport - SimpleComplexArithmetic - SimpleDictManipulation - SimpleFloatArithmetic - SimpleIntFloatArithmetic - SimpleIntegerArithmetic - SimpleListManipulation - SimpleLongArithmetic - SmallLists - SmallTuples - SpecialClassAttribute - SpecialInstanceAttribute - StringMappings - StringPredicates - StringSlicing - TryExcept - TryRaiseExcept - TupleSlicing - UnicodeMappings - UnicodePredicates - UnicodeProperties - UnicodeSlicing - -Running 10 round(s) of the suite: +""" +------------------------------------------------------------------------------- +PYBENCH 2.0 +------------------------------------------------------------------------------- +* using Python 2.4.2 +* disabled garbage collection +* system check interval set to maximum: 2147483647 +* using timer: time.time -... +Calibrating tests. Please wait... - Round 10 real abs overhead - BuiltinFunctionCalls: 0.030r 0.030a 0.000o - BuiltinMethodLookup: 0.059r 0.060a 0.001o - CompareFloats: 0.050r 0.050a 0.000o - CompareFloatsIntegers: 0.050r 0.050a 0.000o - CompareIntegers: 0.070r 0.070a 0.000o - CompareInternedStrings: 0.039r 0.040a 0.001o - CompareLongs: 0.050r 0.050a 0.000o - CompareStrings: 0.060r 0.060a 0.000o - CompareUnicode: 0.060r 0.060a 0.000o - ConcatStrings: 0.040r 0.040a 0.000o - ConcatUnicode: 0.050r 0.050a 0.000o - CreateInstances: 0.050r 0.050a 0.000o - CreateStringsWithConcat: 0.029r 0.030a 0.001o - CreateUnicodeWithConcat: 0.060r 0.060a 0.000o - DictCreation: 0.040r 0.040a 0.000o - DictWithFloatKeys: 0.089r 0.090a 0.000o - DictWithIntegerKeys: 0.059r 0.060a 0.001o - DictWithStringKeys: 0.070r 0.070a 0.001o - ForLoops: 0.050r 0.050a 0.000o - IfThenElse: 0.070r 0.070a 0.000o - ListSlicing: 0.030r 0.030a 0.000o - NestedForLoops: 0.030r 0.030a 0.000o - NormalClassAttribute: 0.060r 0.060a 0.000o - NormalInstanceAttribute: 0.060r 0.060a 0.000o - PythonFunctionCalls: 0.060r 0.060a 0.000o - PythonMethodCalls: 0.050r 0.050a 0.000o - Recursion: 0.050r 0.050a 0.000o - SecondImport: 0.030r 0.030a 0.000o - SecondPackageImport: 0.030r 0.030a 0.000o - SecondSubmoduleImport: 0.040r 0.040a 0.000o - SimpleComplexArithmetic: 0.030r 0.030a 0.000o - SimpleDictManipulation: 0.040r 0.040a 0.000o - SimpleFloatArithmetic: 0.050r 0.050a 0.001o - SimpleIntFloatArithmetic: 0.060r 0.060a 0.000o - SimpleIntegerArithmetic: 0.060r 0.060a 0.000o - SimpleListManipulation: 0.030r 0.030a 0.000o - SimpleLongArithmetic: 0.030r 0.030a 0.000o - SmallLists: 0.050r 0.050a 0.000o - SmallTuples: 0.050r 0.050a 0.000o - SpecialClassAttribute: 0.060r 0.060a 0.000o - SpecialInstanceAttribute: 0.079r 0.080a 0.001o - StringMappings: 0.060r 0.060a 0.000o - StringPredicates: 0.049r 0.050a 0.001o - StringSlicing: 0.039r 0.040a 0.000o - TryExcept: 0.079r 0.080a 0.001o - TryRaiseExcept: 0.059r 0.060a 0.001o - TupleSlicing: 0.050r 0.050a 0.000o - UnicodeMappings: 0.070r 0.070a 0.001o - UnicodePredicates: 0.059r 0.060a 0.001o - UnicodeProperties: 0.059r 0.060a 0.001o - UnicodeSlicing: 0.050r 0.050a 0.000o - ---------------------- - Average round time: 2.937 seconds - - -Tests: per run per oper. overhead ------------------------------------------------------------------------- - BuiltinFunctionCalls: 29.85 ms 0.23 us 0.00 ms - BuiltinMethodLookup: 66.85 ms 0.13 us 0.50 ms - CompareFloats: 43.00 ms 0.10 us 0.00 ms - CompareFloatsIntegers: 51.80 ms 0.12 us 0.00 ms - CompareIntegers: 70.70 ms 0.08 us 0.50 ms - CompareInternedStrings: 41.40 ms 0.08 us 0.50 ms - CompareLongs: 47.90 ms 0.11 us 0.00 ms - CompareStrings: 58.50 ms 0.12 us 0.50 ms - CompareUnicode: 56.55 ms 0.15 us 0.50 ms - ConcatStrings: 44.75 ms 0.30 us 0.00 ms - ConcatUnicode: 54.55 ms 0.36 us 0.50 ms - CreateInstances: 50.95 ms 1.21 us 0.00 ms - CreateStringsWithConcat: 28.85 ms 0.14 us 0.50 ms - CreateUnicodeWithConcat: 53.75 ms 0.27 us 0.00 ms - DictCreation: 41.90 ms 0.28 us 0.00 ms - DictWithFloatKeys: 88.50 ms 0.15 us 0.50 ms - DictWithIntegerKeys: 62.55 ms 0.10 us 0.50 ms - DictWithStringKeys: 60.50 ms 0.10 us 0.50 ms - ForLoops: 46.90 ms 4.69 us 0.00 ms - IfThenElse: 60.55 ms 0.09 us 0.00 ms - ListSlicing: 29.90 ms 8.54 us 0.00 ms - NestedForLoops: 33.95 ms 0.10 us 0.00 ms - NormalClassAttribute: 62.75 ms 0.10 us 0.50 ms - NormalInstanceAttribute: 61.80 ms 0.10 us 0.50 ms - PythonFunctionCalls: 60.00 ms 0.36 us 0.00 ms - PythonMethodCalls: 50.00 ms 0.67 us 0.00 ms - Recursion: 46.85 ms 3.75 us 0.00 ms - SecondImport: 35.00 ms 1.40 us 0.00 ms - SecondPackageImport: 32.00 ms 1.28 us 0.00 ms - SecondSubmoduleImport: 38.00 ms 1.52 us 0.00 ms - SimpleComplexArithmetic: 26.85 ms 0.12 us 0.00 ms - SimpleDictManipulation: 40.85 ms 0.14 us 0.00 ms - SimpleFloatArithmetic: 48.70 ms 0.09 us 0.50 ms - SimpleIntFloatArithmetic: 57.70 ms 0.09 us 0.00 ms - SimpleIntegerArithmetic: 58.75 ms 0.09 us 0.50 ms - SimpleListManipulation: 34.80 ms 0.13 us 0.00 ms - SimpleLongArithmetic: 30.95 ms 0.19 us 0.50 ms - SmallLists: 47.60 ms 0.19 us 0.00 ms - SmallTuples: 48.80 ms 0.20 us 0.50 ms - SpecialClassAttribute: 61.70 ms 0.10 us 0.00 ms - SpecialInstanceAttribute: 76.70 ms 0.13 us 0.50 ms - StringMappings: 58.70 ms 0.47 us 0.00 ms - StringPredicates: 50.00 ms 0.18 us 1.00 ms - StringSlicing: 39.65 ms 0.23 us 0.50 ms - TryExcept: 84.45 ms 0.06 us 0.50 ms - TryRaiseExcept: 61.75 ms 4.12 us 0.50 ms - TupleSlicing: 48.95 ms 0.47 us 0.00 ms - UnicodeMappings: 71.50 ms 3.97 us 0.50 ms - UnicodePredicates: 52.75 ms 0.23 us 1.00 ms - UnicodeProperties: 61.90 ms 0.31 us 1.00 ms - UnicodeSlicing: 53.75 ms 0.31 us 0.50 ms ------------------------------------------------------------------------- - Average round time: 2937.00 ms +Running 10 round(s) of the suite at warp factor 10: +* Round 1 done in 6.388 seconds. +* Round 2 done in 6.485 seconds. +* Round 3 done in 6.786 seconds. +... +* Round 10 done in 6.546 seconds. + +------------------------------------------------------------------------------- +Benchmark: 2006-06-12 12:09:25 +------------------------------------------------------------------------------- + + Rounds: 10 + Warp: 10 + Timer: time.time + + Machine Details: + Platform ID: Linux-2.6.8-24.19-default-x86_64-with-SuSE-9.2-x86-64 + Processor: x86_64 + + Python: + Executable: /usr/local/bin/python + Version: 2.4.2 + Compiler: GCC 3.3.4 (pre 3.3.5 20040809) + Bits: 64bit + Build: Oct 1 2005 15:24:35 (#1) + Unicode: UCS2 + + +Test minimum average operation overhead +------------------------------------------------------------------------------- + BuiltinFunctionCalls: 126ms 145ms 0.28us 0.274ms + BuiltinMethodLookup: 124ms 130ms 0.12us 0.316ms + CompareFloats: 109ms 110ms 0.09us 0.361ms + CompareFloatsIntegers: 100ms 104ms 0.12us 0.271ms + CompareIntegers: 137ms 138ms 0.08us 0.542ms + CompareInternedStrings: 124ms 127ms 0.08us 1.367ms + CompareLongs: 100ms 104ms 0.10us 0.316ms + CompareStrings: 111ms 115ms 0.12us 0.929ms + CompareUnicode: 108ms 128ms 0.17us 0.693ms + ConcatStrings: 142ms 155ms 0.31us 0.562ms + ConcatUnicode: 119ms 127ms 0.42us 0.384ms + CreateInstances: 123ms 128ms 1.14us 0.367ms + CreateNewInstances: 121ms 126ms 1.49us 0.335ms + CreateStringsWithConcat: 130ms 135ms 0.14us 0.916ms + CreateUnicodeWithConcat: 130ms 135ms 0.34us 0.361ms + DictCreation: 108ms 109ms 0.27us 0.361ms + DictWithFloatKeys: 149ms 153ms 0.17us 0.678ms + DictWithIntegerKeys: 124ms 126ms 0.11us 0.915ms + DictWithStringKeys: 114ms 117ms 0.10us 0.905ms + ForLoops: 110ms 111ms 4.46us 0.063ms + IfThenElse: 118ms 119ms 0.09us 0.685ms + ListSlicing: 116ms 120ms 8.59us 0.103ms + NestedForLoops: 125ms 137ms 0.09us 0.019ms + NormalClassAttribute: 124ms 136ms 0.11us 0.457ms + NormalInstanceAttribute: 110ms 117ms 0.10us 0.454ms + PythonFunctionCalls: 107ms 113ms 0.34us 0.271ms + PythonMethodCalls: 140ms 149ms 0.66us 0.141ms + Recursion: 156ms 166ms 3.32us 0.452ms + SecondImport: 112ms 118ms 1.18us 0.180ms + SecondPackageImport: 118ms 127ms 1.27us 0.180ms + SecondSubmoduleImport: 140ms 151ms 1.51us 0.180ms + SimpleComplexArithmetic: 128ms 139ms 0.16us 0.361ms + SimpleDictManipulation: 134ms 136ms 0.11us 0.452ms + SimpleFloatArithmetic: 110ms 113ms 0.09us 0.571ms + SimpleIntFloatArithmetic: 106ms 111ms 0.08us 0.548ms + SimpleIntegerArithmetic: 106ms 109ms 0.08us 0.544ms + SimpleListManipulation: 103ms 113ms 0.10us 0.587ms + SimpleLongArithmetic: 112ms 118ms 0.18us 0.271ms + SmallLists: 105ms 116ms 0.17us 0.366ms + SmallTuples: 108ms 128ms 0.24us 0.406ms + SpecialClassAttribute: 119ms 136ms 0.11us 0.453ms + SpecialInstanceAttribute: 143ms 155ms 0.13us 0.454ms + StringMappings: 115ms 121ms 0.48us 0.405ms + StringPredicates: 120ms 129ms 0.18us 2.064ms + StringSlicing: 111ms 127ms 0.23us 0.781ms + TryExcept: 125ms 126ms 0.06us 0.681ms + TryRaiseExcept: 133ms 137ms 2.14us 0.361ms + TupleSlicing: 117ms 120ms 0.46us 0.066ms + UnicodeMappings: 156ms 160ms 4.44us 0.429ms + UnicodePredicates: 117ms 121ms 0.22us 2.487ms + UnicodeProperties: 115ms 153ms 0.38us 2.070ms + UnicodeSlicing: 126ms 129ms 0.26us 0.689ms +------------------------------------------------------------------------------- +Totals: 6283ms 6673ms +""" ________________________________________________________________________ Writing New Tests @@ -293,7 +243,7 @@ class IntegerCounting(Test): # Number of rounds to execute per test run. This should be # adjusted to a figure that results in a test run-time of between - # 20-50 seconds. + # 1-2 seconds (at warp 1). rounds = 100000 def test(self): @@ -377,6 +327,41 @@ longer strictly comparable with previous runs, the '.version' class variable should be updated. Therefafter, comparisons with previous versions of the test will list as "n/a" to reflect the change. + +Version History +--------------- + + 2.0: rewrote parts of pybench which resulted in more repeatable + timings: + - made timer a parameter + - changed the platform default timer to use high-resolution + timers rather than process timers (which have a much lower + resolution) + - added option to select timer + - added process time timer (using systimes.py) + - changed to use min() as timing estimator (average + is still taken as well to provide an idea of the difference) + - garbage collection is turned off per default + - sys check interval is set to the highest possible value + - calibration is now a separate step and done using + a different strategy that allows measuring the test + overhead more accurately + - modified the tests to each give a run-time of between + 100-200ms using warp 10 + - changed default warp factor to 10 (from 20) + - compared results with timeit.py and confirmed measurements + - bumped all test versions to 2.0 + - updated platform.py to the latest version + - changed the output format a bit to make it look + nicer + - refactored the APIs somewhat + 1.3+: Steve Holden added the NewInstances test and the filtering + option during the NeedForSpeed sprint; this also triggered a long + discussion on how to improve benchmark timing and finally + resulted in the release of 2.0 + 1.3: initial checkin into the Python SVN repository + + Have fun, -- Marc-Andre Lemburg diff --git a/Tools/pybench/Setup.py b/Tools/pybench/Setup.py index f5c5190..f1417e6 100644 --- a/Tools/pybench/Setup.py +++ b/Tools/pybench/Setup.py @@ -14,7 +14,7 @@ # Defaults Number_of_rounds = 10 -Warp_factor = 20 +Warp_factor = 10 # Import tests from Arithmetic import * @@ -24,8 +24,8 @@ from Lookups import * from Instances import * try: from NewInstances import * -except: - print "Cannot test new-style objects" +except ImportError: + pass from Lists import * from Tuples import * from Dict import * diff --git a/Tools/pybench/Strings.py b/Tools/pybench/Strings.py index b01843a..3be8b35 100644 --- a/Tools/pybench/Strings.py +++ b/Tools/pybench/Strings.py @@ -3,9 +3,9 @@ from string import join class ConcatStrings(Test): - version = 0.1 + version = 2.0 operations = 10 * 5 - rounds = 60000 + rounds = 100000 def test(self): @@ -85,7 +85,7 @@ class ConcatStrings(Test): class CompareStrings(Test): - version = 0.2 + version = 2.0 operations = 10 * 5 rounds = 200000 @@ -167,9 +167,9 @@ class CompareStrings(Test): class CompareInternedStrings(Test): - version = 0.1 + version = 2.0 operations = 10 * 5 - rounds = 200000 + rounds = 300000 def test(self): @@ -249,9 +249,9 @@ class CompareInternedStrings(Test): class CreateStringsWithConcat(Test): - version = 0.1 + version = 2.0 operations = 10 * 5 - rounds = 80000 + rounds = 200000 def test(self): @@ -324,9 +324,9 @@ class CreateStringsWithConcat(Test): class StringSlicing(Test): - version = 0.1 + version = 2.0 operations = 5 * 7 - rounds = 100000 + rounds = 160000 def test(self): @@ -387,7 +387,7 @@ if hasattr('', 'lower'): class StringMappings(Test): - version = 0.1 + version = 2.0 operations = 3 * (5 + 4 + 2 + 1) rounds = 70000 @@ -460,9 +460,9 @@ if hasattr('', 'lower'): class StringPredicates(Test): - version = 0.1 + version = 2.0 operations = 10 * 7 - rounds = 80000 + rounds = 100000 def test(self): diff --git a/Tools/pybench/Tuples.py b/Tools/pybench/Tuples.py index e84ea53..8e46989 100644 --- a/Tools/pybench/Tuples.py +++ b/Tools/pybench/Tuples.py @@ -2,18 +2,17 @@ from pybench import Test class TupleSlicing(Test): - version = 0.31 + version = 2.0 operations = 3 * 25 * 10 * 7 - rounds = 400 + rounds = 500 def test(self): r = range(25) + t = tuple(range(100)) for i in xrange(self.rounds): - t = tuple(range(100)) - for j in r: m = t[50:] @@ -259,20 +258,17 @@ class TupleSlicing(Test): def calibrate(self): r = range(25) + t = tuple(range(100)) for i in xrange(self.rounds): - - t = tuple(range(100)) - for j in r: - pass class SmallTuples(Test): - version = 0.3 + version = 2.0 operations = 5*(1 + 3 + 6 + 2) - rounds = 80000 + rounds = 90000 def test(self): diff --git a/Tools/pybench/Unicode.py b/Tools/pybench/Unicode.py index 366f171..153a91e 100644 --- a/Tools/pybench/Unicode.py +++ b/Tools/pybench/Unicode.py @@ -8,7 +8,7 @@ from string import join class ConcatUnicode(Test): - version = 0.1 + version = 2.0 operations = 10 * 5 rounds = 60000 @@ -90,7 +90,7 @@ class ConcatUnicode(Test): class CompareUnicode(Test): - version = 0.1 + version = 2.0 operations = 10 * 5 rounds = 150000 @@ -172,7 +172,7 @@ class CompareUnicode(Test): class CreateUnicodeWithConcat(Test): - version = 0.1 + version = 2.0 operations = 10 * 5 rounds = 80000 @@ -247,9 +247,9 @@ class CreateUnicodeWithConcat(Test): class UnicodeSlicing(Test): - version = 0.1 + version = 2.0 operations = 5 * 7 - rounds = 100000 + rounds = 140000 def test(self): @@ -308,7 +308,7 @@ class UnicodeSlicing(Test): class UnicodeMappings(Test): - version = 0.1 + version = 2.0 operations = 3 * (5 + 4 + 2 + 1) rounds = 10000 @@ -381,9 +381,9 @@ class UnicodeMappings(Test): class UnicodePredicates(Test): - version = 0.1 + version = 2.0 operations = 5 * 9 - rounds = 100000 + rounds = 120000 def test(self): @@ -458,7 +458,7 @@ except ImportError: else: class UnicodeProperties(Test): - version = 0.1 + version = 2.0 operations = 5 * 8 rounds = 100000 diff --git a/Tools/pybench/clockres.py b/Tools/pybench/clockres.py new file mode 100644 index 0000000..64095b3 --- /dev/null +++ b/Tools/pybench/clockres.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +""" clockres - calculates the resolution in seconds of a given timer. + + Copyright (c) 2006, Marc-Andre Lemburg (mal@egenix.com). See the + documentation for further information on copyrights, or contact + the author. All Rights Reserved. + +""" +import time + +TEST_TIME = 1.0 + +def clockres(timer): + d = {} + wallclock = time.time + start = wallclock() + stop = wallclock() + TEST_TIME + spin_loops = range(1000) + while 1: + now = wallclock() + if now >= stop: + break + for i in spin_loops: + d[timer()] = 1 + values = d.keys() + values.sort() + min_diff = TEST_TIME + for i in range(len(values) - 1): + diff = values[i+1] - values[i] + if diff < min_diff: + min_diff = diff + return min_diff + +if __name__ == '__main__': + print 'Clock resolution of various timer implementations:' + print 'time.clock: %10.3fus' % (clockres(time.clock) * 1e6) + print 'time.time: %10.3fus' % (clockres(time.time) * 1e6) + try: + import systimes + print 'systimes.processtime: %10.3fus' % (clockres(systimes.processtime) * 1e6) + except ImportError: + pass diff --git a/Tools/pybench/pybench.py b/Tools/pybench/pybench.py index e0110d0..7d90ba1 100755 --- a/Tools/pybench/pybench.py +++ b/Tools/pybench/pybench.py @@ -34,20 +34,7 @@ NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE ! """ -# Version number -__version__ = '1.3' - -# -# NOTE: Use xrange for all test loops unless you want to face -# a 20MB process ! -# -# All tests should have rounds set to values so that a run() -# takes between 20-50 seconds. This is to get fairly good -# clock() values. You can use option -w to speedup the tests -# by a fixed integer factor (the "warp factor"). -# - -import sys,time,operator +import sys, time, operator, string from CommandLine import * try: @@ -56,6 +43,111 @@ try: except ImportError: import pickle +# Version number; version history: see README file ! +__version__ = '2.0' + +### Constants + +# Second fractions +MILLI_SECONDS = 1e3 +MICRO_SECONDS = 1e6 + +# Percent unit +PERCENT = 100 + +# Horizontal line length +LINE = 79 + +# Minimum test run-time +MIN_TEST_RUNTIME = 1e-3 + +# Number of calibration runs to use for calibrating the tests +CALIBRATION_RUNS = 20 + +# Number of calibration loops to run for each calibration run +CALIBRATION_LOOPS = 20 + +# Allow skipping calibration ? +ALLOW_SKIPPING_CALIBRATION = 1 + +# Timer types +TIMER_TIME_TIME = 'time.time' +TIMER_TIME_CLOCK = 'time.clock' +TIMER_SYSTIMES_PROCESSTIME = 'systimes.processtime' + +# Choose platform default timer +if sys.platform[:3] == 'win': + # On WinXP this has 2.5ms resolution + TIMER_PLATFORM_DEFAULT = TIMER_TIME_CLOCK +else: + # On Linux this has 1ms resolution + TIMER_PLATFORM_DEFAULT = TIMER_TIME_TIME + +# Print debug information ? +_debug = 0 + +### Helpers + +def get_timer(timertype): + + if timertype == TIMER_TIME_TIME: + return time.time + elif timertype == TIMER_TIME_CLOCK: + return time.clock + elif timertype == TIMER_SYSTIMES_PROCESSTIME: + import systimes + return systimes.processtime + else: + raise TypeError('unknown timer type: %s' % timertype) + +def get_machine_details(): + + import platform + if _debug: + print 'Getting machine details...' + buildno, builddate = platform.python_build() + python = platform.python_version() + if python > '2.0': + try: + unichr(100000) + except ValueError: + # UCS2 build (standard) + unicode = 'UCS2' + else: + # UCS4 build (most recent Linux distros) + unicode = 'UCS4' + else: + unicode = None + bits, linkage = platform.architecture() + return { + 'platform': platform.platform(), + 'processor': platform.processor(), + 'executable': sys.executable, + 'python': platform.python_version(), + 'compiler': platform.python_compiler(), + 'buildno': buildno, + 'builddate': builddate, + 'unicode': unicode, + 'bits': bits, + } + +def print_machine_details(d, indent=''): + + l = ['Machine Details:', + ' Platform ID: %s' % d.get('platform', 'n/a'), + ' Processor: %s' % d.get('processor', 'n/a'), + '', + 'Python:', + ' Executable: %s' % d.get('executable', 'n/a'), + ' Version: %s' % d.get('python', 'n/a'), + ' Compiler: %s' % d.get('compiler', 'n/a'), + ' Bits: %s' % d.get('bits', 'n/a'), + ' Build: %s (#%s)' % (d.get('builddate', 'n/a'), + d.get('buildno', 'n/a')), + ' Unicode: %s' % d.get('unicode', 'n/a'), + ] + print indent + string.join(l, '\n' + indent) + '\n' + ### Test baseclass class Test: @@ -84,7 +176,7 @@ class Test: # Version number of the test as float (x.yy); this is important # for comparisons of benchmark runs - tests with unequal version # number will not get compared. - version = 1.0 + version = 2.0 # The number of abstract operations done in each round of the # test. An operation is the basic unit of what you want to @@ -97,36 +189,125 @@ class Test: # Number of rounds to execute per test run. This should be # adjusted to a figure that results in a test run-time of between - # 20-50 seconds. - rounds = 10000 + # 1-2 seconds. + rounds = 100000 ### Internal variables # Mark this class as implementing a test is_a_test = 1 - # Misc. internal variables - last_timing = (0,0,0) # last timing (real,run,calibration) - warp = 1 # warp factor this test uses - cruns = 20 # number of calibration runs - overhead = None # list of calibration timings + # Last timing: (real, run, overhead) + last_timing = (0.0, 0.0, 0.0) + + # Warp factor to use for this test + warp = 1 - def __init__(self,warp=1): + # Number of calibration runs to use + calibration_runs = CALIBRATION_RUNS - if warp > 1: - self.rounds = self.rounds / warp + # List of calibration timings + overhead_times = None + + # List of test run timings + times = [] + + # Timer used for the benchmark + timer = TIMER_PLATFORM_DEFAULT + + def __init__(self, warp=None, calibration_runs=None, timer=None): + + # Set parameters + if warp is not None: + self.rounds = int(self.rounds / warp) if self.rounds == 0: - self.rounds = 1 + raise ValueError('warp factor set too high') self.warp = warp + if calibration_runs is not None: + if (not ALLOW_SKIPPING_CALIBRATION and + calibration_runs < 1): + raise ValueError('at least one calibration run is required') + self.calibration_runs = calibration_runs + if timer is not None: + timer = timer + + # Init variables self.times = [] - self.overhead = [] + self.overhead_times = [] + # We want these to be in the instance dict, so that pickle # saves them self.version = self.version self.operations = self.operations self.rounds = self.rounds - def run(self, cruns): + def get_timer(self): + + """ Return the timer function to use for the test. + + """ + return get_timer(self.timer) + + def compatible(self, other): + + """ Return 1/0 depending on whether the test is compatible + with the other Test instance or not. + + """ + if self.version != other.version: + return 0 + if self.rounds != other.rounds: + return 0 + return 1 + + def calibrate_test(self): + + if self.calibration_runs == 0: + self.overhead_times = [0.0] + return + + calibrate = self.calibrate + timer = self.get_timer() + calibration_loops = range(CALIBRATION_LOOPS) + + # Time the calibration loop overhead + prep_times = [] + for i in range(self.calibration_runs): + t = timer() + for i in calibration_loops: + pass + t = timer() - t + prep_times.append(t) + min_prep_time = min(prep_times) + if _debug: + print + print 'Calib. prep time = %.6fms' % ( + min_prep_time * MILLI_SECONDS) + + # Time the calibration runs (doing CALIBRATION_LOOPS loops of + # .calibrate() method calls each) + for i in range(self.calibration_runs): + t = timer() + for i in calibration_loops: + calibrate() + t = timer() - t + self.overhead_times.append(t / CALIBRATION_LOOPS + - min_prep_time) + + # Check the measured times + min_overhead = min(self.overhead_times) + max_overhead = max(self.overhead_times) + if _debug: + print 'Calib. overhead time = %.6fms' % ( + min_overhead * MILLI_SECONDS) + if min_overhead < 0.0: + raise ValueError('calibration setup did not work') + if max_overhead - min_overhead > 0.1: + raise ValueError( + 'overhead calibration timing range too inaccurate: ' + '%r - %r' % (min_overhead, max_overhead)) + + def run(self): """ Run the test in two phases: first calibrate, then do the actual test. Be careful to keep the calibration @@ -134,27 +315,23 @@ class Test: """ test = self.test - calibrate = self.calibrate - clock = time.clock - # first calibrate - t = clock() - calibrate() - offset = clock() - t - if cruns: - for i in range(cruns-1): - t = clock() - calibrate() - t = clock() - t - if t < offset: - offset = t - # now the real thing - t = clock() + timer = self.get_timer() + + # Get calibration + min_overhead = min(self.overhead_times) + + # Test run + t = timer() test() - t = clock() - t - if t < 0.01: - sys.exit("Lower warp required: test times < 10 ms are unreliable") - self.last_timing = (t-offset,t,offset) - self.times.append(t-offset) + t = timer() - t + if t < MIN_TEST_RUNTIME: + raise ValueError('warp factor too high: ' + 'test times are < 10ms') + eff_time = t - min_overhead + if eff_time < 0: + raise ValueError('wrong calibration') + self.last_timing = (eff_time, t, min_overhead) + self.times.append(eff_time) def calibrate(self): @@ -176,33 +353,33 @@ class Test: self.operations number of operations each. """ - # do some tests return def stat(self): - """ Returns four values: - minimum round time - average time per round - average time per operation - average overhead time + """ Return test run statistics as tuple: + + (minimum run time, + average run time, + total run time, + average time per operation, + minimum overhead time) - XXX Should this take warp factors into account? """ runs = len(self.times) if runs == 0: - return 0,0 - mintime = min(self.times) - totaltime = reduce(operator.add,self.times,0.0) - avg = totaltime / float(runs) - op_avg = totaltime / float(runs * self.rounds * self.operations) - if self.overhead: - totaloverhead = reduce(operator.add,self.overhead,0.0) - ov_avg = totaloverhead / float(runs) + return 0.0, 0.0, 0.0, 0.0 + min_time = min(self.times) + total_time = reduce(operator.add, self.times, 0.0) + avg_time = total_time / float(runs) + operation_avg = total_time / float(runs + * self.rounds + * self.operations) + if self.overhead_times: + min_overhead = min(self.overhead_times) else: - # use self.last_timing - not too accurate - ov_avg = self.last_timing[2] - return mintime, avg, op_avg, ov_avg + min_overhead = self.last_timing[2] + return min_time, avg_time, total_time, operation_avg, min_overhead ### Load Setup @@ -215,153 +392,353 @@ import Setup class Benchmark: - name = '?' # Name of the benchmark - rounds = 1 # Number of rounds to run + # Name of the benchmark + name = '' + + # Number of benchmark rounds to run + rounds = 1 + + # Warp factor use to run the tests warp = 1 # Warp factor - roundtime = 0 # Average round time - version = None # Benchmark version number (see __init__) - # as float x.yy - def __init__(self): + # Average benchmark round time + roundtime = 0 - self.tests = {} - self.version = 0.31 + # Benchmark version number as float x.yy + version = 2.0 - def load_tests(self, setupmod, warp=1, limitnames="", verbose=0): + # Produce verbose output ? + verbose = 0 - self.warp = warp - if limitnames: - limitnames = re.compile(limitnames, re.I) + # Dictionary with the machine details + machine_details = None + + # Timer used for the benchmark + timer = TIMER_PLATFORM_DEFAULT + + def __init__(self, name, verbose=None, timer=None, warp=None, + calibration_runs=None): + + if name: + self.name = name else: - limitnames = None - tests = self.tests - if verbose: - print 'Searching for tests ...', - setupmod.__dict__.values() - for c in setupmod.__dict__.values(): - if not hasattr(c,'is_a_test'): + self.name = '%04i-%02i-%02i %02i:%02i:%02i' % \ + (time.localtime(time.time())[:6]) + if verbose is not None: + self.verbose = verbose + if timer is not None: + self.timer = timer + if warp is not None: + self.warp = warp + if calibration_runs is not None: + self.calibration_runs = calibration_runs + + # Init vars + self.tests = {} + if _debug: + print 'Getting machine details...' + self.machine_details = get_machine_details() + + # Make .version an instance attribute to have it saved in the + # Benchmark pickle + self.version = self.version + + def get_timer(self): + + """ Return the timer function to use for the test. + + """ + return get_timer(self.timer) + + def compatible(self, other): + + """ Return 1/0 depending on whether the benchmark is + compatible with the other Benchmark instance or not. + + """ + if self.version != other.version: + return 0 + if (self.machine_details == other.machine_details and + self.timer != other.timer): + return 0 + if (self.calibration_runs == 0 and + other.calibration_runs != 0): + return 0 + if (self.calibration_runs != 0 and + other.calibration_runs == 0): + return 0 + return 1 + + def load_tests(self, setupmod, limitnames=None): + + # Add tests + if self.verbose: + print 'Searching for tests ...' + print '--------------------------------------' + for testclass in setupmod.__dict__.values(): + if not hasattr(testclass, 'is_a_test'): continue - name = c.__name__ + name = testclass.__name__ if name == 'Test': continue - if limitnames is not None and limitnames.search(name) is None: + if (limitnames is not None and + limitnames.search(name) is None): continue - tests[name] = c(warp) - l = tests.keys() + self.tests[name] = testclass( + warp=self.warp, + calibration_runs=self.calibration_runs, + timer=self.timer) + l = self.tests.keys() l.sort() - if verbose: + if self.verbose: + for name in l: + print ' %s' % name + print '--------------------------------------' + print ' %i tests found' % len(l) print - for t in l: - print ' ', t - print len(l), "tests found" + + def calibrate(self): + + print 'Calibrating tests. Please wait...' + if self.verbose: + print + print 'Test min max' + print '-' * LINE + tests = self.tests.items() + tests.sort() + for i in range(len(tests)): + name, test = tests[i] + test.calibrate_test() + if self.verbose: + print '%30s: %6.3fms %6.3fms' % \ + (name, + min(test.overhead_times) * MILLI_SECONDS, + max(test.overhead_times) * MILLI_SECONDS) print - def run(self, verbose, cruns): + def run(self): tests = self.tests.items() tests.sort() - clock = time.clock - print 'Running %i round(s) of the suite at warp factor %i:' % (self.rounds, self.warp) + timer = self.get_timer() + print 'Running %i round(s) of the suite at warp factor %i:' % \ + (self.rounds, self.warp) print - roundtime = clock() + self.roundtimes = [] for i in range(self.rounds): - roundstarttime = clock() - if verbose: - print ' Round %-25i real abs overhead' % (i+1) + if self.verbose: + print ' Round %-25i effective absolute overhead' % (i+1) + total_eff_time = 0.0 for j in range(len(tests)): - name, t = tests[j] - if verbose: + name, test = tests[j] + if self.verbose: print '%30s:' % name, - t.run(cruns) - if verbose: - print ' %.3fr %.3fa %.3fo' % t.last_timing - if verbose: - print ' ----------------------' - print ' Average round time: %.3f seconds' % \ - ((clock() - roundtime)/(i+1)) + test.run() + (eff_time, abs_time, min_overhead) = test.last_timing + total_eff_time = total_eff_time + eff_time + if self.verbose: + print ' %5.0fms %5.0fms %7.3fms' % \ + (eff_time * MILLI_SECONDS, + abs_time * MILLI_SECONDS, + min_overhead * MILLI_SECONDS) + self.roundtimes.append(total_eff_time) + if self.verbose: + print (' ' + ' ------------------------------') + print (' ' + ' Totals: %6.0fms' % + (total_eff_time * MILLI_SECONDS)) print else: - print '%d done in %.3f seconds' % (i+1, (clock() - roundstarttime)) - self.roundtime = (clock() - roundtime) / self.rounds + print '* Round %i done in %.3f seconds.' % (i+1, + total_eff_time) print - def print_stat(self, compare_to=None, hidenoise=0): - - if not compare_to: - print '%-30s min run avg run per oprn overhead' % 'Tests:' - print '-'*77 - tests = self.tests.items() - tests.sort() - totalmintime = 0 - for name,t in tests: - mintime,avg,op_avg,ov_avg = t.stat() - totalmintime += mintime - print '%30s: %9.2f ms %9.2f ms %6.2f us %6.2f' % \ - (name,mintime*1000.0,avg*1000.0,op_avg*1000000.0,ov_avg*1000.0) - print '-'*77 - print '%30s: %9.2f ms' % \ - ('Notional minimum round time', totalmintime * 1000.0) + def stat(self): + + """ Return benchmark run statistics as tuple: - else: - print 'Comparing with: %s (rounds=%i, warp=%i)' % \ - (compare_to.name,compare_to.rounds,compare_to.warp) - print '%-30s min run cmp run avg run diff' % \ - 'Tests:' - print '-'*77 - tests = self.tests.items() - tests.sort() - compatible = 1 - totalmintime = other_totalmintime = 0 - for name, t in tests: - mintime, avg, op_avg, ov_avg = t.stat() - totalmintime += mintime - try: - other = compare_to.tests[name] - except KeyError: - other = None - if other and other.version == t.version and \ - other.operations == t.operations: - mintime1, avg1, op_avg1, ov_avg1 = other.stat() - other_totalmintime += mintime1 - diff = ((mintime*self.warp)/(mintime1*other.warp) - 1.0)*100.0 - if hidenoise and abs(qop_avg) < 10: - diff = '' + (minimum round time, + average round time, + maximum round time) + + XXX Currently not used, since the benchmark does test + statistics across all rounds. + + """ + runs = len(self.roundtimes) + if runs == 0: + return 0.0, 0.0 + min_time = min(self.roundtimes) + total_time = reduce(operator.add, self.roundtimes, 0.0) + avg_time = total_time / float(runs) + max_time = max(self.roundtimes) + return (min_time, avg_time, max_time) + + def print_header(self, title='Benchmark'): + + print '-' * LINE + print '%s: %s' % (title, self.name) + print '-' * LINE + print + print ' Rounds: %s' % self.rounds + print ' Warp: %s' % self.warp + print ' Timer: %s' % self.timer + print + if self.machine_details: + print_machine_details(self.machine_details, indent=' ') + print + + def print_benchmark(self, hidenoise=0, limitnames=None): + + print ('Test ' + ' minimum average operation overhead') + print '-' * LINE + tests = self.tests.items() + tests.sort() + total_min_time = 0.0 + total_avg_time = 0.0 + for name, test in tests: + if (limitnames is not None and + limitnames.search(name) is None): + continue + (min_time, + avg_time, + total_time, + op_avg, + min_overhead) = test.stat() + total_min_time = total_min_time + min_time + total_avg_time = total_avg_time + avg_time + print '%30s: %5.0fms %5.0fms %6.2fus %7.3fms' % \ + (name, + min_time * MILLI_SECONDS, + avg_time * MILLI_SECONDS, + op_avg * MICRO_SECONDS, + min_overhead *MILLI_SECONDS) + print '-' * LINE + print ('Totals: ' + ' %6.0fms %6.0fms' % + (total_min_time * MILLI_SECONDS, + total_avg_time * MILLI_SECONDS, + )) + print + + def print_comparison(self, compare_to, hidenoise=0, limitnames=None): + + # Check benchmark versions + if compare_to.version != self.version: + print ('* Benchmark versions differ: ' + 'cannot compare this benchmark to "%s" !' % + compare_to.name) + print + self.print_benchmark(hidenoise=hidenoise, + limitnames=limitnames) + return + + # Print header + compare_to.print_header('Comparing with') + print ('Test ' + ' minimum run-time average run-time') + print (' ' + ' this other diff this other diff') + print '-' * LINE + + # Print test comparisons + tests = self.tests.items() + tests.sort() + total_min_time = other_total_min_time = 0.0 + total_avg_time = other_total_avg_time = 0.0 + benchmarks_compatible = self.compatible(compare_to) + tests_compatible = 1 + for name, test in tests: + if (limitnames is not None and + limitnames.search(name) is None): + continue + (min_time, + avg_time, + total_time, + op_avg, + min_overhead) = test.stat() + total_min_time = total_min_time + min_time + total_avg_time = total_avg_time + avg_time + try: + other = compare_to.tests[name] + except KeyError: + other = None + if other is None: + # Other benchmark doesn't include the given test + min_diff, avg_diff = 'n/a', 'n/a' + other_min_time = 0.0 + other_avg_time = 0.0 + tests_compatible = 0 + else: + (other_min_time, + other_avg_time, + other_total_time, + other_op_avg, + other_min_overhead) = other.stat() + other_total_min_time = other_total_min_time + other_min_time + other_total_avg_time = other_total_avg_time + other_avg_time + if (benchmarks_compatible and + test.compatible(other)): + # Both benchmark and tests are comparible + min_diff = ((min_time * self.warp) / + (other_min_time * other.warp) - 1.0) + avg_diff = ((avg_time * self.warp) / + (other_avg_time * other.warp) - 1.0) + if hidenoise and abs(min_diff) < 10.0: + min_diff = '' else: - diff = '%+7.2f%%' % diff + min_diff = '%+5.1f%%' % (min_diff * PERCENT) + if hidenoise and abs(avg_diff) < 10.0: + avg_diff = '' + else: + avg_diff = '%+5.1f%%' % (avg_diff * PERCENT) else: - qavg, diff = 'n/a', 'n/a' - compatible = 0 - print '%30s: %8.2f ms %8.2f ms %8.2f ms %8s' % \ - (name,mintime*1000.0,mintime1*1000.0 * compare_to.warp/self.warp, avg*1000.0,diff) - print '-'*77 - # - # Summarise test results - # - if compatible and compare_to.roundtime > 0 and \ - compare_to.version == self.version: - print '%30s: %8.2f ms %8.2f ms %+7.2f%%' % \ - ('Notional minimum round time', totalmintime * 1000.0, - other_totalmintime * 1000.0 * compare_to.warp/self.warp, - ((totalmintime*self.warp)/ - (other_totalmintime*compare_to.warp)-1.0)*100.0) + # Benchmark or tests are not comparible + min_diff, avg_diff = 'n/a', 'n/a' + tests_compatible = 0 + print '%30s: %5.0fms %5.0fms %7s %5.0fms %5.0fms %7s' % \ + (name, + min_time * MILLI_SECONDS, + other_min_time * MILLI_SECONDS * compare_to.warp / self.warp, + min_diff, + avg_time * MILLI_SECONDS, + other_avg_time * MILLI_SECONDS * compare_to.warp / self.warp, + avg_diff) + print '-' * LINE + + # Summarise test results + if not benchmarks_compatible or not tests_compatible: + min_diff, avg_diff = 'n/a', 'n/a' + else: + if other_total_min_time != 0.0: + min_diff = '%+5.1f%%' % ( + ((total_min_time * self.warp) / + (other_total_min_time * compare_to.warp) - 1.0) * PERCENT) + else: + min_diff = 'n/a' + if other_total_avg_time != 0.0: + avg_diff = '%+5.1f%%' % ( + ((total_avg_time * self.warp) / + (other_total_avg_time * compare_to.warp) - 1.0) * PERCENT) else: - print '%30s: %9.2f ms n/a' % \ - ('Notional minimum round time', totalmintime * 1000.0) + avg_diff = 'n/a' + print ('Totals: ' + ' %5.0fms %5.0fms %7s %5.0fms %5.0fms %7s' % + (total_min_time * MILLI_SECONDS, + (other_total_min_time * compare_to.warp/self.warp + * MILLI_SECONDS), + min_diff, + total_avg_time * MILLI_SECONDS, + (other_total_avg_time * compare_to.warp/self.warp + * MILLI_SECONDS), + avg_diff + )) + print + print '(this=%s, other=%s)' % (self.name, + compare_to.name) print - -def print_machine(): - - import platform - print 'Machine Details:' - print ' Platform ID: %s' % platform.platform() - print ' Executable: %s' % sys.executable - # There's a bug in Python 2.2b1+... - if sys.version[:6] == '2.2b1+': - return - print ' Python: %s' % platform.python_version() - print ' Compiler: %s' % platform.python_compiler() - buildno, builddate = platform.python_build() - print ' Build: %s (#%s)' % (builddate, buildno) class PyBenchCmdline(Application): @@ -370,50 +747,64 @@ class PyBenchCmdline(Application): version = __version__ - options = [ArgumentOption('-n','number of rounds',Setup.Number_of_rounds), - ArgumentOption('-f','save benchmark to file arg',''), - ArgumentOption('-c','compare benchmark with the one in file arg',''), - ArgumentOption('-s','show benchmark in file arg, then exit',''), - SwitchOption('-S','show statistics of benchmarks',0), - ArgumentOption('-w','set warp factor to arg',Setup.Warp_factor), - SwitchOption('-d','hide noise in compares', 0), - SwitchOption('-v','verbose output (not recommended)', 0), - SwitchOption('--no-gc','disable garbage collection', 0), - SwitchOption('--no-syscheck', - '"disable" sys check interval (set to sys.maxint)', 0), - ArgumentOption('-t', 'tests containing substring', ''), - ArgumentOption('-C', 'number of calibration runs', 20) + debug = _debug + + options = [ArgumentOption('-n', + 'number of rounds', + Setup.Number_of_rounds), + ArgumentOption('-f', + 'save benchmark to file arg', + ''), + ArgumentOption('-c', + 'compare benchmark with the one in file arg', + ''), + ArgumentOption('-s', + 'show benchmark in file arg, then exit', + ''), + ArgumentOption('-w', + 'set warp factor to arg', + Setup.Warp_factor), + ArgumentOption('-t', + 'run only tests with names matching arg', + ''), + ArgumentOption('-C', + 'set the number of calibration runs to arg', + CALIBRATION_RUNS), + SwitchOption('-d', + 'hide noise in comparisons', + 0), + SwitchOption('-v', + 'verbose output (not recommended)', + 0), + SwitchOption('--with-gc', + 'enable garbage collection', + 0), + SwitchOption('--with-syscheck', + 'use default sys check interval', + 0), + ArgumentOption('--timer', + 'use given timer', + TIMER_PLATFORM_DEFAULT), ] about = """\ The normal operation is to run the suite and display the -results. Use -f to save them for later reuse or comparisms. +results. Use -f to save them for later reuse or comparisons. + +Available timers: + + time.time + time.clock + systimes.processtime Examples: -python1.5 pybench.py -w 100 -f p15 -python1.4 pybench.py -w 100 -f p14 -python pybench.py -s p15 -c p14 +python2.1 pybench.py -f p21.pybench +python2.5 pybench.py -f p25.pybench +python pybench.py -s p25.pybench -c p21.pybench """ copyright = __copyright__ - def handle_S(self, value): - - """ Display one line stats for each benchmark file given on the - command line. - - """ - for benchmark in self.files: - try: - f = open(benchmark, 'rb') - bench = pickle.load(f) - f.close() - except IOError: - print '* Error opening/reading file %s' % repr(benchmark) - else: - print '%s,%-.2f,ms' % (benchmark, bench.roundtime*1000.0) - return 0 - def main(self): rounds = self.values['-n'] @@ -421,38 +812,52 @@ python pybench.py -s p15 -c p14 show_bench = self.values['-s'] compare_to = self.values['-c'] hidenoise = self.values['-d'] - warp = self.values['-w'] - nogc = self.values['--no-gc'] + warp = int(self.values['-w']) + withgc = self.values['--with-gc'] limitnames = self.values['-t'] + if limitnames: + if _debug: + print '* limiting test names to one with substring "%s"' % \ + limitnames + limitnames = re.compile(limitnames, re.I) + else: + limitnames = None verbose = self.verbose - nosyscheck = self.values['--no-syscheck'] - cruns = self.values['-C'] - print "CRUNS:", cruns + withsyscheck = self.values['--with-syscheck'] + calibration_runs = self.values['-C'] + timer = self.values['--timer'] - print 'PYBENCH',__version__ + print '-' * LINE + print 'PYBENCH %s' % __version__ + print '-' * LINE + print '* using Python %s' % (string.split(sys.version)[0]) - # Switch off GC - if nogc: + # Switch off garbage collection + if not withgc: try: import gc except ImportError: - nogc = 0 + print '* Python version doesn\'t support garbage collection' else: - if self.values['--no-gc']: - gc.disable() - print 'NO GC' - - # maximise sys check interval - if nosyscheck: - sys.setcheckinterval(sys.maxint) - print 'CHECKINTERVAL =', sys.maxint + gc.disable() + print '* disabled garbage collection' + + # "Disable" sys check interval + if not withsyscheck: + # Too bad the check interval uses an int instead of a long... + value = 2147483647 + sys.setcheckinterval(value) + print '* system check interval set to maximum: %s' % value + + if timer == TIMER_SYSTIMES_PROCESSTIME: + import systimes + print '* using timer: systimes.processtime (%s)' % \ + systimes.SYSTIMES_IMPLEMENTATION + else: + print '* using timer: %s' % timer print - if not compare_to: - print_machine() - print - if compare_to: try: f = open(compare_to,'rb') @@ -460,8 +865,10 @@ python pybench.py -s p15 -c p14 bench.name = compare_to f.close() compare_to = bench - except IOError: - print '* Error opening/reading file',compare_to + except IOError, reason: + print '* Error opening/reading file %s: %s' % ( + repr(compare_to), + reason) compare_to = None if show_bench: @@ -470,37 +877,52 @@ python pybench.py -s p15 -c p14 bench = pickle.load(f) bench.name = show_bench f.close() - print 'Benchmark: %s (rounds=%i, warp=%i)' % \ - (bench.name,bench.rounds,bench.warp) - print - bench.print_stat(compare_to, hidenoise) + bench.print_header() + if compare_to: + bench.print_comparison(compare_to, + hidenoise=hidenoise, + limitnames=limitnames) + else: + bench.print_benchmark(hidenoise=hidenoise, + limitnames=limitnames) except IOError: - print '* Error opening/reading file',show_bench + print '* Error opening/reading file %s: %s' % ( + repr(show_bench), + reason) print return if reportfile: - if nogc: - print 'Benchmark: %s (rounds=%i, warp=%i, no GC)' % \ - (reportfile,rounds,warp) - else: - print 'Benchmark: %s (rounds=%i, warp=%i)' % \ - (reportfile,rounds,warp) + print 'Creating benchmark: %s (rounds=%i, warp=%i)' % \ + (reportfile, rounds, warp) print # Create benchmark object - bench = Benchmark() + bench = Benchmark(reportfile, + verbose=verbose, + timer=timer, + warp=warp, + calibration_runs=calibration_runs) bench.rounds = rounds - bench.load_tests(Setup, warp, limitnames, verbose) + bench.load_tests(Setup, limitnames=limitnames) try: - bench.run(verbose, cruns) + bench.calibrate() + bench.run() except KeyboardInterrupt: print print '*** KeyboardInterrupt -- Aborting' print return - bench.print_stat(compare_to) - # ring bell + bench.print_header() + if compare_to: + bench.print_comparison(compare_to, + hidenoise=hidenoise, + limitnames=limitnames) + else: + bench.print_benchmark(hidenoise=hidenoise, + limitnames=limitnames) + + # Ring bell sys.stderr.write('\007') if reportfile: diff --git a/Tools/pybench/systimes.py b/Tools/pybench/systimes.py index 79d249f..bf07e36 100644 --- a/Tools/pybench/systimes.py +++ b/Tools/pybench/systimes.py @@ -16,7 +16,7 @@ platforms. If no supported timing methods based on process time can be found, - the module reverts to the highest resolution wall-time timer + the module reverts to the highest resolution wall-clock timer instead. The system time part will then always be 0.0. The module exports one public API: @@ -52,8 +52,8 @@ USE_CTYPES_GETPROCESSTIMES = 'cytpes GetProcessTimes() wrapper' USE_WIN32PROCESS_GETPROCESSTIMES = 'win32process.GetProcessTimes()' USE_RESOURCE_GETRUSAGE = 'resource.getrusage()' USE_PROCESS_TIME_CLOCK = 'time.clock() (process time)' -USE_WALL_TIME_CLOCK = 'time.clock() (wall-time)' -USE_WALL_TIME_TIME = 'time.time() (wall-time)' +USE_WALL_TIME_CLOCK = 'time.clock() (wall-clock)' +USE_WALL_TIME_TIME = 'time.time() (wall-clock)' if sys.platform[:3] == 'win': # Windows platform @@ -63,7 +63,7 @@ if sys.platform[:3] == 'win': try: import ctypes except ImportError: - # Use the wall-time implementation time.clock(), since this + # Use the wall-clock implementation time.clock(), since this # is the highest resolution clock available on Windows SYSTIMES_IMPLEMENTATION = USE_WALL_TIME_CLOCK else: @@ -91,7 +91,7 @@ if SYSTIMES_IMPLEMENTATION is None: # time) SYSTIMES_IMPLEMENTATION = USE_PROCESS_TIME_CLOCK else: - # Use wall-time implementation time.time() since this provides + # Use wall-clock implementation time.time() since this provides # the highest resolution clock on most systems SYSTIMES_IMPLEMENTATION = USE_WALL_TIME_TIME @@ -103,24 +103,27 @@ def getrusage_systimes(): def process_time_clock_systimes(): return (time.clock(), 0.0) -def wall_time_clock_systimes(): +def wall_clock_clock_systimes(): return (time.clock(), 0.0) -def wall_time_time_systimes(): +def wall_clock_time_systimes(): return (time.time(), 0.0) # Number of clock ticks per second for the values returned # by GetProcessTimes() on Windows. # -# Note: Ticks returned by GetProcessTimes() are micro-seconds on -# Windows XP (though the docs say 100ns intervals) -WIN32_PROCESS_TIMES_TICKS_PER_SECOND = 10e6 +# Note: Ticks returned by GetProcessTimes() are 100ns intervals on +# Windows XP. However, the process times are only updated with every +# clock tick and the frequency of these is somewhat lower: depending +# on the OS version between 10ms and 15ms. Even worse, the process +# time seems to be allocated to process currently running when the +# clock interrupt arrives, ie. it is possible that the current time +# slice gets accounted to a different process. + +WIN32_PROCESS_TIMES_TICKS_PER_SECOND = 1e7 def win32process_getprocesstimes_systimes(): d = win32process.GetProcessTimes(win32process.GetCurrentProcess()) - # Note: I'm not sure whether KernelTime on Windows is the same as - # system time on Unix - I've yet to see a non-zero value for - # KernelTime on Windows. return (d['UserTime'] / WIN32_PROCESS_TIMES_TICKS_PER_SECOND, d['KernelTime'] / WIN32_PROCESS_TIMES_TICKS_PER_SECOND) @@ -149,10 +152,10 @@ elif SYSTIMES_IMPLEMENTATION is USE_PROCESS_TIME_CLOCK: systimes = process_time_clock_systimes elif SYSTIMES_IMPLEMENTATION is USE_WALL_TIME_CLOCK: - systimes = wall_time_clock_systimes + systimes = wall_clock_clock_systimes elif SYSTIMES_IMPLEMENTATION is USE_WALL_TIME_TIME: - systimes = wall_time_time_systimes + systimes = wall_clock_time_systimes elif SYSTIMES_IMPLEMENTATION is USE_WIN32PROCESS_GETPROCESSTIMES: systimes = win32process_getprocesstimes_systimes @@ -163,6 +166,17 @@ elif SYSTIMES_IMPLEMENTATION is USE_CTYPES_GETPROCESSTIMES: else: raise TypeError('no suitable systimes() implementation found') +def processtime(): + + """ Return the total time spent on the process. + + This is the sum of user and system time as returned by + systimes(). + + """ + user, system = systimes() + return user + system + ### Testing def some_workload(): diff --git a/Tools/scripts/README b/Tools/scripts/README index 56aa391..9c7b40e 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -27,7 +27,6 @@ fixheader.py Add some cpp magic to a C include file fixnotice.py Fix the copyright notice in source files fixps.py Fix Python scripts' first line (if #!) ftpmirror.py FTP mirror script -gencodec.py Create Python codecs from Unicode mapping files google.py Open a webbrowser with Google. gprof2html.py Transform gprof(1) output into useful HTML. h2py.py Translate #define's into Python assignments @@ -53,14 +52,14 @@ pindent.py Indent Python code, giving block-closing comments ptags.py Create vi tags file for Python modules pydoc Python documentation browser. pysource.py Find Python source files -redemo.py Basic regular expression demostration facility +redemo.py Basic regular expression demonstration facility reindent.py Change .py files to use 4-space indents. rgrep.py Reverse grep through a file (useful for big logfiles) setup.py Install all scripts listed here. suff.py Sort a list of files by suffix +svneol.py Sets svn:eol-style on all files in directory. texcheck.py Validate Python LaTeX formatting (Raymond Hettinger) texi2html.py Convert GNU texinfo files into HTML -trace.py Trace Python program or function execution treesync.py Synchronize source trees (very ideosyncratic) untabify.py Replace tabs with spaces in argument files which.py Find a program in $PATH diff --git a/Tools/webchecker/webchecker.py b/Tools/webchecker/webchecker.py index 7199da8..923e8e6 100755 --- a/Tools/webchecker/webchecker.py +++ b/Tools/webchecker/webchecker.py @@ -760,7 +760,8 @@ class MyURLopener(urllib.FancyURLopener): try: names = os.listdir(path) except os.error, msg: - raise IOError, msg, sys.exc_traceback + exc_type, exc_value, exc_tb = sys.exc_info() + raise IOError, msg, exc_tb names.sort() s = MyStringIO("file:"+url, {'content-type': 'text/html'}) s.write('<BASE HREF="file:%s">\n' % |