New regression testing harness.

(testall.out): obsolete.  see individual test output files in the
Output directory.

(testall.py): now contains only the list of all tests

(autotest.py): The new testing harness.  Supports -g option to
generate output files instead of verifying; -w option to warn about
non-tested optional modules; additional command line arguments to
generate or test individual tests (useful for generating output file
of new tests).

3 files changed, 130 insertions, 242 deletions

diff --git a/Lib/test/autotest.py b/Lib/test/autotest.py
index 79d9b11..0d2fa8e 100644
--- a/Lib/test/autotest.py
+++ b/Lib/test/autotest.py
@@ -1,34 +1,58 @@
-# Automatic Python regression test.
-#
-# Some essential parts of the Python interpreter are tested by the module
-# 'testall'.  (Despite its name, it doesn't test everything -- that would
-# be a truly Herculean task!)  When a test fails, 'testall' raises an
-# exception.  When all tests succeed, it produces quite a lot of output.
-#
-# For a normal regression test, this output is never looked at unless
-# something goes wrong.  Thus, it would be wise to suppress the output
-# normally.  This module does that, but it doesn't just throw the output
-# from 'testall' away -- it compares it with the output from a previous
-# run.  If a difference is noticed it raises an exception; if all is well,
-# it prints nothing except 'All tests OK.' at the very end.
-#
-# The output from a previous run is supposed to be in a file 'testall.out'
-# somewhere on the search path for modules (sys.path, initialized from
-# $PYTHONPATH plus some default places).
-#
-# Of course, if the normal output of the tests is changed because the
-# tests have been changed (rather than a test producing the wrong output),
-# 'autotest' will fail as well.  In this case, run 'testall' manually
-# and direct its output to 'testall.out'.
-#
-# The comparison uses (and demonstrates!) a rather new Python feature:
-# program output that normally goes to stdout (by 'print' statements
-# or by writing directly to sys.stdout) can be redirected to an
-# arbitrary class instance as long as it has a 'write' method.
+"""
+Automatic Python regression test.
+
+The list of individual tests is contained in the `testall' module.
+These test some (but not all) essential parts of the Python
+interpreter and built-in modules.  When a test fails, an exception is
+raised and testing halts.  When a test succeeds, it can produce quite
+a lot of output, which is compared against the output from a previous
+run.  If a difference is noticed it raises an exception; if all is
+well, it prints nothing except 'All tests OK.' at the very end.
+
+The output from a previous run is supposed to be contained in separate
+files (one per test) in the `Output' subdirectory somewhere on the
+search path for modules (sys.path, initialized from $PYTHONPATH plus
+some default places).
+
+Of course, if the normal output of the tests is changed because the
+tests have been changed (rather than a test producing the wrong
+output), 'autotest' will fail as well.  In this case, run 'autotest'
+with the -g option.
+
+Usage:
+
+    %s [-g] [-w] [-h] [test1 [test2 ...]]
+
+Options:
+
+    -g, --generate : generate the output files instead of verifying
+                     the results
+
+    -w, --warn     : warn about un-importable tests
+
+    -h, --help     : print this message
+
+If individual tests are provided on the command line, only those tests
+will be performed or generated.  Otherwise, all tests (as contained in
+testall.py) will be performed.
+
+"""
 
 import os
 import sys
+import getopt
+import traceback
+from test_support import *
+
+# Exception raised when the test failed (not the same as in test_support)
+TestFailed = 'autotest.TestFailed'
+
+# defaults
+generate = 0
+warn = 0
 
+
+
 # Function to find a file somewhere on sys.path
 def findfile(filename):
 	for dirname in sys.path:
@@ -37,9 +61,8 @@ def findfile(filename):
 			return fullname
 	return filename # Will cause exception later
 
-# Exception raised when the test failed (not the same as in test_support)
-TestFailed = 'autotest.TestFailed'
 
+
 # Class substituted for sys.stdout, to compare it with the given file
 class Compare:
 	def __init__(self, filename):
@@ -50,18 +73,73 @@ class Compare:
 			raise TestFailed, \
 				'Writing: '+`data`+', expected: '+`expected`
 	def close(self):
+		leftover = self.fp.read()
+		if leftover:
+			raise TestFailed, 'Unread: '+`leftover`
 		self.fp.close()
 
+
 # The main program
-def main():
-	import sys
-	filename = findfile('testall.out')
+def usage(status):
+	print __doc__ % sys.argv[0]
+	sys.exit(status)
+
+
+
+def do_one_test(t, outdir):
+	filename = os.path.join(outdir, t)
 	real_stdout = sys.stdout
 	try:
-		sys.stdout = Compare(filename)
-		import testall
+		if generate:
+			print 'Generating:', filename
+			sys.stdout = open(filename, 'w')
+		else:
+			sys.stdout = Compare(filename)
+		print t
+		unload(t)
+		try:
+			__import__(t, globals(), locals())
+		except ImportError, msg:
+			if warn:
+				sys.stderr.write(msg+': Un-installed'
+						 ' optional module?\n')
 	finally:
+		sys.stdout.close()
 		sys.stdout = real_stdout
+
+
+
+def main():
+	global generate
+	global warn
+	try:
+		opts, args = getopt.getopt(
+			sys.argv[1:], 'ghw',
+			['generate', 'help', 'warn'])
+	except getopt.error, msg:
+		print msg
+		usage(1)
+	for opt, val in opts:
+		if opt in ['-h', '--help']:
+			usage(0)
+		elif opt in ['-g', '--generate']:
+			generate = 1
+		elif opt in ['-w', '--warn']:
+			warn = 1
+
+	# find the output directory
+	outdir = findfile('Output')
+	if args:
+	    tests = args
+	else:
+	    import testall
+	    tests = testall.tests
+	for test in tests:
+		try:
+			do_one_test(test, outdir)
+		except TestFailed, msg:
+			print 'Failure of test:', test
+			traceback.print_exc()
 	print 'All tests OK.'
 
 main()
diff --git a/Lib/test/testall.out b/Lib/test/testall.out
deleted file mode 100644
index 412cba4..0000000
--- a/Lib/test/testall.out
+++ /dev/null
@@ -1,184 +0,0 @@
-test_grammar
-1. Parser
-1.1 Tokens
-1.1.1 Backslashes
-1.1.2 Numeric literals
-1.1.2.1 Plain integers
-1.1.2.2 Long integers
-1.1.2.3 Floating point
-1.1.3 String literals
-1.2 Grammar
-single_input
-file_input
-expr_input
-eval_input
-funcdef
-simple_stmt
-expr_stmt
-print_stmt
-1 2 3
-1 2 3
-1 1 1
-del_stmt
-pass_stmt
-flow_stmt
-break_stmt
-continue_stmt
-return_stmt
-raise_stmt
-import_stmt
-global_stmt
-exec_stmt
-if_stmt
-while_stmt
-for_stmt
-try_stmt
-suite
-test
-comparison
-binary mask ops
-shift ops
-additive ops
-multiplicative ops
-unary ops
-selectors
-atoms
-classdef
-test_opcodes
-2. Opcodes
-XXX Not yet fully implemented
-2.1 try inside for loop
-2.2 raise class exceptions
-test_operations
-3. Operations
-XXX Not yet implemented
-test_builtin
-4. Built-in functions
-test_b1
-__import__
-abs
-apply
-callable
-chr
-cmp
-coerce
-compile
-delattr
-dir
-divmod
-eval
-execfile
-filter
-float
-getattr
-hasattr
-hash
-hex
-id
-int
-len
-long
-map
-max
-min
-test_b2
-oct
-open
-ord
-pow
-range
-input and raw_input
-testing
-testing
-reduce
-reload
-repr
-round
-setattr
-str
-tuple
-type
-vars
-xrange
-test_exceptions
-5. Built-in exceptions
-AttributeError
-EOFError
-IOError
-ImportError
-IndexError
-KeyError
-KeyboardInterrupt
-(not testable in a script)
-MemoryError
-(not safe to test)
-NameError
-OverflowError
-RuntimeError
-(not used any more?)
-SyntaxError
-SystemError
-(hard to reproduce)
-SystemExit
-TypeError
-ValueError
-ZeroDivisionError
-test_types
-6. Built-in types
-6.1 Truth value testing
-6.2 Boolean operations
-6.3 Comparisons
-6.4 Numeric types (mostly conversions)
-6.4.1 32-bit integers
-6.4.2 Long integers
-6.4.3 Floating point numbers
-6.5 Sequence types
-6.5.1 Strings
-6.5.2 Tuples
-6.5.3 Lists
-6.5.3a Additional list operations
-6.6 Mappings == Dictionaries
-test_math
-math module, testing with eps 1e-05
-constants
-acos
-asin
-atan
-atan2
-ceil
-cos
-cosh
-exp
-fabs
-floor
-fmod
-frexp
-hypot
-ldexp
-log
-log10
-modf
-pow
-sin
-sinh
-sqrt
-tan
-tanh
-test_time
-test_array
-test_strop
-test_md5
-MD5 test suite:
-MD5 ("") = d41d8cd98f00b204e9800998ecf8427e
-MD5 ("a") = 0cc175b9c0f1b6a831c399e269772661
-MD5 ("abc") = 900150983cd24fb0d6963f7d28e17f72
-MD5 ("message digest") = f96b697d7cb7938d525a2f31aaf161d0
-MD5 ("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b
-MD5 ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f
-MD5 ("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a
-test_cmath
-test_crypt
-Test encryption:  abeTm2pJKypwA
-test_dbm
-test_new
-Passed all tests.
diff --git a/Lib/test/testall.py b/Lib/test/testall.py
index b48accc..a132bfe 100644
--- a/Lib/test/testall.py
+++ b/Lib/test/testall.py
@@ -1,25 +1,19 @@
-# testall.py -- a regression test for the Python interpreter.
-# To run the tests, execute "import testall" in a clean interpreter.
-# It is a good idea to do this whenever you build a new interpreter.
-# Remember to add new tests when new features are added!
+# This file now contains only the list of separate regression tests.
+# All of the testing harness is now contained in autotest.py.
 
-import sys
-from test_support import *
-
-print 'test_grammar'
-forget('test_grammar')
-import test_grammar
-
-for t in ['test_opcodes', 'test_operations', 'test_builtin',
-	  'test_exceptions', 'test_types', 'test_math', 'test_time',
-	  'test_array', 'test_strop', 'test_md5', 'test_cmath',
-	  'test_crypt', 'test_dbm', 'test_new',
-	  ]:
-    print t
-    unload(t)
-    try:
-	__import__(t, globals(), locals())
-    except ImportError, msg:
-	sys.stderr.write('%s.  Uninstalled optional module?\n' % msg)
-
-print 'Passed all tests.'
+tests = ['test_grammar',
+	 'test_opcodes',
+	 'test_operations',
+	 'test_builtin',
+	 'test_exceptions',
+	 'test_types',
+	 'test_math',
+	 'test_time',
+	 'test_array',
+	 'test_strop',
+	 'test_md5',
+	 'test_cmath',
+	 'test_crypt',
+	 'test_dbm',
+	 'test_new',
+	 ]