diff options
Diffstat (limited to 'Lib/test/regrtest.py')
| -rwxr-xr-x | Lib/test/regrtest.py | 287 |
1 files changed, 219 insertions, 68 deletions
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 0b93b52..8e61b03 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -20,6 +20,11 @@ python -E -Wd -m test [options] [test_name1 ...] Options: -h/--help -- print this text and exit +--timeout TIMEOUT + -- dump the traceback and exit if a test takes more + than TIMEOUT seconds; disabled if TIMEOUT is negative + or equals to zero +--wait -- wait for user input, e.g., allow a debugger to be attached Verbosity @@ -44,6 +49,9 @@ Selecting tests -- specify which special resource intensive tests to run -M/--memlimit LIMIT -- run very large memory-consuming tests + --testdir DIR + -- execute test files in the specified directory (instead + of the Python stdlib test suite) Special runs @@ -125,6 +133,8 @@ resources to test. Currently only the following are defined: all - Enable all special resources. + none - Disable all special resources (this is the default). + audio - Tests that use the audio device. (There are known cases of broken audio drivers that can crash Python or even the Linux kernel.) @@ -156,16 +166,19 @@ option '-uall,-gui'. """ import builtins -import errno +import faulthandler import getopt import io import json import logging import os +import packaging.command +import packaging.database import platform import random import re import shutil +import signal import sys import sysconfig import tempfile @@ -225,6 +238,7 @@ ENV_CHANGED = -1 SKIPPED = -2 RESOURCE_DENIED = -3 INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process from test import support @@ -268,6 +282,18 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, on the command line. """ + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + replace_stdout() support.record_original_stdout(sys.stdout) @@ -278,7 +304,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, 'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir', 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', - 'start=', 'nowindows', 'header', 'failfast', 'match']) + 'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait', + 'failfast', 'match']) except getopt.error as msg: usage(msg) @@ -289,6 +316,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, use_resources = [] debug = False start = None + timeout = None for o, a in opts: if o in ('-h', '--help'): print(__doc__) @@ -332,7 +360,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, elif o in ('-T', '--coverage'): trace = True elif o in ('-D', '--coverdir'): - coverdir = os.path.join(os.getcwd(), a) + # CWD is replaced with a temporary dir before calling main(), so we + # need join it with the saved CWD so it goes where the user expects. + coverdir = os.path.join(support.SAVEDCWD, a) elif o in ('-N', '--nocoverdir'): coverdir = None elif o in ('-R', '--huntrleaks'): @@ -361,6 +391,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if r == 'all': use_resources[:] = RESOURCE_NAMES continue + if r == 'none': + del use_resources[:] + continue remove = False if r[0] == '-': remove = True @@ -391,6 +424,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, forever = True elif o in ('-j', '--multiprocess'): use_mp = int(a) + if use_mp <= 0: + try: + import multiprocessing + # Use all cores + extras for tests that like to sleep + use_mp = 2 + multiprocessing.cpu_count() + except (ImportError, NotImplementedError): + use_mp = 3 + if use_mp == 1: + use_mp = None elif o == '--header': header = True elif o == '--slaveargs': @@ -403,6 +445,21 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, print() # Force a newline (just in case) print(json.dumps(result)) sys.exit(0) + elif o == '--testdir': + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + testdir = os.path.join(support.SAVEDCWD, a) + elif o == '--timeout': + if hasattr(faulthandler, 'dump_tracebacks_later'): + timeout = float(a) + if timeout <= 0: + timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_tracebacks_later") + timeout = None + elif o == '--wait': + input("Press any key to continue...") else: print(("No handler for option {}. Please report this as a bug " "at http://bugs.python.org.").format(o), file=sys.stderr) @@ -481,7 +538,13 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, print("== ", os.getcwd()) print("Testing with flags:", sys.flags) - alltests = findtests(testdir, stdtests, nottests) + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if testdir: + alltests = findtests(testdir, list(), set()) + else: + alltests = findtests(testdir, stdtests, nottests) + selected = tests or args or alltests if single: selected = selected[:1] @@ -556,7 +619,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, (test, verbose, quiet), dict(huntrleaks=huntrleaks, use_resources=use_resources, debug=debug, output_on_failure=verbose3, - failfast=failfast, match_tests=match_tests) + timeout=timeout, failfast=failfast, + match_tests=match_tests) ) yield (test, args_tuple) pending = tests_and_args() @@ -577,10 +641,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, universal_newlines=True, close_fds=(os.name != 'nt')) stdout, stderr = popen.communicate() + retcode = popen.wait() # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return if not result: output.put((None, None, None, None)) return @@ -613,6 +682,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if result[0] == INTERRUPTED: assert result[1] == 'KeyboardInterrupt' raise KeyboardInterrupt # What else? + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) test_index += 1 except KeyboardInterrupt: interrupted = True @@ -629,13 +700,14 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - tracer.runctx('runtest(test, verbose, quiet)', + tracer.runctx('runtest(test, verbose, quiet, timeout=timeout)', globals=globals(), locals=vars()) else: try: result = runtest(test, verbose, quiet, huntrleaks, debug, output_on_failure=verbose3, - failfast=failfast, match_tests=match_tests) + timeout=timeout, failfast=failfast, + match_tests=match_tests) accumulate_result(test, result) except KeyboardInterrupt: interrupted = True @@ -706,7 +778,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, sys.stdout.flush() try: verbose = True - ok = runtest(test, True, quiet, huntrleaks, debug) + ok = runtest(test, True, quiet, huntrleaks, debug, timeout=timeout) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -731,6 +803,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, sys.exit(len(bad) > 0 or interrupted) +# small set of tests to determine if we have a basically functioning interpreter +# (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ 'test_grammar', 'test_opcodes', @@ -741,12 +815,11 @@ STDTESTS = [ 'test_unittest', 'test_doctest', 'test_doctest2', + 'test_support' ] -NOTTESTS = { - 'test_future1', - 'test_future2', -} +# set of tests that we don't want to be executed when using regrtest +NOTTESTS = set() def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): """Return a list of all applicable test modules.""" @@ -755,25 +828,22 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): tests = [] others = set(stdtests) | nottests for name in names: - modname, ext = os.path.splitext(name) - if modname[:5] == "test_" and ext == ".py" and modname not in others: - tests.append(modname) + mod, ext = os.path.splitext(name) + if mod[:5] == "test_" and ext in (".py", "") and mod not in others: + tests.append(mod) return stdtests + sorted(tests) def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" - if os.name == "nt": - # Replace sys.stdout breaks the stdout newlines on Windows: issue #8533 - return - import atexit stdout = sys.stdout sys.stdout = open(stdout.fileno(), 'w', encoding=stdout.encoding, errors="backslashreplace", - closefd=False) + closefd=False, + newline='\n') def restore_stdout(): sys.stdout.close() @@ -782,7 +852,8 @@ def replace_stdout(): def runtest(test, verbose, quiet, huntrleaks=False, debug=False, use_resources=None, - output_on_failure=False, failfast=False, match_tests=None): + output_on_failure=False, failfast=False, match_tests=None, + timeout=None): """Run a single test. test -- the name of the test @@ -792,6 +863,8 @@ def runtest(test, verbose, quiet, huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments output_on_failure -- if true, display test output on failure + timeout -- dump the traceback and exit if a test takes more than + timeout seconds Returns one of the test result constants: INTERRUPTED KeyboardInterrupt when run under -j @@ -804,6 +877,9 @@ def runtest(test, verbose, quiet, if use_resources is not None: support.use_resources = use_resources + use_timeout = (timeout is not None) + if use_timeout: + faulthandler.dump_tracebacks_later(timeout, exit=True) try: support.match_tests = match_tests if failfast: @@ -842,6 +918,8 @@ def runtest(test, verbose, quiet, display_failure=not verbose) return result finally: + if use_timeout: + faulthandler.cancel_dump_tracebacks_later() cleanup_test_droppings(test, verbose) runtest.stringio = None @@ -887,10 +965,12 @@ class saved_test_environment: resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', 'warnings.filters', 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', - 'shutil.archive_formats', 'shutil.unpack_formats', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', 'sys.warnoptions', 'threading._dangling', - 'multiprocessing.process._dangling') + 'multiprocessing.process._dangling', + 'sysconfig._CONFIG_VARS', 'sysconfig._SCHEMES', + 'packaging.command._COMMANDS', 'packaging.database_caches', + ) def get_sys_argv(self): return id(sys.argv), sys.argv, sys.argv[:] @@ -937,6 +1017,11 @@ class saved_test_environment: sys.path_hooks = saved_hooks[1] sys.path_hooks[:] = saved_hooks[2] + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) + def get___import__(self): return builtins.__import__ def restore___import__(self, import_): @@ -989,6 +1074,44 @@ class saved_test_environment: # Can't easily revert the logging state pass + def get_packaging_command__COMMANDS(self): + # registry mapping command names to full dotted path or to the actual + # class (resolved on demand); this check only looks at the names, not + # the types of the values (IOW, if a value changes from a string + # (dotted path) to a class it's okay but if a key (i.e. command class) + # is added we complain) + id_ = id(packaging.command._COMMANDS) + keys = set(packaging.command._COMMANDS) + return id_, keys + def restore_packaging_command__COMMANDS(self, saved): + # if command._COMMANDS was bound to another dict object, we can't + # restore the previous object and contents, because the get_ method + # above does not return the dict object (to ignore changes in values) + for key in packaging.command._COMMANDS.keys() - saved[1]: + del packaging.command._COMMANDS[key] + + def get_packaging_database_caches(self): + # caching system used by the PEP 376 implementation + # we have one boolean and four dictionaries, initially empty + switch = packaging.database._cache_enabled + saved = [] + for name in ('_cache_name', '_cache_name_egg', + '_cache_path', '_cache_path_egg'): + cache = getattr(packaging.database, name) + saved.append((id(cache), cache, cache.copy())) + return switch, saved + def restore_packaging_database_caches(self, saved): + switch, saved_caches = saved + packaging.database._cache_enabled = switch + for offset, name in enumerate(('_cache_name', '_cache_name_egg', + '_cache_path', '_cache_path_egg')): + _, cache, items = saved_caches[offset] + # put back the same object in place + setattr(packaging.database, name, cache) + # now restore its items + cache.clear() + cache.update(items) + def get_sys_warnoptions(self): return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] def restore_sys_warnoptions(self, saved_options): @@ -1020,6 +1143,27 @@ class saved_test_environment: multiprocessing.process._dangling.clear() multiprocessing.process._dangling.update(saved) + def get_sysconfig__CONFIG_VARS(self): + # make sure the dict is initialized + sysconfig.get_config_var('prefix') + return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, + dict(sysconfig._CONFIG_VARS)) + def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig._CONFIG_VARS = saved[1] + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(saved[2]) + + def get_sysconfig__SCHEMES(self): + # it's mildly evil to look at the internal attribute, but it's easier + # than copying a RawConfigParser object + return (id(sysconfig._SCHEMES), sysconfig._SCHEMES._sections, + sysconfig._SCHEMES._sections.copy()) + def restore_sysconfig__SCHEMES(self, saved): + sysconfig._SCHEMES._sections = saved[1] + sysconfig._SCHEMES._sections.clear() + sysconfig._SCHEMES._sections.update(saved[2]) + + def resource_info(self): for name in self.resources: method_suffix = name.replace('.', '_') @@ -1159,7 +1303,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): False if the test didn't leak references; True if we detected refleaks. """ # This code is hackish and inelegant, but it seems to do the job. - import copyreg, _abcoll + import copyreg + import collections.abc if not hasattr(sys, 'gettotalrefcount'): raise Exception("Tracking reference leaks requires a debug build " @@ -1176,7 +1321,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): else: zdc = zipimport._zip_directory_cache.copy() abcs = {} - for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: @@ -1222,7 +1367,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): import gc, copyreg import _strptime, linecache import urllib.parse, urllib.request, mimetypes, doctest - import struct, filecmp, _abcoll + import struct, filecmp, collections.abc from distutils.dir_util import _path_created from weakref import WeakSet @@ -1249,7 +1394,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: @@ -1335,13 +1480,14 @@ def printlist(x, width=70, indent=4): # Tests that are expected to be skipped everywhere except on one platform # are also handled separately. -_expectations = { - 'win32': +_expectations = ( + ('win32', """ test__locale test_crypt test_curses test_dbm + test_devpoll test_fcntl test_fork1 test_epoll @@ -1364,15 +1510,16 @@ _expectations = { test_threadsignals test_wait3 test_wait4 - """, - 'linux2': + """), + ('linux', """ test_curses + test_devpoll test_largefile test_kqueue test_ossaudiodev - """, - 'unixware7': + """), + ('unixware', """ test_epoll test_largefile @@ -1382,8 +1529,8 @@ _expectations = { test_pyexpat test_sax test_sundry - """, - 'openunix8': + """), + ('openunix', """ test_epoll test_largefile @@ -1393,8 +1540,8 @@ _expectations = { test_pyexpat test_sax test_sundry - """, - 'sco_sv3': + """), + ('sco_sv', """ test_asynchat test_fork1 @@ -1413,11 +1560,12 @@ _expectations = { test_threaded_import test_threadedtempfile test_threading - """, - 'darwin': + """), + ('darwin', """ test__locale test_curses + test_devpoll test_epoll test_dbm_gnu test_gdb @@ -1426,8 +1574,8 @@ _expectations = { test_minidom test_ossaudiodev test_poll - """, - 'sunos5': + """), + ('sunos', """ test_curses test_dbm @@ -1438,8 +1586,8 @@ _expectations = { test_openpty test_zipfile test_zlib - """, - 'hp-ux11': + """), + ('hp-ux', """ test_curses test_epoll @@ -1454,11 +1602,12 @@ _expectations = { test_sax test_zipfile test_zlib - """, - 'cygwin': + """), + ('cygwin', """ test_curses test_dbm + test_devpoll test_epoll test_ioctl test_kqueue @@ -1466,8 +1615,8 @@ _expectations = { test_locale test_ossaudiodev test_socketserver - """, - 'os2emx': + """), + ('os2emx', """ test_audioop test_curses @@ -1480,9 +1629,10 @@ _expectations = { test_pty test_resource test_signal - """, - 'freebsd4': + """), + ('freebsd', """ + test_devpoll test_epoll test_dbm_gnu test_locale @@ -1497,8 +1647,8 @@ _expectations = { test_timeout test_urllibnet test_multiprocessing - """, - 'aix5': + """), + ('aix', """ test_bz2 test_epoll @@ -1512,10 +1662,11 @@ _expectations = { test_ttk_textonly test_zipimport test_zlib - """, - 'openbsd3': + """), + ('openbsd', """ test_ctypes + test_devpoll test_epoll test_dbm_gnu test_locale @@ -1527,11 +1678,12 @@ _expectations = { test_ttk_guionly test_ttk_textonly test_multiprocessing - """, - 'netbsd3': + """), + ('netbsd', """ test_ctypes test_curses + test_devpoll test_epoll test_dbm_gnu test_locale @@ -1542,12 +1694,8 @@ _expectations = { test_ttk_guionly test_ttk_textonly test_multiprocessing - """, -} -_expectations['freebsd5'] = _expectations['freebsd4'] -_expectations['freebsd6'] = _expectations['freebsd4'] -_expectations['freebsd7'] = _expectations['freebsd4'] -_expectations['freebsd8'] = _expectations['freebsd4'] + """), +) class _ExpectedSkips: def __init__(self): @@ -1555,9 +1703,13 @@ class _ExpectedSkips: from test import test_timeout self.valid = False - if sys.platform in _expectations: - s = _expectations[sys.platform] - self.expected = set(s.split()) + expected = None + for item in _expectations: + if sys.platform.startswith(item[0]): + expected = item[1] + break + if expected is not None: + self.expected = set(expected.split()) # These are broken tests, for now skipped on every platform. # XXX Fix these! @@ -1617,9 +1769,8 @@ def _make_temp_dir_for_build(TEMPDIR): TEMPDIR = os.path.abspath(TEMPDIR) try: os.mkdir(TEMPDIR) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass # Define a writable temp dir that will be used as cwd while running # the tests. The name of the dir includes the pid to allow parallel |