diff options
Diffstat (limited to 'Lib/test/regrtest.py')
| -rwxr-xr-x | Lib/test/regrtest.py | 914 |
1 files changed, 592 insertions, 322 deletions
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 71cc866..238f276 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1,14 +1,21 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 -"""Regression test. +""" +Usage: + +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] + + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: -This will find all modules whose name is "test_*" in the test -directory, and run them. Various command line options provide -additional facilities. +python -E -Wd -m test [options] [test_name1 ...] -If non-option arguments are present, they are names for tests to run, -unless -x is given, in which case they are names for tests not to run. -If no test names are given, all tests are run. Options: @@ -18,6 +25,7 @@ Verbosity -v/--verbose -- run tests in verbose mode with output to stdout -w/--verbose2 -- re-run failed tests in verbose mode +-W/--verbose3 -- re-run failed tests in verbose mode immediately -d/--debug -- print traceback for failed tests -q/--quiet -- no output unless one or more tests fail -S/--slow -- print the slowest 10 tests @@ -41,6 +49,8 @@ Special runs -L/--runleaks -- run the leaks(1) command just before exit -R/--huntrleaks RUNCOUNTS -- search for reference leaks (needs debug build, v. slow) +-j/--multiprocess PROCESSES + -- run PROCESSES processes at once -T/--coverage -- turn on code coverage tracing using the trace module -D/--coverdir DIRECTORY -- Directory where coverage files are put @@ -48,6 +58,7 @@ Special runs -t/--threshold THRESHOLD -- call gc.set_threshold(THRESHOLD) -n/--nowindows -- suppress error message boxes on Windows +-F/--forever -- run the specified tests in a loop, until an error happens Additional Option Details: @@ -56,19 +67,16 @@ Additional Option Details: int seed value for the randomizer; this is useful for reproducing troublesome test orders. --T turns on code coverage tracing with the trace module. - --D specifies the directory where coverage files are put. - --N Put coverage files alongside modules. - --s means to run only a single test and exit. This is useful when -doing memory analysis on the Python interpreter (which tend to consume -too many resources to run the full regression test non-stop). The -file /tmp/pynexttest is read to find the next test to run. If this -file is missing, the first test_*.py file in testdir or on the command -line is used. (actually tempfile.gettempdir() is used instead of -/tmp). +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. -S is used to continue running tests after an aborted run. It will maintain the order a standard run (ie, this assumes -r is not used). @@ -145,36 +153,41 @@ example, to run all the tests except for the gui tests, give the option '-uall,-gui'. """ +import builtins import getopt +import json import os import random import re import io import sys import time -import platform import traceback import warnings import unittest from inspect import isabstract +import tempfile +import platform +import sysconfig +import logging + + +# Some times __path__ and __file__ are not absolute (e.g. while running from +# Lib/) and, if we change the CWD to run the tests in a temporary dir, some +# imports might fail. This affects only the modules imported before os.chdir(). +# These modules are searched first in sys.path[0] (so '' -- the CWD) and if +# they are found in the CWD their __file__ and __path__ will be relative (this +# happens before the chdir). All the modules imported after the chdir, are +# not found in the CWD, and since the other paths in sys.path[1:] are absolute +# (site.py absolutize them), the __file__ and __path__ will be absolute too. +# Therefore it is necessary to absolutize manually the __file__ and __path__ of +# the packages to prevent later imports to fail when the CWD is different. +for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) -# I see no other way to suppress these warnings; -# putting them in test_grammar.py has no effect: -warnings.filterwarnings("ignore", "hex/oct constants", FutureWarning, - ".*test.test_grammar$") -if sys.maxsize > 0x7fffffff: - # Also suppress them in <string>, because for 64-bit platforms, - # that's where test_grammar.py hides them. - warnings.filterwarnings("ignore", "hex/oct constants", FutureWarning, - "<string>") - -# Ignore ImportWarnings that only occur in the source tree, -# (because of modules with the same name as source-directories in Modules/) -for mod in ("ctypes", "gzip", "zipfile", "tarfile", "encodings.zlib_codec", - "test.test_zipimport", "test.test_zlib", "test.test_zipfile", - "test.test_codecs", "test.string_tests"): - warnings.filterwarnings(module=".*%s$" % (mod,), - action="ignore", category=ImportWarning) # MacOSX (a.k.a. Darwin) has a default stack size that is too small # for deeply recursive regular expressions. We see this as crashes in @@ -192,11 +205,20 @@ if sys.platform == 'darwin': newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) +# Test result constants. +PASSED = 1 +FAILED = 0 +ENV_CHANGED = -1 +SKIPPED = -2 +RESOURCE_DENIED = -3 +INTERRUPTED = -4 + from test import support RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') +TEMPDIR = os.path.abspath(tempfile.gettempdir()) def usage(msg): print(msg, file=sys.stderr) @@ -204,11 +226,12 @@ def usage(msg): sys.exit(2) -def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, +def main(tests=None, testdir=None, verbose=0, quiet=False, exclude=False, single=False, randomize=False, fromfile=None, findleaks=False, use_resources=None, trace=False, coverdir='coverage', runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, header=False): + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False): """Execute a test suite. This also parses command-line options and modifies its behavior @@ -225,7 +248,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, command-line will be used. If that's empty, too, then all *.py files beginning with test_ will be used. - The other default arguments (verbose, quiet, generate, exclude, + The other default arguments (verbose, quiet, exclude, single, randomize, findleaks, use_resources, trace, coverdir, print_slow, and random_seed) allow programmers calling main() directly to set the values that would normally be set by flags @@ -236,14 +259,13 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, support.record_original_stdout(sys.stdout) try: - opts, args = getopt.getopt(sys.argv[1:], 'hvgqxsSrf:lu:t:TD:NLR:wM:n', - ['help', 'verbose', 'quiet', 'exclude', - 'single', 'slow', 'random', 'fromfile', - 'findleaks', 'use=', 'threshold=', 'trace', - 'coverdir=', 'nocoverdir', 'runleaks', - 'huntrleaks=', 'verbose2', 'memlimit=', - 'debug', 'start=', 'nowindows', - 'randseed=', 'header']) + opts, args = getopt.getopt(sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:FwWM:nj:', + ['help', 'verbose', 'verbose2', 'verbose3', 'quiet', + 'exclude', 'single', 'slow', 'random', 'fromfile', 'findleaks', + 'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir', + 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', + 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', + 'start=', 'nowindows', 'header']) except getopt.error as msg: usage(msg) @@ -264,6 +286,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, verbose2 = True elif o in ('-d', '--debug'): debug = True + elif o in ('-W', '--verbose3'): + verbose3 = True elif o in ('-q', '--quiet'): quiet = True; verbose = 0 @@ -346,30 +370,39 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + elif o in ('-F', '--forever'): + forever = True + elif o in ('-j', '--multiprocess'): + use_mp = int(a) elif o == '--header': header = True + elif o == '--slaveargs': + args, kwargs = json.loads(a) + try: + result = runtest(*args, **kwargs) + except BaseException as e: + result = INTERRUPTED, e.__class__.__name__ + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) else: print(("No handler for option {}. Please report this as a bug " - "at http://bugs.python.org.").format(o), file=sys.stderr) + "at http://bugs.python.org.").format(o), file=sys.stderr) sys.exit(1) - if generate and verbose: - usage("-g and -v don't go together!") if single and fromfile: usage("-s and -f don't go together!") + if use_mp and trace: + usage("-T and -j don't go together!") + if use_mp and findleaks: + usage("-l and -j don't go together!") good = [] bad = [] skipped = [] resource_denieds = [] - - # For a partial run, we do not need to clutter the output. - if verbose or header or not (quiet or single or tests or args): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) + environment_changed = [] + interrupted = False if findleaks: try: @@ -385,30 +418,29 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, found_garbage = [] if single: - from tempfile import gettempdir - filename = os.path.join(gettempdir(), 'pynexttest') + filename = os.path.join(TEMPDIR, 'pynexttest') try: fp = open(filename, 'r') - next = fp.read().strip() - tests = [next] + next_test = fp.read().strip() + tests = [next_test] fp.close() except IOError: pass if fromfile: tests = [] - fp = open(fromfile) + fp = open(os.path.join(support.SAVEDCWD, fromfile)) + count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') for line in fp: + line = count_pat.sub('', line) guts = line.split() # assuming no test has whitespace in its name if guts and not guts[0].startswith('#'): tests.extend(guts) fp.close() # Strip .py extensions. - if args: - args = list(map(removepy, args)) - if tests: - tests = list(map(removepy, tests)) + removepy(args) + removepy(tests) stdtests = STDTESTS[:] nottests = NOTTESTS.copy() @@ -418,9 +450,24 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, stdtests.remove(arg) nottests.add(arg) args = [] - tests = tests or args or findtests(testdir, stdtests, nottests) + + # For a partial run, we do not need to clutter the output. + if verbose or header or not (quiet or single or tests or args): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + alltests = findtests(testdir, stdtests, nottests) + selected = tests or args or alltests if single: - tests = tests[:1] + selected = selected[:1] + try: + next_single_test = alltests[alltests.index(selected[0])+1] + except IndexError: + next_single_test = None # Remove all the tests that precede start if it's set. if start: try: @@ -430,73 +477,187 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, if randomize: random.seed(random_seed) print("Using random seed", random_seed) - random.shuffle(tests) + random.shuffle(selected) if trace: import trace, tempfile tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix, tempfile.gettempdir()], trace=False, count=True) + test_times = [] support.verbose = verbose # Tell tests to be moderately quiet support.use_resources = use_resources save_modules = sys.modules.keys() - for test in tests: - if not quiet: - print(test) - sys.stdout.flush() - if trace: - # If we're tracing code coverage, then we don't exit with status - # if on a false return value from main. - tracer.runctx('runtest(test, generate, verbose, quiet,' - ' test_times, testdir)', - globals=globals(), locals=vars()) - else: + + def accumulate_result(test, result): + ok, test_time = result + test_times.append((test_time, test)) + if ok == PASSED: + good.append(test) + elif ok == FAILED: + bad.append(test) + elif ok == ENV_CHANGED: + bad.append(test) + environment_changed.append(test) + elif ok == SKIPPED: + skipped.append(test) + elif ok == RESOURCE_DENIED: + skipped.append(test) + resource_denieds.append(test) + + if forever: + def test_forever(tests=list(selected)): + while True: + for test in tests: + yield test + if bad: + return + tests = test_forever() + test_count = '' + test_count_width = 3 + else: + tests = iter(selected) + test_count = '/{}'.format(len(selected)) + test_count_width = len(test_count) - 1 + + if use_mp: + try: + from threading import Thread + except ImportError: + print("Multiprocess option requires thread support") + sys.exit(2) + from queue import Queue + from subprocess import Popen, PIPE + debug_output_pat = re.compile(r"\[\d+ refs\]$") + output = Queue() + def tests_and_args(): + for test in tests: + args_tuple = ( + (test, verbose, quiet), + dict(huntrleaks=huntrleaks, use_resources=use_resources, + debug=debug) + ) + yield (test, args_tuple) + pending = tests_and_args() + opt_args = support.args_from_interpreter_flags() + base_cmd = [sys.executable] + opt_args + ['-m', 'test.regrtest'] + def work(): + # A worker thread. try: - ok = runtest(test, generate, verbose, quiet, test_times, - testdir, huntrleaks) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - except: + while True: + try: + test, args_tuple = next(pending) + except StopIteration: + output.put((None, None, None, None)) + return + # -E is needed by some tests, e.g. test_import + popen = Popen(base_cmd + ['--slaveargs', json.dumps(args_tuple)], + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt')) + stdout, stderr = popen.communicate() + # Strip last refcount output line if it exists, since it + # comes from the shutdown of the interpreter in the subcommand. + stderr = debug_output_pat.sub("", stderr) + stdout, _, result = stdout.strip().rpartition("\n") + if not result: + output.put((None, None, None, None)) + return + result = json.loads(result) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + except BaseException: + output.put((None, None, None, None)) raise - if ok > 0: - good.append(test) - elif ok == 0: - bad.append(test) + workers = [Thread(target=work) for i in range(use_mp)] + for worker in workers: + worker.start() + finished = 0 + test_index = 1 + try: + while finished < use_mp: + test, stdout, stderr, result = output.get() + if test is None: + finished += 1 + continue + if not quiet: + print("[{1:{0}}{2}] {3}".format( + test_count_width, test_index, test_count, test)) + if stdout: + print(stdout) + if stderr: + print(stderr, file=sys.stderr) + if result[0] == INTERRUPTED: + assert result[1] == 'KeyboardInterrupt' + raise KeyboardInterrupt # What else? + accumulate_result(test, result) + test_index += 1 + except KeyboardInterrupt: + interrupted = True + pending.close() + for worker in workers: + worker.join() + else: + for test_index, test in enumerate(tests, 1): + if not quiet: + print("[{1:{0}}{2}] {3}".format( + test_count_width, test_index, test_count, test)) + sys.stdout.flush() + if trace: + # If we're tracing code coverage, then we don't exit with status + # if on a false return value from main. + tracer.runctx('runtest(test, verbose, quiet)', + globals=globals(), locals=vars()) else: - skipped.append(test) - if ok == -2: - resource_denieds.append(test) - if findleaks: - gc.collect() - if gc.garbage: - print("Warning: test created", len(gc.garbage), end=' ') - print("uncollectable object(s).") - # move the uncollectable objects somewhere so we don't see - # them again - found_garbage.extend(gc.garbage) - del gc.garbage[:] - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) - + try: + result = runtest(test, verbose, quiet, huntrleaks, debug) + accumulate_result(test, result) + if verbose3 and result[0] == FAILED: + print("Re-running test {} in verbose mode".format(test)) + runtest(test, True, quiet, huntrleaks, debug) + except KeyboardInterrupt: + interrupted = True + break + except: + raise + if findleaks: + gc.collect() + if gc.garbage: + print("Warning: test created", len(gc.garbage), end=' ') + print("uncollectable object(s).") + # move the uncollectable objects somewhere so we don't see + # them again + found_garbage.extend(gc.garbage) + del gc.garbage[:] + # Unload the newly imported modules (best effort finalization) + for module in sys.modules.keys(): + if module not in save_modules and module.startswith("test."): + support.unload(module) + + if interrupted: + # print a newline after ^C + print() + print("Test suite interrupted by signal SIGINT.") + omitted = set(selected) - set(good) - set(bad) - set(skipped) + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) if good and not quiet: - if not bad and not skipped and len(good) > 1: + if not bad and not skipped and not interrupted and len(good) > 1: print("All", end=' ') print(count(len(good), "test"), "OK.") - if verbose: - print("CAUTION: stdout isn't compared in verbose mode:") - print("a test that passes in verbose mode may fail without it.") if print_slow: test_times.sort(reverse=True) print("10 slowest tests:") for time, test in test_times[:10]: print("%s: %.1fs" % (test, time)) if bad: - print(count(len(bad), "test"), "failed:") - printlist(bad) + bad = sorted(set(bad) - set(environment_changed)) + if bad: + print(count(len(bad), "test"), "failed:") + printlist(bad) + if environment_changed: + print("{} altered the execution environment:".format( + count(len(environment_changed), "test"))) + printlist(environment_changed) if skipped and not quiet: print(count(len(skipped), "test"), "skipped:") printlist(skipped) @@ -521,9 +682,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, print("Re-running test %r in verbose mode" % test) sys.stdout.flush() try: - support.verbose = True - ok = runtest(test, generate, True, quiet, test_times, testdir, - huntrleaks, debug) + verbose = True + ok = runtest(test, True, quiet, huntrleaks, debug) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -532,16 +692,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, raise if single: - alltests = findtests(testdir, stdtests, nottests) - for i in range(len(alltests)): - if tests[0] == alltests[i]: - if i == len(alltests) - 1: - os.unlink(filename) - else: - fp = open(filename, 'w') - fp.write(alltests[i+1] + '\n') - fp.close() - break + if next_single_test: + with open(filename, 'w') as fp: + fp.write(next_single_test + '\n') else: os.unlink(filename) @@ -552,7 +705,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, if runleaks: os.system("leaks %d" % os.getpid()) - sys.exit(len(bad) > 0) + sys.exit(len(bad) > 0 or interrupted) STDTESTS = [ @@ -574,16 +727,15 @@ NOTTESTS = { def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): """Return a list of all applicable test modules.""" - if not testdir: testdir = findtestdir() + testdir = findtestdir(testdir) names = os.listdir(testdir) tests = [] + others = set(stdtests) | nottests for name in names: - if name[:5] == "test_" and name[-3:] == ".py": - modname = name[:-3] - if modname not in stdtests and modname not in nottests: - tests.append(modname) - tests.sort() - return stdtests + tests + modname, ext = os.path.splitext(name) + if modname[:5] == "test_" and ext == ".py" and modname not in others: + tests.append(modname) + return stdtests + sorted(tests) def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error @@ -591,63 +743,228 @@ def replace_stdout(): if os.name == "nt": # Replace sys.stdout breaks the stdout newlines on Windows: issue #8533 return + + import atexit + stdout = sys.stdout sys.stdout = open(stdout.fileno(), 'w', encoding=stdout.encoding, - errors="backslashreplace") + errors="backslashreplace", + closefd=False) -def runtest(test, generate, verbose, quiet, test_times, - testdir=None, huntrleaks=False, debug=False): + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) + +def runtest(test, verbose, quiet, + huntrleaks=False, debug=False, use_resources=None): """Run a single test. test -- the name of the test verbose -- if true, print more messages quiet -- if true, don't print 'skipped' messages (probably redundant) test_times -- a list of (time, test_name) pairs - testdir -- test directory huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments - debug -- if true, print tracebacks for failed tests regardless of - verbose setting - Return: - -2 test skipped because resource denied - -1 test skipped for some other reason - 0 test failed - 1 test passed + + Returns one of the test result constants: + INTERRUPTED KeyboardInterrupt when run under -j + RESOURCE_DENIED test skipped because resource denied + SKIPPED test skipped for some other reason + ENV_CHANGED test failed because it changed the execution environment + FAILED test failed + PASSED test passed """ + support.verbose = verbose # Tell tests to be moderately quiet + if use_resources is not None: + support.use_resources = use_resources try: - return runtest_inner(test, generate, verbose, quiet, test_times, - testdir, huntrleaks) + return runtest_inner(test, verbose, quiet, huntrleaks, debug) finally: cleanup_test_droppings(test, verbose) -def runtest_inner(test, generate, verbose, quiet, test_times, - testdir=None, huntrleaks=False, debug=False): +# Unit tests are supposed to leave the execution environment unchanged +# once they complete. But sometimes tests have bugs, especially when +# tests fail, and the changes to environment go on to mess up other +# tests. This can cause issues with buildbot stability, since tests +# are run in random order and so problems may appear to come and go. +# There are a few things we can save and restore to mitigate this, and +# the following context manager handles this task. + +class saved_test_environment: + """Save bits of the test environment and restore them at block exit. + + with saved_test_environment(testname, verbose, quiet): + #stuff + + Unless quiet is True, a warning is printed to stderr if any of + the saved items was changed by the test. The attribute 'changed' + is initially False, but is set to True if a change is detected. + + If verbose is more than 1, the before and after state of changed + items is also printed. + """ + + changed = False + + def __init__(self, testname, verbose=0, quiet=False): + self.testname = testname + self.verbose = verbose + self.quiet = quiet + + # To add things to save and restore, add a name XXX to the resources list + # and add corresponding get_XXX/restore_XXX functions. get_XXX should + # return the value to be saved and compared against a second call to the + # get function when test execution completes. restore_XXX should accept + # the saved value and restore the resource using it. It will be called if + # and only if a change in the value is detected. + # + # Note: XXX will have any '.' replaced with '_' characters when determining + # the corresponding method names. + + resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', + 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', + 'warnings.filters', 'asyncore.socket_map', + 'logging._handlers', 'logging._handlerList', + 'sys.warnoptions') + + def get_sys_argv(self): + return id(sys.argv), sys.argv, sys.argv[:] + def restore_sys_argv(self, saved_argv): + sys.argv = saved_argv[1] + sys.argv[:] = saved_argv[2] + + def get_cwd(self): + return os.getcwd() + def restore_cwd(self, saved_cwd): + os.chdir(saved_cwd) + + def get_sys_stdout(self): + return sys.stdout + def restore_sys_stdout(self, saved_stdout): + sys.stdout = saved_stdout + + def get_sys_stderr(self): + return sys.stderr + def restore_sys_stderr(self, saved_stderr): + sys.stderr = saved_stderr + + def get_sys_stdin(self): + return sys.stdin + def restore_sys_stdin(self, saved_stdin): + sys.stdin = saved_stdin + + def get_os_environ(self): + return id(os.environ), os.environ, dict(os.environ) + def restore_os_environ(self, saved_environ): + os.environ = saved_environ[1] + os.environ.clear() + os.environ.update(saved_environ[2]) + + def get_sys_path(self): + return id(sys.path), sys.path, sys.path[:] + def restore_sys_path(self, saved_path): + sys.path = saved_path[1] + sys.path[:] = saved_path[2] + + def get_sys_path_hooks(self): + return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] + def restore_sys_path_hooks(self, saved_hooks): + sys.path_hooks = saved_hooks[1] + sys.path_hooks[:] = saved_hooks[2] + + def get___import__(self): + return builtins.__import__ + def restore___import__(self, import_): + builtins.__import__ = import_ + + def get_warnings_filters(self): + return id(warnings.filters), warnings.filters, warnings.filters[:] + def restore_warnings_filters(self, saved_filters): + warnings.filters = saved_filters[1] + warnings.filters[:] = saved_filters[2] + + def get_asyncore_socket_map(self): + asyncore = sys.modules.get('asyncore') + # XXX Making a copy keeps objects alive until __exit__ gets called. + return asyncore and asyncore.socket_map.copy() or {} + def restore_asyncore_socket_map(self, saved_map): + asyncore = sys.modules.get('asyncore') + if asyncore is not None: + asyncore.close_all(ignore_all=True) + asyncore.socket_map.update(saved_map) + + def get_logging__handlers(self): + # _handlers is a WeakValueDictionary + return id(logging._handlers), logging._handlers, logging._handlers.copy() + def restore_logging__handlers(self, saved_handlers): + # Can't easily revert the logging state + pass + + def get_logging__handlerList(self): + # _handlerList is a list of weakrefs to handlers + return id(logging._handlerList), logging._handlerList, logging._handlerList[:] + def restore_logging__handlerList(self, saved_handlerList): + # Can't easily revert the logging state + pass + + def get_sys_warnoptions(self): + return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] + def restore_sys_warnoptions(self, saved_options): + sys.warnoptions = saved_options[1] + sys.warnoptions[:] = saved_options[2] + + def resource_info(self): + for name in self.resources: + method_suffix = name.replace('.', '_') + get_name = 'get_' + method_suffix + restore_name = 'restore_' + method_suffix + yield name, getattr(self, get_name), getattr(self, restore_name) + + def __enter__(self): + self.saved_values = dict((name, get()) for name, get, restore + in self.resource_info()) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + saved_values = self.saved_values + del self.saved_values + for name, get, restore in self.resource_info(): + current = get() + original = saved_values.pop(name) + # Check for changes to the resource's value + if current != original: + self.changed = True + restore(original) + if not self.quiet: + print("Warning -- {} was modified by {}".format( + name, self.testname), + file=sys.stderr) + if self.verbose > 1: + print(" Before: {}\n After: {} ".format( + original, current), + file=sys.stderr) + return False + + +def runtest_inner(test, verbose, quiet, huntrleaks=False, debug=False): support.unload(test) - if not testdir: - testdir = findtestdir() if verbose: - cfp = None + capture_stdout = None else: - cfp = io.StringIO() # XXX Should use io.StringIO() + capture_stdout = io.StringIO() + test_time = 0.0 refleak = False # True if the test leaked references. try: - save_stdout = sys.stdout - # Save various things that tests may mess up so we can restore - # them afterward. - save_environ = dict(os.environ) - save_argv = sys.argv[:] - try: - if cfp: - sys.stdout = cfp - print(test) # Output file starts with test name - if test.startswith('test.'): - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test + if test.startswith('test.'): + abstest = test + else: + # Always import it from the test package + abstest = 'test.' + test + with saved_test_environment(test, verbose, quiet) as environment: start_time = time.time() the_package = __import__(abstest, globals(), locals(), []) the_module = getattr(the_package, test) @@ -658,60 +975,36 @@ def runtest_inner(test, generate, verbose, quiet, test_times, if indirect_test is not None: indirect_test() if huntrleaks: - refleak = dash_R(the_module, test, indirect_test, huntrleaks) + refleak = dash_R(the_module, test, indirect_test, + huntrleaks) test_time = time.time() - start_time - test_times.append((test_time, test)) - finally: - sys.stdout = save_stdout - # Restore what we saved if needed, but also complain if the test - # changed it so that the test may eventually get fixed. - if not os.environ == save_environ: - if not quiet: - print("Warning: os.environ was modified by", test) - os.environ.clear() - os.environ.update(save_environ) - if not sys.argv == save_argv: - if not quiet: - print("Warning: argv was modified by", test) - sys.argv[:] = save_argv except support.ResourceDenied as msg: if not quiet: print(test, "skipped --", msg) sys.stdout.flush() - return -2 + return RESOURCE_DENIED, test_time except unittest.SkipTest as msg: if not quiet: print(test, "skipped --", msg) sys.stdout.flush() - return -1 + return SKIPPED, test_time except KeyboardInterrupt: raise except support.TestFailed as msg: print("test", test, "failed --", msg, file=sys.stderr) sys.stderr.flush() - return 0 + return FAILED, test_time except: - type, value = sys.exc_info()[:2] - print("test", test, "crashed --", str(type) + ":", value, file=sys.stderr) + msg = traceback.format_exc() + print("test", test, "crashed --", msg, file=sys.stderr) sys.stderr.flush() - if verbose or debug: - traceback.print_exc(file=sys.stderr) - sys.stderr.flush() - return 0 + return FAILED, test_time else: if refleak: - return 0 - if not cfp: - return 1 - output = cfp.getvalue() - expected = test + "\n" - if output == expected or huntrleaks: - return 1 - print("test", test, "produced unexpected output:") - sys.stdout.flush() - reportdiff(expected, output) - sys.stdout.flush() - return 0 + return FAILED, test_time + if environment.changed: + return ENV_CHANGED, test_time + return PASSED, test_time def cleanup_test_droppings(testname, verbose): import shutil @@ -719,6 +1012,8 @@ def cleanup_test_droppings(testname, verbose): import gc # First kill any dangling references to open files etc. + # This can also issue some ResourceWarnings which would otherwise get + # triggered during the following test run, and possibly produce failures. gc.collect() # Try to clean up junk commonly left behind. While tests shouldn't leave @@ -770,6 +1065,12 @@ def dash_R(the_module, test, indirect_test, huntrleaks): fs = warnings.filters[:] ps = copyreg.dispatch_table.copy() pic = sys.path_importer_cache.copy() + try: + import zipimport + except ImportError: + zdc = None # Run unmodified on platforms without zipimport support + else: + zdc = zipimport._zip_directory_cache.copy() abcs = {} for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: if not isabstract(abc): @@ -787,29 +1088,33 @@ def dash_R(the_module, test, indirect_test, huntrleaks): deltas = [] nwarmup, ntracked, fname = huntrleaks + fname = os.path.join(support.SAVEDCWD, fname) repcount = nwarmup + ntracked print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) - dash_R_cleanup(fs, ps, pic, abcs) + sys.stderr.flush() + dash_R_cleanup(fs, ps, pic, zdc, abcs) for i in range(repcount): - rc = sys.gettotalrefcount() + rc_before = sys.gettotalrefcount() run_the_test() sys.stderr.write('.') sys.stderr.flush() - dash_R_cleanup(fs, ps, pic, abcs) + dash_R_cleanup(fs, ps, pic, zdc, abcs) + rc_after = sys.gettotalrefcount() if i >= nwarmup: - deltas.append(sys.gettotalrefcount() - rc - 2) + deltas.append(rc_after - rc_before) print(file=sys.stderr) if any(deltas): msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas)) print(msg, file=sys.stderr) - refrep = open(fname, "a") - print(msg, file=refrep) - refrep.close() + sys.stderr.flush() + with open(fname, "a") as refrep: + print(msg, file=refrep) + refrep.flush() return True return False -def dash_R_cleanup(fs, ps, pic, abcs): +def dash_R_cleanup(fs, ps, pic, zdc, abcs): import gc, copyreg import _strptime, linecache import urllib.parse, urllib.request, mimetypes, doctest @@ -828,6 +1133,13 @@ def dash_R_cleanup(fs, ps, pic, abcs): copyreg.dispatch_table.update(ps) sys.path_importer_cache.clear() sys.path_importer_cache.update(pic) + try: + import zipimport + except ImportError: + pass # Run unmodified on platforms without zipimport support + else: + zipimport._zip_directory_cache.clear() + zipimport._zip_directory_cache.update(zdc) # clear type cache sys._clear_type_cache() @@ -841,6 +1153,12 @@ def dash_R_cleanup(fs, ps, pic, abcs): obj._abc_cache.clear() obj._abc_negative_cache.clear() + # Flush standard output, so that buffered data is sent to the OS and + # associated Python objects are reclaimed. + for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): + if stream is not None: + stream.flush() + # Clear assorted module caches. _path_created.clear() re.purge() @@ -861,60 +1179,16 @@ def warm_char_cache(): for i in range(256): s[i:i+1] -def reportdiff(expected, output): - import difflib - print("*" * 70) - a = expected.splitlines(1) - b = output.splitlines(1) - sm = difflib.SequenceMatcher(a=a, b=b) - tuples = sm.get_opcodes() - - def pair(x0, x1): - # x0:x1 are 0-based slice indices; convert to 1-based line indices. - x0 += 1 - if x0 >= x1: - return "line " + str(x0) - else: - return "lines %d-%d" % (x0, x1) - - for op, a0, a1, b0, b1 in tuples: - if op == 'equal': - pass - - elif op == 'delete': - print("***", pair(a0, a1), "of expected output missing:") - for line in a[a0:a1]: - print("-", line, end='') +def findtestdir(path=None): + return path or os.path.dirname(__file__) or os.curdir - elif op == 'replace': - print("*** mismatch between", pair(a0, a1), "of expected", \ - "output and", pair(b0, b1), "of actual output:") - for line in difflib.ndiff(a[a0:a1], b[b0:b1]): - print(line, end='') - - elif op == 'insert': - print("***", pair(b0, b1), "of actual output doesn't appear", \ - "in expected output after line", str(a1)+":") - for line in b[b0:b1]: - print("+", line, end='') - - else: - print("get_opcodes() returned bad tuple?!?!", (op, a0, a1, b0, b1)) - - print("*" * 70) - -def findtestdir(): - if __name__ == '__main__': - file = sys.argv[0] - else: - file = __file__ - testdir = os.path.dirname(file) or os.curdir - return testdir - -def removepy(name): - if name.endswith(".py"): - name = name[:-3] - return name +def removepy(names): + if not names: + return + for idx, name in enumerate(names): + basename, ext = os.path.splitext(name) + if ext == '.py': + names[idx] = basename def count(n, word): if n == 1: @@ -987,34 +1261,6 @@ _expectations = { test_kqueue test_ossaudiodev """, - 'mac': - """ - test_atexit - test_bz2 - test_crypt - test_curses - test_dbm - test_fcntl - test_fork1 - test_epoll - test_grp - test_ioctl - test_largefile - test_locale - test_kqueue - test_mmap - test_openpty - test_ossaudiodev - test_poll - test_popen - test_posix - test_pty - test_pwd - test_resource - test_signal - test_sundry - test_tarfile - """, 'unixware7': """ test_epoll @@ -1063,6 +1309,7 @@ _expectations = { test_curses test_epoll test_dbm_gnu + test_gdb test_largefile test_locale test_minidom @@ -1097,19 +1344,6 @@ _expectations = { test_zipfile test_zlib """, - 'atheos': - """ - test_curses - test_dbm_gnu - test_epoll - test_largefile - test_locale - test_kqueue - test_mhlib - test_mmap - test_poll - test_resource - """, 'cygwin': """ test_curses @@ -1237,15 +1471,17 @@ class _ExpectedSkips: if sys.platform != "win32": # test_sqlite is only reliable on Windows where the library # is distributed with Python - WIN_ONLY = ["test_unicode_file", "test_winreg", + WIN_ONLY = {"test_unicode_file", "test_winreg", "test_winsound", "test_startfile", - "test_sqlite"] - for skip in WIN_ONLY: - self.expected.add(skip) + "test_sqlite"} + self.expected |= WIN_ONLY if sys.platform != 'sunos5': self.expected.add('test_nis') + if support.python_is_optimized(): + self.expected.add("test_gdb") + self.valid = True def isvalid(self): @@ -1261,17 +1497,51 @@ class _ExpectedSkips: assert self.isvalid() return self.expected +def _make_temp_dir_for_build(TEMPDIR): + # When tests are run from the Python build directory, it is best practice + # to keep the test files in a subfolder. It eases the cleanup of leftover + # files using command "make distclean". + if sysconfig.is_python_build(): + TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') + TEMPDIR = os.path.abspath(TEMPDIR) + if not os.path.exists(TEMPDIR): + os.mkdir(TEMPDIR) + + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + TESTCWD = 'test_python_{}'.format(os.getpid()) + + TESTCWD = os.path.join(TEMPDIR, TESTCWD) + return TEMPDIR, TESTCWD + if __name__ == '__main__': - # Remove regrtest.py's own directory from the module search path. This - # prevents relative imports from working, and relative imports will screw - # up the testing framework. E.g. if both test.support and - # support are imported, they will not contain the same globals, and - # much of the testing framework relies on the globals in the - # test.support module. + # Remove regrtest.py's own directory from the module search path. Despite + # the elimination of implicit relative imports, this is still needed to + # ensure that submodules of the test package do not inappropriately appear + # as top-level modules even when people (or buildbots!) invoke regrtest.py + # directly instead of using the -m switch mydir = os.path.abspath(os.path.normpath(os.path.dirname(sys.argv[0]))) i = len(sys.path) while i >= 0: i -= 1 if os.path.abspath(os.path.normpath(sys.path[i])) == mydir: del sys.path[i] - main() + + # findtestdir() gets the dirname out of __file__, so we have to make it + # absolute before changing the working directory. + # For example __file__ may be relative when running trace or profile. + # See issue #9323. + __file__ = os.path.abspath(__file__) + + # sanity check + assert __file__ == os.path.abspath(sys.argv[0]) + + TEMPDIR, TESTCWD = _make_temp_dir_for_build(TEMPDIR) + + # Run the tests in a context manager that temporary changes the CWD to a + # temporary and writable directory. If it's not possible to create or + # change the CWD, the original CWD will be used. The original CWD is + # available from support.SAVEDCWD. + with support.temp_cwd(TESTCWD, quiet=True): + main() |