diff options
Diffstat (limited to 'Lib/test/support')
| -rw-r--r-- | Lib/test/support/__init__.py | 188 | ||||
| -rw-r--r-- | Lib/test/support/socket_helper.py | 177 |
2 files changed, 182 insertions, 183 deletions
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2b3a414..15cf45d 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -16,12 +16,10 @@ import importlib import importlib.util import locale import logging.handlers -import nntplib import os import platform import re import shutil -import socket import stat import struct import subprocess @@ -33,17 +31,11 @@ import threading import time import types import unittest -import urllib.error import warnings from .testresult import get_test_runner try: - import multiprocessing.process -except ImportError: - multiprocessing = None - -try: import zlib except ImportError: zlib = None @@ -98,14 +90,13 @@ __all__ = [ "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", - "check__all__", "skip_unless_bind_unix_socket", "skip_if_buggy_ucrt_strfptime", + "check__all__", "skip_if_buggy_ucrt_strfptime", "ignore_warnings", # sys "is_jython", "is_android", "check_impl_detail", "unix_shell", "setswitchinterval", # network - "HOST", "IPV6_ENABLED", "find_unused_port", "bind_port", "open_urlresource", - "bind_unix_socket", + "open_urlresource", # processes 'temp_umask', "reap_children", # logging @@ -727,135 +718,6 @@ def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): return decorator -HOST = "localhost" -HOSTv4 = "127.0.0.1" -HOSTv6 = "::1" - - -def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): - """Returns an unused port that should be suitable for binding. This is - achieved by creating a temporary socket with the same family and type as - the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to - the specified host address (defaults to 0.0.0.0) with the port set to 0, - eliciting an unused ephemeral port from the OS. The temporary socket is - then closed and deleted, and the ephemeral port is returned. - - Either this method or bind_port() should be used for any tests where a - server socket needs to be bound to a particular port for the duration of - the test. Which one to use depends on whether the calling code is creating - a python socket, or if an unused port needs to be provided in a constructor - or passed to an external program (i.e. the -accept argument to openssl's - s_server mode). Always prefer bind_port() over find_unused_port() where - possible. Hard coded ports should *NEVER* be used. As soon as a server - socket is bound to a hard coded port, the ability to run multiple instances - of the test simultaneously on the same host is compromised, which makes the - test a ticking time bomb in a buildbot environment. On Unix buildbots, this - may simply manifest as a failed test, which can be recovered from without - intervention in most cases, but on Windows, the entire python process can - completely and utterly wedge, requiring someone to log in to the buildbot - and manually kill the affected process. - - (This is easy to reproduce on Windows, unfortunately, and can be traced to - the SO_REUSEADDR socket option having different semantics on Windows versus - Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind, - listen and then accept connections on identical host/ports. An EADDRINUSE - OSError will be raised at some point (depending on the platform and - the order bind and listen were called on each socket). - - However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE - will ever be raised when attempting to bind two identical host/ports. When - accept() is called on each socket, the second caller's process will steal - the port from the first caller, leaving them both in an awkwardly wedged - state where they'll no longer respond to any signals or graceful kills, and - must be forcibly killed via OpenProcess()/TerminateProcess(). - - The solution on Windows is to use the SO_EXCLUSIVEADDRUSE socket option - instead of SO_REUSEADDR, which effectively affords the same semantics as - SO_REUSEADDR on Unix. Given the propensity of Unix developers in the Open - Source world compared to Windows ones, this is a common mistake. A quick - look over OpenSSL's 0.9.8g source shows that they use SO_REUSEADDR when - openssl.exe is called with the 's_server' option, for example. See - http://bugs.python.org/issue2550 for more info. The following site also - has a very thorough description about the implications of both REUSEADDR - and EXCLUSIVEADDRUSE on Windows: - http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx) - - XXX: although this approach is a vast improvement on previous attempts to - elicit unused ports, it rests heavily on the assumption that the ephemeral - port returned to us by the OS won't immediately be dished back out to some - other process when we close and delete our temporary socket but before our - calling code has a chance to bind the returned port. We can deal with this - issue if/when we come across it. - """ - - with socket.socket(family, socktype) as tempsock: - port = bind_port(tempsock) - del tempsock - return port - -def bind_port(sock, host=HOST): - """Bind the socket to a free port and return the port number. Relies on - ephemeral ports in order to ensure we are using an unbound port. This is - important as many tests may be running simultaneously, especially in a - buildbot environment. This method raises an exception if the sock.family - is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR - or SO_REUSEPORT set on it. Tests should *never* set these socket options - for TCP/IP sockets. The only case for setting these options is testing - multicasting via multiple UDP sockets. - - Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e. - on Windows), it will be set on the socket. This will prevent anyone else - from bind()'ing to our host/port for the duration of the test. - """ - - if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM: - if hasattr(socket, 'SO_REUSEADDR'): - if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1: - raise TestFailed("tests should never set the SO_REUSEADDR " \ - "socket option on TCP/IP sockets!") - if hasattr(socket, 'SO_REUSEPORT'): - try: - if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1: - raise TestFailed("tests should never set the SO_REUSEPORT " \ - "socket option on TCP/IP sockets!") - except OSError: - # Python's socket module was compiled using modern headers - # thus defining SO_REUSEPORT but this process is running - # under an older kernel that does not support SO_REUSEPORT. - pass - if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): - sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) - - sock.bind((host, 0)) - port = sock.getsockname()[1] - return port - -def bind_unix_socket(sock, addr): - """Bind a unix socket, raising SkipTest if PermissionError is raised.""" - assert sock.family == socket.AF_UNIX - try: - sock.bind(addr) - except PermissionError: - sock.close() - raise unittest.SkipTest('cannot bind AF_UNIX sockets') - -def _is_ipv6_enabled(): - """Check whether IPv6 is enabled on this host.""" - if socket.has_ipv6: - sock = None - try: - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - sock.bind((HOSTv6, 0)) - return True - except OSError: - pass - finally: - if sock: - sock.close() - return False - -IPV6_ENABLED = _is_ipv6_enabled() - def system_must_validate_cert(f): """Skip the test on TLS certificate validation failures.""" @functools.wraps(f) @@ -1563,31 +1425,13 @@ socket_peer_reset = TransientResource(OSError, errno=errno.ECONNRESET) ioerror_peer_reset = TransientResource(OSError, errno=errno.ECONNRESET) -def get_socket_conn_refused_errs(): - """ - Get the different socket error numbers ('errno') which can be received - when a connection is refused. - """ - errors = [errno.ECONNREFUSED] - if hasattr(errno, 'ENETUNREACH'): - # On Solaris, ENETUNREACH is returned sometimes instead of ECONNREFUSED - errors.append(errno.ENETUNREACH) - if hasattr(errno, 'EADDRNOTAVAIL'): - # bpo-31910: socket.create_connection() fails randomly - # with EADDRNOTAVAIL on Travis CI - errors.append(errno.EADDRNOTAVAIL) - if hasattr(errno, 'EHOSTUNREACH'): - # bpo-37583: The destination host cannot be reached - errors.append(errno.EHOSTUNREACH) - if not IPV6_ENABLED: - errors.append(errno.EAFNOSUPPORT) - return errors - - @contextlib.contextmanager def transient_internet(resource_name, *, timeout=_NOT_SET, errnos=()): """Return a context manager that raises ResourceDenied when various issues with the Internet connection manifest themselves as exceptions.""" + import socket + import nntplib + import urllib.error if timeout is _NOT_SET: timeout = INTERNET_TIMEOUT @@ -2754,28 +2598,6 @@ def skip_if_pgo_task(test): msg = "Not run for (non-extended) PGO task" return test if ok else unittest.skip(msg)(test) -_bind_nix_socket_error = None -def skip_unless_bind_unix_socket(test): - """Decorator for tests requiring a functional bind() for unix sockets.""" - if not hasattr(socket, 'AF_UNIX'): - return unittest.skip('No UNIX Sockets')(test) - global _bind_nix_socket_error - if _bind_nix_socket_error is None: - path = TESTFN + "can_bind_unix_socket" - with socket.socket(socket.AF_UNIX) as sock: - try: - sock.bind(path) - _bind_nix_socket_error = False - except OSError as e: - _bind_nix_socket_error = e - finally: - unlink(path) - if _bind_nix_socket_error: - msg = 'Requires a functional unix bind(): %s' % _bind_nix_socket_error - return unittest.skip(msg)(test) - else: - return test - def fs_is_case_insensitive(directory): """Detects if the file system for the specified directory is case-insensitive.""" diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py new file mode 100644 index 0000000..5f4a7f1 --- /dev/null +++ b/Lib/test/support/socket_helper.py @@ -0,0 +1,177 @@ +import errno +import socket +import unittest + +HOST = "localhost" +HOSTv4 = "127.0.0.1" +HOSTv6 = "::1" + + +def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): + """Returns an unused port that should be suitable for binding. This is + achieved by creating a temporary socket with the same family and type as + the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to + the specified host address (defaults to 0.0.0.0) with the port set to 0, + eliciting an unused ephemeral port from the OS. The temporary socket is + then closed and deleted, and the ephemeral port is returned. + + Either this method or bind_port() should be used for any tests where a + server socket needs to be bound to a particular port for the duration of + the test. Which one to use depends on whether the calling code is creating + a python socket, or if an unused port needs to be provided in a constructor + or passed to an external program (i.e. the -accept argument to openssl's + s_server mode). Always prefer bind_port() over find_unused_port() where + possible. Hard coded ports should *NEVER* be used. As soon as a server + socket is bound to a hard coded port, the ability to run multiple instances + of the test simultaneously on the same host is compromised, which makes the + test a ticking time bomb in a buildbot environment. On Unix buildbots, this + may simply manifest as a failed test, which can be recovered from without + intervention in most cases, but on Windows, the entire python process can + completely and utterly wedge, requiring someone to log in to the buildbot + and manually kill the affected process. + + (This is easy to reproduce on Windows, unfortunately, and can be traced to + the SO_REUSEADDR socket option having different semantics on Windows versus + Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind, + listen and then accept connections on identical host/ports. An EADDRINUSE + OSError will be raised at some point (depending on the platform and + the order bind and listen were called on each socket). + + However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE + will ever be raised when attempting to bind two identical host/ports. When + accept() is called on each socket, the second caller's process will steal + the port from the first caller, leaving them both in an awkwardly wedged + state where they'll no longer respond to any signals or graceful kills, and + must be forcibly killed via OpenProcess()/TerminateProcess(). + + The solution on Windows is to use the SO_EXCLUSIVEADDRUSE socket option + instead of SO_REUSEADDR, which effectively affords the same semantics as + SO_REUSEADDR on Unix. Given the propensity of Unix developers in the Open + Source world compared to Windows ones, this is a common mistake. A quick + look over OpenSSL's 0.9.8g source shows that they use SO_REUSEADDR when + openssl.exe is called with the 's_server' option, for example. See + http://bugs.python.org/issue2550 for more info. The following site also + has a very thorough description about the implications of both REUSEADDR + and EXCLUSIVEADDRUSE on Windows: + http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx) + + XXX: although this approach is a vast improvement on previous attempts to + elicit unused ports, it rests heavily on the assumption that the ephemeral + port returned to us by the OS won't immediately be dished back out to some + other process when we close and delete our temporary socket but before our + calling code has a chance to bind the returned port. We can deal with this + issue if/when we come across it. + """ + + with socket.socket(family, socktype) as tempsock: + port = bind_port(tempsock) + del tempsock + return port + +def bind_port(sock, host=HOST): + """Bind the socket to a free port and return the port number. Relies on + ephemeral ports in order to ensure we are using an unbound port. This is + important as many tests may be running simultaneously, especially in a + buildbot environment. This method raises an exception if the sock.family + is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR + or SO_REUSEPORT set on it. Tests should *never* set these socket options + for TCP/IP sockets. The only case for setting these options is testing + multicasting via multiple UDP sockets. + + Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e. + on Windows), it will be set on the socket. This will prevent anyone else + from bind()'ing to our host/port for the duration of the test. + """ + + if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM: + if hasattr(socket, 'SO_REUSEADDR'): + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1: + raise TestFailed("tests should never set the SO_REUSEADDR " \ + "socket option on TCP/IP sockets!") + if hasattr(socket, 'SO_REUSEPORT'): + try: + if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1: + raise TestFailed("tests should never set the SO_REUSEPORT " \ + "socket option on TCP/IP sockets!") + except OSError: + # Python's socket module was compiled using modern headers + # thus defining SO_REUSEPORT but this process is running + # under an older kernel that does not support SO_REUSEPORT. + pass + if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) + + sock.bind((host, 0)) + port = sock.getsockname()[1] + return port + +def bind_unix_socket(sock, addr): + """Bind a unix socket, raising SkipTest if PermissionError is raised.""" + assert sock.family == socket.AF_UNIX + try: + sock.bind(addr) + except PermissionError: + sock.close() + raise unittest.SkipTest('cannot bind AF_UNIX sockets') + +def _is_ipv6_enabled(): + """Check whether IPv6 is enabled on this host.""" + if socket.has_ipv6: + sock = None + try: + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.bind((HOSTv6, 0)) + return True + except OSError: + pass + finally: + if sock: + sock.close() + return False + +IPV6_ENABLED = _is_ipv6_enabled() + + +_bind_nix_socket_error = None +def skip_unless_bind_unix_socket(test): + """Decorator for tests requiring a functional bind() for unix sockets.""" + if not hasattr(socket, 'AF_UNIX'): + return unittest.skip('No UNIX Sockets')(test) + global _bind_nix_socket_error + if _bind_nix_socket_error is None: + from test.support import TESTFN, unlink + path = TESTFN + "can_bind_unix_socket" + with socket.socket(socket.AF_UNIX) as sock: + try: + sock.bind(path) + _bind_nix_socket_error = False + except OSError as e: + _bind_nix_socket_error = e + finally: + unlink(path) + if _bind_nix_socket_error: + msg = 'Requires a functional unix bind(): %s' % _bind_nix_socket_error + return unittest.skip(msg)(test) + else: + return test + + +def get_socket_conn_refused_errs(): + """ + Get the different socket error numbers ('errno') which can be received + when a connection is refused. + """ + errors = [errno.ECONNREFUSED] + if hasattr(errno, 'ENETUNREACH'): + # On Solaris, ENETUNREACH is returned sometimes instead of ECONNREFUSED + errors.append(errno.ENETUNREACH) + if hasattr(errno, 'EADDRNOTAVAIL'): + # bpo-31910: socket.create_connection() fails randomly + # with EADDRNOTAVAIL on Travis CI + errors.append(errno.EADDRNOTAVAIL) + if hasattr(errno, 'EHOSTUNREACH'): + # bpo-37583: The destination host cannot be reached + errors.append(errno.EHOSTUNREACH) + if not IPV6_ENABLED: + errors.append(errno.EAFNOSUPPORT) + return errors |