From 529525fb5a8fd9b96ab4021311a598c77588b918 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 23 May 2018 22:24:45 +0200 Subject: bpo-33618: Enable TLS 1.3 in tests (GH-7079) TLS 1.3 behaves slightly different than TLS 1.2. Session tickets and TLS client cert auth are now handled after the initialy handshake. Tests now either send/recv data to trigger session and client certs. Or tests ignore ConnectionResetError / BrokenPipeError on the server side to handle clients that force-close the socket fd. To test TLS 1.3, OpenSSL 1.1.1-pre7-dev (git master + OpenSSL PR https://github.com/openssl/openssl/pull/6340) is required. Signed-off-by: Christian Heimes --- Doc/library/ssl.rst | 28 ++++++- Doc/whatsnew/3.7.rst | 12 ++- Lib/test/test_asyncio/test_sslproto.py | 2 + Lib/test/test_asyncio/utils.py | 2 - Lib/test/test_ftplib.py | 10 +-- Lib/test/test_poplib.py | 9 +-- Lib/test/test_ssl.py | 90 ++++++++++++++++++---- .../2018-05-23-20-14-34.bpo-33618.xU39lr.rst | 2 + Tools/ssl/multissltests.py | 33 ++++++-- 9 files changed, 142 insertions(+), 46 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 2ccea13..14eac2c 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2587,7 +2587,33 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or :func:`~ssl.RAND_pseudo_bytes` is sufficient. -.. ssl-libressl: +.. _ssl-tlsv1_3: + +TLS 1.3 +------- + +.. versionadded:: 3.7 + +Python has provisional and experimental support for TLS 1.3 with OpenSSL +1.1.1. The new protocol behaves slightly differently than previous version +of TLS/SSL. Some new TLS 1.3 features are not yet available. + +- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and + ChaCha20 cipher suites are enabled by default. The method + :meth:`SSLContext.set_ciphers` cannot enable or disable any TLS 1.3 + ciphers yet, but :meth:`SSLContext.get_cipers` returns them. +- Session tickets are no longer sent as part of the initial handshake and + are handled differently. :attr:`SSLSocket.session` and :class:`SSLSession` + are not compatible with TLS 1.3. +- Client-side certificates are also no longer verified during the initial + handshake. A server can request a certificate at any time. Clients + process certificate requests while they send or receive application data + from the server. +- TLS 1.3 features like early data, deferred TLS client cert request, + signature algorithm configuration, and rekeying are not supported yet. + + +.. _ssl-libressl: LibreSSL support ---------------- diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index af2aad9..46015af 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1244,8 +1244,8 @@ Host name validation can be customized with .. note:: The improved host name check requires a *libssl* implementation compatible with OpenSSL 1.0.2 or 1.1. Consequently, OpenSSL 0.9.8 and 1.0.1 are no - longer supported and LibreSSL is temporarily not supported until it gains - the necessary OpenSSL 1.0.2 APIs. + longer supported. The ssl module is mostly compatible with LibreSSL 2.7.2 + and newer. The ``ssl`` module no longer sends IP addresses in SNI TLS extension. (Contributed by Christian Heimes in :issue:`32185`.) @@ -1270,8 +1270,12 @@ rather than the U-label form (``"pythön.org"``). (Contributed by Nathaniel J. Smith and Christian Heimes in :issue:`28414`.) The ``ssl`` module has preliminary and experimental support for TLS 1.3 and -OpenSSL 1.1.1. (Contributed by Christian Heimes in :issue:`32947`, -:issue:`20995`, :issue:`29136`, and :issue:`30622`) +OpenSSL 1.1.1. At the time of Python 3.7.0 release, OpenSSL 1.1.1 is still +under development and TLS 1.3 hasn't been finalized yet. The TLS 1.3 +handshake and protocol behaves slightly differently than TLS 1.2 and earlier, +see :ref:`ssl-tlsv1_3`. +(Contributed by Christian Heimes in :issue:`32947`, :issue:`20995`, +:issue:`29136`, :issue:`30622` and :issue:`33618`) :class:`~ssl.SSLSocket` and :class:`~ssl.SSLObject` no longer have a public constructor. Direct instantiation was never a documented and supported diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index 7b27f4c..c534a34 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -251,6 +251,8 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): server_context = test_utils.simple_server_sslcontext() client_context = test_utils.simple_client_sslcontext() + # TODO: fix TLSv1.3 support + client_context.options |= ssl.OP_NO_TLSv1_3 def client(sock, addr): time.sleep(0.5) diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 711085f..96dfe2f 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -74,8 +74,6 @@ def simple_server_sslcontext(): server_context.load_cert_chain(ONLYCERT, ONLYKEY) server_context.check_hostname = False server_context.verify_mode = ssl.CERT_NONE - # TODO: fix TLSv1.3 support - server_context.options |= ssl.OP_NO_TLSv1_3 return server_context diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 1a8e2f9..f9488a9 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -257,6 +257,7 @@ class DummyFTPServer(asyncore.dispatcher, threading.Thread): def __init__(self, address, af=socket.AF_INET): threading.Thread.__init__(self) asyncore.dispatcher.__init__(self) + self.daemon = True self.create_socket(af, socket.SOCK_STREAM) self.bind(address) self.listen(5) @@ -312,8 +313,6 @@ if ssl is not None: def secure_connection(self): context = ssl.SSLContext() - # TODO: fix TLSv1.3 support - context.options |= ssl.OP_NO_TLSv1_3 context.load_cert_chain(CERTFILE) socket = context.wrap_socket(self.socket, suppress_ragged_eofs=False, @@ -405,7 +404,7 @@ if ssl is not None: def close(self): if (isinstance(self.socket, ssl.SSLSocket) and - self.socket._sslobj is not None): + self.socket._sslobj is not None): self._do_ssl_shutdown() else: super(SSLConnection, self).close() @@ -910,8 +909,6 @@ class TestTLS_FTPClass(TestCase): def test_context(self): self.client.quit() ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE, @@ -944,8 +941,6 @@ class TestTLS_FTPClass(TestCase): def test_check_hostname(self): self.client.quit() ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) self.assertEqual(ctx.check_hostname, True) ctx.load_verify_locations(CAFILE) @@ -982,6 +977,7 @@ class TestTimeouts(TestCase): self.sock.settimeout(20) self.port = support.bind_port(self.sock) self.server_thread = threading.Thread(target=self.server) + self.server_thread.daemon = True self.server_thread.start() # Wait for the server to be ready. self.evt.wait() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index bbedbbd..20d4eea 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -153,8 +153,6 @@ class DummyPOP3Handler(asynchat.async_chat): if self.tls_active is False: self.push('+OK Begin TLS negotiation') context = ssl.SSLContext() - # TODO: fix TLSv1.3 support - context.options |= ssl.OP_NO_TLSv1_3 context.load_cert_chain(CERTFILE) tls_sock = context.wrap_socket(self.socket, server_side=True, @@ -206,6 +204,7 @@ class DummyPOP3Server(asyncore.dispatcher, threading.Thread): def __init__(self, address, af=socket.AF_INET): threading.Thread.__init__(self) asyncore.dispatcher.__init__(self) + self.daemon = True self.create_socket(af, socket.SOCK_STREAM) self.bind(address) self.listen(5) @@ -370,8 +369,6 @@ class TestPOP3Class(TestCase): def test_stls_context(self): expected = b'+OK Begin TLS negotiation' ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.load_verify_locations(CAFILE) self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) self.assertEqual(ctx.check_hostname, True) @@ -412,8 +409,6 @@ class TestPOP3_SSLClass(TestPOP3Class): def test_context(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE self.assertRaises(ValueError, poplib.POP3_SSL, self.server.host, @@ -482,7 +477,7 @@ class TestTimeouts(TestCase): self.sock.settimeout(60) # Safety net. Look issue 11812 self.port = test_support.bind_port(self.sock) self.thread = threading.Thread(target=self.server, args=(self.evt,self.sock)) - self.thread.setDaemon(True) + self.thread.daemon = True self.thread.start() self.evt.wait() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 03952b1..73d3e3b 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1826,6 +1826,7 @@ class SimpleBackgroundTests(unittest.TestCase): s.connect(self.server_addr) cert = s.getpeercert() self.assertTrue(cert) + # Same with a bytes `capath` argument ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) ctx.verify_mode = ssl.CERT_REQUIRED @@ -1841,8 +1842,6 @@ class SimpleBackgroundTests(unittest.TestCase): der = ssl.PEM_cert_to_DER_cert(pem) ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) ctx.verify_mode = ssl.CERT_REQUIRED - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.load_verify_locations(cadata=pem) with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: s.connect(self.server_addr) @@ -1852,8 +1851,6 @@ class SimpleBackgroundTests(unittest.TestCase): # same with DER ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) ctx.verify_mode = ssl.CERT_REQUIRED - # TODO: fix TLSv1.3 support - ctx.options |= ssl.OP_NO_TLSv1_3 ctx.load_verify_locations(cadata=der) with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: s.connect(self.server_addr) @@ -2109,11 +2106,21 @@ class ThreadedEchoServer(threading.Thread): self.sock, server_side=True) self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol()) self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) - except (ssl.SSLError, ConnectionResetError, OSError) as e: + except (ConnectionResetError, BrokenPipeError) as e: # We treat ConnectionResetError as though it were an # SSLError - OpenSSL on Ubuntu abruptly closes the # connection when asked to use an unsupported protocol. # + # BrokenPipeError is raised in TLS 1.3 mode, when OpenSSL + # tries to send session tickets after handshake. + # https://github.com/openssl/openssl/issues/6342 + self.server.conn_errors.append(str(e)) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.close() + return False + except (ssl.SSLError, OSError) as e: # OSError may occur with wrong protocols, e.g. both # sides use PROTOCOL_TLS_SERVER. # @@ -2220,11 +2227,22 @@ class ThreadedEchoServer(threading.Thread): sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" % (msg, ctype, msg.lower(), ctype)) self.write(msg.lower()) + except ConnectionResetError: + # XXX: OpenSSL 1.1.1 sometimes raises ConnectionResetError + # when connection is not shut down gracefully. + if self.server.chatty and support.verbose: + sys.stdout.write( + " Connection reset by peer: {}\n".format( + self.addr) + ) + self.close() + self.running = False except OSError: if self.server.chatty: handle_error("Test server failure:\n") self.close() self.running = False + # normally, we'd just stop here, but for the test # harness, we want to stop the server self.server.stop() @@ -2299,6 +2317,11 @@ class ThreadedEchoServer(threading.Thread): pass except KeyboardInterrupt: self.stop() + except BaseException as e: + if support.verbose and self.chatty: + sys.stdout.write( + ' connection handling failed: ' + repr(e) + '\n') + self.sock.close() def stop(self): @@ -2745,8 +2768,6 @@ class ThreadedTests(unittest.TestCase): server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) server_context.load_cert_chain(IDNSANSFILE) - # TODO: fix TLSv1.3 support - server_context.options |= ssl.OP_NO_TLSv1_3 context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) context.verify_mode = ssl.CERT_REQUIRED @@ -2797,7 +2818,7 @@ class ThreadedTests(unittest.TestCase): with self.assertRaises(ssl.CertificateError): s.connect((HOST, server.port)) - def test_wrong_cert(self): + def test_wrong_cert_tls12(self): """Connecting when the server rejects the client's certificate Launch a server with CERT_REQUIRED, and check that trying to @@ -2808,9 +2829,8 @@ class ThreadedTests(unittest.TestCase): client_context.load_cert_chain(WRONG_CERT) # require TLS client authentication server_context.verify_mode = ssl.CERT_REQUIRED - # TODO: fix TLSv1.3 support - # With TLS 1.3, test fails with exception in server thread - server_context.options |= ssl.OP_NO_TLSv1_3 + # TLS 1.3 has different handshake + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 server = ThreadedEchoServer( context=server_context, chatty=True, connectionchatty=True, @@ -2835,6 +2855,36 @@ class ThreadedTests(unittest.TestCase): else: self.fail("Use of invalid cert should have failed!") + @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3") + def test_wrong_cert_tls13(self): + client_context, server_context, hostname = testing_context() + client_context.load_cert_chain(WRONG_CERT) + server_context.verify_mode = ssl.CERT_REQUIRED + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + + server = ThreadedEchoServer( + context=server_context, chatty=True, connectionchatty=True, + ) + with server, \ + client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # TLS 1.3 perform client cert exchange after handshake + s.connect((HOST, server.port)) + try: + s.write(b'data') + s.read(4) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except OSError as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + def test_rude_shutdown(self): """A brutal shutdown of an SSL server should raise an OSError in the client when attempting handshake. @@ -3405,7 +3455,7 @@ class ThreadedTests(unittest.TestCase): # Block on the accept and wait on the connection to close. evt.set() remote, peer = server.accept() - remote.recv(1) + remote.send(remote.recv(4)) t = threading.Thread(target=serve) t.start() @@ -3413,6 +3463,8 @@ class ThreadedTests(unittest.TestCase): evt.wait() client = context.wrap_socket(socket.socket()) client.connect((host, port)) + client.send(b'data') + client.recv() client_addr = client.getsockname() client.close() t.join() @@ -3465,7 +3517,7 @@ class ThreadedTests(unittest.TestCase): self.assertIs(s.version(), None) self.assertIs(s._sslobj, None) s.connect((HOST, server.port)) - if ssl.OPENSSL_VERSION_INFO >= (1, 1, 1): + if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3: self.assertEqual(s.version(), 'TLSv1.3') elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2): self.assertEqual(s.version(), 'TLSv1.2') @@ -3574,8 +3626,6 @@ class ThreadedTests(unittest.TestCase): sys.stdout.write("\n") client_context, server_context, hostname = testing_context() - # TODO: fix TLSv1.3 support - client_context.options |= ssl.OP_NO_TLSv1_3 server = ThreadedEchoServer(context=server_context, chatty=True, @@ -3594,7 +3644,10 @@ class ThreadedTests(unittest.TestCase): # check if it is sane self.assertIsNotNone(cb_data) - self.assertEqual(len(cb_data), 12) # True for TLSv1 + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 # and compare with the peers version s.write(b"CB tls-unique\n") @@ -3616,7 +3669,10 @@ class ThreadedTests(unittest.TestCase): # is it really unique self.assertNotEqual(cb_data, new_cb_data) self.assertIsNotNone(cb_data) - self.assertEqual(len(cb_data), 12) # True for TLSv1 + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 s.write(b"CB tls-unique\n") peer_data_repr = s.read().strip() self.assertEqual(peer_data_repr, diff --git a/Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst b/Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst new file mode 100644 index 0000000..6cc2452 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst @@ -0,0 +1,2 @@ +Finalize and document preliminary and experimental TLS 1.3 support with +OpenSSL 1.1.1 diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index bbc5c66..c4ebe31 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -47,7 +47,7 @@ OPENSSL_OLD_VERSIONS = [ OPENSSL_RECENT_VERSIONS = [ "1.0.2o", "1.1.0h", - "1.1.1-pre6", + # "1.1.1-pre7", ] LIBRESSL_OLD_VERSIONS = [ @@ -73,7 +73,7 @@ parser = argparse.ArgumentParser( parser.add_argument( '--debug', action='store_true', - help="Enable debug mode", + help="Enable debug logging", ) parser.add_argument( '--disable-ancient', @@ -130,6 +130,18 @@ parser.add_argument( default='', help="Override the automatic system type detection." ) +parser.add_argument( + '--force', + action='store_true', + dest='force', + help="Force build and installation." +) +parser.add_argument( + '--keep-sources', + action='store_true', + dest='keep_sources', + help="Keep original sources for debugging." +) class AbstractBuilder(object): @@ -260,26 +272,31 @@ class AbstractBuilder(object): """Now build openssl""" log.info("Running build in {}".format(self.build_dir)) cwd = self.build_dir - cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)] - env = None + cmd = [ + "./config", + "shared", "--debug", + "--prefix={}".format(self.install_dir) + ] + env = os.environ.copy() + # set rpath + env["LD_RUN_PATH"] = self.lib_dir if self.system: - env = os.environ.copy() env['SYSTEM'] = self.system self._subprocess_call(cmd, cwd=cwd, env=env) # Old OpenSSL versions do not support parallel builds. self._subprocess_call(["make", "-j1"], cwd=cwd, env=env) - def _make_install(self, remove=True): + def _make_install(self): self._subprocess_call( ["make", "-j1", self.install_target], cwd=self.build_dir ) - if remove: + if not self.args.keep_sources: shutil.rmtree(self.build_dir) def install(self): log.info(self.openssl_cli) - if not self.has_openssl: + if not self.has_openssl or self.args.force: if not self.has_src: self._download_src() else: -- cgit v0.12