bpo-33618: Enable TLS 1.3 in tests (GH-7079)

TLS 1.3 behaves slightly different than TLS 1.2. Session tickets and TLS
client cert auth are now handled after the initialy handshake. Tests now
either send/recv data to trigger session and client certs. Or tests
ignore ConnectionResetError / BrokenPipeError on the server side to
handle clients that force-close the socket fd.

To test TLS 1.3, OpenSSL 1.1.1-pre7-dev (git master + OpenSSL PR
https://github.com/openssl/openssl/pull/6340) is required.

Signed-off-by: Christian Heimes <christian@python.org>

5 files changed, 80 insertions, 33 deletions

diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py
index 7b27f4c..c534a34 100644
--- a/Lib/test/test_asyncio/test_sslproto.py
+++ b/Lib/test/test_asyncio/test_sslproto.py
@@ -251,6 +251,8 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin):
 
         server_context = test_utils.simple_server_sslcontext()
         client_context = test_utils.simple_client_sslcontext()
+        # TODO: fix TLSv1.3 support
+        client_context.options |= ssl.OP_NO_TLSv1_3
 
         def client(sock, addr):
             time.sleep(0.5)
diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py
index 711085f..96dfe2f 100644
--- a/Lib/test/test_asyncio/utils.py
+++ b/Lib/test/test_asyncio/utils.py
@@ -74,8 +74,6 @@ def simple_server_sslcontext():
     server_context.load_cert_chain(ONLYCERT, ONLYKEY)
     server_context.check_hostname = False
     server_context.verify_mode = ssl.CERT_NONE
-    # TODO: fix TLSv1.3 support
-    server_context.options |= ssl.OP_NO_TLSv1_3
     return server_context
 
 
diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py
index 1a8e2f9..f9488a9 100644
--- a/Lib/test/test_ftplib.py
+++ b/Lib/test/test_ftplib.py
@@ -257,6 +257,7 @@ class DummyFTPServer(asyncore.dispatcher, threading.Thread):
     def __init__(self, address, af=socket.AF_INET):
         threading.Thread.__init__(self)
         asyncore.dispatcher.__init__(self)
+        self.daemon = True
         self.create_socket(af, socket.SOCK_STREAM)
         self.bind(address)
         self.listen(5)
@@ -312,8 +313,6 @@ if ssl is not None:
 
         def secure_connection(self):
             context = ssl.SSLContext()
-            # TODO: fix TLSv1.3 support
-            context.options |= ssl.OP_NO_TLSv1_3
             context.load_cert_chain(CERTFILE)
             socket = context.wrap_socket(self.socket,
                                          suppress_ragged_eofs=False,
@@ -405,7 +404,7 @@ if ssl is not None:
 
         def close(self):
             if (isinstance(self.socket, ssl.SSLSocket) and
-                self.socket._sslobj is not None):
+                    self.socket._sslobj is not None):
                 self._do_ssl_shutdown()
             else:
                 super(SSLConnection, self).close()
@@ -910,8 +909,6 @@ class TestTLS_FTPClass(TestCase):
     def test_context(self):
         self.client.quit()
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.check_hostname = False
         ctx.verify_mode = ssl.CERT_NONE
         self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE,
@@ -944,8 +941,6 @@ class TestTLS_FTPClass(TestCase):
     def test_check_hostname(self):
         self.client.quit()
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED)
         self.assertEqual(ctx.check_hostname, True)
         ctx.load_verify_locations(CAFILE)
@@ -982,6 +977,7 @@ class TestTimeouts(TestCase):
         self.sock.settimeout(20)
         self.port = support.bind_port(self.sock)
         self.server_thread = threading.Thread(target=self.server)
+        self.server_thread.daemon = True
         self.server_thread.start()
         # Wait for the server to be ready.
         self.evt.wait()
diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py
index bbedbbd..20d4eea 100644
--- a/Lib/test/test_poplib.py
+++ b/Lib/test/test_poplib.py
@@ -153,8 +153,6 @@ class DummyPOP3Handler(asynchat.async_chat):
             if self.tls_active is False:
                 self.push('+OK Begin TLS negotiation')
                 context = ssl.SSLContext()
-                # TODO: fix TLSv1.3 support
-                context.options |= ssl.OP_NO_TLSv1_3
                 context.load_cert_chain(CERTFILE)
                 tls_sock = context.wrap_socket(self.socket,
                                                server_side=True,
@@ -206,6 +204,7 @@ class DummyPOP3Server(asyncore.dispatcher, threading.Thread):
     def __init__(self, address, af=socket.AF_INET):
         threading.Thread.__init__(self)
         asyncore.dispatcher.__init__(self)
+        self.daemon = True
         self.create_socket(af, socket.SOCK_STREAM)
         self.bind(address)
         self.listen(5)
@@ -370,8 +369,6 @@ class TestPOP3Class(TestCase):
     def test_stls_context(self):
         expected = b'+OK Begin TLS negotiation'
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.load_verify_locations(CAFILE)
         self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED)
         self.assertEqual(ctx.check_hostname, True)
@@ -412,8 +409,6 @@ class TestPOP3_SSLClass(TestPOP3Class):
 
     def test_context(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.check_hostname = False
         ctx.verify_mode = ssl.CERT_NONE
         self.assertRaises(ValueError, poplib.POP3_SSL, self.server.host,
@@ -482,7 +477,7 @@ class TestTimeouts(TestCase):
         self.sock.settimeout(60)  # Safety net. Look issue 11812
         self.port = test_support.bind_port(self.sock)
         self.thread = threading.Thread(target=self.server, args=(self.evt,self.sock))
-        self.thread.setDaemon(True)
+        self.thread.daemon = True
         self.thread.start()
         self.evt.wait()
 
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 03952b1..73d3e3b 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -1826,6 +1826,7 @@ class SimpleBackgroundTests(unittest.TestCase):
             s.connect(self.server_addr)
             cert = s.getpeercert()
             self.assertTrue(cert)
+
         # Same with a bytes `capath` argument
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
         ctx.verify_mode = ssl.CERT_REQUIRED
@@ -1841,8 +1842,6 @@ class SimpleBackgroundTests(unittest.TestCase):
         der = ssl.PEM_cert_to_DER_cert(pem)
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
         ctx.verify_mode = ssl.CERT_REQUIRED
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.load_verify_locations(cadata=pem)
         with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
             s.connect(self.server_addr)
@@ -1852,8 +1851,6 @@ class SimpleBackgroundTests(unittest.TestCase):
         # same with DER
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
         ctx.verify_mode = ssl.CERT_REQUIRED
-        # TODO: fix TLSv1.3 support
-        ctx.options |= ssl.OP_NO_TLSv1_3
         ctx.load_verify_locations(cadata=der)
         with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
             s.connect(self.server_addr)
@@ -2109,11 +2106,21 @@ class ThreadedEchoServer(threading.Thread):
                     self.sock, server_side=True)
                 self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol())
                 self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol())
-            except (ssl.SSLError, ConnectionResetError, OSError) as e:
+            except (ConnectionResetError, BrokenPipeError) as e:
                 # We treat ConnectionResetError as though it were an
                 # SSLError - OpenSSL on Ubuntu abruptly closes the
                 # connection when asked to use an unsupported protocol.
                 #
+                # BrokenPipeError is raised in TLS 1.3 mode, when OpenSSL
+                # tries to send session tickets after handshake.
+                # https://github.com/openssl/openssl/issues/6342
+                self.server.conn_errors.append(str(e))
+                if self.server.chatty:
+                    handle_error("\n server:  bad connection attempt from " + repr(self.addr) + ":\n")
+                self.running = False
+                self.close()
+                return False
+            except (ssl.SSLError, OSError) as e:
                 # OSError may occur with wrong protocols, e.g. both
                 # sides use PROTOCOL_TLS_SERVER.
                 #
@@ -2220,11 +2227,22 @@ class ThreadedEchoServer(threading.Thread):
                             sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n"
                                              % (msg, ctype, msg.lower(), ctype))
                         self.write(msg.lower())
+                except ConnectionResetError:
+                    # XXX: OpenSSL 1.1.1 sometimes raises ConnectionResetError
+                    # when connection is not shut down gracefully.
+                    if self.server.chatty and support.verbose:
+                        sys.stdout.write(
+                            " Connection reset by peer: {}\n".format(
+                                self.addr)
+                        )
+                    self.close()
+                    self.running = False
                 except OSError:
                     if self.server.chatty:
                         handle_error("Test server failure:\n")
                     self.close()
                     self.running = False
+
                     # normally, we'd just stop here, but for the test
                     # harness, we want to stop the server
                     self.server.stop()
@@ -2299,6 +2317,11 @@ class ThreadedEchoServer(threading.Thread):
                 pass
             except KeyboardInterrupt:
                 self.stop()
+            except BaseException as e:
+                if support.verbose and self.chatty:
+                    sys.stdout.write(
+                        ' connection handling failed: ' + repr(e) + '\n')
+
         self.sock.close()
 
     def stop(self):
@@ -2745,8 +2768,6 @@ class ThreadedTests(unittest.TestCase):
 
         server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         server_context.load_cert_chain(IDNSANSFILE)
-        # TODO: fix TLSv1.3 support
-        server_context.options |= ssl.OP_NO_TLSv1_3
 
         context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
         context.verify_mode = ssl.CERT_REQUIRED
@@ -2797,7 +2818,7 @@ class ThreadedTests(unittest.TestCase):
                 with self.assertRaises(ssl.CertificateError):
                     s.connect((HOST, server.port))
 
-    def test_wrong_cert(self):
+    def test_wrong_cert_tls12(self):
         """Connecting when the server rejects the client's certificate
 
         Launch a server with CERT_REQUIRED, and check that trying to
@@ -2808,9 +2829,8 @@ class ThreadedTests(unittest.TestCase):
         client_context.load_cert_chain(WRONG_CERT)
         # require TLS client authentication
         server_context.verify_mode = ssl.CERT_REQUIRED
-        # TODO: fix TLSv1.3 support
-        # With TLS 1.3, test fails with exception in server thread
-        server_context.options |= ssl.OP_NO_TLSv1_3
+        # TLS 1.3 has different handshake
+        client_context.maximum_version = ssl.TLSVersion.TLSv1_2
 
         server = ThreadedEchoServer(
             context=server_context, chatty=True, connectionchatty=True,
@@ -2835,6 +2855,36 @@ class ThreadedTests(unittest.TestCase):
             else:
                 self.fail("Use of invalid cert should have failed!")
 
+    @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3")
+    def test_wrong_cert_tls13(self):
+        client_context, server_context, hostname = testing_context()
+        client_context.load_cert_chain(WRONG_CERT)
+        server_context.verify_mode = ssl.CERT_REQUIRED
+        server_context.minimum_version = ssl.TLSVersion.TLSv1_3
+        client_context.minimum_version = ssl.TLSVersion.TLSv1_3
+
+        server = ThreadedEchoServer(
+            context=server_context, chatty=True, connectionchatty=True,
+        )
+        with server, \
+             client_context.wrap_socket(socket.socket(),
+                                        server_hostname=hostname) as s:
+            # TLS 1.3 perform client cert exchange after handshake
+            s.connect((HOST, server.port))
+            try:
+                s.write(b'data')
+                s.read(4)
+            except ssl.SSLError as e:
+                if support.verbose:
+                    sys.stdout.write("\nSSLError is %r\n" % e)
+            except OSError as e:
+                if e.errno != errno.ECONNRESET:
+                    raise
+                if support.verbose:
+                    sys.stdout.write("\nsocket.error is %r\n" % e)
+            else:
+                self.fail("Use of invalid cert should have failed!")
+
     def test_rude_shutdown(self):
         """A brutal shutdown of an SSL server should raise an OSError
         in the client when attempting handshake.
@@ -3405,7 +3455,7 @@ class ThreadedTests(unittest.TestCase):
             # Block on the accept and wait on the connection to close.
             evt.set()
             remote, peer = server.accept()
-            remote.recv(1)
+            remote.send(remote.recv(4))
 
         t = threading.Thread(target=serve)
         t.start()
@@ -3413,6 +3463,8 @@ class ThreadedTests(unittest.TestCase):
         evt.wait()
         client = context.wrap_socket(socket.socket())
         client.connect((host, port))
+        client.send(b'data')
+        client.recv()
         client_addr = client.getsockname()
         client.close()
         t.join()
@@ -3465,7 +3517,7 @@ class ThreadedTests(unittest.TestCase):
                 self.assertIs(s.version(), None)
                 self.assertIs(s._sslobj, None)
                 s.connect((HOST, server.port))
-                if ssl.OPENSSL_VERSION_INFO >= (1, 1, 1):
+                if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3:
                     self.assertEqual(s.version(), 'TLSv1.3')
                 elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
                     self.assertEqual(s.version(), 'TLSv1.2')
@@ -3574,8 +3626,6 @@ class ThreadedTests(unittest.TestCase):
             sys.stdout.write("\n")
 
         client_context, server_context, hostname = testing_context()
-        # TODO: fix TLSv1.3 support
-        client_context.options |= ssl.OP_NO_TLSv1_3
 
         server = ThreadedEchoServer(context=server_context,
                                     chatty=True,
@@ -3594,7 +3644,10 @@ class ThreadedTests(unittest.TestCase):
 
                 # check if it is sane
                 self.assertIsNotNone(cb_data)
-                self.assertEqual(len(cb_data), 12) # True for TLSv1
+                if s.version() == 'TLSv1.3':
+                    self.assertEqual(len(cb_data), 48)
+                else:
+                    self.assertEqual(len(cb_data), 12)  # True for TLSv1
 
                 # and compare with the peers version
                 s.write(b"CB tls-unique\n")
@@ -3616,7 +3669,10 @@ class ThreadedTests(unittest.TestCase):
                 # is it really unique
                 self.assertNotEqual(cb_data, new_cb_data)
                 self.assertIsNotNone(cb_data)
-                self.assertEqual(len(cb_data), 12) # True for TLSv1
+                if s.version() == 'TLSv1.3':
+                    self.assertEqual(len(cb_data), 48)
+                else:
+                    self.assertEqual(len(cb_data), 12)  # True for TLSv1
                 s.write(b"CB tls-unique\n")
                 peer_data_repr = s.read().strip()
                 self.assertEqual(peer_data_repr,