bpo-33570: TLS 1.3 ciphers for OpenSSL 1.1.1 (GH-6976) (GH-8760)

Change TLS 1.3 cipher suite settings for compatibility with OpenSSL
1.1.1-pre6 and newer. OpenSSL 1.1.1 will have TLS 1.3 cipers enabled by
default.

Also update multissltests to test with latest OpenSSL.

Signed-off-by: Christian Heimes <christian@python.org>

4 files changed, 134 insertions, 89 deletions

diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index e12cfc6..376d467 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -300,11 +300,6 @@ purposes.
 
      3DES was dropped from the default cipher string.
 
-   .. versionchanged:: 3.6.3
-
-     TLS 1.3 cipher suites TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384,
-     and TLS_CHACHA20_POLY1305_SHA256 were added to the default cipher string.
-
 
 Random generation
 ^^^^^^^^^^^^^^^^^
@@ -1483,6 +1478,9 @@ to speed up repeated connections from the same clients.
       when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will
       give the currently selected cipher.
 
+      OpenSSL 1.1.1 has TLS 1.3 cipher suites enabled by default. The suites
+      cannot be disabled with :meth:`~SSLContext.set_ciphers`.
+
 .. method:: SSLContext.set_alpn_protocols(protocols)
 
    Specify which protocols the socket should advertise during the SSL/TLS
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 6b3cd5b..5bcec2d 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -3089,17 +3089,22 @@ if _have_threads:
                     sock.do_handshake()
                 self.assertEqual(cm.exception.errno, errno.ENOTCONN)
 
-        def test_default_ciphers(self):
-            context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
-            try:
-                # Force a set of weak ciphers on our client context
-                context.set_ciphers("DES")
-            except ssl.SSLError:
-                self.skipTest("no DES cipher available")
-            with ThreadedEchoServer(CERTFILE,
-                                    ssl_version=ssl.PROTOCOL_SSLv23,
-                                    chatty=False) as server:
-                with context.wrap_socket(socket.socket()) as s:
+        def test_no_shared_ciphers(self):
+            server_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+            server_context.load_cert_chain(SIGNED_CERTFILE)
+            client_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+            client_context.verify_mode = ssl.CERT_REQUIRED
+            client_context.check_hostname = True
+
+            # OpenSSL enables all TLS 1.3 ciphers, enforce TLS 1.2 for test
+            client_context.options |= ssl.OP_NO_TLSv1_3
+            # Force different suites on client and master
+            client_context.set_ciphers("AES128")
+            server_context.set_ciphers("AES256")
+            with ThreadedEchoServer(context=server_context) as server:
+                with client_context.wrap_socket(
+                        socket.socket(),
+                        server_hostname="localhost") as s:
                     with self.assertRaises(OSError):
                         s.connect((HOST, server.port))
             self.assertIn("no shared cipher", server.conn_errors[0])
@@ -3132,9 +3137,9 @@ if _have_threads:
                 with context.wrap_socket(socket.socket()) as s:
                     s.connect((HOST, server.port))
                     self.assertIn(s.cipher()[0], [
-                        'TLS13-AES-256-GCM-SHA384',
-                        'TLS13-CHACHA20-POLY1305-SHA256',
-                        'TLS13-AES-128-GCM-SHA256',
+                        'TLS_AES_256_GCM_SHA384',
+                        'TLS_CHACHA20_POLY1305_SHA256',
+                        'TLS_AES_128_GCM_SHA256',
                     ])
 
         @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL")
@@ -3460,19 +3465,25 @@ if _have_threads:
             if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
                 client_context.set_ciphers("AES128:AES256")
                 server_context.set_ciphers("AES256")
-                alg1 = "AES256"
-                alg2 = "AES-256"
+                expected_algs = [
+                    "AES256", "AES-256"
+                ]
             else:
                 client_context.set_ciphers("AES:3DES")
                 server_context.set_ciphers("3DES")
-                alg1 = "3DES"
-                alg2 = "DES-CBC3"
+                expected_algs = [
+                    "3DES", "DES-CBC3"
+                ]
+
+            if ssl.HAS_TLSv1_3:
+                # TLS 1.3 ciphers are always enabled
+                expected_algs.extend(["TLS_CHACHA20", "TLS_AES"])
 
             stats = server_params_test(client_context, server_context)
             ciphers = stats['server_shared_ciphers'][0]
             self.assertGreater(len(ciphers), 0)
             for name, tls_version, bits in ciphers:
-                if not alg1 in name.split("-") and alg2 not in name:
+                if not any(alg in name for alg in expected_algs):
                     self.fail(name)
 
         def test_read_write_after_close_raises_valuerror(self):
diff --git a/Misc/NEWS.d/next/Library/2018-05-18-21-50-47.bpo-33570.7CZy4t.rst b/Misc/NEWS.d/next/Library/2018-05-18-21-50-47.bpo-33570.7CZy4t.rst
new file mode 100644
index 0000000..bd719a4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-05-18-21-50-47.bpo-33570.7CZy4t.rst
@@ -0,0 +1,3 @@
+Change TLS 1.3 cipher suite settings for compatibility with OpenSSL
+1.1.1-pre6 and newer. OpenSSL 1.1.1 will have TLS 1.3 cipers enabled by
+default.
diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index f3241cd..9d668d4 100755
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -41,30 +41,31 @@ import tarfile
 log = logging.getLogger("multissl")
 
 OPENSSL_OLD_VERSIONS = [
-     "0.9.8zh",
-     "1.0.1u",
+    "0.9.8zh",
+    "1.0.1u",
+    "1.0.2",
 ]
 
 OPENSSL_RECENT_VERSIONS = [
-     "1.0.2",
-     "1.0.2m",
-     "1.1.0g",
+    "1.0.2o",
+    "1.1.0h",
+    # "1.1.1-pre7",
 ]
 
 LIBRESSL_OLD_VERSIONS = [
-    "2.3.10",
-    "2.4.5",
+    "2.5.5",
+    "2.6.4",
 ]
 
 LIBRESSL_RECENT_VERSIONS = [
-    "2.5.5",
-    "2.6.4",
-    "2.7.1",
+    "2.7.3",
 ]
 
 # store files in ../multissl
-HERE = os.path.abspath(os.getcwd())
-MULTISSL_DIR = os.path.abspath(os.path.join(HERE, '..', 'multissl'))
+HERE = os.path.dirname(os.path.abspath(__file__))
+PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..'))
+MULTISSL_DIR = os.path.abspath(os.path.join(PYTHONROOT, '..', 'multissl'))
+
 
 parser = argparse.ArgumentParser(
     prog='multissl',
@@ -76,7 +77,7 @@ parser = argparse.ArgumentParser(
 parser.add_argument(
     '--debug',
     action='store_true',
-    help="Enable debug mode",
+    help="Enable debug logging",
 )
 parser.add_argument(
     '--disable-ancient',
@@ -119,15 +120,32 @@ parser.add_argument(
     help="Disable network tests."
 )
 parser.add_argument(
-    '--compile-only',
-    action='store_true',
-    help="Don't run tests, only compile _ssl.c and _hashopenssl.c."
+    '--steps',
+    choices=['library', 'modules', 'tests'],
+    default='tests',
+    help=(
+        "Which steps to perform. 'library' downloads and compiles OpenSSL "
+        "or LibreSSL. 'module' also compiles Python modules. 'tests' builds "
+        "all and runs the test suite."
+    )
 )
 parser.add_argument(
     '--system',
     default='',
     help="Override the automatic system type detection."
 )
+parser.add_argument(
+    '--force',
+    action='store_true',
+    dest='force',
+    help="Force build and installation."
+)
+parser.add_argument(
+    '--keep-sources',
+    action='store_true',
+    dest='keep_sources',
+    help="Keep original sources for debugging."
+)
 
 
 class AbstractBuilder(object):
@@ -135,21 +153,21 @@ class AbstractBuilder(object):
     url_template = None
     src_template = None
     build_template = None
+    install_target = 'install'
 
     module_files = ("Modules/_ssl.c",
                     "Modules/_hashopenssl.c")
     module_libs = ("_ssl", "_hashlib")
 
-    def __init__(self, version, compile_args=(),
-                 basedir=MULTISSL_DIR):
+    def __init__(self, version, args):
         self.version = version
-        self.compile_args = compile_args
+        self.args = args
         # installation directory
         self.install_dir = os.path.join(
-            os.path.join(basedir, self.library.lower()), version
+            os.path.join(args.base_directory, self.library.lower()), version
         )
         # source file
-        self.src_dir = os.path.join(basedir, 'src')
+        self.src_dir = os.path.join(args.base_directory, 'src')
         self.src_file = os.path.join(
             self.src_dir, self.src_template.format(version))
         # build directory (removed after install)
@@ -258,24 +276,31 @@ class AbstractBuilder(object):
         """Now build openssl"""
         log.info("Running build in {}".format(self.build_dir))
         cwd = self.build_dir
-        cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)]
-        cmd.extend(self.compile_args)
-        env = None
+        cmd = [
+            "./config",
+            "shared", "--debug",
+            "--prefix={}".format(self.install_dir)
+        ]
+        env = os.environ.copy()
+        # set rpath
+        env["LD_RUN_PATH"] = self.lib_dir
         if self.system:
-            env = os.environ.copy()
             env['SYSTEM'] = self.system
         self._subprocess_call(cmd, cwd=cwd, env=env)
         # Old OpenSSL versions do not support parallel builds.
         self._subprocess_call(["make", "-j1"], cwd=cwd, env=env)
 
-    def _make_install(self, remove=True):
-        self._subprocess_call(["make", "-j1", "install"], cwd=self.build_dir)
-        if remove:
+    def _make_install(self):
+        self._subprocess_call(
+            ["make", "-j1", self.install_target],
+            cwd=self.build_dir
+        )
+        if not self.args.keep_sources:
             shutil.rmtree(self.build_dir)
 
     def install(self):
         log.info(self.openssl_cli)
-        if not self.has_openssl:
+        if not self.has_openssl or self.args.force:
             if not self.has_src:
                 self._download_src()
             else:
@@ -341,6 +366,8 @@ class BuildOpenSSL(AbstractBuilder):
     url_template = "https://www.openssl.org/source/openssl-{}.tar.gz"
     src_template = "openssl-{}.tar.gz"
     build_template = "openssl-{}"
+    # only install software, skip docs
+    install_target = 'install_sw'
 
 
 class BuildLibreSSL(AbstractBuilder):
@@ -379,57 +406,63 @@ def main():
 
     start = datetime.now()
 
-    for name in ['python', 'setup.py', 'Modules/_ssl.c']:
-        if not os.path.isfile(name):
+    if args.steps in {'modules', 'tests'}:
+        for name in ['setup.py', 'Modules/_ssl.c']:
+            if not os.path.isfile(os.path.join(PYTHONROOT, name)):
+                parser.error(
+                    "Must be executed from CPython build dir"
+                )
+        if not os.path.samefile('python', sys.executable):
             parser.error(
-                "Must be executed from CPython build dir"
+                "Must be executed with ./python from CPython build dir"
             )
-    if not os.path.samefile('python', sys.executable):
-        parser.error(
-            "Must be executed with ./python from CPython build dir"
-        )
-
-    # check for configure and run make
-    configure_make()
+        # check for configure and run make
+        configure_make()
 
     # download and register builder
     builds = []
 
     for version in args.openssl:
-        build = BuildOpenSSL(version)
+        build = BuildOpenSSL(
+            version,
+            args
+        )
         build.install()
         builds.append(build)
 
     for version in args.libressl:
-        build = BuildLibreSSL(version)
+        build = BuildLibreSSL(
+            version,
+            args
+        )
         build.install()
         builds.append(build)
 
-    for build in builds:
-        try:
-            build.recompile_pymods()
-            build.check_pyssl()
-            if not args.compile_only:
-                build.run_python_tests(
-                    tests=args.tests,
-                    network=args.network,
-                )
-        except Exception as e:
-            log.exception("%s failed", build)
-            print("{} failed: {}".format(build, e), file=sys.stderr)
-            sys.exit(2)
-
-    print("\n{} finished in {}".format(
-        "Tests" if not args.compile_only else "Builds",
-        datetime.now() - start
-    ))
+    if args.steps in {'modules', 'tests'}:
+        for build in builds:
+            try:
+                build.recompile_pymods()
+                build.check_pyssl()
+                if args.steps == 'tests':
+                    build.run_python_tests(
+                        tests=args.tests,
+                        network=args.network,
+                    )
+            except Exception as e:
+                log.exception("%s failed", build)
+                print("{} failed: {}".format(build, e), file=sys.stderr)
+                sys.exit(2)
+
+    log.info("\n{} finished in {}".format(
+            args.steps.capitalize(),
+            datetime.now() - start
+        ))
     print('Python: ', sys.version)
-    if args.compile_only:
-        print('Build only')
-    elif args.tests:
-        print('Executed Tests:', ' '.join(args.tests))
-    else:
-        print('Executed all SSL tests.')
+    if args.steps == 'tests':
+        if args.tests:
+            print('Executed Tests:', ' '.join(args.tests))
+        else:
+            print('Executed all SSL tests.')
 
     print('OpenSSL / LibreSSL versions:')
     for build in builds: