diff options
Diffstat (limited to 'Tools')
| -rwxr-xr-x | Tools/ssl/multissltests.py | 430 | ||||
| -rw-r--r-- | Tools/ssl/test_multiple_versions.py | 241 |
2 files changed, 430 insertions, 241 deletions
diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py new file mode 100755 index 0000000..994e420 --- /dev/null +++ b/Tools/ssl/multissltests.py @@ -0,0 +1,430 @@ +#!./python +"""Run Python tests against multiple installations of OpenSSL and LibreSSL + +The script + + (1) downloads OpenSSL / LibreSSL tar bundle + (2) extracts it to ./src + (3) compiles OpenSSL / LibreSSL + (4) installs OpenSSL / LibreSSL into ../multissl/$LIB/$VERSION/ + (5) forces a recompilation of Python modules using the + header and library files from ../multissl/$LIB/$VERSION/ + (6) runs Python's test suite + +The script must be run with Python's build directory as current working +directory. + +The script uses LD_RUN_PATH, LD_LIBRARY_PATH, CPPFLAGS and LDFLAGS to bend +search paths for header files and shared libraries. It's known to work on +Linux with GCC and clang. + +Please keep this script compatible with Python 2.7, and 3.4 to 3.7. + +(c) 2013-2017 Christian Heimes <christian@python.org> +""" +from __future__ import print_function + +import argparse +from datetime import datetime +import logging +import os +try: + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen +import subprocess +import shutil +import sys +import tarfile + + +log = logging.getLogger("multissl") + +OPENSSL_OLD_VERSIONS = [ + "0.9.8zc", + "0.9.8zh", + "1.0.1u", +] + +OPENSSL_RECENT_VERSIONS = [ + "1.0.2", + "1.0.2l", + "1.1.0f", +] + +LIBRESSL_OLD_VERSIONS = [ + "2.3.10", + "2.4.5", +] + +LIBRESSL_RECENT_VERSIONS = [ + "2.5.3", + "2.5.5", +] + +# store files in ../multissl +HERE = os.path.abspath(os.getcwd()) +MULTISSL_DIR = os.path.abspath(os.path.join(HERE, '..', 'multissl')) + +parser = argparse.ArgumentParser( + prog='multissl', + description=( + "Run CPython tests with multiple OpenSSL and LibreSSL " + "versions." + ) +) +parser.add_argument( + '--debug', + action='store_true', + help="Enable debug mode", +) +parser.add_argument( + '--disable-ancient', + action='store_true', + help="Don't test OpenSSL < 1.0.2 and LibreSSL < 2.5.3.", +) +parser.add_argument( + '--openssl', + nargs='+', + default=(), + help=( + "OpenSSL versions, defaults to '{}' (ancient: '{}') if no " + "OpenSSL and LibreSSL versions are given." + ).format(OPENSSL_RECENT_VERSIONS, OPENSSL_OLD_VERSIONS) +) +parser.add_argument( + '--libressl', + nargs='+', + default=(), + help=( + "LibreSSL versions, defaults to '{}' (ancient: '{}') if no " + "OpenSSL and LibreSSL versions are given." + ).format(LIBRESSL_RECENT_VERSIONS, LIBRESSL_OLD_VERSIONS) +) +parser.add_argument( + '--tests', + nargs='*', + default=(), + help="Python tests to run, defaults to all SSL related tests.", +) +parser.add_argument( + '--base-directory', + default=MULTISSL_DIR, + help="Base directory for OpenSSL / LibreSSL sources and builds." +) +parser.add_argument( + '--no-network', + action='store_false', + dest='network', + help="Disable network tests." +) +parser.add_argument( + '--compile-only', + action='store_true', + help="Don't run tests, only compile _ssl.c and _hashopenssl.c." +) + + +class AbstractBuilder(object): + library = None + url_template = None + src_template = None + build_template = None + + module_files = ("Modules/_ssl.c", + "Modules/_hashopenssl.c") + module_libs = ("_ssl", "_hashlib") + + def __init__(self, version, compile_args=(), + basedir=MULTISSL_DIR): + self.version = version + self.compile_args = compile_args + # installation directory + self.install_dir = os.path.join( + os.path.join(basedir, self.library.lower()), version + ) + # source file + self.src_dir = os.path.join(basedir, 'src') + self.src_file = os.path.join( + self.src_dir, self.src_template.format(version)) + # build directory (removed after install) + self.build_dir = os.path.join( + self.src_dir, self.build_template.format(version)) + + def __str__(self): + return "<{0.__class__.__name__} for {0.version}>".format(self) + + def __eq__(self, other): + if not isinstance(other, AbstractBuilder): + return NotImplemented + return ( + self.library == other.library + and self.version == other.version + ) + + def __hash__(self): + return hash((self.library, self.version)) + + @property + def openssl_cli(self): + """openssl CLI binary""" + return os.path.join(self.install_dir, "bin", "openssl") + + @property + def openssl_version(self): + """output of 'bin/openssl version'""" + cmd = [self.openssl_cli, "version"] + return self._subprocess_output(cmd) + + @property + def pyssl_version(self): + """Value of ssl.OPENSSL_VERSION""" + cmd = [ + sys.executable, + '-c', 'import ssl; print(ssl.OPENSSL_VERSION)' + ] + return self._subprocess_output(cmd) + + @property + def include_dir(self): + return os.path.join(self.install_dir, "include") + + @property + def lib_dir(self): + return os.path.join(self.install_dir, "lib") + + @property + def has_openssl(self): + return os.path.isfile(self.openssl_cli) + + @property + def has_src(self): + return os.path.isfile(self.src_file) + + def _subprocess_call(self, cmd, env=None, **kwargs): + log.debug("Call '{}'".format(" ".join(cmd))) + return subprocess.check_call(cmd, env=env, **kwargs) + + def _subprocess_output(self, cmd, env=None, **kwargs): + log.debug("Call '{}'".format(" ".join(cmd))) + if env is None: + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = self.lib_dir + out = subprocess.check_output(cmd, env=env, **kwargs) + return out.strip().decode("utf-8") + + def _download_src(self): + """Download sources""" + src_dir = os.path.dirname(self.src_file) + if not os.path.isdir(src_dir): + os.makedirs(src_dir) + url = self.url_template.format(self.version) + log.info("Downloading from {}".format(url)) + req = urlopen(url) + # KISS, read all, write all + data = req.read() + log.info("Storing {}".format(self.src_file)) + with open(self.src_file, "wb") as f: + f.write(data) + + def _unpack_src(self): + """Unpack tar.gz bundle""" + # cleanup + if os.path.isdir(self.build_dir): + shutil.rmtree(self.build_dir) + os.makedirs(self.build_dir) + + tf = tarfile.open(self.src_file) + name = self.build_template.format(self.version) + base = name + '/' + # force extraction into build dir + members = tf.getmembers() + for member in list(members): + if member.name == name: + members.remove(member) + elif not member.name.startswith(base): + raise ValueError(member.name, base) + member.name = member.name[len(base):].lstrip('/') + log.info("Unpacking files to {}".format(self.build_dir)) + tf.extractall(self.build_dir, members) + + def _build_src(self): + """Now build openssl""" + log.info("Running build in {}".format(self.build_dir)) + cwd = self.build_dir + cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)] + cmd.extend(self.compile_args) + self._subprocess_call(cmd, cwd=cwd) + # Old OpenSSL versions do not support parallel builds. + self._subprocess_call(["make", "-j1"], cwd=cwd) + + def _make_install(self, remove=True): + self._subprocess_call(["make", "-j1", "install"], cwd=self.build_dir) + if remove: + shutil.rmtree(self.build_dir) + + def install(self): + log.info(self.openssl_cli) + if not self.has_openssl: + if not self.has_src: + self._download_src() + else: + log.debug("Already has src {}".format(self.src_file)) + self._unpack_src() + self._build_src() + self._make_install() + else: + log.info("Already has installation {}".format(self.install_dir)) + # validate installation + version = self.openssl_version + if self.version not in version: + raise ValueError(version) + + def recompile_pymods(self): + log.warning("Using build from {}".format(self.build_dir)) + # force a rebuild of all modules that use OpenSSL APIs + for fname in self.module_files: + os.utime(fname, None) + # remove all build artefacts + for root, dirs, files in os.walk('build'): + for filename in files: + if filename.startswith(self.module_libs): + os.unlink(os.path.join(root, filename)) + + # overwrite header and library search paths + env = os.environ.copy() + env["CPPFLAGS"] = "-I{}".format(self.include_dir) + env["LDFLAGS"] = "-L{}".format(self.lib_dir) + # set rpath + env["LD_RUN_PATH"] = self.lib_dir + + log.info("Rebuilding Python modules") + cmd = [sys.executable, "setup.py", "build"] + self._subprocess_call(cmd, env=env) + self.check_imports() + + def check_imports(self): + cmd = [sys.executable, "-c", "import _ssl; import _hashlib"] + self._subprocess_call(cmd) + + def check_pyssl(self): + version = self.pyssl_version + if self.version not in version: + raise ValueError(version) + + def run_python_tests(self, tests, network=True): + if not tests: + cmd = [sys.executable, 'Lib/test/ssltests.py', '-j0'] + elif sys.version_info < (3, 3): + cmd = [sys.executable, '-m', 'test.regrtest'] + else: + cmd = [sys.executable, '-m', 'test', '-j0'] + if network: + cmd.extend(['-u', 'network', '-u', 'urlfetch']) + cmd.extend(['-w', '-r']) + cmd.extend(tests) + self._subprocess_call(cmd, stdout=None) + + +class BuildOpenSSL(AbstractBuilder): + library = "OpenSSL" + url_template = "https://www.openssl.org/source/openssl-{}.tar.gz" + src_template = "openssl-{}.tar.gz" + build_template = "openssl-{}" + + +class BuildLibreSSL(AbstractBuilder): + library = "LibreSSL" + url_template = ( + "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-{}.tar.gz") + src_template = "libressl-{}.tar.gz" + build_template = "libressl-{}" + + +def configure_make(): + if not os.path.isfile('Makefile'): + log.info('Running ./configure') + subprocess.check_call([ + './configure', '--config-cache', '--quiet', + '--with-pydebug' + ]) + + log.info('Running make') + subprocess.check_call(['make', '--quiet']) + + +def main(): + args = parser.parse_args() + if not args.openssl and not args.libressl: + args.openssl = list(OPENSSL_RECENT_VERSIONS) + args.libressl = list(LIBRESSL_RECENT_VERSIONS) + if not args.disable_ancient: + args.openssl.extend(OPENSSL_OLD_VERSIONS) + args.libressl.extend(LIBRESSL_OLD_VERSIONS) + + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format="*** %(levelname)s %(message)s" + ) + + start = datetime.now() + + for name in ['python', 'setup.py', 'Modules/_ssl.c']: + if not os.path.isfile(name): + parser.error( + "Must be executed from CPython build dir" + ) + if not os.path.samefile('python', sys.executable): + parser.error( + "Must be executed with ./python from CPython build dir" + ) + + # check for configure and run make + configure_make() + + # download and register builder + builds = [] + + for version in args.openssl: + build = BuildOpenSSL(version) + build.install() + builds.append(build) + + for version in args.libressl: + build = BuildLibreSSL(version) + build.install() + builds.append(build) + + for build in builds: + try: + build.recompile_pymods() + build.check_pyssl() + if not args.compile_only: + build.run_python_tests( + tests=args.tests, + network=args.network, + ) + except Exception as e: + log.exception("%s failed", build) + print("{} failed: {}".format(build, e), file=sys.stderr) + sys.exit(2) + + print("\n{} finished in {}".format( + "Tests" if not args.compile_only else "Builds", + datetime.now() - start + )) + print('Python: ', sys.version) + if args.compile_only: + print('Build only') + elif args.tests: + print('Executed Tests:', ' '.join(args.tests)) + else: + print('Executed all SSL tests.') + + print('OpenSSL / LibreSSL versions:') + for build in builds: + print(" * {0.library} {0.version}".format(build)) + + +if __name__ == "__main__": + main() diff --git a/Tools/ssl/test_multiple_versions.py b/Tools/ssl/test_multiple_versions.py deleted file mode 100644 index 30d5fcf..0000000 --- a/Tools/ssl/test_multiple_versions.py +++ /dev/null @@ -1,241 +0,0 @@ -#./python -"""Run Python tests with multiple installations of OpenSSL - -The script - - (1) downloads OpenSSL tar bundle - (2) extracts it to ../openssl/src/openssl-VERSION/ - (3) compiles OpenSSL - (4) installs OpenSSL into ../openssl/VERSION/ - (5) forces a recompilation of Python modules using the - header and library files from ../openssl/VERSION/ - (6) runs Python's test suite - -The script must be run with Python's build directory as current working -directory: - - ./python Tools/ssl/test_multiple_versions.py - -The script uses LD_RUN_PATH, LD_LIBRARY_PATH, CPPFLAGS and LDFLAGS to bend -search paths for header files and shared libraries. It's known to work on -Linux with GCC 4.x. - -(c) 2013 Christian Heimes <christian@python.org> -""" -import logging -import os -import tarfile -import shutil -import subprocess -import sys -from urllib.request import urlopen - -log = logging.getLogger("multissl") - -OPENSSL_VERSIONS = [ - "0.9.7m", "0.9.8i", "0.9.8l", "0.9.8m", "0.9.8y", "1.0.0k", "1.0.1e" -] -FULL_TESTS = [ - "test_asyncio", "test_ftplib", "test_hashlib", "test_httplib", - "test_imaplib", "test_nntplib", "test_poplib", "test_smtplib", - "test_smtpnet", "test_urllib2_localnet", "test_venv" -] -MINIMAL_TESTS = ["test_ssl", "test_hashlib"] -CADEFAULT = True -HERE = os.path.abspath(os.getcwd()) -DEST_DIR = os.path.abspath(os.path.join(HERE, os.pardir, "openssl")) - - -class BuildSSL: - url_template = "https://www.openssl.org/source/openssl-{}.tar.gz" - - module_files = ["Modules/_ssl.c", - "Modules/socketmodule.c", - "Modules/_hashopenssl.c"] - - def __init__(self, version, openssl_compile_args=(), destdir=DEST_DIR): - self._check_python_builddir() - self.version = version - self.openssl_compile_args = openssl_compile_args - # installation directory - self.install_dir = os.path.join(destdir, version) - # source file - self.src_file = os.path.join(destdir, "src", - "openssl-{}.tar.gz".format(version)) - # build directory (removed after install) - self.build_dir = os.path.join(destdir, "src", - "openssl-{}".format(version)) - - @property - def openssl_cli(self): - """openssl CLI binary""" - return os.path.join(self.install_dir, "bin", "openssl") - - @property - def openssl_version(self): - """output of 'bin/openssl version'""" - env = os.environ.copy() - env["LD_LIBRARY_PATH"] = self.lib_dir - cmd = [self.openssl_cli, "version"] - return self._subprocess_output(cmd, env=env) - - @property - def pyssl_version(self): - """Value of ssl.OPENSSL_VERSION""" - env = os.environ.copy() - env["LD_LIBRARY_PATH"] = self.lib_dir - cmd = ["./python", "-c", "import ssl; print(ssl.OPENSSL_VERSION)"] - return self._subprocess_output(cmd, env=env) - - @property - def include_dir(self): - return os.path.join(self.install_dir, "include") - - @property - def lib_dir(self): - return os.path.join(self.install_dir, "lib") - - @property - def has_openssl(self): - return os.path.isfile(self.openssl_cli) - - @property - def has_src(self): - return os.path.isfile(self.src_file) - - def _subprocess_call(self, cmd, stdout=subprocess.DEVNULL, env=None, - **kwargs): - log.debug("Call '%s'", " ".join(cmd)) - return subprocess.check_call(cmd, stdout=stdout, env=env, **kwargs) - - def _subprocess_output(self, cmd, env=None, **kwargs): - log.debug("Call '%s'", " ".join(cmd)) - out = subprocess.check_output(cmd, env=env) - return out.strip().decode("utf-8") - - def _check_python_builddir(self): - if not os.path.isfile("python") or not os.path.isfile("setup.py"): - raise ValueError("Script must be run in Python build directory") - - def _download_openssl(self): - """Download OpenSSL source dist""" - src_dir = os.path.dirname(self.src_file) - if not os.path.isdir(src_dir): - os.makedirs(src_dir) - url = self.url_template.format(self.version) - log.info("Downloading OpenSSL from {}".format(url)) - req = urlopen(url, cadefault=CADEFAULT) - # KISS, read all, write all - data = req.read() - log.info("Storing {}".format(self.src_file)) - with open(self.src_file, "wb") as f: - f.write(data) - - def _unpack_openssl(self): - """Unpack tar.gz bundle""" - # cleanup - if os.path.isdir(self.build_dir): - shutil.rmtree(self.build_dir) - os.makedirs(self.build_dir) - - tf = tarfile.open(self.src_file) - base = "openssl-{}/".format(self.version) - # force extraction into build dir - members = tf.getmembers() - for member in members: - if not member.name.startswith(base): - raise ValueError(member.name) - member.name = member.name[len(base):] - log.info("Unpacking files to {}".format(self.build_dir)) - tf.extractall(self.build_dir, members) - - def _build_openssl(self): - """Now build openssl""" - log.info("Running build in {}".format(self.install_dir)) - cwd = self.build_dir - cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)] - cmd.extend(self.openssl_compile_args) - self._subprocess_call(cmd, cwd=cwd) - self._subprocess_call(["make"], cwd=cwd) - - def _install_openssl(self, remove=True): - self._subprocess_call(["make", "install"], cwd=self.build_dir) - if remove: - shutil.rmtree(self.build_dir) - - def install_openssl(self): - if not self.has_openssl: - if not self.has_src: - self._download_openssl() - else: - log.debug("Already has src %s", self.src_file) - self._unpack_openssl() - self._build_openssl() - self._install_openssl() - else: - log.info("Already has installation {}".format(self.install_dir)) - # validate installation - version = self.openssl_version - if self.version not in version: - raise ValueError(version) - - def touch_pymods(self): - # force a rebuild of all modules that use OpenSSL APIs - for fname in self.module_files: - os.utime(fname) - - def recompile_pymods(self): - log.info("Using OpenSSL build from {}".format(self.build_dir)) - # overwrite header and library search paths - env = os.environ.copy() - env["CPPFLAGS"] = "-I{}".format(self.include_dir) - env["LDFLAGS"] = "-L{}".format(self.lib_dir) - # set rpath - env["LD_RUN_PATH"] = self.lib_dir - - log.info("Rebuilding Python modules") - self.touch_pymods() - cmd = ["./python", "setup.py", "build"] - self._subprocess_call(cmd, env=env) - - def check_pyssl(self): - version = self.pyssl_version - if self.version not in version: - raise ValueError(version) - - def run_pytests(self, *args): - cmd = ["./python", "-m", "test"] - cmd.extend(args) - self._subprocess_call(cmd, stdout=None) - - def run_python_tests(self, *args): - self.recompile_pymods() - self.check_pyssl() - self.run_pytests(*args) - - -def main(*args): - builders = [] - for version in OPENSSL_VERSIONS: - if version in ("0.9.8i", "0.9.8l"): - openssl_compile_args = ("no-asm",) - else: - openssl_compile_args = () - builder = BuildSSL(version, openssl_compile_args) - builder.install_openssl() - builders.append(builder) - - for builder in builders: - builder.run_python_tests(*args) - # final touch - builder.touch_pymods() - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, - format="*** %(levelname)s %(message)s") - args = sys.argv[1:] - if not args: - args = ["-unetwork", "-v"] - args.extend(FULL_TESTS) - main(*args) |