diff options
Diffstat (limited to 'Tools/scripts/check_extension_modules.py')
| -rw-r--r-- | Tools/scripts/check_extension_modules.py | 489 |
1 files changed, 489 insertions, 0 deletions
diff --git a/Tools/scripts/check_extension_modules.py b/Tools/scripts/check_extension_modules.py new file mode 100644 index 0000000..b3f43ab --- /dev/null +++ b/Tools/scripts/check_extension_modules.py @@ -0,0 +1,489 @@ +"""Check extension modules + +The script checks shared and built-in extension modules. It verifies that the +modules have been built and that they can be imported successfully. Missing +modules and failed imports are reported to the user. Shared extension +files are renamed on failed import. + +Module information is parsed from several sources: + +- core modules hard-coded in Modules/config.c.in +- Windows-specific modules that are hard-coded in PC/config.c +- MODULE_{name}_STATE entries in Makefile (provided through sysconfig) +- Various makesetup files: + - $(srcdir)/Modules/Setup + - Modules/Setup.[local|bootstrap|stdlib] files, which are generated + from $(srcdir)/Modules/Setup.*.in files + +See --help for more information +""" +import argparse +import collections +import enum +import logging +import os +import pathlib +import re +import sys +import sysconfig +import warnings + +from importlib._bootstrap import _load as bootstrap_load +from importlib.machinery import BuiltinImporter, ExtensionFileLoader, ModuleSpec +from importlib.util import spec_from_file_location, spec_from_loader +from typing import Iterable + +SRC_DIR = pathlib.Path(__file__).parent.parent.parent + +# core modules, hard-coded in Modules/config.h.in +CORE_MODULES = { + "_ast", + "_imp", + "_string", + "_tokenize", + "_warnings", + "builtins", + "gc", + "marshal", + "sys", +} + +# Windows-only modules +WINDOWS_MODULES = { + "_msi", + "_overlapped", + "_testconsole", + "_winapi", + "msvcrt", + "nt", + "winreg", + "winsound", +} + + +logger = logging.getLogger(__name__) + +parser = argparse.ArgumentParser( + prog="check_extension_modules", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, +) + +parser.add_argument( + "--verbose", + action="store_true", + help="Verbose, report builtin, shared, and unavailable modules", +) + +parser.add_argument( + "--debug", + action="store_true", + help="Enable debug logging", +) + +parser.add_argument( + "--strict", + action=argparse.BooleanOptionalAction, + help=( + "Strict check, fail when a module is missing or fails to import" + "(default: no, unless env var PYTHONSTRICTEXTENSIONBUILD is set)" + ), + default=bool(os.environ.get("PYTHONSTRICTEXTENSIONBUILD")), +) + +parser.add_argument( + "--cross-compiling", + action=argparse.BooleanOptionalAction, + help=( + "Use cross-compiling checks " + "(default: no, unless env var _PYTHON_HOST_PLATFORM is set)." + ), + default="_PYTHON_HOST_PLATFORM" in os.environ, +) + +parser.add_argument( + "--list-module-names", + action="store_true", + help="Print a list of module names to stdout and exit", +) + + +class ModuleState(enum.Enum): + # Makefile state "yes" + BUILTIN = "builtin" + SHARED = "shared" + + DISABLED = "disabled" + MISSING = "missing" + NA = "n/a" + # disabled by Setup / makesetup rule + DISABLED_SETUP = "disabled_setup" + + def __bool__(self): + return self.value in {"builtin", "shared"} + + +ModuleInfo = collections.namedtuple("ModuleInfo", "name state") + + +class ModuleChecker: + pybuilddir_txt = "pybuilddir.txt" + + setup_files = ( + SRC_DIR / "Modules/Setup", + "Modules/Setup.local", + "Modules/Setup.bootstrap", + "Modules/Setup.stdlib", + ) + + def __init__(self, cross_compiling: bool = False, strict: bool = False): + self.cross_compiling = cross_compiling + self.strict_extensions_build = strict + self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") + self.platform = sysconfig.get_platform() + self.builddir = self.get_builddir() + self.modules = self.get_modules() + + self.builtin_ok = [] + self.shared_ok = [] + self.failed_on_import = [] + self.missing = [] + self.disabled_configure = [] + self.disabled_setup = [] + self.notavailable = [] + + def check(self): + for modinfo in self.modules: + logger.debug("Checking '%s' (%s)", modinfo.name, self.get_location(modinfo)) + if modinfo.state == ModuleState.DISABLED: + self.disabled_configure.append(modinfo) + elif modinfo.state == ModuleState.DISABLED_SETUP: + self.disabled_setup.append(modinfo) + elif modinfo.state == ModuleState.MISSING: + self.missing.append(modinfo) + elif modinfo.state == ModuleState.NA: + self.notavailable.append(modinfo) + else: + try: + if self.cross_compiling: + self.check_module_cross(modinfo) + else: + self.check_module_import(modinfo) + except (ImportError, FileNotFoundError): + self.rename_module(modinfo) + self.failed_on_import.append(modinfo) + else: + if modinfo.state == ModuleState.BUILTIN: + self.builtin_ok.append(modinfo) + else: + assert modinfo.state == ModuleState.SHARED + self.shared_ok.append(modinfo) + + def summary(self, *, verbose: bool = False): + longest = max([len(e.name) for e in self.modules], default=0) + + def print_three_column(modinfos: list[ModuleInfo]): + names = [modinfo.name for modinfo in modinfos] + names.sort(key=str.lower) + # guarantee zip() doesn't drop anything + while len(names) % 3: + names.append("") + for l, m, r in zip(names[::3], names[1::3], names[2::3]): + print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r)) + + if verbose and self.builtin_ok: + print("The following *built-in* modules have been successfully built:") + print_three_column(self.builtin_ok) + print() + + if verbose and self.shared_ok: + print("The following *shared* modules have been successfully built:") + print_three_column(self.shared_ok) + print() + + if self.disabled_configure: + print("The following modules are *disabled* in configure script:") + print_three_column(self.disabled_configure) + print() + + if self.disabled_setup: + print("The following modules are *disabled* in Modules/Setup files:") + print_three_column(self.disabled_setup) + print() + + if verbose and self.notavailable: + print( + f"The following modules are not available on platform '{self.platform}':" + ) + print_three_column(self.notavailable) + print() + + if self.missing: + print("The necessary bits to build these optional modules were not found:") + print_three_column(self.missing) + print("To find the necessary bits, look in configure.ac and config.log.") + print() + + if self.failed_on_import: + print( + "Following modules built successfully " + "but were removed because they could not be imported:" + ) + print_three_column(self.failed_on_import) + print() + + if any( + modinfo.name == "_ssl" for modinfo in self.missing + self.failed_on_import + ): + print("Could not build the ssl module!") + print("Python requires a OpenSSL 1.1.1 or newer") + if sysconfig.get_config_var("OPENSSL_LDFLAGS"): + print("Custom linker flags may require --with-openssl-rpath=auto") + print() + + disabled = len(self.disabled_configure) + len(self.disabled_setup) + print( + f"Checked {len(self.modules)} modules (" + f"{len(self.builtin_ok)} built-in, " + f"{len(self.shared_ok)} shared, " + f"{len(self.notavailable)} n/a on {self.platform}, " + f"{disabled} disabled, " + f"{len(self.missing)} missing, " + f"{len(self.failed_on_import)} failed on import)" + ) + + def check_strict_build(self): + """Fail if modules are missing and it's a strict build""" + if self.strict_extensions_build and (self.failed_on_import or self.missing): + raise RuntimeError("Failed to build some stdlib modules") + + def list_module_names(self, *, all: bool = False) -> set: + names = {modinfo.name for modinfo in self.modules} + if all: + names.update(WINDOWS_MODULES) + return names + + def get_builddir(self) -> pathlib.Path: + try: + with open(self.pybuilddir_txt, encoding="utf-8") as f: + builddir = f.read() + except FileNotFoundError: + logger.error("%s must be run from the top build directory", __file__) + raise + builddir = pathlib.Path(builddir) + logger.debug("%s: %s", self.pybuilddir_txt, builddir) + return builddir + + def get_modules(self) -> list[ModuleInfo]: + """Get module info from sysconfig and Modules/Setup* files""" + seen = set() + modules = [] + # parsing order is important, first entry wins + for modinfo in self.get_core_modules(): + modules.append(modinfo) + seen.add(modinfo.name) + for setup_file in self.setup_files: + for modinfo in self.parse_setup_file(setup_file): + if modinfo.name not in seen: + modules.append(modinfo) + seen.add(modinfo.name) + for modinfo in self.get_sysconfig_modules(): + if modinfo.name not in seen: + modules.append(modinfo) + seen.add(modinfo.name) + logger.debug("Found %i modules in total", len(modules)) + modules.sort() + return modules + + def get_core_modules(self) -> Iterable[ModuleInfo]: + """Get hard-coded core modules""" + for name in CORE_MODULES: + modinfo = ModuleInfo(name, ModuleState.BUILTIN) + logger.debug("Found core module %s", modinfo) + yield modinfo + + def get_sysconfig_modules(self) -> Iterable[ModuleInfo]: + """Get modules defined in Makefile through sysconfig + + MODBUILT_NAMES: modules in *static* block + MODSHARED_NAMES: modules in *shared* block + MODDISABLED_NAMES: modules in *disabled* block + + Modules built by setup.py addext() have a MODULE_{modname}_STATE entry, + but are not listed in MODSHARED_NAMES. + + Modules built by old-style setup.py add() have neither a MODULE_{modname} + entry nor an entry in MODSHARED_NAMES. + """ + moddisabled = set(sysconfig.get_config_var("MODDISABLED_NAMES").split()) + if self.cross_compiling: + modbuiltin = set(sysconfig.get_config_var("MODBUILT_NAMES").split()) + else: + modbuiltin = set(sys.builtin_module_names) + + for key, value in sysconfig.get_config_vars().items(): + if not key.startswith("MODULE_") or not key.endswith("_STATE"): + continue + if value not in {"yes", "disabled", "missing", "n/a"}: + raise ValueError(f"Unsupported value '{value}' for {key}") + + modname = key[7:-6].lower() + if modname in moddisabled: + # Setup "*disabled*" rule + state = ModuleState.DISABLED_SETUP + elif value in {"disabled", "missing", "n/a"}: + state = ModuleState(value) + elif modname in modbuiltin: + assert value == "yes" + state = ModuleState.BUILTIN + else: + assert value == "yes" + state = ModuleState.SHARED + + modinfo = ModuleInfo(modname, state) + logger.debug("Found %s in Makefile", modinfo) + yield modinfo + + def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: + """Parse a Modules/Setup file""" + assign_var = re.compile(r"^\w+=") # EGG_SPAM=foo + # default to static module + state = ModuleState.BUILTIN + logger.debug("Parsing Setup file %s", setup_file) + with open(setup_file, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or assign_var.match(line): + continue + match line.split(): + case ["*shared*"]: + state = ModuleState.SHARED + case ["*static*"]: + state = ModuleState.BUILTIN + case ["*disabled*"]: + state = ModuleState.DISABLED + case ["*noconfig*"]: + state = None + case [*items]: + if state == ModuleState.DISABLED: + # *disabled* can disable multiple modules per line + for item in items: + modinfo = ModuleInfo(item, state) + logger.debug("Found %s in %s", modinfo, setup_file) + yield modinfo + elif state in {ModuleState.SHARED, ModuleState.BUILTIN}: + # *shared* and *static*, first item is the name of the module. + modinfo = ModuleInfo(items[0], state) + logger.debug("Found %s in %s", modinfo, setup_file) + yield modinfo + + def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec: + """Get ModuleSpec for builtin or extension module""" + if modinfo.state == ModuleState.SHARED: + location = os.fspath(self.get_location(modinfo)) + loader = ExtensionFileLoader(modinfo.name, location) + return spec_from_file_location(modinfo.name, location, loader=loader) + elif modinfo.state == ModuleState.BUILTIN: + return spec_from_loader(modinfo.name, loader=BuiltinImporter) + else: + raise ValueError(modinfo) + + def get_location(self, modinfo: ModuleInfo) -> pathlib.Path: + """Get shared library location in build directory""" + if modinfo.state == ModuleState.SHARED: + return self.builddir / f"{modinfo.name}{self.ext_suffix}" + else: + return None + + def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): + """Check that the module file is present and not empty""" + if spec.loader is BuiltinImporter: + return + try: + st = os.stat(spec.origin) + except FileNotFoundError: + logger.error("%s (%s) is missing", modinfo.name, spec.origin) + raise + if not st.st_size: + raise ImportError(f"{spec.origin} is an empty file") + + def check_module_import(self, modinfo: ModuleInfo): + """Attempt to import module and report errors""" + spec = self.get_spec(modinfo) + self._check_file(modinfo, spec) + try: + with warnings.catch_warnings(): + # ignore deprecation warning from deprecated modules + warnings.simplefilter("ignore", DeprecationWarning) + bootstrap_load(spec) + except ImportError as e: + logger.error("%s failed to import: %s", modinfo.name, e) + raise + except Exception as e: + logger.exception("Importing extension '%s' failed!", modinfo.name) + raise + + def check_module_cross(self, modinfo: ModuleInfo): + """Sanity check for cross compiling""" + spec = self.get_spec(modinfo) + self._check_file(modinfo, spec) + + def rename_module(self, modinfo: ModuleInfo) -> None: + """Rename module file""" + if modinfo.state == ModuleState.BUILTIN: + logger.error("Cannot mark builtin module '%s' as failed!", modinfo.name) + return + + failed_name = f"{modinfo.name}_failed{self.ext_suffix}" + builddir_path = self.get_location(modinfo) + if builddir_path.is_symlink(): + symlink = builddir_path + module_path = builddir_path.resolve().relative_to(os.getcwd()) + failed_path = module_path.parent / failed_name + else: + symlink = None + module_path = builddir_path + failed_path = self.builddir / failed_name + + # remove old failed file + failed_path.unlink(missing_ok=True) + # remove symlink + if symlink is not None: + symlink.unlink(missing_ok=True) + # rename shared extension file + try: + module_path.rename(failed_path) + except FileNotFoundError: + logger.debug("Shared extension file '%s' does not exist.", module_path) + else: + logger.debug("Rename '%s' -> '%s'", module_path, failed_path) + + +def main(): + args = parser.parse_args() + if args.debug: + args.verbose = True + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format="[%(levelname)s] %(message)s", + ) + + checker = ModuleChecker( + cross_compiling=args.cross_compiling, + strict=args.strict, + ) + if args.list_module_names: + names = checker.list_module_names(all=True) + for name in sorted(names): + print(name) + else: + checker.check() + checker.summary(verbose=args.verbose) + try: + checker.check_strict_build() + except RuntimeError as e: + parser.exit(1, f"\nError: {e}\n") + + +if __name__ == "__main__": + main() |