From 1eb3ade6e5dbc3ee3a75cc11b802b9b0b81ce384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Thu, 13 Feb 2025 13:58:00 +0000 Subject: GH-107956: install build-details.json (PEP 739) (#130069) --- Lib/sysconfig/__init__.py | 58 +++---- Lib/test/test_build_details.py | 128 ++++++++++++++ Makefile.pre.in | 6 +- .../2025-02-13-02-39-42.gh-issue-107956.dLguDW.rst | 2 + Tools/build/generate-build-details.py | 192 +++++++++++++++++++++ 5 files changed, 356 insertions(+), 30 deletions(-) create mode 100644 Lib/test/test_build_details.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-02-39-42.gh-issue-107956.dLguDW.rst create mode 100644 Tools/build/generate-build-details.py diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 69f7245..18e6b8d 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -666,34 +666,34 @@ def get_platform(): # Set for cross builds explicitly if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - - # Try to distinguish various flavours of Unix - osname, host, release, version, machine = os.uname() - - # Convert the OS name to lowercase, remove '/' characters, and translate - # spaces (for "Power Macintosh") - osname = osname.lower().replace('/', '') - machine = machine.replace(' ', '_') - machine = machine.replace('/', '-') - - if osname[:5] == "linux": - if sys.platform == "android": - osname = "android" - release = get_config_var("ANDROID_API_LEVEL") - - # Wheel tags use the ABI names from Android's own tools. - machine = { - "x86_64": "x86_64", - "i686": "x86", - "aarch64": "arm64_v8a", - "armv7l": "armeabi_v7a", - }[machine] - else: - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + osname, _, machine = os.environ["_PYTHON_HOST_PLATFORM"].partition('-') + release = None + else: + # Try to distinguish various flavours of Unix + osname, host, release, version, machine = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace('/', '') + machine = machine.replace(' ', '_') + machine = machine.replace('/', '-') + + if osname == "android" or sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + elif osname == "linux": + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" @@ -725,7 +725,7 @@ def get_platform(): get_config_vars(), osname, release, machine) - return f"{osname}-{release}-{machine}" + return '-'.join(map(str, filter(None, (osname, release, machine)))) def get_python_version(): diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py new file mode 100644 index 0000000..05ce163 --- /dev/null +++ b/Lib/test/test_build_details.py @@ -0,0 +1,128 @@ +import json +import os +import sys +import sysconfig +import string +import unittest + +from test.support import is_android, is_apple_mobile, is_emscripten, is_wasi + + +class FormatTestsBase: + @property + def contents(self): + """Install details file contents. Should be overriden by subclasses.""" + raise NotImplementedError + + @property + def data(self): + """Parsed install details file data, as a Python object.""" + return json.loads(self.contents) + + def key(self, name): + """Helper to fetch subsection entries. + + It takes the entry name, allowing the usage of a dot to separate the + different subsection names (eg. specifying 'a.b.c' as the key will + return the value of self.data['a']['b']['c']). + """ + value = self.data + for part in name.split('.'): + value = value[part] + return value + + def test_parse(self): + self.data + + def test_top_level_container(self): + self.assertIsInstance(self.data, dict) + for key, value in self.data.items(): + with self.subTest(key=key): + if key in ('schema_version', 'base_prefix', 'base_interpreter', 'platform'): + self.assertIsInstance(value, str) + elif key in ('language', 'implementation', 'abi', 'suffixes', 'libpython', 'c_api', 'arbitrary_data'): + self.assertIsInstance(value, dict) + + def test_base_prefix(self): + self.assertIsInstance(self.key('base_prefix'), str) + + def test_base_interpreter(self): + """Test the base_interpreter entry. + + The generic test wants the key to be missing. If your implementation + provides a value for it, you should override this test. + """ + with self.assertRaises(KeyError): + self.key('base_interpreter') + + def test_platform(self): + self.assertEqual(self.key('platform'), sysconfig.get_platform()) + + def test_language_version(self): + allowed_characters = string.digits + string.ascii_letters + '.' + value = self.key('language.version') + + self.assertLessEqual(set(value), set(allowed_characters)) + self.assertTrue(sys.version.startswith(value)) + + def test_language_version_info(self): + value = self.key('language.version_info') + + self.assertEqual(len(value), sys.version_info.n_fields) + for part_name, part_value in value.items(): + with self.subTest(part=part_name): + self.assertEqual(part_value, getattr(sys.version_info, part_name)) + + def test_implementation(self): + for key, value in self.key('implementation').items(): + with self.subTest(part=key): + if key == 'version': + self.assertEqual(len(value), len(sys.implementation.version)) + for part_name, part_value in value.items(): + self.assertEqual(getattr(sys.implementation.version, part_name), part_value) + else: + self.assertEqual(getattr(sys.implementation, key), value) + + +needs_installed_python = unittest.skipIf( + sysconfig.is_python_build(), + 'This test can only run in an installed Python', +) + + +@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') +@unittest.skipIf(is_wasi or is_emscripten, 'Feature not available on WebAssembly builds') +class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase): + """Test CPython's install details file implementation.""" + + @property + def location(self): + if sysconfig.is_python_build(): + projectdir = sysconfig.get_config_var('projectbase') + with open(os.path.join(projectdir, 'pybuilddir.txt')) as f: + dirname = os.path.join(projectdir, f.read()) + else: + dirname = sysconfig.get_path('stdlib') + return os.path.join(dirname, 'build-details.json') + + @property + def contents(self): + with open(self.location, 'r') as f: + return f.read() + + @needs_installed_python + def test_location(self): + self.assertTrue(os.path.isfile(self.location)) + + # Override generic format tests with tests for our specific implemenation. + + @needs_installed_python + @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS run tests via a custom testbed method that changes sys.executable') + def test_base_interpreter(self): + value = self.key('base_interpreter') + + self.assertEqual(os.path.realpath(value), os.path.realpath(sys.executable)) + + +if __name__ == '__main__': + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 67acf0f..43355e2e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -728,7 +728,7 @@ list-targets: .PHONY: build_all build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil build-details.json .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ @@ -934,6 +934,9 @@ pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS) exit 1 ; \ fi +build-details.json: pybuilddir.txt + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/build-details.json + # Build static library $(LIBRARY): $(LIBRARY_OBJS) -rm -f $@ @@ -2644,6 +2647,7 @@ libinstall: all $(srcdir)/Modules/xxmodule.c done $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/build-details.json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the @ # installed library code. The patch has been previously validated against diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-02-39-42.gh-issue-107956.dLguDW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-02-39-42.gh-issue-107956.dLguDW.rst new file mode 100644 index 0000000..737ecdc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-02-13-02-39-42.gh-issue-107956.dLguDW.rst @@ -0,0 +1,2 @@ +A ``build-details.json`` file is now install in the platform-independent +standard library directory (:pep:`739` implementation). diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py new file mode 100644 index 0000000..edf6d3e --- /dev/null +++ b/Tools/build/generate-build-details.py @@ -0,0 +1,192 @@ +"""Generate build-details.json (see PEP 739).""" + +# Script initially imported from: +# https://github.com/FFY00/python-instrospection/blob/main/python_introspection/scripts/generate-build-details.py + +import argparse +import collections +import importlib.machinery +import json +import os +import sys +import sysconfig +import traceback +import warnings + + +if False: # TYPE_CHECKING + pass + + +def version_info_to_dict(obj): # (object) -> dict[str, Any] + field_names = ('major', 'minor', 'micro', 'releaselevel', 'serial') + return {field: getattr(obj, field) for field in field_names} + + +def get_dict_key(container, key): # (dict[str, Any], str) -> dict[str, Any] + for part in key.split('.'): + container = container[part] + return container + + +def generate_data(schema_version): + """Generate the build-details.json data (PEP 739). + + :param schema_version: The schema version of the data we want to generate. + """ + + if schema_version != '1.0': + raise ValueError(f'Unsupported schema_version: {schema_version}') + + data = collections.defaultdict(lambda: collections.defaultdict(dict)) + + data['schema_version'] = schema_version + + data['base_prefix'] = sysconfig.get_config_var('installed_base') + #data['base_interpreter'] = sys._base_executable + data['base_interpreter'] = os.path.join( + sysconfig.get_path('scripts'), + 'python' + sysconfig.get_config_var('VERSION'), + ) + data['platform'] = sysconfig.get_platform() + + data['language']['version'] = sysconfig.get_python_version() + data['language']['version_info'] = version_info_to_dict(sys.version_info) + + data['implementation'] = vars(sys.implementation) + data['implementation']['version'] = version_info_to_dict(sys.implementation.version) + # Fix cross-compilation + if '_multiarch' in data['implementation']: + data['implementation']['_multiarch'] = sysconfig.get_config_var('MULTIARCH') + + data['abi']['flags'] = list(sys.abiflags) + + data['suffixes']['source'] = importlib.machinery.SOURCE_SUFFIXES + data['suffixes']['bytecode'] = importlib.machinery.BYTECODE_SUFFIXES + #data['suffixes']['optimized_bytecode'] = importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES + #data['suffixes']['debug_bytecode'] = importlib.machinery.DEBUG_BYTECODE_SUFFIXES + data['suffixes']['extensions'] = importlib.machinery.EXTENSION_SUFFIXES + + LIBDIR = sysconfig.get_config_var('LIBDIR') + LDLIBRARY = sysconfig.get_config_var('LDLIBRARY') + LIBRARY = sysconfig.get_config_var('LIBRARY') + PY3LIBRARY = sysconfig.get_config_var('PY3LIBRARY') + LIBPYTHON = sysconfig.get_config_var('LIBPYTHON') + LIBPC = sysconfig.get_config_var('LIBPC') + INCLUDEDIR = sysconfig.get_config_var('INCLUDEDIR') + + if os.name == 'posix': + # On POSIX, LIBRARY is always the static library, while LDLIBRARY is the + # dynamic library if enabled, otherwise it's the static library. + # If LIBRARY != LDLIBRARY, support for the dynamic library is enabled. + has_dynamic_library = LDLIBRARY != LIBRARY + has_static_library = sysconfig.get_config_var('STATIC_LIBPYTHON') + elif os.name == 'nt': + # Windows can only use a dynamic library or a static library. + # If it's using a dynamic library, sys.dllhandle will be set. + # Static builds on Windows are not really well supported, though. + # More context: https://github.com/python/cpython/issues/110234 + has_dynamic_library = hasattr(sys, 'dllhandle') + has_static_library = not has_dynamic_library + else: + raise NotADirectoryError(f'Unknown platform: {os.name}') + + # On POSIX, EXT_SUFFIX is set regardless if extension modules are supported + # or not, and on Windows older versions of CPython only set EXT_SUFFIX when + # extension modules are supported, but newer versions of CPython set it + # regardless. + # + # We only want to set abi.extension_suffix and stable_abi_suffix if + # extension modules are supported. + if has_dynamic_library: + data['abi']['extension_suffix'] = sysconfig.get_config_var('EXT_SUFFIX') + + # EXTENSION_SUFFIXES has been constant for a long time, and currently we + # don't have a better information source to find the stable ABI suffix. + for suffix in importlib.machinery.EXTENSION_SUFFIXES: + if suffix.startswith('.abi'): + data['abi']['stable_abi_suffix'] = suffix + break + + data['libpython']['dynamic'] = os.path.join(LIBDIR, LDLIBRARY) + # FIXME: Not sure if windows has a different dll for the stable ABI, and + # even if it does, currently we don't have a way to get its name. + if PY3LIBRARY: + data['libpython']['dynamic_stableabi'] = os.path.join(LIBDIR, PY3LIBRARY) + + # Os POSIX, this is defined by the LIBPYTHON Makefile variable not being + # empty. On Windows, don't link extensions — LIBPYTHON won't be defined, + data['libpython']['link_extensions'] = bool(LIBPYTHON) + + if has_static_library: + data['libpython']['static'] = os.path.join(LIBDIR, LIBRARY) + + data['c_api']['include'] = INCLUDEDIR + if LIBPC: + data['c_api']['pkgconfig_path'] = LIBPC + + return data + + +def make_paths_relative(data, config_path=None): # (dict[str, Any], str | None) -> None + # Make base_prefix relative to the config_path directory + if config_path: + data['base_prefix'] = os.path.relpath(data['base_prefix'], os.path.dirname(config_path)) + # Update path values to make them relative to base_prefix + PATH_KEYS = [ + 'base_interpreter', + 'libpython.dynamic', + 'libpython.dynamic_stableabi', + 'libpython.static', + 'c_api.headers', + 'c_api.pkgconfig_path', + ] + for entry in PATH_KEYS: + parent, _, child = entry.rpartition('.') + # Get the key container object + try: + container = data + for part in parent.split('.'): + container = container[part] + current_path = container[child] + except KeyError: + continue + # Get the relative path + new_path = os.path.relpath(current_path, data['base_prefix']) + # Join '.' so that the path is formated as './path' instead of 'path' + new_path = os.path.join('.', new_path) + container[child] = new_path + + +def main(): # () -> None + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('location') + parser.add_argument( + '--schema-version', + default='1.0', + help='Schema version of the build-details.json file to generate.', + ) + parser.add_argument( + '--relative-paths', + action='store_true', + help='Whether to specify paths as absolute, or as relative paths to ``base_prefix``.', + ) + parser.add_argument( + '--config-file-path', + default=None, + help='If specified, ``base_prefix`` will be set as a relative path to the given config file path.', + ) + + args = parser.parse_args() + + data = generate_data(args.schema_version) + if args.relative_paths: + make_paths_relative(data, args.config_file_path) + + json_output = json.dumps(data, indent=2) + with open(args.location, 'w') as f: + print(json_output, file=f) + + +if __name__ == '__main__': + main() -- cgit v0.12