From 2950bc50af8fc2539e64731359bfb39b335a614d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns=20=F0=9F=87=B5=F0=9F=87=B8?= Date: Mon, 2 Dec 2024 07:12:36 +0000 Subject: GH-127429: fix sysconfig data generation on cross-builds (#127430) --- .github/workflows/jit.yml | 2 +- Lib/sysconfig/__init__.py | 49 ++++++++++++++++------ Lib/sysconfig/__main__.py | 7 +++- .../2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst | 3 ++ configure | 6 +-- configure.ac | 2 +- 6 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 7dbbe71..4ef543d 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -104,7 +104,7 @@ jobs: # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - name: Native macOS if: runner.os == 'macOS' diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index ee52700..ad86609 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -318,14 +318,22 @@ def get_default_scheme(): def get_makefile_filename(): """Return the path of the Makefile.""" + + # GH-127429: When cross-compiling, use the Makefile from the target, instead of the host Python. + if cross_base := os.environ.get('_PYTHON_PROJECT_BASE'): + return os.path.join(cross_base, 'Makefile') + if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") + if hasattr(sys, 'abiflags'): config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: config_dir_name = 'config' + if hasattr(sys.implementation, '_multiarch'): config_dir_name += f'-{sys.implementation._multiarch}' + return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') @@ -464,27 +472,44 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} + + prefix = _PREFIX + exec_prefix = _EXEC_PREFIX + base_prefix = _BASE_PREFIX + base_exec_prefix = _BASE_EXEC_PREFIX + + try: + abiflags = sys.abiflags + except AttributeError: + abiflags = '' + + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + # If we are cross-compiling, load the prefixes from the Makefile instead. + if '_PYTHON_PROJECT_BASE' in os.environ: + prefix = _CONFIG_VARS['prefix'] + exec_prefix = _CONFIG_VARS['exec_prefix'] + base_prefix = _CONFIG_VARS['prefix'] + base_exec_prefix = _CONFIG_VARS['exec_prefix'] + abiflags = _CONFIG_VARS['ABIFLAGS'] + # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX + _CONFIG_VARS['prefix'] = prefix + _CONFIG_VARS['exec_prefix'] = exec_prefix _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['installed_base'] = base_prefix + _CONFIG_VARS['base'] = prefix + _CONFIG_VARS['installed_platbase'] = base_exec_prefix + _CONFIG_VARS['platbase'] = exec_prefix _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir _CONFIG_VARS['implementation'] = _get_implementation() _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' + _CONFIG_VARS['abiflags'] = abiflags try: _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: @@ -493,8 +518,6 @@ def _init_config_vars(): if os.name == 'nt': _init_non_posix(_CONFIG_VARS) _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 5660a6c..10728c7 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -7,6 +7,7 @@ from sysconfig import ( _PYTHON_BUILD, _get_sysconfigdata_name, get_config_h_filename, + get_config_var, get_config_vars, get_default_scheme, get_makefile_filename, @@ -161,7 +162,7 @@ def _print_config_dict(d, stream): def _get_pybuilddir(): pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): + if get_config_var('Py_DEBUG') == '1': pybuilddir += '-pydebug' return pybuilddir @@ -229,11 +230,15 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + print(f'Written {destfile}') + # Write a JSON file with the output of sysconfig.get_config_vars jsonfile = os.path.join(pybuilddir, _get_json_data_name()) with open(jsonfile, 'w') as f: json.dump(get_config_vars(), f, indent=2) + print(f'Written {jsonfile}') + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) diff --git a/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst new file mode 100644 index 0000000..708c1a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst @@ -0,0 +1,3 @@ +Fixed bug where, on cross-builds, the :mod:`sysconfig` POSIX data was being +generated with the host Python's ``Makefile``. The data is now generated from +current build's ``Makefile``. diff --git a/configure b/configure index 84b74ac..4e40432 100755 --- a/configure +++ b/configure @@ -944,8 +944,8 @@ AR LINK_PYTHON_OBJS LINK_PYTHON_DEPS LIBRARY_DEPS -NODE HOSTRUNNER +NODE STATIC_LIBPYTHON GNULD EXPORTSFROM @@ -1147,7 +1147,6 @@ LDFLAGS LIBS CPPFLAGS CPP -HOSTRUNNER PROFILE_TASK BOLT_INSTRUMENT_FLAGS BOLT_APPLY_FLAGS @@ -1968,7 +1967,6 @@ Some influential environment variables: CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory CPP C preprocessor - HOSTRUNNER Program to run CPython for the host platform PROFILE_TASK Python args for PGO generation task BOLT_INSTRUMENT_FLAGS @@ -7622,9 +7620,9 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 printf %s "checking HOSTRUNNER... " >&6; } - if test -z "$HOSTRUNNER" then case $ac_sys_system in #( diff --git a/configure.ac b/configure.ac index 8fa6cb6..4cfced1 100644 --- a/configure.ac +++ b/configure.ac @@ -1609,8 +1609,8 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform AC_MSG_CHECKING([HOSTRUNNER]) -AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then AS_CASE([$ac_sys_system], -- cgit v0.12