bpo-45881: configure --with-freeze-module --with-build-python (GH-29835)

Co-authored-by: Erlend Egeberg Aasland <erlend.aasland@innova.no>
Co-authored-by: Ethan Smith <ethan@ethanhs.me>

5 files changed, 202 insertions, 52 deletions

diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst
index c6aa4af..5720f01 100644
--- a/Doc/using/configure.rst
+++ b/Doc/using/configure.rst
@@ -513,6 +513,56 @@ See ``Mac/README.rst``.
    :option:`--enable-framework` is set (default: ``Python``).
 
 
+Cross Compiling Options
+-----------------------
+
+Cross compiling, also known as cross building, can be used to build Python
+for another CPU architecture or platform. Cross compiling requires a Python
+interpreter and the :program:`_freeze_module` binary from another build. The
+version of the build Python and :program:`_freeze_module` command must be
+the same as the cross compiled host Python.
+
+.. cmdoption:: --build=BUILD
+
+   configure for building on BUILD, usually guessed by :program:`config.guess`.
+
+.. cmdoption:: --host=HOST
+
+   cross-compile to build programs to run on HOST (target platform)
+
+.. cmdoption:: --with-freeze-module=Programs/_freeze_module
+
+   path to ``_freeze_module`` binary for cross compiling.
+
+   .. versionadded:: 3.11
+
+.. cmdoption:: --with-build-python=python3.xx
+
+   path to build ``python`` binary for cross compiling
+
+   .. versionadded:: 3.11
+
+.. cmdoption:: CONFIG_SITE=file
+
+   An environment variable that points to a file with configure overrides.
+
+   Example *config.site* file::
+
+      # config.site-aarch64
+      ac_cv_buggy_getaddrinfo=no
+      ac_cv_file__dev_ptmx=yes
+      ac_cv_file__dev_ptc=no
+
+
+Cross compiling example::
+
+   CONFIG_SITE=config.site-aarch64 ../configure \
+       --build=x86_64-pc-linux-gnu \
+       --host=aarch64-unknown-linux-gnu \
+       --with-freeze-module=../x86_64/Programs/_freeze_module \
+       --with-build-python=../x86_64/python
+
+
 Python Build System
 ===================
 
diff --git a/Makefile.pre.in b/Makefile.pre.in
index fc8ab99..d5be6ed 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -282,6 +282,8 @@ PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
 _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
 BUILD_GNU_TYPE=	@build@
 HOST_GNU_TYPE=	@host@
+# Allow developers to override freeze_module command for cross building (bpo-45886)
+FREEZE_MODULE?=@FREEZE_MODULE@
 
 # Tcl and Tk config info from --with-tcltk-includes and -libs options
 TCLTK_INCLUDES=	@TCLTK_INCLUDES@
@@ -1011,9 +1013,6 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ
 ############################################################################
 # frozen modules (including importlib)
 
-# Allow developers to override freeze_module command for cross building (bpo-45886)
-FREEZE_MODULE?=Programs/_freeze_module
-
 # FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py.
 FROZEN_FILES_IN = \
 		Lib/importlib/_bootstrap.py \
diff --git a/Misc/NEWS.d/next/Build/2021-11-29-16-32-55.bpo-45881.7597J6.rst b/Misc/NEWS.d/next/Build/2021-11-29-16-32-55.bpo-45881.7597J6.rst
new file mode 100644
index 0000000..7ee57d0
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-11-29-16-32-55.bpo-45881.7597J6.rst
@@ -0,0 +1,2 @@
+The :program:`configure` script now accepts ``--with-build-python`` and
+``--with-freeze-module`` options to make cross compiling easier.
diff --git a/configure b/configure
index 29c65a6..382e030 100755
--- a/configure
+++ b/configure
@@ -931,8 +931,9 @@ PKG_CONFIG
 CONFIG_ARGS
 SOVERSION
 VERSION
-PYTHON_FOR_BUILD
 PYTHON_FOR_REGEN
+PYTHON_FOR_BUILD
+FREEZE_MODULE
 host_os
 host_vendor
 host_cpu
@@ -988,6 +989,8 @@ SHELL'
 ac_subst_files=''
 ac_user_opts='
 enable_option_checking
+with_freeze_module
+with_build_python
 with_pkg_config
 enable_universalsdk
 with_universal_archs
@@ -1721,6 +1724,11 @@ Optional Features:
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
   --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-freeze-module=Programs/_freeze_module
+                          path to _freeze_module binary for cross compiling
+  --with-build-python=python3.11
+                          path to build python binary for cross compiling
+                          (default: python3.11)
   --with-pkg-config=[yes|no|check]
                           use pkg-config to detect build options (default is
                           check)
@@ -3177,9 +3185,88 @@ case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac
 
 
 
+if test "x$cross_compiling" = xmaybe; then :
+  as_fn_error $? "Cross compiling required --host=HOST-TUPLE and --build=ARCH" "$LINENO" 5
+
+fi
+
 # pybuilddir.txt will be created by --generate-posix-vars in the Makefile
 rm -f pybuilddir.txt
 
+
+# Check whether --with-freeze-module was given.
+if test "${with_freeze_module+set}" = set; then :
+  withval=$with_freeze_module;
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-freeze-module" >&5
+$as_echo_n "checking for --with-freeze-module... " >&6; }
+    if test "x$cross_compiling" = xno; then :
+  as_fn_error $? "--with-freeze-module only applies to cross compiling" "$LINENO" 5
+fi
+    if test "$with_freeze_module" = yes -o "$with_freeze_module" = no; then
+      as_fn_error $? "invalid --with-freeze-module option: expected path, not \"$with_freeze_module\"" "$LINENO" 5
+    fi
+    if ! $(command -v "$with_freeze_module" >/dev/null 2>&1); then
+      as_fn_error $? "invalid or missing freeze module binary \"$with_freeze_module\"" "$LINENO" 5
+    fi
+    FREEZE_MODULE="$with_freeze_module"
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: $FREEZE_MODULE" >&5
+$as_echo "$FREEZE_MODULE" >&6; }
+
+else
+
+    if test "x$cross_compiling" = xyes; then :
+  as_fn_error $? "Cross compiling requires --with-freeze-module" "$LINENO" 5
+
+fi
+    FREEZE_MODULE=Programs/_freeze_module
+
+
+fi
+
+
+
+
+# Check whether --with-build-python was given.
+if test "${with_build_python+set}" = set; then :
+  withval=$with_build_python;
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-build-python" >&5
+$as_echo_n "checking for --with-build-python... " >&6; }
+
+    if test "x$cross_compiling" = xno; then :
+  as_fn_error $? "--with-build-python only applies to cross compiling" "$LINENO" 5
+fi
+    if test "x$with_build_python" = xyes; then :
+  with_build_python=python$PACKAGE_VERSION
+fi
+    if test "x$with_build_python" = xno; then :
+  as_fn_error $? "invalid --with-build-python option: expected path, not \"no\"" "$LINENO" 5
+fi
+
+    if ! $(command -v "$with_build_python" >/dev/null 2>&1); then
+      as_fn_error $? "invalid or missing build python binary \"$with_build_python\"" "$LINENO" 5
+    fi
+    build_python_ver=$($with_build_python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+    if test "$build_python_ver" != "$PACKAGE_VERSION"; then
+      as_fn_error $? "\"$with_build_python\" has incompatible version $build_python_ver (expected: $PACKAGE_VERSION)" "$LINENO" 5
+    fi
+        ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python
+    PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_build_python" >&5
+$as_echo "$with_build_python" >&6; }
+
+else
+
+    if test "x$cross_compiling" = xyes; then :
+  as_fn_error $? "Cross compiling requires --with-build-python" "$LINENO" 5
+
+fi
+    PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E'
+
+
+fi
+
+
+
 for ac_prog in python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python
 do
   # Extract the first word of "$ac_prog", so it can be a program name with args.
@@ -3235,31 +3322,6 @@ else
 $as_echo "missing" >&6; }
 fi
 
-if test "$cross_compiling" = yes; then
-    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for python interpreter for cross build" >&5
-$as_echo_n "checking for python interpreter for cross build... " >&6; }
-    if test -z "$PYTHON_FOR_BUILD"; then
-        for interp in python$PACKAGE_VERSION python3 python; do
-	    which $interp >/dev/null 2>&1 || continue
-	    if $interp -c "import sys;sys.exit(not '.'.join(str(n) for n in sys.version_info[:2]) == '$PACKAGE_VERSION')"; then
-	        break
-	    fi
-            interp=
-	done
-        if test x$interp = x; then
-	    as_fn_error $? "python$PACKAGE_VERSION interpreter not found" "$LINENO" 5
-	fi
-        { $as_echo "$as_me:${as_lineno-$LINENO}: result: $interp" >&5
-$as_echo "$interp" >&6; }
-	PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp
-    fi
-elif test "$cross_compiling" = maybe; then
-    as_fn_error $? "Cross compiling required --host=HOST-TUPLE and --build=ARCH" "$LINENO" 5
-else
-    PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E'
-fi
-
-
 
 if test "$prefix" != "/"; then
     prefix=`echo "$prefix" | sed -e 's/\/$//g'`
diff --git a/configure.ac b/configure.ac
index 80f2560..1e5fef3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -93,9 +93,69 @@ AC_CANONICAL_HOST
 AC_SUBST(build)
 AC_SUBST(host)
 
+AS_VAR_IF([cross_compiling], [maybe],
+ [AC_MSG_ERROR([Cross compiling required --host=HOST-TUPLE and --build=ARCH])]
+)
+
 # pybuilddir.txt will be created by --generate-posix-vars in the Makefile
 rm -f pybuilddir.txt
 
+dnl cross-compiling needs a freeze_module binary for build platform
+AC_ARG_WITH(
+  [freeze-module],
+  [AS_HELP_STRING([--with-freeze-module=Programs/_freeze_module],
+                  [path to _freeze_module binary for cross compiling])],
+  [
+    AC_MSG_CHECKING([for --with-freeze-module])
+    AS_VAR_IF([cross_compiling], [no], AC_MSG_ERROR([--with-freeze-module only applies to cross compiling]))
+    if test "$with_freeze_module" = yes -o "$with_freeze_module" = no; then
+      AC_MSG_ERROR([invalid --with-freeze-module option: expected path, not "$with_freeze_module"])
+    fi
+    if ! $(command -v "$with_freeze_module" >/dev/null 2>&1); then
+      AC_MSG_ERROR([invalid or missing freeze module binary "$with_freeze_module"])
+    fi
+    FREEZE_MODULE="$with_freeze_module"
+    AC_MSG_RESULT([$FREEZE_MODULE])
+  ], [
+    AS_VAR_IF([cross_compiling], [yes],
+      [AC_MSG_ERROR([Cross compiling requires --with-freeze-module])]
+    )
+    FREEZE_MODULE=Programs/_freeze_module
+  ]
+)
+AC_SUBST([FREEZE_MODULE])
+
+AC_ARG_WITH(
+  [build-python],
+  [AS_HELP_STRING([--with-build-python=python]PYTHON_VERSION,
+                  [path to build python binary for cross compiling (default: python]PYTHON_VERSION[)])],
+  [
+    AC_MSG_CHECKING([for --with-build-python])
+
+    AS_VAR_IF([cross_compiling], [no], AC_MSG_ERROR([--with-build-python only applies to cross compiling]))
+    AS_VAR_IF([with_build_python], [yes], [with_build_python=python$PACKAGE_VERSION])
+    AS_VAR_IF([with_build_python], [no], [AC_MSG_ERROR([invalid --with-build-python option: expected path, not "no"])])
+
+    if ! $(command -v "$with_build_python" >/dev/null 2>&1); then
+      AC_MSG_ERROR([invalid or missing build python binary "$with_build_python"])
+    fi
+    build_python_ver=$($with_build_python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+    if test "$build_python_ver" != "$PACKAGE_VERSION"; then
+      AC_MSG_ERROR(["$with_build_python" has incompatible version $build_python_ver (expected: $PACKAGE_VERSION)])
+    fi
+    dnl use build Python for regeneration, too
+    ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python
+    PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python
+    AC_MSG_RESULT([$with_build_python])
+  ], [
+    AS_VAR_IF([cross_compiling], [yes],
+      [AC_MSG_ERROR([Cross compiling requires --with-build-python])]
+    )
+    PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E'
+  ]
+)
+AC_SUBST([PYTHON_FOR_BUILD])
+
 AC_CHECK_PROGS([PYTHON_FOR_REGEN],
   [python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python],
   [python3])
@@ -108,29 +168,6 @@ else
   AC_MSG_RESULT([missing])
 fi
 
-if test "$cross_compiling" = yes; then
-    AC_MSG_CHECKING([for python interpreter for cross build])
-    if test -z "$PYTHON_FOR_BUILD"; then
-        for interp in python$PACKAGE_VERSION python3 python; do
-	    which $interp >/dev/null 2>&1 || continue
-	    if $interp -c "import sys;sys.exit(not '.'.join(str(n) for n in sys.version_info@<:@:2@:>@) == '$PACKAGE_VERSION')"; then
-	        break
-	    fi
-            interp=
-	done
-        if test x$interp = x; then
-	    AC_MSG_ERROR([python$PACKAGE_VERSION interpreter not found])
-	fi
-        AC_MSG_RESULT($interp)
-	PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$interp
-    fi
-elif test "$cross_compiling" = maybe; then
-    AC_MSG_ERROR([Cross compiling required --host=HOST-TUPLE and --build=ARCH])
-else
-    PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E'
-fi
-AC_SUBST(PYTHON_FOR_BUILD)
-
 dnl Ensure that if prefix is specified, it does not end in a slash. If
 dnl it does, we get path names containing '//' which is both ugly and
 dnl can cause trouble.