bpo-45949: Pure Python freeze module for cross builds (GH-29899)

1 files changed, 65 insertions, 36 deletions

diff --git a/Makefile.pre.in b/Makefile.pre.in
index a182786..fb11497 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -283,14 +283,19 @@ BUILDPYTHON=	python$(BUILDEXE)
 PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
 UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
 PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
-# Standard builds use _bootstrap_python for freezing, cross compiling
-# uses build Python, which must have the same version and bytecode,
-PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@
+
+# Normal builds use Programs/_freeze_module.c for bootstrapping and
+# ./_bootstrap_python Programs/_freeze_module.py for remaining modules
+# Cross builds use an external "build Python" for all modules.
+PYTHON_FOR_FREEZE=@PYTHON_FOR_FREEZE@
+FREEZE_MODULE_BOOTSTRAP=@FREEZE_MODULE_BOOTSTRAP@
+FREEZE_MODULE_BOOTSTRAP_DEPS=@FREEZE_MODULE_BOOTSTRAP_DEPS@
+FREEZE_MODULE=@FREEZE_MODULE@
+FREEZE_MODULE_DEPS=@FREEZE_MODULE_DEPS@
+
 _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
 BUILD_GNU_TYPE=	@build@
 HOST_GNU_TYPE=	@host@
-# Allow developers to override freeze_module command for cross building (bpo-45886)
-FREEZE_MODULE?=@FREEZE_MODULE@
 
 # Tcl and Tk config info from --with-tcltk-includes and -libs options
 TCLTK_INCLUDES=	@TCLTK_INCLUDES@
@@ -967,7 +972,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu
 .PHONY: regen-deepfreeze
 regen-deepfreeze: $(DEEPFREEZE_OBJS)
 
-DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py _bootstrap_python
+DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS)
 
 # BEGIN: deepfreeze modules
 
@@ -1044,6 +1049,30 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ
 
 ############################################################################
 # frozen modules (including importlib)
+#
+# Freezing is a multi step process. It works differently for standard builds
+# and cross builds. Standard builds use Programs/_freeze_module and
+# _bootstrap_python for freezing and deepfreezing, so users can build Python
+# without an existing Python installation. Cross builds cannot execute
+# compiled binaries and therefore rely on an external build Python
+# interpreter. The build interpreter must have same version and same bytecode
+# as the host (target) binary.
+#
+# Standard build process:
+# 1) compile minimal core objects for Py_Compile*() and PyMarshal_Write*().
+# 2) build Programs/_freeze_module binary.
+# 3) create frozen module headers for importlib and getpath.
+# 4) build _bootstrap_python binary.
+# 5) create remaining frozen module headers with
+#    ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python
+#    script is used to test the cross compile code path.
+# 6) deepfreeze modules with _bootstrap_python
+#
+# Cross compile process:
+# 1) create all frozen module headers with external build Python and
+#    Programs/_freeze_module.py script.
+# 2) deepfreeze modules with external build Python.
+#
 
 # FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py.
 FROZEN_FILES_IN = \
@@ -1104,83 +1133,83 @@ Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile
 Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN)
 	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
+# We manually freeze getpath.py rather than through freeze_modules
+Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
+	$(FREEZE_MODULE_BOOTSTRAP) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h
+
 # BEGIN: freezing modules
 
-Python/frozen_modules/importlib._bootstrap.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap.py
-	$(FREEZE_MODULE) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h
+Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
+	$(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h
 
-Python/frozen_modules/importlib._bootstrap_external.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap_external.py
-	$(FREEZE_MODULE) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h
+Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
+	$(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h
 
-Python/frozen_modules/zipimport.h: $(FREEZE_MODULE) Lib/zipimport.py
-	$(FREEZE_MODULE) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h
+Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
+	$(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h
 
-Python/frozen_modules/abc.h: $(FREEZE_MODULE) Lib/abc.py
+Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h
 
-Python/frozen_modules/codecs.h: $(FREEZE_MODULE) Lib/codecs.py
+Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h
 
-Python/frozen_modules/io.h: $(FREEZE_MODULE) Lib/io.py
+Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h
 
-Python/frozen_modules/_collections_abc.h: $(FREEZE_MODULE) Lib/_collections_abc.py
+Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h
 
-Python/frozen_modules/_sitebuiltins.h: $(FREEZE_MODULE) Lib/_sitebuiltins.py
+Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h
 
-Python/frozen_modules/genericpath.h: $(FREEZE_MODULE) Lib/genericpath.py
+Python/frozen_modules/genericpath.h: Lib/genericpath.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h
 
-Python/frozen_modules/ntpath.h: $(FREEZE_MODULE) Lib/ntpath.py
+Python/frozen_modules/ntpath.h: Lib/ntpath.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h
 
-Python/frozen_modules/posixpath.h: $(FREEZE_MODULE) Lib/posixpath.py
+Python/frozen_modules/posixpath.h: Lib/posixpath.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h
 
-Python/frozen_modules/os.h: $(FREEZE_MODULE) Lib/os.py
+Python/frozen_modules/os.h: Lib/os.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h
 
-Python/frozen_modules/site.h: $(FREEZE_MODULE) Lib/site.py
+Python/frozen_modules/site.h: Lib/site.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h
 
-Python/frozen_modules/stat.h: $(FREEZE_MODULE) Lib/stat.py
+Python/frozen_modules/stat.h: Lib/stat.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h
 
-Python/frozen_modules/importlib.util.h: $(FREEZE_MODULE) Lib/importlib/util.py
+Python/frozen_modules/importlib.util.h: Lib/importlib/util.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h
 
-Python/frozen_modules/importlib.machinery.h: $(FREEZE_MODULE) Lib/importlib/machinery.py
+Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h
 
-Python/frozen_modules/runpy.h: $(FREEZE_MODULE) Lib/runpy.py
+Python/frozen_modules/runpy.h: Lib/runpy.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h
 
-Python/frozen_modules/__hello__.h: $(FREEZE_MODULE) Lib/__hello__.py
+Python/frozen_modules/__hello__.h: Lib/__hello__.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h
 
-Python/frozen_modules/__phello__.h: $(FREEZE_MODULE) Lib/__phello__/__init__.py
+Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h
 
-Python/frozen_modules/__phello__.ham.h: $(FREEZE_MODULE) Lib/__phello__/ham/__init__.py
+Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h
 
-Python/frozen_modules/__phello__.ham.eggs.h: $(FREEZE_MODULE) Lib/__phello__/ham/eggs.py
+Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h
 
-Python/frozen_modules/__phello__.spam.h: $(FREEZE_MODULE) Lib/__phello__/spam.py
+Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h
 
-Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py
+Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py $(FREEZE_MODULE_DEPS)
 	$(FREEZE_MODULE) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h
 
 # END: freezing modules
 
-# We manually freeze getpath.py rather than through freeze_modules
-Python/frozen_modules/getpath.h: $(FREEZE_MODULE) Modules/getpath.py
-	$(FREEZE_MODULE) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h
-
 Tools/scripts/freeze_modules.py: $(FREEZE_MODULE)
 
 .PHONY: regen-frozen