bpo-45696: Deep-freeze selected modules (GH-29118)

This gains 10% or more in startup time for `python -c pass` on UNIX-ish systems.

The Makefile.pre.in generating code builds on Eric's work for bpo-45020, but the .c file generator is new.

Windows version TBD.

1 files changed, 182 insertions, 3 deletions

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 7e959b0..6968ae4 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -259,6 +259,7 @@ LIBOBJS=	@LIBOBJS@
 
 PYTHON=		python$(EXE)
 BUILDPYTHON=	python$(BUILDEXE)
+BOOTSTRAP= _bootstrap_python
 
 PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
 UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
@@ -448,6 +449,30 @@ OBJECT_OBJS=	\
 		Objects/unionobject.o \
 		Objects/weakrefobject.o
 
+# DEEPFREEZE_OBJS is auto-generated by Tools/scripts/freeze_modules.py.
+DEEPFREEZE_OBJS = \
+		Python/deepfreeze/importlib._bootstrap.o \
+		Python/deepfreeze/importlib._bootstrap_external.o \
+		Python/deepfreeze/zipimport.o \
+		Python/deepfreeze/abc.o \
+		Python/deepfreeze/codecs.o \
+		Python/deepfreeze/io.o \
+		Python/deepfreeze/_collections_abc.o \
+		Python/deepfreeze/_sitebuiltins.o \
+		Python/deepfreeze/genericpath.o \
+		Python/deepfreeze/ntpath.o \
+		Python/deepfreeze/posixpath.o \
+		Python/deepfreeze/os.o \
+		Python/deepfreeze/site.o \
+		Python/deepfreeze/stat.o \
+		Python/deepfreeze/__hello__.o \
+		Python/deepfreeze/__phello__.o \
+		Python/deepfreeze/__phello__.ham.o \
+		Python/deepfreeze/__phello__.ham.eggs.o \
+		Python/deepfreeze/__phello__.spam.o \
+		Python/deepfreeze/frozen_only.o
+# End DEEPFREEZE_OBJS
+
 ##########################################################################
 # objects that get linked into the Python library
 LIBRARY_OBJS_OMIT_FROZEN=	\
@@ -460,6 +485,7 @@ LIBRARY_OBJS_OMIT_FROZEN=	\
 
 LIBRARY_OBJS=	\
 		$(LIBRARY_OBJS_OMIT_FROZEN) \
+		$(DEEPFREEZE_OBJS) \
 		Python/frozen.o
 
 ##########################################################################
@@ -602,9 +628,9 @@ platform: $(BUILDPYTHON) pybuilddir.txt
 # problems by creating a dummy pybuilddir.txt just to allow interpreter
 # initialization to succeed.  It will be overwritten by generate-posix-vars
 # or removed in case of failure.
-pybuilddir.txt: $(BUILDPYTHON)
+pybuilddir.txt: $(BOOTSTRAP)
 	@echo "none" > ./pybuilddir.txt
-	$(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\
+	./$(BOOTSTRAP) -S -m sysconfig --generate-posix-vars ;\
 	if test $$? -ne 0 ; then \
 		echo "generate-posix-vars failed" ; \
 		rm -f ./pybuilddir.txt ; \
@@ -739,6 +765,158 @@ Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
 	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
 ############################################################################
+# "Bootstrap Python" used to run deepfreeze.py
+
+BOOTSTRAP_HEADERS = \
+	Python/frozen_modules/importlib._bootstrap.h \
+	Python/frozen_modules/importlib._bootstrap_external.h \
+	Python/frozen_modules/zipimport.h
+
+Python/bootstrap_frozen.o: Python/bootstrap_frozen.c Include/cpython/import.h $(BOOTSTRAP_HEADERS)
+
+$(BOOTSTRAP): $(LIBRARY_OBJS_OMIT_FROZEN) \
+		Python/bootstrap_frozen.o Programs/python.o
+	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
+		Python/bootstrap_frozen.o \
+		Programs/python.o \
+		$(LIBS) $(MODLIBS) $(SYSLIBS)
+
+############################################################################
+# Deepfreeze targets
+
+.PHONY: regen-deepfreeze
+regen-deepfreeze: $(DEEPFREEZE_OBJS)
+
+DEEPFREEZE_DEPS = \
+	$(BOOTSTRAP) \
+	pybuilddir.txt \
+	$(srcdir)/Tools/scripts/deepfreeze.py
+
+# BEGIN: deepfreeze modules
+
+Python/deepfreeze/importlib._bootstrap.c: $(srcdir)/Lib/importlib/_bootstrap.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/importlib._bootstrap.c from Lib/importlib/_bootstrap.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/importlib/_bootstrap.py -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
+
+Python/deepfreeze/importlib._bootstrap_external.c: $(srcdir)/Lib/importlib/_bootstrap_external.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/importlib._bootstrap_external.c from Lib/importlib/_bootstrap_external.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/importlib/_bootstrap_external.py -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
+
+Python/deepfreeze/zipimport.c: $(srcdir)/Lib/zipimport.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/zipimport.c from Lib/zipimport.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/zipimport.py -m zipimport -o Python/deepfreeze/zipimport.c
+
+Python/deepfreeze/abc.c: $(srcdir)/Lib/abc.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/abc.c from Lib/abc.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/abc.py -m abc -o Python/deepfreeze/abc.c
+
+Python/deepfreeze/codecs.c: $(srcdir)/Lib/codecs.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/codecs.c from Lib/codecs.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/codecs.py -m codecs -o Python/deepfreeze/codecs.c
+
+Python/deepfreeze/io.c: $(srcdir)/Lib/io.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/io.c from Lib/io.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/io.py -m io -o Python/deepfreeze/io.c
+
+Python/deepfreeze/_collections_abc.c: $(srcdir)/Lib/_collections_abc.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/_collections_abc.c from Lib/_collections_abc.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/_collections_abc.py -m _collections_abc -o Python/deepfreeze/_collections_abc.c
+
+Python/deepfreeze/_sitebuiltins.c: $(srcdir)/Lib/_sitebuiltins.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/_sitebuiltins.c from Lib/_sitebuiltins.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/_sitebuiltins.py -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
+
+Python/deepfreeze/genericpath.c: $(srcdir)/Lib/genericpath.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/genericpath.c from Lib/genericpath.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/genericpath.py -m genericpath -o Python/deepfreeze/genericpath.c
+
+Python/deepfreeze/ntpath.c: $(srcdir)/Lib/ntpath.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/ntpath.c from Lib/ntpath.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/ntpath.py -m ntpath -o Python/deepfreeze/ntpath.c
+
+Python/deepfreeze/posixpath.c: $(srcdir)/Lib/posixpath.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/posixpath.c from Lib/posixpath.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/posixpath.py -m posixpath -o Python/deepfreeze/posixpath.c
+
+Python/deepfreeze/os.c: $(srcdir)/Lib/os.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/os.c from Lib/os.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/os.py -m os -o Python/deepfreeze/os.c
+
+Python/deepfreeze/site.c: $(srcdir)/Lib/site.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/site.c from Lib/site.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/site.py -m site -o Python/deepfreeze/site.c
+
+Python/deepfreeze/stat.c: $(srcdir)/Lib/stat.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/stat.c from Lib/stat.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/stat.py -m stat -o Python/deepfreeze/stat.c
+
+Python/deepfreeze/__hello__.c: $(srcdir)/Lib/__hello__.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__hello__.c from Lib/__hello__.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__hello__.py -m __hello__ -o Python/deepfreeze/__hello__.c
+
+Python/deepfreeze/__phello__.c: $(srcdir)/Lib/__phello__/__init__.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__phello__.c from Lib/__phello__/__init__.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__phello__/__init__.py -m __phello__ -o Python/deepfreeze/__phello__.c
+
+Python/deepfreeze/__phello__.ham.c: $(srcdir)/Lib/__phello__/ham/__init__.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__phello__.ham.c from Lib/__phello__/ham/__init__.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__phello__/ham/__init__.py -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
+
+Python/deepfreeze/__phello__.ham.eggs.c: $(srcdir)/Lib/__phello__/ham/eggs.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__phello__.ham.eggs.c from Lib/__phello__/ham/eggs.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__phello__/ham/eggs.py -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
+
+Python/deepfreeze/__phello__.spam.c: $(srcdir)/Lib/__phello__/spam.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__phello__.spam.c from Lib/__phello__/spam.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__phello__/spam.py -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
+
+Python/deepfreeze/frozen_only.c: $(srcdir)/Tools/freeze/flag.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/frozen_only.c from Tools/freeze/flag.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Tools/freeze/flag.py -m frozen_only -o Python/deepfreeze/frozen_only.c
+
+# END: deepfreeze modules
+
+############################################################################
 # frozen modules (including importlib)
 
 # FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py.
@@ -2017,7 +2195,8 @@ clean-retain-profile: pycremoval
 	find build -name '*.py[co]' -exec rm -f {} ';' || true
 	-rm -f pybuilddir.txt
 	-rm -f Lib/lib2to3/*Grammar*.pickle
-	-rm -f Programs/_testembed Programs/_freeze_module
+	-rm -f Programs/_testembed Programs/_freeze_module $(BOOTSTRAP)
+	-rm -f Python/deepfreeze/*.[co]
 	-find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
 	-rm -f Include/pydtrace_probes.h
 	-rm -f profile-gen-stamp