bpo-45696: Deep-freeze selected modules (GH-29118)

This gains 10% or more in startup time for `python -c pass` on UNIX-ish systems.

The Makefile.pre.in generating code builds on Eric's work for bpo-45020, but the .c file generator is new.

Windows version TBD.

4 files changed, 120 insertions, 32 deletions

diff --git a/Python/bootstrap_frozen.c b/Python/bootstrap_frozen.c
new file mode 100644
index 0000000..68ba147
--- /dev/null
+++ b/Python/bootstrap_frozen.c
@@ -0,0 +1,45 @@
+
+/* Frozen modules bootstrap */
+
+/* This file is linked with "bootstrap Python"
+   which is used (only) to run Tools/scripts/deepfreeze.py. */
+
+#include "Python.h"
+#include "pycore_import.h"
+
+/* Includes for frozen modules: */
+#include "frozen_modules/importlib._bootstrap.h"
+#include "frozen_modules/importlib._bootstrap_external.h"
+#include "frozen_modules/zipimport.h"
+/* End includes */
+
+/* Note that a negative size indicates a package. */
+
+static const struct _frozen bootstrap_modules[] = {
+    {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)},
+    {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)},
+    {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
+    {0, 0, 0} /* bootstrap sentinel */
+};
+static const struct _frozen stdlib_modules[] = {
+    {0, 0, 0} /* stdlib sentinel */
+};
+static const struct _frozen test_modules[] = {
+    {0, 0, 0} /* test sentinel */
+};
+const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;
+const struct _frozen *_PyImport_FrozenStdlib = stdlib_modules;
+const struct _frozen *_PyImport_FrozenTest = test_modules;
+
+static const struct _module_alias aliases[] = {
+    {"_frozen_importlib", "importlib._bootstrap"},
+    {"_frozen_importlib_external", "importlib._bootstrap_external"},
+    {0, 0} /* aliases sentinel */
+};
+const struct _module_alias *_PyImport_FrozenAliases = aliases;
+
+
+/* Embedding apps may change this pointer to point to their favorite
+   collection of frozen modules: */
+
+const struct _frozen *PyImport_FrozenModules = NULL;
diff --git a/Python/deepfreeze/README.txt b/Python/deepfreeze/README.txt
new file mode 100644
index 0000000..da55d4e
--- /dev/null
+++ b/Python/deepfreeze/README.txt
@@ -0,0 +1,6 @@
+This directory contains the generated .c files for all the deep-frozen
+modules.  Python/frozen.c depends on these files.
+
+None of these files are committed into the repo.
+
+See Tools/scripts/freeze_modules.py for more info.
diff --git a/Python/frozen.c b/Python/frozen.c
index 15baa97..1565c9a 100644
--- a/Python/frozen.c
+++ b/Python/frozen.c
@@ -61,50 +61,80 @@
 #include "frozen_modules/frozen_only.h"
 /* End includes */
 
+#ifdef MS_WINDOWS
+/* Deepfreeze isn't supported on Windows yet. */
+#define GET_CODE(name) NULL
+#else
+#define GET_CODE(name) _Py_get_##name##_toplevel
+#endif
+
+/* Start extern declarations */
+extern PyObject *_Py_get_importlib__bootstrap_toplevel(void);
+extern PyObject *_Py_get_importlib__bootstrap_external_toplevel(void);
+extern PyObject *_Py_get_zipimport_toplevel(void);
+extern PyObject *_Py_get_abc_toplevel(void);
+extern PyObject *_Py_get_codecs_toplevel(void);
+extern PyObject *_Py_get_io_toplevel(void);
+extern PyObject *_Py_get__collections_abc_toplevel(void);
+extern PyObject *_Py_get__sitebuiltins_toplevel(void);
+extern PyObject *_Py_get_genericpath_toplevel(void);
+extern PyObject *_Py_get_ntpath_toplevel(void);
+extern PyObject *_Py_get_posixpath_toplevel(void);
+extern PyObject *_Py_get_posixpath_toplevel(void);
+extern PyObject *_Py_get_os_toplevel(void);
+extern PyObject *_Py_get_site_toplevel(void);
+extern PyObject *_Py_get_stat_toplevel(void);
+extern PyObject *_Py_get___hello___toplevel(void);
+extern PyObject *_Py_get___hello___toplevel(void);
+extern PyObject *_Py_get___hello___toplevel(void);
+extern PyObject *_Py_get___hello___toplevel(void);
+extern PyObject *_Py_get___phello___toplevel(void);
+extern PyObject *_Py_get___phello___toplevel(void);
+extern PyObject *_Py_get___phello___ham_toplevel(void);
+extern PyObject *_Py_get___phello___ham_toplevel(void);
+extern PyObject *_Py_get___phello___ham_eggs_toplevel(void);
+extern PyObject *_Py_get___phello___spam_toplevel(void);
+extern PyObject *_Py_get_frozen_only_toplevel(void);
+/* End extern declarations */
+
 /* Note that a negative size indicates a package. */
 
 static const struct _frozen bootstrap_modules[] = {
-    {"_frozen_importlib", _Py_M__importlib__bootstrap,
-        (int)sizeof(_Py_M__importlib__bootstrap)},
-    {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external,
-        (int)sizeof(_Py_M__importlib__bootstrap_external)},
-    {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
+    {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), GET_CODE(importlib__bootstrap)},
+    {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), GET_CODE(importlib__bootstrap_external)},
+    {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), GET_CODE(zipimport)},
     {0, 0, 0} /* bootstrap sentinel */
 };
 static const struct _frozen stdlib_modules[] = {
     /* stdlib - startup, without site (python -S) */
-    {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc)},
-    {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs)},
-    {"io", _Py_M__io, (int)sizeof(_Py_M__io)},
+    {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), GET_CODE(abc)},
+    {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), GET_CODE(codecs)},
+    {"io", _Py_M__io, (int)sizeof(_Py_M__io), GET_CODE(io)},
 
     /* stdlib - startup, with site */
-    {"_collections_abc", _Py_M___collections_abc,
-        (int)sizeof(_Py_M___collections_abc)},
-    {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins)},
-    {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath)},
-    {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath)},
-    {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
-    {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
-    {"os", _Py_M__os, (int)sizeof(_Py_M__os)},
-    {"site", _Py_M__site, (int)sizeof(_Py_M__site)},
-    {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat)},
+    {"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc), GET_CODE(_collections_abc)},
+    {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins), GET_CODE(_sitebuiltins)},
+    {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), GET_CODE(genericpath)},
+    {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), GET_CODE(ntpath)},
+    {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)},
+    {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)},
+    {"os", _Py_M__os, (int)sizeof(_Py_M__os), GET_CODE(os)},
+    {"site", _Py_M__site, (int)sizeof(_Py_M__site), GET_CODE(site)},
+    {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), GET_CODE(stat)},
     {0, 0, 0} /* stdlib sentinel */
 };
 static const struct _frozen test_modules[] = {
-    {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__)},
-    {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__)},
-    {"__phello_alias__", _Py_M____hello__, -(int)sizeof(_Py_M____hello__)},
-    {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__)},
-    {"__phello__", _Py_M____phello__, -(int)sizeof(_Py_M____phello__)},
-    {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__)},
-    {"__phello__.ham", _Py_M____phello___ham, -(int)sizeof(_Py_M____phello___ham)},
-    {"__phello__.ham.__init__", _Py_M____phello___ham,
-        (int)sizeof(_Py_M____phello___ham)},
-    {"__phello__.ham.eggs", _Py_M____phello___ham_eggs,
-        (int)sizeof(_Py_M____phello___ham_eggs)},
-    {"__phello__.spam", _Py_M____phello___spam,
-        (int)sizeof(_Py_M____phello___spam)},
-    {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only)},
+    {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), GET_CODE(__hello__)},
+    {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), GET_CODE(__hello__)},
+    {"__phello_alias__", _Py_M____hello__, -(int)sizeof(_Py_M____hello__), GET_CODE(__hello__)},
+    {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__), GET_CODE(__hello__)},
+    {"__phello__", _Py_M____phello__, -(int)sizeof(_Py_M____phello__), GET_CODE(__phello__)},
+    {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), GET_CODE(__phello__)},
+    {"__phello__.ham", _Py_M____phello___ham, -(int)sizeof(_Py_M____phello___ham), GET_CODE(__phello___ham)},
+    {"__phello__.ham.__init__", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), GET_CODE(__phello___ham)},
+    {"__phello__.ham.eggs", _Py_M____phello___ham_eggs, (int)sizeof(_Py_M____phello___ham_eggs), GET_CODE(__phello___ham_eggs)},
+    {"__phello__.spam", _Py_M____phello___spam, (int)sizeof(_Py_M____phello___spam), GET_CODE(__phello___spam)},
+    {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only), GET_CODE(frozen_only)},
     {0, 0, 0} /* test sentinel */
 };
 const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;
diff --git a/Python/import.c b/Python/import.c
index cdcb903..225fbf4 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -1262,6 +1262,7 @@ look_up_frozen(const char *name)
 struct frozen_info {
     PyObject *nameobj;
     const char *data;
+    PyObject *(*get_code)(void);
     Py_ssize_t size;
     bool is_package;
     bool is_alias;
@@ -1295,6 +1296,7 @@ find_frozen(PyObject *nameobj, struct frozen_info *info)
     if (info != NULL) {
         info->nameobj = nameobj;  // borrowed
         info->data = (const char *)p->code;
+        info->get_code = p->get_code;
         info->size = p->size < 0 ? -(p->size) : p->size;
         info->is_package = p->size < 0 ? true : false;
         info->origname = name;
@@ -1316,6 +1318,11 @@ find_frozen(PyObject *nameobj, struct frozen_info *info)
 static PyObject *
 unmarshal_frozen_code(struct frozen_info *info)
 {
+    if (info->get_code) {
+        PyObject *code = info->get_code();
+        assert(code != NULL);
+        return code;
+    }
     PyObject *co = PyMarshal_ReadObjectFromString(info->data, info->size);
     if (co == NULL) {
         /* Does not contain executable code. */