bpo-45019: Add a tool to generate list of modules to include for frozen modules (gh-27980)

Frozen modules must be added to several files in order to work properly. Before this change this had to be done manually. Here we add a tool to generate the relevant lines in those files instead. This helps us avoid mistakes and omissions.

https://bugs.python.org/issue45019

5 files changed, 57 insertions, 29 deletions

diff --git a/Python/frozen.c b/Python/frozen.c
index 7f433ff..67aff2e 100644
--- a/Python/frozen.c
+++ b/Python/frozen.c
@@ -1,35 +1,63 @@
 
-/* Frozen modules initializer */
-
-#include "Python.h"
-#include "importlib.h"
-#include "importlib_external.h"
-#include "importlib_zipimport.h"
+/* Frozen modules initializer
+ *
+ * Frozen modules are written to header files by Programs/_freeze_module.
+ * These files are typically put in Python/frozen_modules/.  Each holds
+ * an array of bytes named "_Py_M__<module>", which is used below.
+ *
+ * These files must be regenerated any time the corresponding .pyc
+ * file would change (including with changes to the compiler, bytecode
+ * format, marshal format).  This can be done with "make regen-frozen".
+ * That make target just runs Tools/scripts/freeze_modules.py.
+ *
+ * The freeze_modules.py script also determines which modules get
+ * frozen.  Update the list at the top of the script to add, remove,
+ * or modify the target modules.  Then run the script
+ * (or run "make regen-frozen").
+ *
+ * The script does the following:
+ *
+ * 1. run Programs/_freeze_module on the target modules
+ * 2. update the includes and _PyImport_FrozenModules[] in this file
+ * 3. update the FROZEN_FILES variable in Makefile.pre.in
+ * 4. update the per-module targets in Makefile.pre.in
+ * 5. update the lists of modules in PCbuild/_freeze_module.vcxproj and
+ *    PCbuild/_freeze_module.vcxproj.filters
+ *
+ * (Note that most of the data in this file is auto-generated by the script.)
+ *
+ * Those steps can also be done manually, though this is not recommended.
+ * Expect such manual changes to be removed the next time
+ * freeze_modules.py runs.
+ * */
 
 /* In order to test the support for frozen modules, by default we
-   define a single frozen module, __hello__.  Loading it will print
-   some famous words... */
+   define some simple frozen modules: __hello__, __phello__ (a package),
+   and __phello__.spam.  Loading any will print some famous words... */
 
-/* Run "make regen-frozen" to regen the file below (e.g. after a bytecode
- * format change).  The include file defines _Py_M__hello as an array of bytes.
- */
-#include "frozen_hello.h"
+#include "Python.h"
 
-#define SIZE (int)sizeof(_Py_M__hello)
+/* Includes for frozen modules: */
+#include "frozen_modules/importlib__bootstrap.h"
+#include "frozen_modules/importlib__bootstrap_external.h"
+#include "frozen_modules/zipimport.h"
+#include "frozen_modules/hello.h"
+/* End includes */
+
+/* Note that a negative size indicates a package. */
 
 static const struct _frozen _PyImport_FrozenModules[] = {
     /* importlib */
-    {"_frozen_importlib", _Py_M__importlib_bootstrap,
-        (int)sizeof(_Py_M__importlib_bootstrap)},
-    {"_frozen_importlib_external", _Py_M__importlib_bootstrap_external,
-        (int)sizeof(_Py_M__importlib_bootstrap_external)},
-    {"zipimport", _Py_M__zipimport,
-        (int)sizeof(_Py_M__zipimport)},
+    {"_frozen_importlib", _Py_M__importlib__bootstrap,
+        (int)sizeof(_Py_M__importlib__bootstrap)},
+    {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external,
+        (int)sizeof(_Py_M__importlib__bootstrap_external)},
+    {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
+
     /* Test module */
-    {"__hello__", _Py_M__hello, SIZE},
-    /* Test package (negative size indicates package-ness) */
-    {"__phello__", _Py_M__hello, -SIZE},
-    {"__phello__.spam", _Py_M__hello, SIZE},
+    {"__hello__", _Py_M__hello, (int)sizeof(_Py_M__hello)},
+    {"__phello__", _Py_M__hello, -(int)sizeof(_Py_M__hello)},
+    {"__phello__.spam", _Py_M__hello, (int)sizeof(_Py_M__hello)},
     {0, 0, 0} /* sentinel */
 };
 
diff --git a/Python/frozen_hello.h b/Python/frozen_modules/hello.h
index c65c661..2658c05 100644
--- a/Python/frozen_hello.h
+++ b/Python/frozen_modules/hello.h
@@ -1,4 +1,4 @@
-/* Auto-generated by Programs/_freeze_importlib.c */
+/* Auto-generated by Programs/_freeze_module.c */
 const unsigned char _Py_M__hello[] = {
     99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,
     0,0,0,0,0,115,16,0,0,0,100,0,90,0,101,1,
diff --git a/Python/importlib.h b/Python/frozen_modules/importlib__bootstrap.h
index 69bd972..2716896 100644
--- a/Python/importlib.h
+++ b/Python/frozen_modules/importlib__bootstrap.h
@@ -1,5 +1,5 @@
-/* Auto-generated by Programs/_freeze_importlib.c */
-const unsigned char _Py_M__importlib_bootstrap[] = {
+/* Auto-generated by Programs/_freeze_module.c */
+const unsigned char _Py_M__importlib__bootstrap[] = {
     99,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,
     0,0,0,0,0,115,130,1,0,0,100,0,90,0,100,1,
     132,0,90,1,100,2,90,2,100,2,90,3,100,2,90,4,
diff --git a/Python/importlib_external.h b/Python/frozen_modules/importlib__bootstrap_external.h
index c49fa55..7a34100 100644
--- a/Python/importlib_external.h
+++ b/Python/frozen_modules/importlib__bootstrap_external.h
@@ -1,5 +1,5 @@
-/* Auto-generated by Programs/_freeze_importlib.c */
-const unsigned char _Py_M__importlib_bootstrap_external[] = {
+/* Auto-generated by Programs/_freeze_module.c */
+const unsigned char _Py_M__importlib__bootstrap_external[] = {
     99,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,
     0,0,0,0,0,115,158,2,0,0,100,0,90,0,100,1,
     97,1,100,2,100,1,108,2,90,2,100,2,100,1,108,3,
diff --git a/Python/importlib_zipimport.h b/Python/frozen_modules/zipimport.h
index c12ed52..b4e2e85 100644
--- a/Python/importlib_zipimport.h
+++ b/Python/frozen_modules/zipimport.h
@@ -1,4 +1,4 @@
-/* Auto-generated by Programs/_freeze_importlib.c */
+/* Auto-generated by Programs/_freeze_module.c */
 const unsigned char _Py_M__zipimport[] = {
     99,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,
     0,0,0,0,0,115,48,1,0,0,100,0,90,0,100,1,