bpo-46429: Merge all deepfrozen files into one (GH-30572)

2 files changed, 49 insertions, 55 deletions

diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py
index 49638b8..a7546a8 100644
--- a/Tools/scripts/deepfreeze.py
+++ b/Tools/scripts/deepfreeze.py
@@ -10,10 +10,9 @@ import collections
 import contextlib
 import os
 import re
-import sys
 import time
 import types
-from typing import Dict, FrozenSet, Tuple, TextIO
+from typing import Dict, FrozenSet, TextIO, Tuple
 
 import umarshal
 
@@ -104,10 +103,10 @@ def removesuffix(base: str, suffix: str) -> str:
 
 class Printer:
 
-    def __init__(self, file: TextIO):
+    def __init__(self, file: TextIO) -> None:
         self.level = 0
         self.file = file
-        self.cache: Dict[Tuple[type, object], str] = {}
+        self.cache: Dict[tuple[type, object, str], str] = {}
         self.hits, self.misses = 0, 0
         self.patchups: list[str] = []
         self.write('#include "Python.h"')
@@ -349,6 +348,15 @@ class Printer:
         self.write("// TODO: The above tuple should be a frozenset")
         return ret
 
+    def generate_file(self, module: str, code: object)-> None:
+        module = module.replace(".", "_")
+        self.generate(f"{module}_toplevel", code)
+        with self.block(f"static void {module}_do_patchups(void)"):
+            for p in self.patchups:
+                self.write(p)
+        self.patchups.clear()
+        self.write(EPILOGUE.replace("%%NAME%%", module))
+
     def generate(self, name: str, obj: object) -> str:
         # Use repr() in the key to distinguish -0.0 from +0.0
         key = (type(obj), obj, repr(obj))
@@ -357,7 +365,7 @@ class Printer:
             # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}")
             return self.cache[key]
         self.misses += 1
-        if isinstance(obj, types.CodeType) or isinstance(obj, umarshal.Code):
+        if isinstance(obj, (types.CodeType, umarshal.Code)) :
             val = self.generate_code(name, obj)
         elif isinstance(obj, tuple):
             val = self.generate_tuple(name, obj)
@@ -393,8 +401,8 @@ EPILOGUE = """
 PyObject *
 _Py_get_%%NAME%%_toplevel(void)
 {
-    do_patchups();
-    return (PyObject *) &toplevel;
+    %%NAME%%_do_patchups();
+    return (PyObject *) &%%NAME%%_toplevel;
 }
 """
 
@@ -419,29 +427,25 @@ def decode_frozen_data(source: str) -> types.CodeType:
     return umarshal.loads(data)
 
 
-def generate(source: str, filename: str, modname: str, file: TextIO) -> None:
-    if is_frozen_header(source):
-        code = decode_frozen_data(source)
-    else:
-        code = compile(source, filename, "exec")
-    printer = Printer(file)
-    printer.generate("toplevel", code)
-    printer.write("")
-    with printer.block("static void do_patchups(void)"):
-        for p in printer.patchups:
-            printer.write(p)
-    here = os.path.dirname(__file__)
-    printer.write(EPILOGUE.replace("%%NAME%%", modname.replace(".", "_")))
+def generate(args: list[str], output: TextIO) -> None:
+    printer = Printer(output)
+    for arg in args:
+        file, modname = arg.rsplit(':', 1)
+        with open(file, "r", encoding="utf8") as fd:
+            source = fd.read()
+            if is_frozen_header(source):
+                code = decode_frozen_data(source)
+            else:
+                code = compile(fd.read(), f"<frozen {modname}>", "exec")
+            printer.generate_file(modname, code)
     if verbose:
         print(f"Cache hits: {printer.hits}, misses: {printer.misses}")
 
 
 parser = argparse.ArgumentParser()
-parser.add_argument("-m", "--module", help="Defaults to basename(file)")
-parser.add_argument("-o", "--output", help="Defaults to MODULE.c")
+parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c", default="deepfreeze.c")
 parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics")
-parser.add_argument("file", help="Input file (required)")
-
+parser.add_argument('args', nargs="+", help="Input file and module name (required) in file:modname format")
 
 @contextlib.contextmanager
 def report_time(label: str):
@@ -458,13 +462,10 @@ def main() -> None:
     global verbose
     args = parser.parse_args()
     verbose = args.verbose
-    with open(args.file, encoding="utf-8") as f:
-        source = f.read()
-    modname = args.module or removesuffix(os.path.basename(args.file), ".py")
-    output = args.output or modname + ".c"
+    output = args.output
     with open(output, "w", encoding="utf-8") as file:
         with report_time("generate"):
-            generate(source, f"<frozen {modname}>", modname, file)
+            generate(args.args, file)
     if verbose:
         print(f"Wrote {os.path.getsize(output)} bytes to {output}")
 
diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py
index cbe8bf1..6d10758 100644
--- a/Tools/scripts/freeze_modules.py
+++ b/Tools/scripts/freeze_modules.py
@@ -575,16 +575,12 @@ def regen_frozen(modules):
 def regen_makefile(modules):
     pyfiles = []
     frozenfiles = []
-    deepfreezefiles = []
     rules = ['']
-    deepfreezerules = ['']
+    deepfreezerules = ["Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)",
+                       "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py \\"]
     for src in _iter_sources(modules):
         frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
-        deepfreeze_header = relpath_for_posix_display(src.deepfreezefile, ROOT_DIR)
         frozenfiles.append(f'\t\t{frozen_header} \\')
-        cfile = deepfreeze_header[:-2] + ".c"
-        ofile = deepfreeze_header[:-2] + ".o"
-        deepfreezefiles.append(f"\t\t{ofile} \\")
 
         pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
         pyfiles.append(f'\t\t{pyfile} \\')
@@ -603,15 +599,11 @@ def regen_makefile(modules):
             f'\t{freeze}',
             '',
         ])
-        deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)')
-        deepfreezerules.append(
-            f"\t$(PYTHON_FOR_FREEZE) "
-            f"$(srcdir)/Tools/scripts/deepfreeze.py "
-            f"{frozen_header} -m {src.frozenid} -o {cfile}")
-        deepfreezerules.append('')
+        deepfreezerules.append(f"\t{frozen_header}:{src.frozenid} \\")
+    deepfreezerules.append('\t-o Python/deepfreeze/deepfreeze.c')
+    deepfreezerules.append('')
     pyfiles[-1] = pyfiles[-1].rstrip(" \\")
     frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")
-    deepfreezefiles[-1] = deepfreezefiles[-1].rstrip(" \\")
 
     print(f'# Updating {os.path.relpath(MAKEFILE)}')
     with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile):
@@ -632,13 +624,6 @@ def regen_makefile(modules):
         )
         lines = replace_block(
             lines,
-            "DEEPFREEZE_OBJS =",
-            "# End DEEPFREEZE_OBJS",
-            deepfreezefiles,
-            MAKEFILE,
-        )
-        lines = replace_block(
-            lines,
             "# BEGIN: freezing modules",
             "# END: freezing modules",
             rules,
@@ -658,26 +643,24 @@ def regen_pcbuild(modules):
     projlines = []
     filterlines = []
     corelines = []
+    deepfreezerules = ['\t<Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\scripts\\deepfreeze.py" ^']
     for src in _iter_sources(modules):
         pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
         header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
-        deepbase = "df." + src.id
-        deepoutfile = f"Python\\deepfreeze\\{deepbase}.c"
         intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h'
-        deepintfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.c'
         projlines.append(f'    <None Include="..\\{pyfile}">')
         projlines.append(f'      <ModName>{src.frozenid}</ModName>')
         projlines.append(f'      <IntFile>$(IntDir){intfile}</IntFile>')
         projlines.append(f'      <OutFile>$(PySourcePath){header}</OutFile>')
-        projlines.append(f'      <DeepIntFile>$(IntDir){deepintfile}</DeepIntFile>')
-        projlines.append(f'      <DeepOutFile>$(PySourcePath){deepoutfile}</DeepOutFile>')
         projlines.append(f'    </None>')
 
         filterlines.append(f'    <None Include="..\\{pyfile}">')
         filterlines.append('      <Filter>Python Files</Filter>')
         filterlines.append('    </None>')
+        deepfreezerules.append(f'\t\t "$(PySourcePath){header}:{src.frozenid}" ^')
+    deepfreezerules.append('\t\t "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' )
 
-        corelines.append(f'    <ClCompile Include="..\\{deepoutfile}" />')
+    corelines.append(f'    <ClCompile Include="..\\Python\\deepfreeze\\deepfreeze.c" />')
 
     print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}')
     with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
@@ -690,6 +673,16 @@ def regen_pcbuild(modules):
             PCBUILD_PROJECT,
         )
         outfile.writelines(lines)
+    with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
+        lines = infile.readlines()
+        lines = replace_block(
+            lines,
+            '<!-- BEGIN deepfreeze rule -->',
+            '<!-- END deepfreeze rule -->',
+            deepfreezerules,
+            PCBUILD_PROJECT,
+        )
+        outfile.writelines(lines)
     print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}')
     with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile):
         lines = infile.readlines()