From 09b4ad11f323f8702cde795e345b75e0fbb1a9a5 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 15 Sep 2021 18:11:12 +0100 Subject: bpo-45188: Windows now regenerates frozen modules at the start of build instead of late (GH-28322) This will enable us to drop the frozen module header files from the repository. It does currently cause many source files to be built twice, which just takes more time. For whoever comes to fix this in the future, the files shared between freeze_module and pythoncore should be put into a static library that is consumed by both. --- .../Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst | 3 + PC/config_minimal.c | 53 +++++++ PCbuild/_freeze_module.vcxproj | 157 ++++++++++++++++++--- PCbuild/pcbuild.proj | 19 ++- PCbuild/pcbuild.sln | 49 +++---- Programs/_freeze_module.c | 5 - Tools/scripts/freeze_modules.py | 60 ++++++-- 7 files changed, 278 insertions(+), 68 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst create mode 100644 PC/config_minimal.c diff --git a/Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst b/Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst new file mode 100644 index 0000000..df470e8 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-09-14-00-47-57.bpo-45188.MNbo_T.rst @@ -0,0 +1,3 @@ +Windows builds now regenerate frozen modules as the first part of the build. +Previously the regeneration was later in the build, which would require it +to be restarted if any modules had changed. diff --git a/PC/config_minimal.c b/PC/config_minimal.c new file mode 100644 index 0000000..adb1c44 --- /dev/null +++ b/PC/config_minimal.c @@ -0,0 +1,53 @@ +/* Module configuration */ + +/* This file contains the table of built-in modules. + See create_builtin() in import.c. */ + +#include "Python.h" + +extern PyObject* PyInit_faulthandler(void); +extern PyObject* PyInit__tracemalloc(void); +extern PyObject* PyInit_gc(void); +extern PyObject* PyInit_nt(void); +extern PyObject* PyInit__signal(void); +extern PyObject* PyInit_winreg(void); + +extern PyObject* PyInit__ast(void); +extern PyObject* PyInit__io(void); +extern PyObject* PyInit_atexit(void); +extern PyObject* _PyWarnings_Init(void); +extern PyObject* PyInit__string(void); +extern PyObject* PyInit__tokenize(void); + +extern PyObject* PyMarshal_Init(void); +extern PyObject* PyInit__imp(void); + +struct _inittab _PyImport_Inittab[] = { + {"_ast", PyInit__ast}, + {"faulthandler", PyInit_faulthandler}, + {"gc", PyInit_gc}, + {"nt", PyInit_nt}, /* Use the NT os functions, not posix */ + {"_signal", PyInit__signal}, + {"_tokenize", PyInit__tokenize}, + {"_tracemalloc", PyInit__tracemalloc}, + + {"winreg", PyInit_winreg}, + + /* This module "lives in" with marshal.c */ + {"marshal", PyMarshal_Init}, + + /* This lives it with import.c */ + {"_imp", PyInit__imp}, + + /* These entries are here for sys.builtin_module_names */ + {"builtins", NULL}, + {"sys", NULL}, + {"_warnings", _PyWarnings_Init}, + {"_string", PyInit__string}, + + {"_io", PyInit__io}, + {"atexit", PyInit_atexit}, + + /* Sentinel */ + {0, 0} +}; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index a0bedf4..a87dca7 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -88,24 +88,145 @@ - _CONSOLE;%(PreprocessorDefinitions) + Py_NO_ENABLE_SHARED;Py_BUILD_CORE;_CONSOLE;%(PreprocessorDefinitions) Console + version.lib;shlwapi.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) + - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - true - true - false - true - false - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -126,7 +247,7 @@ hello - $(IntDir)ello.g.h + $(IntDir)hello.g.h $(PySourcePath)Python\frozen_modules\hello.h @@ -134,9 +255,9 @@ - + - + @@ -145,18 +266,8 @@ - - - - - - - + diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index f464ad3..b3cbd47 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -14,6 +14,18 @@ + + $(Platform) + Win32 + x64 + $(Configuration) + Release + + Build + Clean + CleanAll + false + $(Platform) $(Configuration) @@ -73,7 +85,7 @@ - + @@ -82,6 +94,11 @@ + ' else: - source = os.path.relpath(self.pyfile, ROOT_DIR) + source = relpath_for_posix_display(self.pyfile, ROOT_DIR) return { 'module': self.name, 'ispkg': self.ispkg, @@ -397,7 +432,7 @@ def replace_block(lines, start_marker, end_marker, replacements, file): raise Exception(f"End marker {end_marker!r} " f"occurs before start marker {start_marker!r} " f"in file {file}") - replacements = [line.rstrip() + os.linesep for line in replacements] + replacements = [line.rstrip() + '\n' for line in replacements] return lines[:start_pos + 1] + replacements + lines[end_pos:] @@ -446,7 +481,7 @@ def regen_frozen(modules): for src in _iter_sources(modules): # Adding a comment to separate sections here doesn't add much, # so we don't. - header = os.path.relpath(src.frozenfile, parentdir) + header = relpath_for_posix_display(src.frozenfile, parentdir) headerlines.append(f'#include "{header}"') deflines = [] @@ -503,11 +538,10 @@ def regen_makefile(modules): frozenfiles = [] rules = [''] for src in _iter_sources(modules): - header = os.path.relpath(src.frozenfile, ROOT_DIR) - relfile = header.replace('\\', '/') - frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') + header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) + frozenfiles.append(f'\t\t$(srcdir)/{header} \\') - pyfile = os.path.relpath(src.pyfile, ROOT_DIR) + pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) # Note that we freeze the module to the target .h file # instead of going through an intermediate file like we used to. rules.append(f'{header}: Programs/_freeze_module {pyfile}') @@ -546,9 +580,9 @@ def regen_pcbuild(modules): # See bpo-45186 and bpo-45188. if src.id not in ESSENTIAL and src.id != 'hello': continue - pyfile = os.path.relpath(src.pyfile, ROOT_DIR).replace('/', '\\') - header = os.path.relpath(src.frozenfile, ROOT_DIR).replace('/', '\\') - intfile = header.split('\\')[-1].strip('.h') + '.g.h' + pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) + header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) + intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h' projlines.append(f' ') projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') @@ -600,7 +634,7 @@ def _freeze_module(frozenid, pyfile, frozenfile): print('#', ' '.join(os.path.relpath(a) for a in argv), flush=True) try: subprocess.run(argv, check=True) - except subprocess.CalledProcessError: + except (FileNotFoundError, subprocess.CalledProcessError): if not os.path.exists(TOOL): sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"') raise # re-raise -- cgit v0.12