bpo-45019: Add a tool to generate list of modules to include for frozen modules (gh-27980)

Frozen modules must be added to several files in order to work properly. Before this change this had to be done manually. Here we add a tool to generate the relevant lines in those files instead. This helps us avoid mistakes and omissions.

https://bugs.python.org/issue45019

5 files changed, 36 insertions, 25 deletions

diff --git a/PCbuild/_freeze_importlib.vcxproj b/PCbuild/_freeze_module.vcxproj
index e437412..a0bedf4 100644
--- a/PCbuild/_freeze_importlib.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -69,7 +69,7 @@
   <PropertyGroup Label="Globals">
     <ProjectGuid>{19C0C13F-47CA-4432-AFF3-799A296A4DDC}</ProjectGuid>
     <Keyword>Win32Proj</Keyword>
-    <RootNamespace>_freeze_importlib</RootNamespace>
+    <RootNamespace>_freeze_module</RootNamespace>
     <SupportPGO>false</SupportPGO>
   </PropertyGroup>
   <Import Project="python.props" />
@@ -95,7 +95,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
-    <ClCompile Include="..\Programs\_freeze_importlib.c" />
+    <ClCompile Include="..\Programs\_freeze_module.c" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="pythoncore.vcxproj">
@@ -108,31 +108,33 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <!-- BEGIN frozen modules -->
     <None Include="..\Lib\importlib\_bootstrap.py">
       <ModName>importlib._bootstrap</ModName>
-      <IntFile>$(IntDir)importlib.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\importlib.h</OutFile>
+      <IntFile>$(IntDir)importlib__bootstrap.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap.h</OutFile>
     </None>
     <None Include="..\Lib\importlib\_bootstrap_external.py">
       <ModName>importlib._bootstrap_external</ModName>
-      <IntFile>$(IntDir)importlib_external.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\importlib_external.h</OutFile>
+      <IntFile>$(IntDir)importlib__bootstrap_external.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap_external.h</OutFile>
     </None>
     <None Include="..\Lib\zipimport.py">
       <ModName>zipimport</ModName>
-      <IntFile>$(IntDir)importlib_zipimport.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\importlib_zipimport.h</OutFile>
+      <IntFile>$(IntDir)zipimport.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\zipimport.h</OutFile>
     </None>
     <None Include="..\Tools\freeze\flag.py">
       <ModName>hello</ModName>
-      <IntFile>$(IntDir)frozen_hello.g.h</IntFile>
-      <OutFile>$(PySourcePath)Python\frozen_hello.h</OutFile>
+      <IntFile>$(IntDir)ello.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\hello.h</OutFile>
     </None>
+    <!-- END frozen modules -->
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-  <Target Name="_RebuildImportLib">
+  <Target Name="_RebuildFrozen">
     <Exec Command='"$(TargetPath)" "%(None.ModName)" "%(None.FullPath)" "%(None.IntFile)"' />
     
     <Copy SourceFiles="%(None.IntFile)"
@@ -143,15 +145,18 @@
 
     <Message Text="Updated files: @(_Updated->'%(Filename)%(Extension)',', ')"
              Condition="'@(_Updated)' != ''" Importance="high" />
-    <Warning Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
+    <Warning Text="Frozen modules (e.g. importlib) were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
              Condition="'@(_Updated)' != '' and $(Configuration) == 'Debug'" />
-    <Error Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
+    <Error Text="Frozen (e.g. importlib) files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
            Condition="'@(_Updated)' != '' and $(Configuration) == 'Release'" />
   </Target>
+  <Target Name="RebuildFrozen" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'"
+          DependsOnTargets="_RebuildFrozen">
+  </Target>
   <Target Name="RebuildImportLib" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'"
-          DependsOnTargets="_RebuildImportLib">
+          DependsOnTargets="_RebuildFrozen">
   </Target>
-  <Target Name="_CleanImportLib" BeforeTargets="CoreClean">
+  <Target Name="_CleanFrozen" BeforeTargets="CoreClean">
     <ItemGroup>
       <Clean Include="%(None.IntFile)" />
     </ItemGroup>
diff --git a/PCbuild/_freeze_importlib.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters
index 3ee9eb7..bed7920 100644
--- a/PCbuild/_freeze_importlib.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -10,19 +10,24 @@
     </Filter>
   </ItemGroup>
   <ItemGroup>
-    <ClCompile Include="..\Programs\_freeze_importlib.c">
+    <ClCompile Include="..\Programs\_freeze_module.c">
       <Filter>Source Files</Filter>
     </ClCompile>
   </ItemGroup>
   <ItemGroup>
+    <!-- BEGIN frozen modules -->
     <None Include="..\Lib\importlib\_bootstrap.py">
-      <Filter>Source Files</Filter>
+      <Filter>Python Files</Filter>
+    </None>
+    <None Include="..\Lib\importlib\_bootstrap_external.py">
+      <Filter>Python Files</Filter>
     </None>
     <None Include="..\Lib\zipimport.py">
       <Filter>Python Files</Filter>
     </None>
-    <None Include="..\Lib\importlib\_bootstrap_external.py">
+    <None Include="..\Tools\freeze\flag.py">
       <Filter>Python Files</Filter>
     </None>
+    <!-- END frozen modules -->
   </ItemGroup>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj
index 8e7088d..f464ad3 100644
--- a/PCbuild/pcbuild.proj
+++ b/PCbuild/pcbuild.proj
@@ -72,8 +72,8 @@
       <BuildInParallel>false</BuildInParallel>
     </Projects>
 
-    <!-- _freeze_importlib -->
-    <Projects2 Condition="$(Platform) != 'ARM' and $(Platform) != 'ARM64'" Include="_freeze_importlib.vcxproj" />
+    <!-- _freeze_module -->
+    <Projects2 Condition="$(Platform) != 'ARM' and $(Platform) != 'ARM64'" Include="_freeze_module.vcxproj" />
     <!-- python[w].exe -->
     <Projects2 Include="python.vcxproj;pythonw.vcxproj" />
     <Projects2 Include="python_uwp.vcxproj;pythonw_uwp.vcxproj" Condition="$(IncludeUwp)" />
diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln
index 3507b97..c774e04 100644
--- a/PCbuild/pcbuild.sln
+++ b/PCbuild/pcbuild.sln
@@ -75,7 +75,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher.
 		{7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446}
 	EndProjectSection
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_importlib", "_freeze_importlib.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}"
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_module", "_freeze_module.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}"
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}"
 EndProject
diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt
index 6c25522..5ecded0 100644
--- a/PCbuild/readme.txt
+++ b/PCbuild/readme.txt
@@ -115,9 +115,10 @@ _testembed
 
 These are miscellaneous sub-projects that don't really fit the other
 categories:
-_freeze_importlib
-    _freeze_importlib.exe, used to regenerate Python\importlib.h after
-    changes have been made to Lib\importlib\_bootstrap.py
+_freeze_module
+    _freeze_module.exe, used to regenerate frozen modules in Python
+    after changes have been made to the corresponding source files
+    (e.g. Lib\importlib\_bootstrap.py).
 pyshellext
     pyshellext.dll, the shell extension deployed with the launcher
 python3dll