bpo-36500: Add --regen option to PCbuild/build.bat so Windows users can regen grammar, opcodes, tokens and symbols (GH-12654)

2 files changed, 241 insertions, 1 deletions

diff --git a/PCbuild/build.bat b/PCbuild/build.bat
index 623409c..5bc5da9 100644
--- a/PCbuild/build.bat
+++ b/PCbuild/build.bat
@@ -32,6 +32,7 @@ echo.      automatically by the pythoncore project)
 echo.  --pgo          Build with Profile-Guided Optimization.  This flag
 echo.                 overrides -c and -d
 echo.  --test-marker  Enable the test marker within the build.
+echo.  --regen        Regenerate all opcodes, grammar and tokens
 echo.
 echo.Available flags to avoid building certain modules.
 echo.These flags have no effect if '-e' is not given:
@@ -76,7 +77,8 @@ if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts
 if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts
 if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts
 if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts
-if "%~1"=="-V" shift & goto :Version
+if "%~1"=="-V" shift & goto Version
+if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts
 rem These use the actual property names used by MSBuild.  We could just let
 rem them in through the environment, but we specify them on the command line
 rem anyway for visibility so set defaults after this
@@ -154,9 +156,22 @@ echo on
  /p:UseTestMarker=%UseTestMarker% %GITProperty%^
  %1 %2 %3 %4 %5 %6 %7 %8 %9
 
+if "%Regen%"=="true" call :Regen
 @echo off
 exit /b %ERRORLEVEL%
 
+:Regen
+echo on
+call "%dir%find_msbuild.bat" %MSBUILD%
+if not ERRORLEVEL 1 %MSBUILD% "%dir%regen.vcxproj" /t:%target% %parallel% %verbose%^
+ /p:IncludeExternals=%IncludeExternals%^
+ /p:Configuration=%conf% /p:Platform=%platf%^
+ /p:UseTestMarker=%UseTestMarker% %GITProperty%^
+ %1 %2 %3 %4 %5 %6 %7 %8 %9
+
+@echo off
+goto :eof
+
 :Version
 rem Display the current build version information
 call "%dir%find_msbuild.bat" %MSBUILD%
diff --git a/PCbuild/regen.vcxproj b/PCbuild/regen.vcxproj
new file mode 100644
index 0000000..876b12b
--- /dev/null
+++ b/PCbuild/regen.vcxproj
@@ -0,0 +1,225 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGInstrument|ARM">
+      <Configuration>PGInstrument</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGInstrument|Win32">
+      <Configuration>PGInstrument</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGInstrument|x64">
+      <Configuration>PGInstrument</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGUpdate|ARM">
+      <Configuration>PGUpdate</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGUpdate|Win32">
+      <Configuration>PGUpdate</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="PGUpdate|x64">
+      <Configuration>PGUpdate</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{21CF2108-2CC9-4005-A6ED-B7965ADE3854}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>regen</RootNamespace>
+    <SupportPGO>false</SupportPGO>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="python.props" />
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Label="Configuration">
+    <ConfigurationType>Utility</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGUpdate|x64'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGUpdate|ARM'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGUpdate|Win32'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGInstrument|ARM'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGInstrument|x64'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Label="Configuration" Condition="'$(Configuration)|$(Platform)'=='PGInstrument|Win32'">
+    <PlatformToolset>v142</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+    <Import Project="pyproject.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\Programs\_freeze_importlib.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="pythoncore.vcxproj">
+      <Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>
+      <Private>true</Private>
+      <ReferenceOutputAssembly>true</ReferenceOutputAssembly>
+      <CopyLocalSatelliteAssemblies>false</CopyLocalSatelliteAssemblies>
+      <LinkLibraryDependencies>true</LinkLibraryDependencies>
+      <UseLibraryDependencyInputs>false</UseLibraryDependencyInputs>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="..\Grammar\Grammar">
+    </None>
+    <None Include="..\Grammar\Tokens">
+    </None>
+    <None Include="..\Include\graminit.h">
+    </None>
+    <None Include="..\Python\graminit.c">
+    </None>
+    <None Include="..\Include\token.h">
+    </None>
+    <None Include="..\Include\opcode.h">
+    </None>
+    <None Include="..\Include\Python-ast.h">
+    </None>
+    <None Include="..\Python\Python-ast.c">
+    </None>
+    <None Include="..\Lib\keyword.py">
+    </None>
+    <None Include="..\Lib\symbol.py">
+    </None>
+    <None Include="..\Doc\library\token-list.inc">
+    </None>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+  <Target Name="_RegenGrammar" BeforeTargets="Build">
+    <!-- Regenerate Include/graminit.h and Python/graminit.c from Grammar/Grammar using pgen-->
+    <Exec Command="&quot;$(PythonExe)&quot; -m Parser.pgen &quot;$(PySourcePath)Grammar\Grammar&quot; &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(IntDir)graminit.h&quot; &quot;$(IntDir)graminit.c&quot;" />
+    <Copy SourceFiles="$(IntDir)graminit.h" DestinationFiles="$(PySourcePath)Include\graminit.h">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
+    </Copy>
+    <Copy SourceFiles="$(IntDir)graminit.c" DestinationFiles="$(PySourcePath)Python\graminit.c">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedC" />
+    </Copy>
+    <Warning Text="Grammar updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
+  </Target>
+  <Target Name="_RegenAST_H" AfterTargets="_RegenGrammar">
+    <!-- Regenerate Include/Python-ast.h using Parser/asdl_c.py -h -->
+    <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -h &quot;$(IntDir)Python-ast.h&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
+    <Copy SourceFiles="$(IntDir)Python-ast.h" DestinationFiles="$(PySourcePath)Include\Python-ast.h">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
+    </Copy>
+    <Warning Text="Python-ast.h updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
+  </Target>
+  <Target Name="_RegenAST_C" AfterTargets="_RegenGrammar">
+    <!-- Regenerate Python/Python-ast.c using Parser/asdl_c.py -c -->
+    <Exec Command="&quot;$(PythonExe)&quot; &quot;$(PySourcePath)Parser\asdl_c.py&quot; -c &quot;$(IntDir)Python-ast.c&quot; &quot;$(PySourcePath)Parser\Python.asdl&quot;" />
+    <Copy SourceFiles="$(IntDir)Python-ast.c" DestinationFiles="$(PySourcePath)Python\Python-ast.c">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedH" />
+    </Copy>
+    <Warning Text="Python-ast.c updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_UpdatedH)' != '' and '@(_UpdatedC)' != ''" />
+  </Target>
+  <Target Name="_RegenOpcodes" AfterTargets="_RegenAST_C">
+    <!-- Regenerate Include/opcode.h from Lib/opcode.py using Tools/scripts/generate_opcode_h.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_opcode_h.py &quot;$(PySourcePath)Lib\opcode.py&quot; &quot;$(IntDir)opcode.h&quot;" />
+    <Copy SourceFiles="$(IntDir)opcode.h" DestinationFiles="$(PySourcePath)Include\opcode.h">
+      <Output TaskParameter="CopiedFiles" ItemName="_Updated" />
+    </Copy>
+    <Warning Text="Opcodes updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_Updated)' != ''" />
+  </Target>
+  <Target Name="_RegenTokens" AfterTargets="_RegenOpcodes">
+    <!-- Regenerate Doc/library/token-list.inc from Grammar/Tokens using Tools/scripts/generate_token.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py rst &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Doc\library\token-list.inc&quot;" />
+    <!-- Regenerate Include/token.h from Grammar/Tokens using Tools/scripts/generate_token.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py h &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Include\token.h&quot;" />
+    <!-- Regenerate Parser/token.c from Grammar/Tokens using Tools/scripts/generate_token.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py c &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Parser\token.c&quot;" />
+    <!-- Regenerate Lib/token.py from Grammar/Tokens using Tools/scripts/generate_token.py -->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_token.py py &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(PySourcePath)Lib\token.py&quot;" />
+  </Target>
+  <Target Name="_RegenKeywords" AfterTargets="_RegenTokens">
+    <!-- Regenerate Lib/keyword.py from Grammar/Grammar and Grammar/Tokens using Parser/pgen-->
+    <Exec Command="&quot;$(PythonExe)&quot; -m Parser.pgen.keywordgen &quot;$(PySourcePath)Grammar\Grammar&quot; &quot;$(PySourcePath)Grammar\Tokens&quot; &quot;$(IntDir)keyword.py&quot;" />
+    <Copy SourceFiles="$(IntDir)keyword.py" DestinationFiles="$(PySourcePath)Lib\keyword.py">
+      <Output TaskParameter="CopiedFiles" ItemName="_Updated" />
+    </Copy>
+    <Warning Text="Keywords updated. You will need to rebuild pythoncore to see the changes." Condition="'@(_Updated)' != ''" />
+  </Target>
+  <Target Name="_RegenSymbols" AfterTargets="_RegenKeywords">
+    <!-- Regenerate Lib/symbol.py from Include/graminit.h using Tools/scripts/generate_symbol_py.py-->
+    <Exec Command="&quot;$(PythonExe)&quot; $(PySourcePath)Tools\scripts\generate_symbol_py.py &quot;$(PySourcePath)Include\graminit.h&quot;  &quot;$(PySourcePath)Lib\symbol.py&quot;" />
+  </Target>
+  <Target Name="_CleanFiles" BeforeTargets="CoreClean">
+    <ItemGroup>
+      <Clean Include="$(IntDir)keyword.py" />
+      <Clean Include="$(IntDir)opcode.h" />
+      <Clean Include="$(IntDir)Python-ast.c" />
+      <Clean Include="$(IntDir)Python-ast.h" />
+      <Clean Include="$(IntDir)graminit.h.new" />
+      <Clean Include="$(IntDir)graminit.c.new" />
+    </ItemGroup>
+  </Target>
+</Project>
\ No newline at end of file