bpo-35947: Update Windows to the current version of libffi (GH-11797)

We now use a pre-built libffi binary from our binaries repository, and no longer vendor the full implementation.

6 files changed, 202 insertions, 44 deletions

diff --git a/PCbuild/_ctypes.vcxproj b/PCbuild/_ctypes.vcxproj
index d4c9f87..a265427 100644
--- a/PCbuild/_ctypes.vcxproj
+++ b/PCbuild/_ctypes.vcxproj
@@ -70,6 +70,7 @@
   <ImportGroup Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
     <Import Project="pyproject.props" />
+    <Import Project="libffi.props" />
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup>
@@ -77,7 +78,7 @@
   </PropertyGroup>
   <ItemDefinitionGroup>
     <ClCompile>
-      <AdditionalIncludeDirectories>..\Modules\_ctypes\libffi_msvc;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>FFI_BUILDING;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
       <AdditionalOptions>/EXPORT:DllGetClassObject,PRIVATE /EXPORT:DllCanUnloadNow,PRIVATE %(AdditionalOptions)</AdditionalOptions>
@@ -86,32 +87,14 @@
   <ItemGroup>
     <ClInclude Include="..\Modules\_ctypes\ctypes.h" />
     <ClInclude Include="..\Modules\_ctypes\ctypes_dlfcn.h" />
-    <ClInclude Include="..\Modules\_ctypes\libffi_msvc\ffi.h" />
-    <ClInclude Include="..\Modules\_ctypes\libffi_msvc\ffi_common.h" />
-    <ClInclude Include="..\Modules\_ctypes\libffi_msvc\fficonfig.h" />
-    <ClInclude Include="..\Modules\_ctypes\libffi_msvc\ffitarget.h" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_ctypes\_ctypes.c" />
     <ClCompile Include="..\Modules\_ctypes\callbacks.c" />
     <ClCompile Include="..\Modules\_ctypes\callproc.c" />
     <ClCompile Include="..\Modules\_ctypes\cfield.c" />
-    <ClCompile Include="..\Modules\_ctypes\libffi_msvc\ffi.c" />
     <ClCompile Include="..\Modules\_ctypes\malloc_closure.c" />
-    <ClCompile Include="..\Modules\_ctypes\libffi_msvc\prep_cif.c">
-      <DisableSpecificWarnings Condition="'$(Platform)'=='x64'">4267;%(DisableSpecificWarnings)</DisableSpecificWarnings>
-    </ClCompile>
     <ClCompile Include="..\Modules\_ctypes\stgdict.c" />
-    <ClCompile Include="..\Modules\_ctypes\libffi_msvc\win32.c">
-      <ExcludedFromBuild Condition="'$(Platform)'=='x64'">true</ExcludedFromBuild>
-    </ClCompile>
-  </ItemGroup>
-  <ItemGroup>
-    <CustomBuild Include="..\Modules\_ctypes\libffi_msvc\win64.asm">
-      <ExcludedFromBuild Condition="'$(Platform)'=='Win32'">true</ExcludedFromBuild>
-      <Command>ml64 /nologo /c /Zi /Fo "$(IntDir)win64.obj" "%(FullPath)"</Command>
-      <Outputs>$(IntDir)win64.obj;%(Outputs)</Outputs>
-    </CustomBuild>
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc" />
@@ -122,4 +105,4 @@
     </ProjectReference>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
-</Project>
\ No newline at end of file
+</Project>
diff --git a/PCbuild/_ctypes.vcxproj.filters b/PCbuild/_ctypes.vcxproj.filters
index 83d7a7b..3123286 100644
--- a/PCbuild/_ctypes.vcxproj.filters
+++ b/PCbuild/_ctypes.vcxproj.filters
@@ -15,18 +15,6 @@
     <ClInclude Include="..\Modules\_ctypes\ctypes_dlfcn.h">
       <Filter>Header Files</Filter>
     </ClInclude>
-    <ClInclude Include="..\Modules\_ctypes\libffi_msvc\ffi.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="..\Modules\_ctypes\libffi_msvc\ffi_common.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="..\Modules\_ctypes\libffi_msvc\fficonfig.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
-    <ClInclude Include="..\Modules\_ctypes\libffi_msvc\ffitarget.h">
-      <Filter>Header Files</Filter>
-    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_ctypes\_ctypes.c">
@@ -41,25 +29,14 @@
     <ClCompile Include="..\Modules\_ctypes\cfield.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_ctypes\libffi_msvc\ffi.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Modules\_ctypes\malloc_closure.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_ctypes\libffi_msvc\prep_cif.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Modules\_ctypes\stgdict.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Modules\_ctypes\libffi_msvc\win32.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
   </ItemGroup>
   <ItemGroup>
-    <CustomBuild Include="..\Modules\_ctypes\libffi_msvc\win64.asm">
-      <Filter>Source Files</Filter>
-    </CustomBuild>
+    <ResourceCompile Include="..\PC\python_nt.rc" />
   </ItemGroup>
 </Project>
\ No newline at end of file
diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index 887fdc9..b82b6e6 100644
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -7,14 +7,17 @@ if NOT DEFINED EXTERNALS_DIR (set EXTERNALS_DIR=%PCBUILD%\..\externals)
 
 set DO_FETCH=true
 set DO_CLEAN=false
+set IncludeLibffiSrc=false
 set IncludeTkinterSrc=false
 set IncludeSSLSrc=false
 
 :CheckOpts
 if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
 if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
+if "%~1"=="--no-libffi" (set IncludeLibffi=false) & shift & goto CheckOpts
 if "%~1"=="--tkinter-src" (set IncludeTkinterSrc=true) & shift & goto CheckOpts
 if "%~1"=="--openssl-src" (set IncludeSSLSrc=true) & shift & goto CheckOpts
+if "%~1"=="--libffi-src" (set IncludeLibffiSrc=true) & shift & goto CheckOpts
 if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts
 if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
 if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
@@ -49,6 +52,7 @@ echo.Fetching external libraries...
 
 set libraries=
 set libraries=%libraries%                                       bzip2-1.0.6
+if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries%  libffi-3.3.0-rc0-r1
 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries%     openssl-1.1.0j
 set libraries=%libraries%                                       sqlite-3.21.0.0
 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.9.0
@@ -72,6 +76,7 @@ for %%e in (%libraries%) do (
 echo.Fetching external binaries...
 
 set binaries=
+if NOT "%IncludeLibffi%"=="false"  set binaries=%binaries% libffi
 if NOT "%IncludeSSL%"=="false"     set binaries=%binaries% openssl-bin-1.1.0j
 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0
 if NOT "%IncludeSSLSrc%"=="false"  set binaries=%binaries% nasm-2.11.06
diff --git a/PCbuild/libffi.props b/PCbuild/libffi.props
new file mode 100644
index 0000000..975c4a0
--- /dev/null
+++ b/PCbuild/libffi.props
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <AdditionalIncludeDirectories>$(libffiIncludeDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+    <Link>
+      <AdditionalLibraryDirectories>$(libffiOutDir);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+      <AdditionalDependencies>libffi-7.lib;%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <_LIBFFIDLL Include="$(libffiOutDir)\libffi-7.dll" />
+  </ItemGroup>
+  <Target Name="_CopyLIBFFIDLL" Inputs="@(_LIBFFIDLL)" Outputs="@(_LIBFFIDLL->'$(OutDir)%(Filename)%(Extension)')" AfterTargets="Build">
+    <Copy SourceFiles="@(_LIBFFIDLL)" DestinationFolder="$(OutDir)" />
+  </Target>
+  <Target Name="_CleanLIBFFIDLL" BeforeTargets="Clean">
+    <Delete Files="@(_LIBFFIDLL->'$(OutDir)%(Filename)%(Extension)')" TreatErrorsAsWarnings="true" />
+  </Target>
+</Project>
\ No newline at end of file
diff --git a/PCbuild/prepare_libffi.bat b/PCbuild/prepare_libffi.bat
new file mode 100644
index 0000000..3df8513
--- /dev/null
+++ b/PCbuild/prepare_libffi.bat
@@ -0,0 +1,169 @@
+@echo off
+goto :Run
+
+:Usage
+echo.
+echo Before running prepare_libffi.bat 
+echo   LIBFFI_SOURCE environment variable must be set to the location of
+echo     of python-source-deps clone of libffi branch
+echo   VCVARSALL must be set to location of vcvarsall.bat
+echo   cygwin must be installed (see below)
+echo   SH environment variable must be set to the location of sh.exe
+echo.
+echo Tested with cygwin-x86 from https://www.cygwin.com/install.html
+echo Select http://mirrors.kernel.org as the download site
+echo Include the following cygwin packages in cygwin configuration:
+echo     make, autoconf, automake, libtool, dejagnu
+echo.
+echo NOTE: dejagnu is only required for running tests.
+echo       set LIBFFI_TEST=1 to run tests (optional)
+echo.
+echo Based on https://github.com/libffi/libffi/blob/master/.appveyor.yml
+echo.
+echo.
+echo.Available flags:
+echo.  -x64    build for x64
+echo.  -x86    build for x86
+echo.  -?      this help
+echo.  --install-cygwin  install cygwin to c:\cygwin
+exit /b 127
+
+:Run
+
+set BUILD_X64=
+set BUILD_X86=
+set INSTALL_CYGWIN=
+
+:CheckOpts
+if "%1"=="" goto :CheckOptsDone
+if /I "%1"=="-x64" (set BUILD_X64=1) & shift & goto :CheckOpts
+if /I "%1"=="-x86" (set BUILD_X86=1) & shift & goto :CheckOpts
+if /I "%1"=="-?" goto :Usage
+if /I "%1"=="--install-cygwin" (set INSTALL_CYGWIN=1) & shift & goto :CheckOpts
+goto :Usage
+
+:CheckOptsDone
+
+if NOT DEFINED BUILD_X64 if NOT DEFINED BUILD_X86 if NOT DEFINED BUILD_ARM32 (
+    set BUILD_X64=1
+    set BUILD_X86=1
+)
+
+if "%INSTALL_CYGWIN%"=="1" call :InstallCygwin
+
+setlocal
+if NOT DEFINED SH if exist c:\cygwin\bin\sh.exe set SH=c:\cygwin\bin\sh.exe
+
+if NOT DEFINED VCVARSALL (
+    if exist "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" (
+        set VCVARSALL="C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat"
+    )
+)
+if ^%VCVARSALL:~0,1% NEQ ^" SET VCVARSALL="%VCVARSALL%"
+
+if NOT DEFINED LIBFFI_SOURCE echo.&&echo ERROR LIBFFI_SOURCE environment variable not set && goto :Usage
+if NOT DEFINED SH echo ERROR SH environment variable not set && goto :Usage
+
+if not exist %SH% echo ERROR %SH% does not exist && goto :Usage
+if not exist %LIBFFI_SOURCE% echo ERROR %LIBFFI_SOURCE% does not exist && goto :Usage
+
+set OLDPWD=%LIBFFI_SOURCE%
+pushd %LIBFFI_SOURCE%
+
+%SH% --login -lc "cygcheck -dc cygwin"
+set GET_MSVCC=%SH% -lc "cd $OLDPWD; export MSVCC=`/usr/bin/find $PWD -name msvcc.sh`; echo ${MSVCC};"
+FOR /F "usebackq delims==" %%i IN (`%GET_MSVCC%`) do @set MSVCC=%%i
+
+echo.
+echo VCVARSALL    : %VCVARSALL%
+echo SH           : %SH%
+echo LIBFFI_SOURCE: %LIBFFI_SOURCE% 
+echo MSVCC        : %MSVCC%
+echo.
+
+if not exist Makefile.in (%SH% -lc "(cd $LIBFFI_SOURCE; ./autogen.sh;)")
+
+call :BuildOne x86 i686-pc-cygwin i686-pc-cygwin
+call :BuildOne x64 x86_64-w64-cygwin x86_64-w64-cygwin
+
+popd
+endlocal
+exit /b 0
+REM all done
+
+
+REM this subroutine is called once for each architecture
+:BuildOne
+
+setlocal
+
+REM Initialize variables
+set VCVARS_PLATFORM=%1
+set BUILD=%2
+set HOST=%3
+set ASSEMBLER=
+set SRC_ARCHITECTURE=x86
+
+if NOT DEFINED VCVARS_PLATFORM echo ERROR bad VCVARS_PLATFORM&&exit /b 123
+
+if /I "%VCVARS_PLATFORM%" EQU "x64" (
+    set ARCH=amd64
+    set ARTIFACTS=%LIBFFI_SOURCE%\x86_64-w64-cygwin
+    set ASSEMBLER=-m64
+    set SRC_ARCHITECTURE=x86
+)
+if /I "%VCVARS_PLATFORM%" EQU "x86" (
+    set ARCH=win32
+    set ARTIFACTS=%LIBFFI_SOURCE%\i686-pc-cygwin
+    set ASSEMBLER=
+    set SRC_ARCHITECTURE=x86
+)
+
+if NOT DEFINED LIBFFI_OUT set LIBFFI_OUT=%~dp0\..\externals\libffi
+set _LIBFFI_OUT=%LIBFFI_OUT%\%ARCH%
+
+echo get VS build environment
+call %VCVARSALL% %VCVARS_PLATFORM%
+
+echo clean %_LIBFFI_OUT%
+if exist %_LIBFFI_OUT% (rd %_LIBFFI_OUT% /s/q)
+
+echo Configure the build to generate fficonfig.h and ffi.h
+%SH% -lc "(cd $OLDPWD; ./configure CC='%MSVCC% %ASSEMBLER%' CXX='%MSVCC% %ASSEMBLER%' LD='link' CPP='cl -nologo -EP' CXXCPP='cl -nologo -EP' CPPFLAGS='-DFFI_BUILDING_DLL' NM='dumpbin -symbols' STRIP=':' --build=$BUILD --host=$HOST;)"
+
+echo Building libffi
+%SH% -lc "(cd $OLDPWD; export PATH=/usr/bin:$PATH; cp src/%SRC_ARCHITECTURE%/ffitarget.h include; make; find .;)"
+
+REM Tests are not needed to produce artifacts
+if "%LIBFFI_TEST%" EQU "1" (
+    echo "Running tests..."
+    %SH% -lc "(cd $OLDPWD; export PATH=/usr/bin:$PATH; cp `find $PWD -name 'libffi-?.dll'` $HOST/testsuite/; make check; cat `find ./ -name libffi.log`)"
+) else (
+    echo "Not running tests"
+)
+
+
+echo copying files to %_LIBFFI_OUT%
+if not exist %_LIBFFI_OUT%\include (md %_LIBFFI_OUT%\include)
+copy %ARTIFACTS%\.libs\libffi-7.dll %_LIBFFI_OUT%
+copy %ARTIFACTS%\.libs\libffi-7.lib %_LIBFFI_OUT%
+copy %ARTIFACTS%\fficonfig.h %_LIBFFI_OUT%\include
+copy %ARTIFACTS%\include\*.h %_LIBFFI_OUT%\include
+
+endlocal
+exit /b
+
+:InstallCygwin
+setlocal
+
+if NOT DEFINED CYG_ROOT (set CYG_ROOT=c:/cygwin)
+if NOT DEFINED CYG_CACHE (set CYG_CACHE=C:/cygwin/var/cache/setup)
+if NOT DEFINED CYG_MIRROR (set CYG_MIRROR=http://mirrors.kernel.org/sourceware/cygwin/)
+
+powershell -c "md $env:CYG_ROOT -ErrorAction SilentlyContinue"
+powershell -c "$setup = $env:CYG_ROOT+'/setup.exe'; if (!(Test-Path $setup)){invoke-webrequest https://cygwin.com/setup-x86.exe -outfile $setup}
+%CYG_ROOT%/setup.exe -qnNdO -R "%CYG_ROOT%" -s "%CYG_MIRROR%" -l "%CYG_CACHE%" -P make -P autoconf -P automake -P libtool -P dejagnu
+
+endlocal
+exit /b
+
diff --git a/PCbuild/python.props b/PCbuild/python.props
index 3a0ddce..52bc99e 100644
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -53,6 +53,9 @@
     <sqlite3Dir>$(ExternalsDir)sqlite-3.21.0.0\</sqlite3Dir>
     <bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
     <lzmaDir>$(ExternalsDir)xz-5.2.2\</lzmaDir>
+    <libffiDir>$(ExternalsDir)libffi\</libffiDir>
+    <libffiOutDir>$(ExternalsDir)libffi\$(ArchName)\</libffiOutDir>
+    <libffiIncludeDir>$(libffiOutDir)include</libffiIncludeDir>
     <opensslDir>$(ExternalsDir)openssl-1.1.0j\</opensslDir>
     <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.0j\$(ArchName)\</opensslOutDir>
     <opensslIncludeDir>$(opensslOutDir)include</opensslIncludeDir>