From 00134f64d982561750f57f1a0bb7bb073f9f237c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 19 Jun 2017 11:12:56 -0700 Subject: bpo-30687: Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat (#2252) (#2280) * Fixes build scripts to find msbuild.exe and stop relying on vcvarsall.bat Also fixes bdist_wininst.vcxproj to use correct version in generated name. (cherry picked from commit 06d6e3d0bb5b8a3d3105289034953a8014356a0b) --- Misc/NEWS | 3 ++ PC/bdist_wininst/bdist_wininst.vcxproj | 4 +++ PC/bdist_wininst/build.bat | 11 +++----- PCbuild/build.bat | 32 ++++++++------------- PCbuild/find_msbuild.bat | 51 ++++++++++++++++++++++++++++++++++ PCbuild/python.vcxproj | 5 +++- Tools/msi/build.bat | 10 +++---- Tools/msi/buildrelease.bat | 15 +++++----- Tools/msi/uploadrelease.bat | 15 +++++----- Tools/nuget/build.bat | 7 +++-- 10 files changed, 102 insertions(+), 51 deletions(-) create mode 100644 PCbuild/find_msbuild.bat diff --git a/Misc/NEWS b/Misc/NEWS index bccdf94..2349eaa 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -327,6 +327,9 @@ Tests Windows ------- +- bpo-30687: Locate msbuild.exe on Windows when building rather than + vcvarsall.bat + - bpo-30450: The build process on Windows no longer depends on Subversion, instead pulling external code from GitHub via a Python script. If Python 3.6 is not found on the system (via ``py -3.6``), NuGet is used to download a diff --git a/PC/bdist_wininst/bdist_wininst.vcxproj b/PC/bdist_wininst/bdist_wininst.vcxproj index 4dc5009..9bbfda2 100644 --- a/PC/bdist_wininst/bdist_wininst.vcxproj +++ b/PC/bdist_wininst/bdist_wininst.vcxproj @@ -58,6 +58,10 @@ $(PySourcePath)lib\distutils\command\ false wininst-$(VisualStudioVersion) + wininst-14.0 + wininst-12.0 + wininst-11.0 + wininst-10.0 $(TargetName)-amd64 .exe diff --git a/PC/bdist_wininst/build.bat b/PC/bdist_wininst/build.bat index ee68567..d5cf752 100644 --- a/PC/bdist_wininst/build.bat +++ b/PC/bdist_wininst/build.bat @@ -7,16 +7,13 @@ set PCBUILD=%~dp0..\..\PCBuild\ echo Building Lib\distutils\command\wininst-xx.0.exe -call "%PCBUILD%env.bat" x86 -if errorlevel 1 goto :eof +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=Win32 if errorlevel 1 goto :eof echo Building Lib\distutils\command\wininst-xx.0-amd64.exe -call "%PCBUILD%env.bat" x86_amd64 -if errorlevel 1 goto :eof - -msbuild "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 +%MSBUILD% "%D%bdist_wininst.vcxproj" "/p:SolutionDir=%PCBUILD%\" /p:Configuration=Release /p:Platform=x64 diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 8e1c168..b63843f 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -48,7 +48,6 @@ exit /b 127 :Run setlocal set platf=Win32 -set vs_platf=x86 set conf=Release set target=Build set dir=%~dp0 @@ -57,10 +56,6 @@ set verbose=/nologo /v:m set kill= set do_pgo= set pgo_job=-m test --pgo -set on_64_bit=true - -rem This may not be 100% accurate, but close enough. -if "%ProgramFiles(x86)%"=="" (set on_64_bit=false) :CheckOpts if "%~1"=="-h" goto Usage @@ -90,18 +85,12 @@ if "%IncludeTkinter%"=="" set IncludeTkinter=true if "%IncludeExternals%"=="true" call "%dir%get_externals.bat" -if "%platf%"=="x64" ( - if "%on_64_bit%"=="true" ( - rem This ought to always be correct these days... - set vs_platf=amd64 - ) else ( - if "%do_pgo%"=="true" ( - echo.ERROR: Cannot cross-compile with PGO - echo. 32bit operating system detected, if this is incorrect, - echo. make sure the ProgramFiles(x86^) environment variable is set - exit /b 1 - ) - set vs_platf=x86_amd64 +if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" ( + if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" ( + echo.ERROR: Cannot cross-compile with PGO + echo. 32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE + echo. and PROCESSOR_ARCHITEW6432 environment variables are correct. + exit /b 1 ) ) @@ -110,7 +99,8 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%" if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty= rem Setup the environment -call "%dir%env.bat" %vs_platf% >nul +call "%dir%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if "%kill%"=="true" call :Kill @@ -129,7 +119,7 @@ if "%do_pgo%"=="true" ( goto Build :Kill echo on -msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ +%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:KillPython=true @@ -141,7 +131,7 @@ rem Call on MSBuild to do the work, echo the command. rem Passing %1-9 is not the preferred option, but argument parsing in rem batch is, shall we say, "lackluster" echo on -msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ +%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^ /p:Configuration=%conf% /p:Platform=%platf%^ /p:IncludeExternals=%IncludeExternals%^ /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^ @@ -153,4 +143,4 @@ goto :eof :Version rem Display the current build version information -msbuild "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 +%MSBUILD% "%dir%python.props" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/PCbuild/find_msbuild.bat b/PCbuild/find_msbuild.bat new file mode 100644 index 0000000..1877906 --- /dev/null +++ b/PCbuild/find_msbuild.bat @@ -0,0 +1,51 @@ +@rem +@rem Searches for MSBuild.exe. This is the only tool we need to initiate +@rem a build, so we no longer search for the full VC toolset. +@rem +@rem This file is supposed to modify the state of the caller (specifically +@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid +@rem changing any other persistent state. +@rem + +@rem No arguments provided means do full search +@if '%1' EQU '' goto :begin_search + +@rem One argument may be the full path. Use a goto so we don't try to +@rem parse the next if statement - incorrect quoting in the multi-arg +@rem case can cause us to break immediately. +@if '%2' EQU '' goto :one_arg + +@rem Entire command line may represent the full path if quoting failed. +@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found +@goto :begin_search + +:one_arg +@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found + +:begin_search +@set MSBUILD= + +@rem If msbuild.exe is on the PATH, assume that the user wants that one. +@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc" +@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found + +@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @( + @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found + +@rem VS 2015 and earlier register MSBuild separately, so we can find it. +@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul +@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @( + @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe") +) +@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found + + +@exit /b 1 + +:found +@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%) +@set _Py_MSBuild_Source= diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 807213f..2786ac2 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -83,7 +83,7 @@ - + ucrtbase ucrtbased @@ -94,6 +94,8 @@ set PYTHONPATH=$(PySourcePath)Lib + <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin + <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64 <_Content>@rem This script invokes the most recently built Python with all arguments @rem passed through to the interpreter. This file is generated by the @rem build process and any changes *will* be thrown away by the next @@ -103,6 +105,7 @@ set PYTHONPATH=$(PySourcePath)Lib @echo Running $(Configuration)^|$(Platform) interpreter... @setlocal @set PYTHONHOME=$(PySourcePath) +$(_PGOPath) @"$(OutDir)python$(PyDebugExt).exe" %* <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat')) diff --git a/Tools/msi/build.bat b/Tools/msi/build.bat index 59e0261..f720ea9 100644 --- a/Tools/msi/build.bat +++ b/Tools/msi/build.bat @@ -22,8 +22,8 @@ if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) call "%D%get_externals.bat" - -call "%PCBUILD%env.bat" x86 +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if defined BUILDX86 ( call "%PCBUILD%build.bat" -d -e %REBUILD% %BUILDTEST% @@ -44,7 +44,7 @@ if defined BUILDDOC ( ) rem Build the launcher MSI separately -msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 +%MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 set BUILD_CMD="%D%bundle\snapshot.wixproj" if defined BUILDTEST ( @@ -58,11 +58,11 @@ if defined REBUILD ( ) if defined BUILDX86 ( - msbuild %BUILD_CMD% + %MSBUILD% %BUILD_CMD% if errorlevel 1 goto :eof ) if defined BUILDX64 ( - msbuild /p:Platform=x64 %BUILD_CMD% + %MSBUILD% /p:Platform=x64 %BUILD_CMD% if errorlevel 1 goto :eof ) diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat index ed930c6..f60fe6a 100644 --- a/Tools/msi/buildrelease.bat +++ b/Tools/msi/buildrelease.bat @@ -69,6 +69,8 @@ if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 call "%D%get_externals.bat" +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) :builddoc if "%SKIPBUILD%" EQU "1" goto skipdoc @@ -165,28 +167,27 @@ if not "%SKIPBUILD%" EQU "1" ( @echo off ) -call "%PCBUILD%env.bat" if "%OUTDIR_PLAT%" EQU "win32" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B ) else if not exist "%PCBUILD%win32\en-us\launcher.msi" ( - msbuild "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% + %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% if errorlevel 1 exit /B ) set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI% -msbuild "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true +%MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true if errorlevel 1 exit /B -msbuild "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false +%MSBUILD% "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false if errorlevel 1 exit /B if defined BUILDZIP ( - msbuild "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" + %MSBUILD% "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" if errorlevel 1 exit /B ) if defined BUILDNUGET ( - msbuild "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" + %MSBUILD% "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" if errorlevel 1 exit /B ) diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat index 0873d55..610b092 100644 --- a/Tools/msi/uploadrelease.bat +++ b/Tools/msi/uploadrelease.bat @@ -49,16 +49,17 @@ if defined NOGPG ( echo Found gpg2.exe at %GPG% ) -call "%PCBUILD%env.bat" > nul 2> nul +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) pushd "%D%" -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% -msbuild /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 %PURGE_OPTION% +%MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false %PURGE_OPTION% if not defined NOTEST ( - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 - msbuild /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x86 + %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=x64 ) -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 -msbuild /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false +%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 +%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false popd exit /B 0 diff --git a/Tools/nuget/build.bat b/Tools/nuget/build.bat index 5068c17..54498ca 100644 --- a/Tools/nuget/build.bat +++ b/Tools/nuget/build.bat @@ -21,7 +21,8 @@ if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckO if not defined BUILDX86 if not defined BUILDX64 (set BUILDX86=1) && (set BUILDX64=1) call "%D%..\msi\get_externals.bat" -call "%PCBUILD%env.bat" x86 +call "%PCBUILD%find_msbuild.bat" %MSBUILD% +if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%" @@ -30,7 +31,7 @@ if defined BUILDX86 ( ) else if not exist "%PCBUILD%win32\python.exe" call "%PCBUILD%build.bat" -e if errorlevel 1 goto :eof - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% + %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% if errorlevel 1 goto :eof ) @@ -39,7 +40,7 @@ if defined BUILDX64 ( ) else if not exist "%PCBUILD%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e if errorlevel 1 goto :eof - msbuild "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% + %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% if errorlevel 1 goto :eof ) -- cgit v0.12 From b0fba8874a4bd6bf4773e6efdbd8fa762e9f05bd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 20 Jun 2017 15:37:24 +0200 Subject: bpo-30500: urllib: Simplify splithost by calling into urlparse. (#1849) (#2289) The current regex based splitting produces a wrong result. For example:: http://abc#@def Web browsers parse that URL as ``http://abc/#@def``, that is, the host is ``abc``, the path is ``/``, and the fragment is ``#@def``. (cherry picked from commit 90e01e50ef8a9e6c91f30d965563c378a4ad26de) (cherry picked from commit 536c1f1246f4faa302f9f5613fc3444e7ae09b4a) --- Lib/test/test_urlparse.py | 51 ++++++++++++++++++++++++++++++++++++----------- Lib/urllib/parse.py | 2 +- Misc/ACKS | 1 + Misc/NEWS | 17 ++++++++++++++++ 4 files changed, 58 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 3c89ed9..e5f6130 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -755,28 +755,35 @@ class UrlParseTestCase(unittest.TestCase): def test_parse_fragments(self): # Exercise the allow_fragments parameter of urlparse() and urlsplit() tests = ( - ("http:#frag", "path"), - ("//example.net#frag", "path"), - ("index.html#frag", "path"), - (";a=b#frag", "params"), - ("?a=b#frag", "query"), - ("#frag", "path"), + ("http:#frag", "path", "frag"), + ("//example.net#frag", "path", "frag"), + ("index.html#frag", "path", "frag"), + (";a=b#frag", "params", "frag"), + ("?a=b#frag", "query", "frag"), + ("#frag", "path", "frag"), + ("abc#@frag", "path", "@frag"), + ("//abc#@frag", "path", "@frag"), + ("//abc:80#@frag", "path", "@frag"), + ("//abc#@frag:80", "path", "@frag:80"), ) - for url, attr in tests: + for url, attr, expected_frag in tests: for func in (urllib.parse.urlparse, urllib.parse.urlsplit): if attr == "params" and func is urllib.parse.urlsplit: attr = "path" with self.subTest(url=url, function=func): result = func(url, allow_fragments=False) self.assertEqual(result.fragment, "") - self.assertTrue(getattr(result, attr).endswith("#frag")) + self.assertTrue( + getattr(result, attr).endswith("#" + expected_frag)) self.assertEqual(func(url, "", False).fragment, "") result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, "frag") - self.assertFalse(getattr(result, attr).endswith("frag")) - self.assertEqual(func(url, "", True).fragment, "frag") - self.assertEqual(func(url).fragment, "frag") + self.assertEqual(result.fragment, expected_frag) + self.assertFalse( + getattr(result, attr).endswith(expected_frag)) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -983,6 +990,26 @@ class Utility_Tests(unittest.TestCase): self.assertEqual(splithost('/foo/bar/baz.html'), (None, '/foo/bar/baz.html')) + # bpo-30500: # starts a fragment. + self.assertEqual(splithost('//127.0.0.1#@host.com'), + ('127.0.0.1', '/#@host.com')) + self.assertEqual(splithost('//127.0.0.1#@host.com:80'), + ('127.0.0.1', '/#@host.com:80')) + self.assertEqual(splithost('//127.0.0.1:80#@host.com'), + ('127.0.0.1:80', '/#@host.com')) + + # Empty host is returned as empty string. + self.assertEqual(splithost("///file"), + ('', '/file')) + + # Trailing semicolon, question mark and hash symbol are kept. + self.assertEqual(splithost("//example.net/file;"), + ('example.net', '/file;')) + self.assertEqual(splithost("//example.net/file?"), + ('example.net', '/file?')) + self.assertEqual(splithost("//example.net/file#"), + ('example.net', '/file#')) + def test_splituser(self): splituser = urllib.parse.splituser self.assertEqual(splituser('User:Pass@www.python.org:080'), diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 888247b..3cab2d1 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -944,7 +944,7 @@ def splithost(url): """splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" global _hostprog if _hostprog is None: - _hostprog = re.compile('//([^/?]*)(.*)', re.DOTALL) + _hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL) match = _hostprog.match(url) if match: diff --git a/Misc/ACKS b/Misc/ACKS index 64484f3..03f9ea6 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1081,6 +1081,7 @@ Max Neunhöffer Anthon van der Neut George Neville-Neil Hieu Nguyen +Nam Nguyen Johannes Nicolai Samuel Nicolary Jonathan Niehof diff --git a/Misc/NEWS b/Misc/NEWS index 2349eaa..3a52a3c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,23 @@ Python News +++++++++++ +What's New in Python 3.6.3 release candidate 1? +=============================================== + +*Release date: XXXX-XX-XX* + +Core and Builtins +----------------- + +Library +------- + +- [Security] bpo-30500: Fix urllib.parse.splithost() to correctly parse + fragments. For example, ``splithost('//127.0.0.1#@evil.com/')`` now + correctly returns the ``127.0.0.1`` host, instead of treating ``@evil.com`` + as the host in an authentification (``login@host``). + + What's New in Python 3.6.2 release candidate 1? =============================================== -- cgit v0.12 From ea1ab803ddc14ab02ffed50ecc5089897f259623 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 21 Jun 2017 16:05:11 +0200 Subject: bpo-30694: Upgrade Modules/expat/ to libexpat 2.2.1 (#2300) (#2313) New file: Modules/expat/siphash.h. (cherry picked from commit 5ff7132313eb651107b179d20218dfe5d4e47f13) (cherry picked from commit 4a66524006852fc982aebafa277f2c043d9ad149) --- Misc/NEWS | 9 + Modules/expat/COPYING | 2 +- Modules/expat/expat.h | 22 +- Modules/expat/expat_external.h | 5 +- Modules/expat/siphash.h | 344 +++++++++++++++++++++++ Modules/expat/winconfig.h | 10 + Modules/expat/xmlparse.c | 609 ++++++++++++++++++++++++++++++++--------- Modules/expat/xmlrole.c | 10 +- Modules/expat/xmltok.c | 34 +-- 9 files changed, 887 insertions(+), 158 deletions(-) create mode 100644 Modules/expat/siphash.h diff --git a/Misc/NEWS b/Misc/NEWS index 3a52a3c..1f6161a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,15 @@ Core and Builtins Library ------- +- [Security] bpo-30694: Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes + of multiple security vulnerabilities including: CVE-2017-9233 (External + entity infinite loop DoS), CVE-2016-9063 (Integer overflow, re-fix), + CVE-2016-0718 (Fix regression bugs from 2.2.0's fix to CVE-2016-0718) + and CVE-2012-0876 (Counter hash flooding with SipHash). + Note: the CVE-2016-5300 (Use os-specific entropy sources like getrandom) + doesn't impact Python, since Python already gets entropy from the OS to set + the expat secret using ``XML_SetHashSalt()``. + - [Security] bpo-30500: Fix urllib.parse.splithost() to correctly parse fragments. For example, ``splithost('//127.0.0.1#@evil.com/')`` now correctly returns the ``127.0.0.1`` host, instead of treating ``@evil.com`` diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index 092c83b..8d288f0 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2016 Expat maintainers +Copyright (c) 2001-2017 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 086e24b..28b0f95 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -95,7 +95,9 @@ enum XML_Error { /* Added in 2.0. */ XML_ERROR_RESERVED_PREFIX_XML, XML_ERROR_RESERVED_PREFIX_XMLNS, - XML_ERROR_RESERVED_NAMESPACE_URI + XML_ERROR_RESERVED_NAMESPACE_URI, + /* Added in 2.2.1. */ + XML_ERROR_INVALID_ARGUMENT }; enum XML_Content_Type { @@ -706,6 +708,7 @@ XML_UseParserAsHandlerArg(XML_Parser parser); be called, despite an external subset being parsed. Note: If XML_DTD is not defined when Expat is compiled, returns XML_ERROR_FEATURE_REQUIRES_XML_DTD. + Note: If parser == NULL, returns XML_ERROR_INVALID_ARGUMENT. */ XMLPARSEAPI(enum XML_Error) XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD); @@ -729,15 +732,16 @@ XML_GetBase(XML_Parser parser); to the XML_StartElementHandler that were specified in the start-tag rather than defaulted. Each attribute/value pair counts as 2; thus this correspondds to an index into the atts array passed to the - XML_StartElementHandler. + XML_StartElementHandler. Returns -1 if parser == NULL. */ XMLPARSEAPI(int) XML_GetSpecifiedAttributeCount(XML_Parser parser); /* Returns the index of the ID attribute passed in the last call to - XML_StartElementHandler, or -1 if there is no ID attribute. Each - attribute/value pair counts as 2; thus this correspondds to an - index into the atts array passed to the XML_StartElementHandler. + XML_StartElementHandler, or -1 if there is no ID attribute or + parser == NULL. Each attribute/value pair counts as 2; thus this + correspondds to an index into the atts array passed to the + XML_StartElementHandler. */ XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); @@ -901,6 +905,7 @@ enum XML_ParamEntityParsing { entities is requested; otherwise it will return non-zero. Note: If XML_SetParamEntityParsing is called after XML_Parse or XML_ParseBuffer, then it has no effect and will always return 0. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetParamEntityParsing(XML_Parser parser, @@ -910,6 +915,7 @@ XML_SetParamEntityParsing(XML_Parser parser, Helps in preventing DoS attacks based on predicting hash function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. + Note: If parser == NULL, the function will do nothing and return 0. */ XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, @@ -936,6 +942,10 @@ XML_GetErrorCode(XML_Parser parser); the location is the location of the character at which the error was detected; otherwise the location is the location of the last parse event, as described above. + + Note: XML_GetCurrentLineNumber and XML_GetCurrentColumnNumber + return 0 to indicate an error. + Note: XML_GetCurrentByteIndex returns -1 to indicate an error. */ XMLPARSEAPI(XML_Size) XML_GetCurrentLineNumber(XML_Parser parser); XMLPARSEAPI(XML_Size) XML_GetCurrentColumnNumber(XML_Parser parser); @@ -1039,7 +1049,7 @@ XML_GetFeatureList(void); */ #define XML_MAJOR_VERSION 2 #define XML_MINOR_VERSION 2 -#define XML_MICRO_VERSION 0 +#define XML_MICRO_VERSION 1 #ifdef __cplusplus } diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 4860c57..4c9e5ea 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -97,7 +97,10 @@ extern "C" { #endif #ifdef XML_UNICODE_WCHAR_T -#define XML_UNICODE +# define XML_UNICODE +# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) +# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" +# endif #endif #ifdef XML_UNICODE /* Information is UTF-16 encoded. */ diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h new file mode 100644 index 0000000..23b56d2 --- /dev/null +++ b/Modules/expat/siphash.h @@ -0,0 +1,344 @@ +/* ========================================================================== + * siphash.h - SipHash-2-4 in a single header file + * -------------------------------------------------------------------------- + * Derived by William Ahern from the reference implementation[1] published[2] + * by Jean-Philippe Aumasson and Daniel J. Berstein. Licensed in kind. + * by Jean-Philippe Aumasson and Daniel J. Berstein. + * Minimal changes by Sebastian Pipping on top, details below. + * Licensed under the CC0 Public Domain Dedication license. + * + * 1. https://www.131002.net/siphash/siphash24.c + * 2. https://www.131002.net/siphash/ + * -------------------------------------------------------------------------- + * HISTORY: + * + * 2017-06-10 (Sebastian Pipping) + * - Clarify license note in the header + * - Address C89 issues: + * - Stop using inline keyword (and let compiler decide) + * - Turn integer suffix ULL to UL + * - Replace _Bool by int + * - Turn macro siphash24 into a function + * - Address invalid conversion (void pointer) by explicit cast + * - Always expose sip24_valid (for self-tests) + * + * 2012-11-04 - Born. (William Ahern) + * -------------------------------------------------------------------------- + * USAGE: + * + * SipHash-2-4 takes as input two 64-bit words as the key, some number of + * message bytes, and outputs a 64-bit word as the message digest. This + * implementation employs two data structures: a struct sipkey for + * representing the key, and a struct siphash for representing the hash + * state. + * + * For converting a 16-byte unsigned char array to a key, use either the + * macro sip_keyof or the routine sip_tokey. The former instantiates a + * compound literal key, while the latter requires a key object as a + * parameter. + * + * unsigned char secret[16]; + * arc4random_buf(secret, sizeof secret); + * struct sipkey *key = sip_keyof(secret); + * + * For hashing a message, use either the convenience macro siphash24 or the + * routines sip24_init, sip24_update, and sip24_final. + * + * struct siphash state; + * void *msg; + * size_t len; + * uint64_t hash; + * + * sip24_init(&state, key); + * sip24_update(&state, msg, len); + * hash = sip24_final(&state); + * + * or + * + * hash = siphash24(msg, len, key); + * + * To convert the 64-bit hash value to a canonical 8-byte little-endian + * binary representation, use either the macro sip_binof or the routine + * sip_tobin. The former instantiates and returns a compound literal array, + * while the latter requires an array object as a parameter. + * -------------------------------------------------------------------------- + * NOTES: + * + * o Neither sip_keyof, sip_binof, nor siphash24 will work with compilers + * lacking compound literal support. Instead, you must use the lower-level + * interfaces which take as parameters the temporary state objects. + * + * o Uppercase macros may evaluate parameters more than once. Lowercase + * macros should not exhibit any such side effects. + * ========================================================================== + */ +#ifndef SIPHASH_H +#define SIPHASH_H + +#include /* size_t */ +#include /* uint64_t uint32_t uint8_t */ + + +#define SIP_ROTL(x, b) (uint64_t)(((x) << (b)) | ( (x) >> (64 - (b)))) + +#define SIP_U32TO8_LE(p, v) \ + (p)[0] = (uint8_t)((v) >> 0); (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); (p)[3] = (uint8_t)((v) >> 24); + +#define SIP_U64TO8_LE(p, v) \ + SIP_U32TO8_LE((p) + 0, (uint32_t)((v) >> 0)); \ + SIP_U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); + +#define SIP_U8TO64_LE(p) \ + (((uint64_t)((p)[0]) << 0) | \ + ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | \ + ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | \ + ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | \ + ((uint64_t)((p)[7]) << 56)) + + +#define SIPHASH_INITIALIZER { 0, 0, 0, 0, { 0 }, 0, 0 } + +struct siphash { + uint64_t v0, v1, v2, v3; + + unsigned char buf[8], *p; + uint64_t c; +}; /* struct siphash */ + + +#define SIP_KEYLEN 16 + +struct sipkey { + uint64_t k[2]; +}; /* struct sipkey */ + +#define sip_keyof(k) sip_tokey(&(struct sipkey){ { 0 } }, (k)) + +static struct sipkey *sip_tokey(struct sipkey *key, const void *src) { + key->k[0] = SIP_U8TO64_LE((const unsigned char *)src); + key->k[1] = SIP_U8TO64_LE((const unsigned char *)src + 8); + return key; +} /* sip_tokey() */ + + +#define sip_binof(v) sip_tobin((unsigned char[8]){ 0 }, (v)) + +static void *sip_tobin(void *dst, uint64_t u64) { + SIP_U64TO8_LE((unsigned char *)dst, u64); + return dst; +} /* sip_tobin() */ + + +static void sip_round(struct siphash *H, const int rounds) { + int i; + + for (i = 0; i < rounds; i++) { + H->v0 += H->v1; + H->v1 = SIP_ROTL(H->v1, 13); + H->v1 ^= H->v0; + H->v0 = SIP_ROTL(H->v0, 32); + + H->v2 += H->v3; + H->v3 = SIP_ROTL(H->v3, 16); + H->v3 ^= H->v2; + + H->v0 += H->v3; + H->v3 = SIP_ROTL(H->v3, 21); + H->v3 ^= H->v0; + + H->v2 += H->v1; + H->v1 = SIP_ROTL(H->v1, 17); + H->v1 ^= H->v2; + H->v2 = SIP_ROTL(H->v2, 32); + } +} /* sip_round() */ + + +static struct siphash *sip24_init(struct siphash *H, const struct sipkey *key) { + H->v0 = 0x736f6d6570736575UL ^ key->k[0]; + H->v1 = 0x646f72616e646f6dUL ^ key->k[1]; + H->v2 = 0x6c7967656e657261UL ^ key->k[0]; + H->v3 = 0x7465646279746573UL ^ key->k[1]; + + H->p = H->buf; + H->c = 0; + + return H; +} /* sip24_init() */ + + +#define sip_endof(a) (&(a)[sizeof (a) / sizeof *(a)]) + +static struct siphash *sip24_update(struct siphash *H, const void *src, size_t len) { + const unsigned char *p = (const unsigned char *)src, *pe = p + len; + uint64_t m; + + do { + while (p < pe && H->p < sip_endof(H->buf)) + *H->p++ = *p++; + + if (H->p < sip_endof(H->buf)) + break; + + m = SIP_U8TO64_LE(H->buf); + H->v3 ^= m; + sip_round(H, 2); + H->v0 ^= m; + + H->p = H->buf; + H->c += 8; + } while (p < pe); + + return H; +} /* sip24_update() */ + + +static uint64_t sip24_final(struct siphash *H) { + char left = H->p - H->buf; + uint64_t b = (H->c + left) << 56; + + switch (left) { + case 7: b |= (uint64_t)H->buf[6] << 48; + case 6: b |= (uint64_t)H->buf[5] << 40; + case 5: b |= (uint64_t)H->buf[4] << 32; + case 4: b |= (uint64_t)H->buf[3] << 24; + case 3: b |= (uint64_t)H->buf[2] << 16; + case 2: b |= (uint64_t)H->buf[1] << 8; + case 1: b |= (uint64_t)H->buf[0] << 0; + case 0: break; + } + + H->v3 ^= b; + sip_round(H, 2); + H->v0 ^= b; + H->v2 ^= 0xff; + sip_round(H, 4); + + return H->v0 ^ H->v1 ^ H->v2 ^ H->v3; +} /* sip24_final() */ + + +static uint64_t siphash24(const void *src, size_t len, const struct sipkey *key) { + struct siphash state = SIPHASH_INITIALIZER; + return sip24_final(sip24_update(sip24_init(&state, key), src, len)); +} /* siphash24() */ + + +/* + * SipHash-2-4 output with + * k = 00 01 02 ... + * and + * in = (empty string) + * in = 00 (1 byte) + * in = 00 01 (2 bytes) + * in = 00 01 02 (3 bytes) + * ... + * in = 00 01 02 ... 3e (63 bytes) + */ +static int sip24_valid(void) { + static const unsigned char vectors[64][8] = { + { 0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72, }, + { 0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74, }, + { 0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d, }, + { 0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85, }, + { 0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf, }, + { 0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18, }, + { 0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb, }, + { 0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab, }, + { 0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93, }, + { 0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e, }, + { 0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a, }, + { 0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4, }, + { 0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75, }, + { 0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14, }, + { 0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7, }, + { 0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1, }, + { 0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f, }, + { 0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69, }, + { 0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b, }, + { 0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb, }, + { 0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe, }, + { 0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0, }, + { 0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93, }, + { 0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8, }, + { 0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8, }, + { 0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc, }, + { 0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17, }, + { 0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f, }, + { 0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde, }, + { 0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6, }, + { 0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad, }, + { 0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32, }, + { 0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71, }, + { 0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7, }, + { 0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12, }, + { 0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15, }, + { 0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31, }, + { 0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02, }, + { 0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca, }, + { 0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a, }, + { 0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e, }, + { 0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad, }, + { 0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18, }, + { 0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4, }, + { 0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9, }, + { 0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9, }, + { 0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb, }, + { 0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0, }, + { 0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6, }, + { 0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7, }, + { 0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee, }, + { 0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1, }, + { 0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a, }, + { 0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81, }, + { 0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f, }, + { 0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24, }, + { 0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7, }, + { 0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea, }, + { 0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60, }, + { 0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66, }, + { 0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c, }, + { 0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f, }, + { 0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5, }, + { 0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95, } + }; + unsigned char in[64]; + struct sipkey k; + size_t i; + + sip_tokey(&k, "\000\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017"); + + for (i = 0; i < sizeof in; ++i) { + in[i] = i; + + if (siphash24(in, i, &k) != SIP_U8TO64_LE(vectors[i])) + return 0; + } + + return 1; +} /* sip24_valid() */ + + +#if SIPHASH_MAIN + +#include + +int main(void) { + int ok = sip24_valid(); + + if (ok) + puts("OK"); + else + puts("FAIL"); + + return !ok; +} /* main() */ + +#endif /* SIPHASH_MAIN */ + + +#endif /* SIPHASH_H */ diff --git a/Modules/expat/winconfig.h b/Modules/expat/winconfig.h index c1b791d..9bf014d 100644 --- a/Modules/expat/winconfig.h +++ b/Modules/expat/winconfig.h @@ -17,6 +17,12 @@ #include #include + +#if defined(HAVE_EXPAT_CONFIG_H) /* e.g. MinGW */ +# include +#else /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #define XML_NS 1 #define XML_DTD 1 #define XML_CONTEXT_BYTES 1024 @@ -27,4 +33,8 @@ /* Windows has memmove() available. */ #define HAVE_MEMMOVE + +#endif /* !defined(HAVE_EXPAT_CONFIG_H) */ + + #endif /* ndef WINCONFIG_H */ diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index b308e67..76f078e 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,13 +1,19 @@ /* Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd See the file COPYING for copying permission. + + 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) */ +#define _GNU_SOURCE /* syscall prototype */ + #include #include /* memset(), memcpy() */ #include #include /* UINT_MAX */ +#include /* fprintf */ +#include /* getenv */ -#ifdef WIN32 +#ifdef _WIN32 #define getpid GetCurrentProcessId #else #include /* gettimeofday() */ @@ -17,20 +23,15 @@ #define XML_BUILDING_EXPAT 1 -#ifdef WIN32 +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #elif defined(HAVE_EXPAT_CONFIG_H) #include -#endif /* ndef WIN32 */ +#endif /* ndef _WIN32 */ #include "ascii.h" #include "expat.h" +#include "siphash.h" #ifdef XML_UNICODE #define XML_ENCODE_MAX XML_UTF16_ENCODE_MAX @@ -109,17 +110,11 @@ typedef struct { const XML_Memory_Handling_Suite *mem; } HASH_TABLE; -/* Basic character hash algorithm, taken from Python's string hash: - h = h * 1000003 ^ character, the constant being a prime number. +static size_t +keylen(KEY s); -*/ -#ifdef XML_UNICODE -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned short)(c)) -#else -#define CHAR_HASH(h, c) \ - (((h) * 0xF4243) ^ (unsigned char)(c)) -#endif +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key); /* For probing (after a collision) we need a step size relative prime to the hash table size, which is a power of 2. We use double-hashing, @@ -355,6 +350,8 @@ doIgnoreSection(XML_Parser parser, const ENCODING *, const char **startPtr, const char *end, const char **nextPtr, XML_Bool haveMore); #endif /* XML_DTD */ +static void +freeBindings(XML_Parser parser, BINDING *bindings); static enum XML_Error storeAtts(XML_Parser parser, const ENCODING *, const char *s, TAG_NAME *tagNamePtr, BINDING **bindingsPtr); @@ -697,10 +694,84 @@ static const XML_Char implicitContext[] = { ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0' }; + +#if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) +# include + +# if defined(HAVE_GETRANDOM) +# include /* getrandom */ +# else +# include /* syscall */ +# include /* SYS_getrandom */ +# endif + +/* Obtain entropy on Linux 3.17+ */ +static int +writeRandomBytes_getrandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + size_t bytesWrittenTotal = 0; + const unsigned int getrandomFlags = 0; + + do { + void * const currentTarget = (void*)((char*)target + bytesWrittenTotal); + const size_t bytesToWrite = count - bytesWrittenTotal; + + const int bytesWrittenMore = +#if defined(HAVE_GETRANDOM) + getrandom(currentTarget, bytesToWrite, getrandomFlags); +#else + syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); +#endif + + if (bytesWrittenMore > 0) { + bytesWrittenTotal += bytesWrittenMore; + if (bytesWrittenTotal >= count) + success = 1; + } + } while (! success && (errno == EINTR || errno == EAGAIN)); + + return success; +} + +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + + +#ifdef _WIN32 + +typedef BOOLEAN (APIENTRY *RTLGENRANDOM_FUNC)(PVOID, ULONG); + +/* Obtain entropy on Windows XP / Windows Server 2003 and later. + * Hint on RtlGenRandom and the following article from libsodioum. + * + * Michael Howard: Cryptographically Secure Random number on Windows without using CryptoAPI + * https://blogs.msdn.microsoft.com/michael_howard/2005/01/14/cryptographically-secure-random-number-on-windows-without-using-cryptoapi/ + */ +static int +writeRandomBytes_RtlGenRandom(void * target, size_t count) { + int success = 0; /* full count bytes written? */ + const HMODULE advapi32 = LoadLibrary("ADVAPI32.DLL"); + + if (advapi32) { + const RTLGENRANDOM_FUNC RtlGenRandom + = (RTLGENRANDOM_FUNC)GetProcAddress(advapi32, "SystemFunction036"); + if (RtlGenRandom) { + if (RtlGenRandom((PVOID)target, (ULONG)count) == TRUE) { + success = 1; + } + } + FreeLibrary(advapi32); + } + + return success; +} + +#endif /* _WIN32 */ + + static unsigned long gather_time_entropy(void) { -#ifdef WIN32 +#ifdef _WIN32 FILETIME ft; GetSystemTimeAsFileTime(&ft); /* never fails */ return ft.dwHighDateTime ^ ft.dwLowDateTime; @@ -716,20 +787,62 @@ gather_time_entropy(void) #endif } +#if defined(HAVE_ARC4RANDOM_BUF) && defined(HAVE_LIBBSD) +# include +#endif + +static unsigned long +ENTROPY_DEBUG(const char * label, unsigned long entropy) { + const char * const EXPAT_ENTROPY_DEBUG = getenv("EXPAT_ENTROPY_DEBUG"); + if (EXPAT_ENTROPY_DEBUG && ! strcmp(EXPAT_ENTROPY_DEBUG, "1")) { + fprintf(stderr, "Entropy: %s --> 0x%0*lx (%lu bytes)\n", + label, + (int)sizeof(entropy) * 2, entropy, + (unsigned long)sizeof(entropy)); + } + return entropy; +} + static unsigned long generate_hash_secret_salt(XML_Parser parser) { - /* Process ID is 0 bits entropy if attacker has local access - * XML_Parser address is few bits of entropy if attacker has local access */ - const unsigned long entropy = - gather_time_entropy() ^ getpid() ^ (unsigned long)parser; + unsigned long entropy; + (void)parser; +#if defined(HAVE_ARC4RANDOM_BUF) || defined(__CloudABI__) + (void)gather_time_entropy; + arc4random_buf(&entropy, sizeof(entropy)); + return ENTROPY_DEBUG("arc4random_buf", entropy); +#else + /* Try high quality providers first .. */ +#ifdef _WIN32 + if (writeRandomBytes_RtlGenRandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("RtlGenRandom", entropy); + } +#elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) + if (writeRandomBytes_getrandom((void *)&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getrandom", entropy); + } +#endif + /* .. and self-made low quality for backup: */ + + /* Process ID is 0 bits entropy if attacker has local access */ + entropy = gather_time_entropy() ^ getpid(); /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ if (sizeof(unsigned long) == 4) { - return entropy * 2147483647; + return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); } else { - return entropy * (unsigned long)2305843009213693951; + return ENTROPY_DEBUG("fallback(8)", + entropy * (unsigned long)2305843009213693951); } +#endif +} + +static unsigned long +get_hash_secret_salt(XML_Parser parser) { + if (parser->m_parentParser != NULL) + return get_hash_secret_salt(parser->m_parentParser); + return parser->m_hash_secret_salt; } static XML_Bool /* only valid for root parser */ @@ -960,6 +1073,10 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; OPEN_INTERNAL_ENTITY *openEntityList; + + if (parser == NULL) + return XML_FALSE; + if (parentParser) return XML_FALSE; /* move tagStack to freeTagList */ @@ -994,6 +1111,8 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) enum XML_Status XMLCALL XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { + if (parser == NULL) + return XML_STATUS_ERROR; /* Block after XML_Parse()/XML_ParseBuffer() has been called. XXX There's no way for the caller to determine which of the XXX possible error cases caused the XML_STATUS_ERROR return. @@ -1017,52 +1136,88 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, { XML_Parser parser = oldParser; DTD *newDtd = NULL; - DTD *oldDtd = _dtd; - XML_StartElementHandler oldStartElementHandler = startElementHandler; - XML_EndElementHandler oldEndElementHandler = endElementHandler; - XML_CharacterDataHandler oldCharacterDataHandler = characterDataHandler; - XML_ProcessingInstructionHandler oldProcessingInstructionHandler - = processingInstructionHandler; - XML_CommentHandler oldCommentHandler = commentHandler; - XML_StartCdataSectionHandler oldStartCdataSectionHandler - = startCdataSectionHandler; - XML_EndCdataSectionHandler oldEndCdataSectionHandler - = endCdataSectionHandler; - XML_DefaultHandler oldDefaultHandler = defaultHandler; - XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler - = unparsedEntityDeclHandler; - XML_NotationDeclHandler oldNotationDeclHandler = notationDeclHandler; - XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler - = startNamespaceDeclHandler; - XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler - = endNamespaceDeclHandler; - XML_NotStandaloneHandler oldNotStandaloneHandler = notStandaloneHandler; - XML_ExternalEntityRefHandler oldExternalEntityRefHandler - = externalEntityRefHandler; - XML_SkippedEntityHandler oldSkippedEntityHandler = skippedEntityHandler; - XML_UnknownEncodingHandler oldUnknownEncodingHandler - = unknownEncodingHandler; - XML_ElementDeclHandler oldElementDeclHandler = elementDeclHandler; - XML_AttlistDeclHandler oldAttlistDeclHandler = attlistDeclHandler; - XML_EntityDeclHandler oldEntityDeclHandler = entityDeclHandler; - XML_XmlDeclHandler oldXmlDeclHandler = xmlDeclHandler; - ELEMENT_TYPE * oldDeclElementType = declElementType; - - void *oldUserData = userData; - void *oldHandlerArg = handlerArg; - XML_Bool oldDefaultExpandInternalEntities = defaultExpandInternalEntities; - XML_Parser oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; + DTD *oldDtd; + XML_StartElementHandler oldStartElementHandler; + XML_EndElementHandler oldEndElementHandler; + XML_CharacterDataHandler oldCharacterDataHandler; + XML_ProcessingInstructionHandler oldProcessingInstructionHandler; + XML_CommentHandler oldCommentHandler; + XML_StartCdataSectionHandler oldStartCdataSectionHandler; + XML_EndCdataSectionHandler oldEndCdataSectionHandler; + XML_DefaultHandler oldDefaultHandler; + XML_UnparsedEntityDeclHandler oldUnparsedEntityDeclHandler; + XML_NotationDeclHandler oldNotationDeclHandler; + XML_StartNamespaceDeclHandler oldStartNamespaceDeclHandler; + XML_EndNamespaceDeclHandler oldEndNamespaceDeclHandler; + XML_NotStandaloneHandler oldNotStandaloneHandler; + XML_ExternalEntityRefHandler oldExternalEntityRefHandler; + XML_SkippedEntityHandler oldSkippedEntityHandler; + XML_UnknownEncodingHandler oldUnknownEncodingHandler; + XML_ElementDeclHandler oldElementDeclHandler; + XML_AttlistDeclHandler oldAttlistDeclHandler; + XML_EntityDeclHandler oldEntityDeclHandler; + XML_XmlDeclHandler oldXmlDeclHandler; + ELEMENT_TYPE * oldDeclElementType; + + void *oldUserData; + void *oldHandlerArg; + XML_Bool oldDefaultExpandInternalEntities; + XML_Parser oldExternalEntityRefHandlerArg; #ifdef XML_DTD - enum XML_ParamEntityParsing oldParamEntityParsing = paramEntityParsing; - int oldInEntityValue = prologState.inEntityValue; + enum XML_ParamEntityParsing oldParamEntityParsing; + int oldInEntityValue; #endif - XML_Bool oldns_triplets = ns_triplets; + XML_Bool oldns_triplets; /* Note that the new parser shares the same hash secret as the old parser, so that dtdCopy and copyEntityTable can lookup values from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt = hash_secret_salt; + unsigned long oldhash_secret_salt; + + /* Validate the oldParser parameter before we pull everything out of it */ + if (oldParser == NULL) + return NULL; + + /* Stash the original parser contents on the stack */ + oldDtd = _dtd; + oldStartElementHandler = startElementHandler; + oldEndElementHandler = endElementHandler; + oldCharacterDataHandler = characterDataHandler; + oldProcessingInstructionHandler = processingInstructionHandler; + oldCommentHandler = commentHandler; + oldStartCdataSectionHandler = startCdataSectionHandler; + oldEndCdataSectionHandler = endCdataSectionHandler; + oldDefaultHandler = defaultHandler; + oldUnparsedEntityDeclHandler = unparsedEntityDeclHandler; + oldNotationDeclHandler = notationDeclHandler; + oldStartNamespaceDeclHandler = startNamespaceDeclHandler; + oldEndNamespaceDeclHandler = endNamespaceDeclHandler; + oldNotStandaloneHandler = notStandaloneHandler; + oldExternalEntityRefHandler = externalEntityRefHandler; + oldSkippedEntityHandler = skippedEntityHandler; + oldUnknownEncodingHandler = unknownEncodingHandler; + oldElementDeclHandler = elementDeclHandler; + oldAttlistDeclHandler = attlistDeclHandler; + oldEntityDeclHandler = entityDeclHandler; + oldXmlDeclHandler = xmlDeclHandler; + oldDeclElementType = declElementType; + + oldUserData = userData; + oldHandlerArg = handlerArg; + oldDefaultExpandInternalEntities = defaultExpandInternalEntities; + oldExternalEntityRefHandlerArg = externalEntityRefHandlerArg; +#ifdef XML_DTD + oldParamEntityParsing = paramEntityParsing; + oldInEntityValue = prologState.inEntityValue; +#endif + oldns_triplets = ns_triplets; + /* Note that the new parser shares the same hash secret as the old + parser, so that dtdCopy and copyEntityTable can lookup values + from hash tables associated with either parser without us having + to worry which hash secrets each table has. + */ + oldhash_secret_salt = hash_secret_salt; #ifdef XML_DTD if (!context) @@ -1228,12 +1383,15 @@ XML_ParserFree(XML_Parser parser) void XMLCALL XML_UseParserAsHandlerArg(XML_Parser parser) { - handlerArg = parser; + if (parser != NULL) + handlerArg = parser; } enum XML_Error XMLCALL XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; #ifdef XML_DTD /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) @@ -1248,6 +1406,8 @@ XML_UseForeignDTD(XML_Parser parser, XML_Bool useDTD) void XMLCALL XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) { + if (parser == NULL) + return; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return; @@ -1257,6 +1417,8 @@ XML_SetReturnNSTriplet(XML_Parser parser, int do_nst) void XMLCALL XML_SetUserData(XML_Parser parser, void *p) { + if (parser == NULL) + return; if (handlerArg == userData) handlerArg = userData = p; else @@ -1266,6 +1428,8 @@ XML_SetUserData(XML_Parser parser, void *p) enum XML_Status XMLCALL XML_SetBase(XML_Parser parser, const XML_Char *p) { + if (parser == NULL) + return XML_STATUS_ERROR; if (p) { p = poolCopyString(&_dtd->pool, p); if (!p) @@ -1280,18 +1444,24 @@ XML_SetBase(XML_Parser parser, const XML_Char *p) const XML_Char * XMLCALL XML_GetBase(XML_Parser parser) { + if (parser == NULL) + return NULL; return curBase; } int XMLCALL XML_GetSpecifiedAttributeCount(XML_Parser parser) { + if (parser == NULL) + return -1; return nSpecifiedAtts; } int XMLCALL XML_GetIdAttributeIndex(XML_Parser parser) { + if (parser == NULL) + return -1; return idAttIndex; } @@ -1299,6 +1469,8 @@ XML_GetIdAttributeIndex(XML_Parser parser) const XML_AttrInfo * XMLCALL XML_GetAttributeInfo(XML_Parser parser) { + if (parser == NULL) + return NULL; return attInfo; } #endif @@ -1308,6 +1480,8 @@ XML_SetElementHandler(XML_Parser parser, XML_StartElementHandler start, XML_EndElementHandler end) { + if (parser == NULL) + return; startElementHandler = start; endElementHandler = end; } @@ -1315,34 +1489,39 @@ XML_SetElementHandler(XML_Parser parser, void XMLCALL XML_SetStartElementHandler(XML_Parser parser, XML_StartElementHandler start) { - startElementHandler = start; + if (parser != NULL) + startElementHandler = start; } void XMLCALL XML_SetEndElementHandler(XML_Parser parser, XML_EndElementHandler end) { - endElementHandler = end; + if (parser != NULL) + endElementHandler = end; } void XMLCALL XML_SetCharacterDataHandler(XML_Parser parser, XML_CharacterDataHandler handler) { - characterDataHandler = handler; + if (parser != NULL) + characterDataHandler = handler; } void XMLCALL XML_SetProcessingInstructionHandler(XML_Parser parser, XML_ProcessingInstructionHandler handler) { - processingInstructionHandler = handler; + if (parser != NULL) + processingInstructionHandler = handler; } void XMLCALL XML_SetCommentHandler(XML_Parser parser, XML_CommentHandler handler) { - commentHandler = handler; + if (parser != NULL) + commentHandler = handler; } void XMLCALL @@ -1350,6 +1529,8 @@ XML_SetCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start, XML_EndCdataSectionHandler end) { + if (parser == NULL) + return; startCdataSectionHandler = start; endCdataSectionHandler = end; } @@ -1357,19 +1538,23 @@ XML_SetCdataSectionHandler(XML_Parser parser, void XMLCALL XML_SetStartCdataSectionHandler(XML_Parser parser, XML_StartCdataSectionHandler start) { - startCdataSectionHandler = start; + if (parser != NULL) + startCdataSectionHandler = start; } void XMLCALL XML_SetEndCdataSectionHandler(XML_Parser parser, XML_EndCdataSectionHandler end) { - endCdataSectionHandler = end; + if (parser != NULL) + endCdataSectionHandler = end; } void XMLCALL XML_SetDefaultHandler(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_FALSE; } @@ -1378,6 +1563,8 @@ void XMLCALL XML_SetDefaultHandlerExpand(XML_Parser parser, XML_DefaultHandler handler) { + if (parser == NULL) + return; defaultHandler = handler; defaultExpandInternalEntities = XML_TRUE; } @@ -1387,6 +1574,8 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start, XML_EndDoctypeDeclHandler end) { + if (parser == NULL) + return; startDoctypeDeclHandler = start; endDoctypeDeclHandler = end; } @@ -1394,27 +1583,31 @@ XML_SetDoctypeDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartDoctypeDeclHandler(XML_Parser parser, XML_StartDoctypeDeclHandler start) { - startDoctypeDeclHandler = start; + if (parser != NULL) + startDoctypeDeclHandler = start; } void XMLCALL XML_SetEndDoctypeDeclHandler(XML_Parser parser, XML_EndDoctypeDeclHandler end) { - endDoctypeDeclHandler = end; + if (parser != NULL) + endDoctypeDeclHandler = end; } void XMLCALL XML_SetUnparsedEntityDeclHandler(XML_Parser parser, XML_UnparsedEntityDeclHandler handler) { - unparsedEntityDeclHandler = handler; + if (parser != NULL) + unparsedEntityDeclHandler = handler; } void XMLCALL XML_SetNotationDeclHandler(XML_Parser parser, XML_NotationDeclHandler handler) { - notationDeclHandler = handler; + if (parser != NULL) + notationDeclHandler = handler; } void XMLCALL @@ -1422,6 +1615,8 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start, XML_EndNamespaceDeclHandler end) { + if (parser == NULL) + return; startNamespaceDeclHandler = start; endNamespaceDeclHandler = end; } @@ -1429,32 +1624,38 @@ XML_SetNamespaceDeclHandler(XML_Parser parser, void XMLCALL XML_SetStartNamespaceDeclHandler(XML_Parser parser, XML_StartNamespaceDeclHandler start) { - startNamespaceDeclHandler = start; + if (parser != NULL) + startNamespaceDeclHandler = start; } void XMLCALL XML_SetEndNamespaceDeclHandler(XML_Parser parser, XML_EndNamespaceDeclHandler end) { - endNamespaceDeclHandler = end; + if (parser != NULL) + endNamespaceDeclHandler = end; } void XMLCALL XML_SetNotStandaloneHandler(XML_Parser parser, XML_NotStandaloneHandler handler) { - notStandaloneHandler = handler; + if (parser != NULL) + notStandaloneHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandler(XML_Parser parser, XML_ExternalEntityRefHandler handler) { - externalEntityRefHandler = handler; + if (parser != NULL) + externalEntityRefHandler = handler; } void XMLCALL XML_SetExternalEntityRefHandlerArg(XML_Parser parser, void *arg) { + if (parser == NULL) + return; if (arg) externalEntityRefHandlerArg = (XML_Parser)arg; else @@ -1465,7 +1666,8 @@ void XMLCALL XML_SetSkippedEntityHandler(XML_Parser parser, XML_SkippedEntityHandler handler) { - skippedEntityHandler = handler; + if (parser != NULL) + skippedEntityHandler = handler; } void XMLCALL @@ -1473,6 +1675,8 @@ XML_SetUnknownEncodingHandler(XML_Parser parser, XML_UnknownEncodingHandler handler, void *data) { + if (parser == NULL) + return; unknownEncodingHandler = handler; unknownEncodingHandlerData = data; } @@ -1481,33 +1685,39 @@ void XMLCALL XML_SetElementDeclHandler(XML_Parser parser, XML_ElementDeclHandler eldecl) { - elementDeclHandler = eldecl; + if (parser != NULL) + elementDeclHandler = eldecl; } void XMLCALL XML_SetAttlistDeclHandler(XML_Parser parser, XML_AttlistDeclHandler attdecl) { - attlistDeclHandler = attdecl; + if (parser != NULL) + attlistDeclHandler = attdecl; } void XMLCALL XML_SetEntityDeclHandler(XML_Parser parser, XML_EntityDeclHandler handler) { - entityDeclHandler = handler; + if (parser != NULL) + entityDeclHandler = handler; } void XMLCALL XML_SetXmlDeclHandler(XML_Parser parser, XML_XmlDeclHandler handler) { - xmlDeclHandler = handler; + if (parser != NULL) + xmlDeclHandler = handler; } int XMLCALL XML_SetParamEntityParsing(XML_Parser parser, enum XML_ParamEntityParsing peParsing) { + if (parser == NULL) + return 0; /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1523,6 +1733,10 @@ int XMLCALL XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { + if (parser == NULL) + return 0; + if (parser->m_parentParser) + return XML_SetHashSalt(parser->m_parentParser, hash_salt); /* block after XML_Parse()/XML_ParseBuffer() has been called */ if (ps_parsing == XML_PARSING || ps_parsing == XML_SUSPENDED) return 0; @@ -1533,6 +1747,10 @@ XML_SetHashSalt(XML_Parser parser, enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { + if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { + errorCode = XML_ERROR_INVALID_ARGUMENT; + return XML_STATUS_ERROR; + } switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1585,6 +1803,13 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) const char *end; int nLeftOver; enum XML_Status result; + /* Detect overflow (a+b > MAX <==> b > MAX-a) */ + if (len > ((XML_Size)-1) / 2 - parseEndByteIndex) { + errorCode = XML_ERROR_NO_MEMORY; + eventPtr = eventEndPtr = NULL; + processor = errorProcessor; + return XML_STATUS_ERROR; + } parseEndByteIndex += len; positionPtr = s; ps_finalBuffer = (XML_Bool)isFinal; @@ -1617,11 +1842,14 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) nLeftOver = s + len - end; if (nLeftOver) { if (buffer == NULL || nLeftOver > bufferLim - buffer) { - /* FIXME avoid integer overflow */ - char *temp; - temp = (buffer == NULL - ? (char *)MALLOC(len * 2) - : (char *)REALLOC(buffer, len * 2)); + /* avoid _signed_ integer overflow */ + char *temp = NULL; + const int bytesToAllocate = (int)((unsigned)len * 2U); + if (bytesToAllocate > 0) { + temp = (buffer == NULL + ? (char *)MALLOC(bytesToAllocate) + : (char *)REALLOC(buffer, bytesToAllocate)); + } if (temp == NULL) { errorCode = XML_ERROR_NO_MEMORY; eventPtr = eventEndPtr = NULL; @@ -1629,7 +1857,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) return XML_STATUS_ERROR; } buffer = temp; - bufferLim = buffer + len * 2; + bufferLim = buffer + bytesToAllocate; } memcpy(buffer, end, nLeftOver); } @@ -1659,6 +1887,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) const char *start; enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: errorCode = XML_ERROR_SUSPENDED; @@ -1712,6 +1942,8 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) void * XMLCALL XML_GetBuffer(XML_Parser parser, int len) { + if (parser == NULL) + return NULL; if (len < 0) { errorCode = XML_ERROR_NO_MEMORY; return NULL; @@ -1808,6 +2040,8 @@ XML_GetBuffer(XML_Parser parser, int len) enum XML_Status XMLCALL XML_StopParser(XML_Parser parser, XML_Bool resumable) { + if (parser == NULL) + return XML_STATUS_ERROR; switch (ps_parsing) { case XML_SUSPENDED: if (resumable) { @@ -1840,6 +2074,8 @@ XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; + if (parser == NULL) + return XML_STATUS_ERROR; if (ps_parsing != XML_SUSPENDED) { errorCode = XML_ERROR_NOT_SUSPENDED; return XML_STATUS_ERROR; @@ -1876,6 +2112,8 @@ XML_ResumeParser(XML_Parser parser) void XMLCALL XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) { + if (parser == NULL) + return; assert(status != NULL); *status = parser->m_parsingStatus; } @@ -1883,12 +2121,16 @@ XML_GetParsingStatus(XML_Parser parser, XML_ParsingStatus *status) enum XML_Error XMLCALL XML_GetErrorCode(XML_Parser parser) { + if (parser == NULL) + return XML_ERROR_INVALID_ARGUMENT; return errorCode; } XML_Index XMLCALL XML_GetCurrentByteIndex(XML_Parser parser) { + if (parser == NULL) + return -1; if (eventPtr) return (XML_Index)(parseEndByteIndex - (parseEndPtr - eventPtr)); return -1; @@ -1897,6 +2139,8 @@ XML_GetCurrentByteIndex(XML_Parser parser) int XMLCALL XML_GetCurrentByteCount(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventEndPtr && eventPtr) return (int)(eventEndPtr - eventPtr); return 0; @@ -1906,11 +2150,19 @@ const char * XMLCALL XML_GetInputContext(XML_Parser parser, int *offset, int *size) { #ifdef XML_CONTEXT_BYTES + if (parser == NULL) + return NULL; if (eventPtr && buffer) { - *offset = (int)(eventPtr - buffer); - *size = (int)(bufferEnd - buffer); + if (offset != NULL) + *offset = (int)(eventPtr - buffer); + if (size != NULL) + *size = (int)(bufferEnd - buffer); return buffer; } +#else + (void)parser; + (void)offset; + (void)size; #endif /* defined XML_CONTEXT_BYTES */ return (char *) 0; } @@ -1918,6 +2170,8 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) XML_Size XMLCALL XML_GetCurrentLineNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1928,6 +2182,8 @@ XML_GetCurrentLineNumber(XML_Parser parser) XML_Size XMLCALL XML_GetCurrentColumnNumber(XML_Parser parser) { + if (parser == NULL) + return 0; if (eventPtr && eventPtr >= positionPtr) { XmlUpdatePosition(encoding, positionPtr, eventPtr, &position); positionPtr = eventPtr; @@ -1938,30 +2194,38 @@ XML_GetCurrentColumnNumber(XML_Parser parser) void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { - FREE(model); + if (parser != NULL) + FREE(model); } void * XMLCALL XML_MemMalloc(XML_Parser parser, size_t size) { + if (parser == NULL) + return NULL; return MALLOC(size); } void * XMLCALL XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) { + if (parser == NULL) + return NULL; return REALLOC(ptr, size); } void XMLCALL XML_MemFree(XML_Parser parser, void *ptr) { - FREE(ptr); + if (parser != NULL) + FREE(ptr); } void XMLCALL XML_DefaultCurrent(XML_Parser parser) { + if (parser == NULL) + return; if (defaultHandler) { if (openInternalEntities) reportDefault(parser, @@ -2468,7 +2732,7 @@ doContent(XML_Parser parser, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); convLen = (int)(toPtr - (XML_Char *)tag->buf); - if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { + if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } @@ -2511,8 +2775,10 @@ doContent(XML_Parser parser, return XML_ERROR_NO_MEMORY; poolFinish(&tempPool); result = storeAtts(parser, enc, s, &name, &bindings); - if (result) + if (result != XML_ERROR_NONE) { + freeBindings(parser, bindings); return result; + } poolFinish(&tempPool); if (startElementHandler) { startElementHandler(handlerArg, name.str, (const XML_Char **)atts); @@ -2527,15 +2793,7 @@ doContent(XML_Parser parser, if (noElmHandlers && defaultHandler) reportDefault(parser, enc, s, next); poolClear(&tempPool); - while (bindings) { - BINDING *b = bindings; - if (endNamespaceDeclHandler) - endNamespaceDeclHandler(handlerArg, b->prefix->name); - bindings = bindings->nextTagBinding; - b->nextTagBinding = freeBindingList; - freeBindingList = b; - b->prefix->binding = b->prevPrefixBinding; - } + freeBindings(parser, bindings); } if (tagLevel == 0) return epilogProcessor(parser, next, end, nextPtr); @@ -2733,6 +2991,29 @@ doContent(XML_Parser parser, /* not reached */ } +/* This function does not call free() on the allocated memory, merely + * moving it to the parser's freeBindingList where it can be freed or + * reused as appropriate. + */ +static void +freeBindings(XML_Parser parser, BINDING *bindings) +{ + while (bindings) { + BINDING *b = bindings; + + /* startNamespaceDeclHandler will have been called for this + * binding in addBindings(), so call the end handler now. + */ + if (endNamespaceDeclHandler) + endNamespaceDeclHandler(handlerArg, b->prefix->name); + + bindings = bindings->nextTagBinding; + b->nextTagBinding = freeBindingList; + freeBindingList = b; + b->prefix->binding = b->prevPrefixBinding; + } +} + /* Precondition: all arguments must be non-NULL; Purpose: - normalize attributes @@ -2957,7 +3238,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, if (s[-1] == 2) { /* prefixed */ ATTRIBUTE_ID *id; const BINDING *b; - unsigned long uriHash = hash_secret_salt; + unsigned long uriHash; + struct siphash sip_state; + struct sipkey sip_key; + + copy_salt_to_sipkey(parser, &sip_key); + sip24_init(&sip_state, &sip_key); + ((XML_Char *)s)[-1] = 0; /* clear flag */ id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (!id || !id->prefix) @@ -2966,22 +3253,26 @@ storeAtts(XML_Parser parser, const ENCODING *enc, if (!b) return XML_ERROR_UNBOUND_PREFIX; - /* as we expand the name we also calculate its hash value */ for (j = 0; j < b->uriLen; j++) { const XML_Char c = b->uri[j]; if (!poolAppendChar(&tempPool, c)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } + + sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); + while (*s++ != XML_T(ASCII_COLON)) ; + + sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); + do { /* copies null terminator */ - const XML_Char c = *s; if (!poolAppendChar(&tempPool, *s)) return XML_ERROR_NO_MEMORY; - uriHash = CHAR_HASH(uriHash, c); } while (*s++); + uriHash = (unsigned long)sip24_final(&sip_state); + { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). */ @@ -3695,6 +3986,14 @@ entityValueInitProcessor(XML_Parser parser, *nextPtr = next; return XML_ERROR_NONE; } + /* If we get this token, we have the start of what might be a + normal tag, but not a declaration (i.e. it doesn't begin with + "internalEventEndPtr = NULL; textStart = (char *)entity->textPtr; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -4916,6 +5217,8 @@ internalEntityProcessor(XML_Parser parser, entity = openEntity->entity; textStart = ((char *)entity->textPtr) + entity->processed; textEnd = (char *)(entity->textPtr + entity->textLen); + /* Set a safe default value in case 'next' does not get set */ + next = textStart; #ifdef XML_DTD if (entity->is_param) { @@ -5876,7 +6179,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, const XML_Memory_H newE->defaultAtts = (DEFAULT_ATTRIBUTE *) ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (!newE->defaultAtts) { - ms->free_fcn(newE); return 0; } } @@ -6011,13 +6313,32 @@ keyeq(KEY s1, KEY s2) return XML_FALSE; } +static size_t +keylen(KEY s) +{ + size_t len = 0; + for (; *s; s++, len++); + return len; +} + +static void +copy_salt_to_sipkey(XML_Parser parser, struct sipkey * key) +{ + key->k[0] = 0; + key->k[1] = get_hash_secret_salt(parser); +} + static unsigned long FASTCALL hash(XML_Parser parser, KEY s) { - unsigned long h = hash_secret_salt; - while (*s) - h = CHAR_HASH(h, *s++); - return h; + struct siphash state; + struct sipkey key; + (void)sip_tobin; + (void)sip24_valid; + copy_salt_to_sipkey(parser, &key); + sip24_init(&state, &key); + sip24_update(&state, s, keylen(s) * sizeof(XML_Char)); + return (unsigned long)sip24_final(&state); } static NAMED * @@ -6260,6 +6581,35 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, return pool->start; } +static size_t +poolBytesToAllocateFor(int blockSize) +{ + /* Unprotected math would be: + ** return offsetof(BLOCK, s) + blockSize * sizeof(XML_Char); + ** + ** Detect overflow, avoiding _signed_ overflow undefined behavior + ** For a + b * c we check b * c in isolation first, so that addition of a + ** on top has no chance of making us accept a small non-negative number + */ + const size_t stretch = sizeof(XML_Char); /* can be 4 bytes */ + + if (blockSize <= 0) + return 0; + + if (blockSize > (int)(INT_MAX / stretch)) + return 0; + + { + const int stretchedBlockSize = blockSize * (int)stretch; + const int bytesToAllocate = (int)( + offsetof(BLOCK, s) + (unsigned)stretchedBlockSize); + if (bytesToAllocate < 0) + return 0; + + return (size_t)bytesToAllocate; + } +} + static XML_Bool FASTCALL poolGrow(STRING_POOL *pool) { @@ -6289,14 +6639,17 @@ poolGrow(STRING_POOL *pool) if (pool->blocks && pool->start == pool->blocks->s) { BLOCK *temp; int blockSize = (int)((unsigned)(pool->end - pool->start)*2U); + size_t bytesToAllocate; if (blockSize < 0) return XML_FALSE; + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + temp = (BLOCK *) - pool->mem->realloc_fcn(pool->blocks, - (offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char))); + pool->mem->realloc_fcn(pool->blocks, (unsigned)bytesToAllocate); if (temp == NULL) return XML_FALSE; pool->blocks = temp; @@ -6308,16 +6661,26 @@ poolGrow(STRING_POOL *pool) else { BLOCK *tem; int blockSize = (int)(pool->end - pool->start); + size_t bytesToAllocate; if (blockSize < 0) return XML_FALSE; if (blockSize < INIT_BLOCK_SIZE) blockSize = INIT_BLOCK_SIZE; - else + else { + /* Detect overflow, avoiding _signed_ overflow undefined behavior */ + if ((int)((unsigned)blockSize * 2U) < 0) { + return XML_FALSE; + } blockSize *= 2; - tem = (BLOCK *)pool->mem->malloc_fcn(offsetof(BLOCK, s) - + blockSize * sizeof(XML_Char)); + } + + bytesToAllocate = poolBytesToAllocateFor(blockSize); + if (bytesToAllocate == 0) + return XML_FALSE; + + tem = (BLOCK *)pool->mem->malloc_fcn(bytesToAllocate); if (!tem) return XML_FALSE; tem->size = blockSize; diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index fcd0dc6..a7c5630 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -4,19 +4,13 @@ #include -#ifdef WIN32 +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef WIN32 */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index a29d9e2..cdf0720 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -4,19 +4,13 @@ #include -#ifdef WIN32 +#ifdef _WIN32 #include "winconfig.h" -#elif defined(MACOS_CLASSIC) -#include "macconfig.h" -#elif defined(__amigaos__) -#include "amigaconfig.h" -#elif defined(__WATCOMC__) -#include "watcomconfig.h" #else #ifdef HAVE_EXPAT_CONFIG_H #include #endif -#endif /* ndef WIN32 */ +#endif /* ndef _WIN32 */ #include "expat_external.h" #include "internal.h" @@ -369,24 +363,24 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), const char **fromP, const char *fromLim, char **toP, const char *toLim) { - enum XML_Convert_Result res = XML_CONVERT_COMPLETED; char *to; const char *from; - if (fromLim - *fromP > toLim - *toP) { - /* Avoid copying partial characters. */ - res = XML_CONVERT_OUTPUT_EXHAUSTED; - fromLim = *fromP + (toLim - *toP); - align_limit_to_full_utf8_characters(*fromP, &fromLim); - } + const char *fromLimInitial = fromLim; + + /* Avoid copying partial characters. */ + align_limit_to_full_utf8_characters(*fromP, &fromLim); + for (to = *toP, from = *fromP; (from < fromLim) && (to < toLim); from++, to++) *to = *from; *fromP = from; *toP = to; - if ((to == toLim) && (from < fromLim)) + if (fromLim < fromLimInitial) + return XML_CONVERT_INPUT_INCOMPLETE; + else if ((to == toLim) && (from < fromLim)) return XML_CONVERT_OUTPUT_EXHAUSTED; else - return res; + return XML_CONVERT_COMPLETED; } static enum XML_Convert_Result PTRCALL @@ -402,7 +396,7 @@ utf8_toUtf16(const ENCODING *enc, case BT_LEAD2: if (fromLim - from < 2) { res = XML_CONVERT_INPUT_INCOMPLETE; - break; + goto after; } *to++ = (unsigned short)(((from[0] & 0x1f) << 6) | (from[1] & 0x3f)); from += 2; @@ -410,7 +404,7 @@ utf8_toUtf16(const ENCODING *enc, case BT_LEAD3: if (fromLim - from < 3) { res = XML_CONVERT_INPUT_INCOMPLETE; - break; + goto after; } *to++ = (unsigned short)(((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f)); @@ -441,6 +435,8 @@ utf8_toUtf16(const ENCODING *enc, break; } } + if (from < fromLim) + res = XML_CONVERT_OUTPUT_EXHAUSTED; after: *fromP = from; *toP = to; -- cgit v0.12 From 5777e79ecbd1f2adf36456e09f210608ee221691 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 23 Jun 2017 15:21:33 +0200 Subject: [3.6] bpo-30726: PCbuild _elementtree: remove duplicate defines (#2348) (#2349) * bpo-30726: PCbuild _elementtree: remove duplicate defines (#2348) bpo-30726, bpo-29591: libexpat 2.2.1 of Modules/expat/ now uses a winconfig.h configuration file which already defines: * XML_NS * XML_DTD * BYTEORDER=1234 * XML_CONTEXT_BYTES=1024 * HAVE_MEMMOVE Remove these defines from PCbuild/_elementtree.vcxproj to prevent compiler warnings. Co-Authored-By: Jeremy Kloth (cherry picked from commit c8fb58bd7917151e63398587a7fc2126db7c26de) * bpo-30726: Fix elementtree warnings on Windows due to expat upgrade (#2319) * bpo-30726: Fix elementtree warnings on Windows Caused by usage of `getenv` which should be safe. And a few integer truncations which should also be ok. * bpo-30726: Don't ignore libexpat warnings which haypo intends to fix upstream (cherry picked from commit 87c65550730a8f85ce339ba197bce4fb7e836619) (cherry picked from commit d32a05953130fb5cc2d3c0c9fcb20ad0859353f3) --- PCbuild/_elementtree.vcxproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 725b5a9..639ba47 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -62,7 +62,7 @@ ..\Modules\expat;%(AdditionalIncludeDirectories) - XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;USE_PYEXPAT_CAPI;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;USE_PYEXPAT_CAPI;XML_STATIC;%(PreprocessorDefinitions) 0x1D100000 @@ -103,4 +103,4 @@ - \ No newline at end of file + -- cgit v0.12 From b6012f982fabed6029d7e2db2a509a8b28b4f6e1 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Tue, 4 Jul 2017 21:38:47 +0300 Subject: [3.6] bpo-30726: Also fix pyexpat.vcxproj (GH-2375) (#2570) (cherry picked from commit 7526cadd64566725ffc56071a7208828a46ddbd8) (cherry picked from commit d02c8416fe1b29b3322004b73133bf6c8a2e353a) --- PCbuild/pyexpat.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index 99c7286..a2763f0 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -59,7 +59,7 @@ $(PySourcePath)Modules\expat;%(AdditionalIncludeDirectories) - PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_NS;XML_DTD;BYTEORDER=1234;XML_CONTEXT_BYTES=1024;XML_STATIC;HAVE_MEMMOVE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;PYEXPAT_EXPORTS;HAVE_EXPAT_H;XML_STATIC;%(PreprocessorDefinitions) -- cgit v0.12 From c794b643c9172d69afa46f85982befd82511d9df Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 28 Jun 2017 02:28:51 +0200 Subject: bpo-30704, bpo-30604: Fix memleak in code_dealloc() (#2455) (#2456) Free also co_extra->ce_extras, not only co_extra. (cherry picked from commit 23e7944eba1968bb8432fdc4cc96d4fdd2c1a230) (cherry picked from commit 26daad4ee14693381d84a5235709d22aed1c22ed) --- Objects/codeobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d38f185..6de697a 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -422,7 +422,8 @@ code_dealloc(PyCodeObject *co) } } - PyMem_FREE(co->co_extra); + PyMem_Free(co_extra->ce_extras); + PyMem_Free(co_extra); } Py_XDECREF(co->co_code); -- cgit v0.12 From a9b16cff35811f88cdfeb4f50758140dfff36ebc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 23 Jun 2017 20:17:38 +0300 Subject: [3.6] bpo-30730: Prevent environment variables injection in subprocess on Windows. (GH-2325) (#2360) Prevent passing other invalid environment variables and command arguments.. (cherry picked from commit d174d24a5d37d1516b885dc7c82f71ecd5930700) (cherry picked from commit e7135751b8e48af80665e40ac8fa6d0073e5affe) --- Lib/subprocess.py | 8 ++++++-- Lib/test/test_subprocess.py | 40 ++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/_winapi.c | 26 +++++++++++++++++++++----- Objects/abstract.c | 4 ++-- 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index e626a8a..b790cbd 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1239,8 +1239,12 @@ class Popen(object): # and pass it to fork_exec() if env is not None: - env_list = [os.fsencode(k) + b'=' + os.fsencode(v) - for k, v in env.items()] + env_list = [] + for k, v in env.items(): + k = os.fsencode(k) + if b'=' in k: + raise ValueError("illegal environment variable name") + env_list.append(k + b'=' + os.fsencode(v)) else: env_list = None # Use execv instead of execve. executable = os.fsencode(executable) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d8d6d82..9a9e3ca 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -644,6 +644,46 @@ class ProcessTestCase(BaseTestCase): # environment b"['__CF_USER_TEXT_ENCODING']")) + def test_invalid_cmd(self): + # null character in the command name + cmd = sys.executable + '\0' + with self.assertRaises(ValueError): + subprocess.Popen([cmd, "-c", "pass"]) + + # null character in the command argument + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass#\0"]) + + def test_invalid_env(self): + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + with subprocess.Popen([sys.executable, "-c", + 'import sys, os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange=lemon") + def test_communicate_stdin(self): p = subprocess.Popen([sys.executable, "-c", 'import sys;' diff --git a/Misc/NEWS b/Misc/NEWS index 1f6161a..136c84f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- [Security] bpo-30730: Prevent environment variables injection in subprocess on + Windows. Prevent passing other environment variables and command arguments. + - [Security] bpo-30694: Upgrade expat copy from 2.2.0 to 2.2.1 to get fixes of multiple security vulnerabilities including: CVE-2017-9233 (External entity infinite loop DoS), CVE-2016-9063 (Integer overflow, re-fix), diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 248f458..71426bc 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -744,6 +744,20 @@ getenvironment(PyObject* environment) "environment can only contain strings"); goto error; } + if (PyUnicode_FindChar(key, '\0', 0, PyUnicode_GET_LENGTH(key), 1) != -1 || + PyUnicode_FindChar(value, '\0', 0, PyUnicode_GET_LENGTH(value), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto error; + } + /* Search from index 1 because on Windows starting '=' is allowed for + defining hidden environment variables. */ + if (PyUnicode_GET_LENGTH(key) == 0 || + PyUnicode_FindChar(key, '=', 1, PyUnicode_GET_LENGTH(key), 1) != -1) + { + PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); + goto error; + } if (totalsize > PY_SSIZE_T_MAX - PyUnicode_GET_LENGTH(key) - 1) { PyErr_SetString(PyExc_OverflowError, "environment too long"); goto error; @@ -830,7 +844,8 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, PROCESS_INFORMATION pi; STARTUPINFOW si; PyObject* environment; - wchar_t *wenvironment; + const wchar_t *wenvironment; + Py_ssize_t wenvironment_size; ZeroMemory(&si, sizeof(si)); si.cb = sizeof(si); @@ -846,12 +861,13 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, if (env_mapping != Py_None) { environment = getenvironment(env_mapping); - if (! environment) + if (environment == NULL) { return NULL; + } + /* contains embedded null characters */ wenvironment = PyUnicode_AsUnicode(environment); - if (wenvironment == NULL) - { - Py_XDECREF(environment); + if (wenvironment == NULL) { + Py_DECREF(environment); return NULL; } } diff --git a/Objects/abstract.c b/Objects/abstract.c index 3585b34..38de774 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -3193,8 +3193,8 @@ _PySequence_BytesToCharpArray(PyObject* self) array[i] = NULL; goto fail; } - data = PyBytes_AsString(item); - if (data == NULL) { + /* check for embedded null bytes */ + if (PyBytes_AsStringAndSize(item, &data, NULL) < 0) { /* NULL terminate before freeing. */ array[i] = NULL; goto fail; -- cgit v0.12 From d1d65015fca44b8d1f0b1df78694310270f03a6d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 24 Jun 2017 16:28:26 +0300 Subject: [3.6] bpo-30745: Fix compiler warnings introduced in bpo-30730. (GH-2376) (#2378) (cherry picked from commit 0ee32c1) (cherry picked from commit 0e1f9e8d3ea82262cbb9a403b70a884da5e6a6ac) --- Modules/_winapi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 71426bc..1606f0d 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -844,8 +844,7 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, PROCESS_INFORMATION pi; STARTUPINFOW si; PyObject* environment; - const wchar_t *wenvironment; - Py_ssize_t wenvironment_size; + wchar_t *wenvironment; ZeroMemory(&si, sizeof(si)); si.cb = sizeof(si); -- cgit v0.12 From bdabd7666032ce356d550da21c35e4bee5b3448c Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 7 Jul 2017 01:31:43 -0400 Subject: bpo-30797, bpo-30694: Avoid _GNU_SOURCE redefined warning in xmlparse.c (#2615) (cherry picked from commit 05b72ede95521b2d897cb4c7b034139b5437c592) --- Modules/expat/xmlparse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 76f078e..daec151 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -4,7 +4,7 @@ 77fea421d361dca90041d0040ecf1dca651167fadf2af79e990e35168d70d933 (2.2.1+) */ -#define _GNU_SOURCE /* syscall prototype */ +#define _GNU_SOURCE 1 /* syscall prototype */ #include #include /* memset(), memcpy() */ -- cgit v0.12 From 891331134516784ff1c5ad8d1720fdc2d056a15f Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 7 Jul 2017 03:16:00 -0400 Subject: Version bump to 3.6.2rc2 --- Include/patchlevel.h | 4 ++-- Misc/NEWS | 6 +++--- README.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 9c22385..91b8ece 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 2 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.6.2rc1" +#define PY_VERSION "3.6.2rc2" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Misc/NEWS b/Misc/NEWS index 136c84f..a76d2a1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1,11 +1,11 @@ -+++++++++++ ++++++++++++ Python News +++++++++++ -What's New in Python 3.6.3 release candidate 1? +What's New in Python 3.6.2 release candidate 2? =============================================== -*Release date: XXXX-XX-XX* +*Release date: 2017-07-07* Core and Builtins ----------------- diff --git a/README.rst b/README.rst index 71eb257..4b56475 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.6.2 release candidate 1 +This is Python version 3.6.2 release candidate 2 ================================================ .. image:: https://travis-ci.org/python/cpython.svg?branch=3.6 -- cgit v0.12