bpo-34977: Add Windows App Store package (GH-10245)

7 files changed, 196 insertions, 258 deletions

diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat
index 4178981..45e189b 100644
--- a/Tools/msi/buildrelease.bat
+++ b/Tools/msi/buildrelease.bat
@@ -37,6 +37,7 @@ set BUILDX64=
 set TARGET=Rebuild
 set TESTTARGETDIR=
 set PGO=-m test -q --pgo
+set BUILDMSI=1
 set BUILDNUGET=1
 set BUILDZIP=1
 
@@ -61,6 +62,7 @@ if "%1" EQU "--pgo" (set PGO=%~2) && shift && shift && goto CheckOpts
 if "%1" EQU "--skip-pgo" (set PGO=) && shift && goto CheckOpts
 if "%1" EQU "--skip-nuget" (set BUILDNUGET=) && shift && goto CheckOpts
 if "%1" EQU "--skip-zip" (set BUILDZIP=) && shift && goto CheckOpts
+if "%1" EQU "--skip-msi" (set BUILDMSI=) && shift && goto CheckOpts
 
 if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1
 
@@ -174,10 +176,12 @@ if "%OUTDIR_PLAT%" EQU "win32" (
 )
 
 set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI%
-%MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true
-if errorlevel 1 exit /B
-%MSBUILD% "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false
-if errorlevel 1 exit /B
+if defined BUILDMSI (
+    %MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true
+    if errorlevel 1 exit /B
+    %MSBUILD% "%D%bundle\releaseweb.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=false
+    if errorlevel 1 exit /B
+)
 
 if defined BUILDZIP (
     %MSBUILD% "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us"
@@ -214,6 +218,7 @@ echo    --skip-build (-B)   Do not build Python (just do the installers)
 echo    --skip-doc (-D)     Do not build documentation
 echo    --pgo               Specify PGO command for x64 installers
 echo    --skip-pgo          Build x64 installers without using PGO
+echo    --skip-msi          Do not build executable/MSI packages
 echo    --skip-nuget        Do not build Nuget packages
 echo    --skip-zip          Do not build embeddable package
 echo    --download          Specify the full download URL for MSIs
diff --git a/Tools/msi/make_appx.ps1 b/Tools/msi/make_appx.ps1
new file mode 100644
index 0000000..b3f190e
--- /dev/null
+++ b/Tools/msi/make_appx.ps1
@@ -0,0 +1,71 @@
+<#
+.Synopsis
+    Compiles and signs an APPX package
+.Description
+    Given the file listing, ensures all the contents are signed
+    and builds and signs the final package.
+.Parameter mapfile
+    The location on disk of the text mapping file.
+.Parameter msix
+    The path and name to store the APPX/MSIX.
+.Parameter sign
+    When set, signs the APPX/MSIX. Packages to be published to
+    the store should not be signed. 
+.Parameter description
+    Description to embed in the signature (optional).
+.Parameter certname
+    The name of the certificate to sign with (optional).
+.Parameter certsha1
+    The SHA1 hash of the certificate to sign with (optional).
+#>
+param(
+    [Parameter(Mandatory=$true)][string]$layout,
+    [Parameter(Mandatory=$true)][string]$msix,
+    [switch]$sign,
+    [string]$description,
+    [string]$certname,
+    [string]$certsha1,
+    [string]$certfile
+)
+
+$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent;
+Import-Module $tools\sdktools.psm1 -WarningAction SilentlyContinue -Force
+
+Set-Alias makeappx (Find-Tool "makeappx.exe") -Scope Script
+Set-Alias makepri (Find-Tool "makepri.exe") -Scope Script
+
+$msixdir = Split-Path $msix -Parent
+if ($msixdir) {
+    $msixdir = (mkdir -Force $msixdir).FullName
+} else {
+    $msixdir = Get-Location
+}
+$msix = Join-Path $msixdir (Split-Path $msix -Leaf)
+
+pushd $layout
+try {
+    if (Test-Path resources.pri) {
+        del resources.pri
+    }
+    $name = ([xml](gc AppxManifest.xml)).Package.Identity.Name
+    makepri new /pr . /mn AppxManifest.xml /in $name /cf _resources.xml /of _resources.pri /mf appx /o
+    if (-not $? -or -not (Test-Path _resources.map.txt)) {
+        throw "makepri step failed"
+    }
+    $lines = gc _resources.map.txt
+    $lines | ?{ -not ($_ -match '"_resources[\w\.]+?"') } | Out-File _resources.map.txt -Encoding utf8
+    makeappx pack /f _resources.map.txt /m AppxManifest.xml /o /p $msix
+    if (-not $?) {
+        throw "makeappx step failed"
+    }
+} finally {
+    popd
+}
+
+if ($sign) {
+    Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files $msix
+
+    if (-not $?) {
+        throw "Package signing failed"
+    }
+}
diff --git a/Tools/msi/make_cat.ps1 b/Tools/msi/make_cat.ps1
new file mode 100644
index 0000000..7074143
--- /dev/null
+++ b/Tools/msi/make_cat.ps1
@@ -0,0 +1,34 @@
+<#
+.Synopsis
+    Compiles and signs a catalog file.
+.Description
+    Given the CDF definition file, builds and signs a catalog.
+.Parameter catalog
+    The path to the catalog definition file to compile and
+    sign. It is assumed that the .cat file will be the same
+    name with a new extension.
+.Parameter description
+    The description to add to the signature (optional).
+.Parameter certname
+    The name of the certificate to sign with (optional).
+.Parameter certsha1
+    The SHA1 hash of the certificate to sign with (optional).
+#>
+param(
+    [Parameter(Mandatory=$true)][string]$catalog,
+    [string]$description,
+    [string]$certname,
+    [string]$certsha1,
+    [string]$certfile
+)
+
+$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent;
+Import-Module $tools\sdktools.psm1 -WarningAction SilentlyContinue -Force
+
+Set-Alias MakeCat (Find-Tool "makecat.exe") -Scope Script
+
+MakeCat $catalog
+if (-not $?) {
+    throw "Catalog compilation failed"
+}
+Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files @($catalog -replace 'cdf$', 'cat')
diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj
index 2141117..125a434 100644
--- a/Tools/msi/make_zip.proj
+++ b/Tools/msi/make_zip.proj
@@ -15,11 +15,12 @@
         <TargetExt>.zip</TargetExt>
         <TargetPath>$(OutputPath)\$(TargetName)$(TargetExt)</TargetPath>
         <CleanCommand>rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)"</CleanCommand>
-        <Arguments>"$(PythonExe)" "$(MSBuildThisFileDirectory)\make_zip.py"</Arguments>
-        <Arguments>$(Arguments) -e -o "$(TargetPath)" -t "$(IntermediateOutputPath)\zip_$(ArchName)" -b "$(BuildPath.TrimEnd(`\`))"</Arguments>
-        <Environment>set DOC_FILENAME=python$(PythonVersion).chm</Environment>
+        <Arguments>"$(PythonExe)" "$(PySourcePath)PC\layout"</Arguments>
+        <Arguments>$(Arguments) -b "$(BuildPath.TrimEnd(`\`))" -s "$(PySourcePath.TrimEnd(`\`))"</Arguments>
+        <Arguments>$(Arguments) -t "$(IntermediateOutputPath)\zip_$(ArchName)"</Arguments>
+        <Arguments>$(Arguments) --zip "$(TargetPath)"</Arguments>
+        <Arguments>$(Arguments) --precompile --zip-lib --include-underpth --include-stable --flat-dlls</Arguments>
         <Environment>$(Environment)%0D%0Aset PYTHONPATH=$(PySourcePath)Lib</Environment>
-        <Environment Condition="Exists($(CRTRedist))">$(Environment)%0D%0Aset VCREDIST_PATH=$(CRTRedist)\$(Platform)</Environment>
     </PropertyGroup>
 
     <Target Name="_Build">
diff --git a/Tools/msi/make_zip.py b/Tools/msi/make_zip.py
deleted file mode 100644
index 58f3b15..0000000
--- a/Tools/msi/make_zip.py
+++ /dev/null
@@ -1,250 +0,0 @@
-import argparse
-import py_compile
-import re
-import sys
-import shutil
-import stat
-import os
-import tempfile
-
-from itertools import chain
-from pathlib import Path
-from zipfile import ZipFile, ZIP_DEFLATED
-
-
-TKTCL_RE = re.compile(r'^(_?tk|tcl).+\.(pyd|dll)', re.IGNORECASE)
-DEBUG_RE = re.compile(r'_d\.(pyd|dll|exe|pdb|lib)$', re.IGNORECASE)
-PYTHON_DLL_RE = re.compile(r'python\d\d?\.dll$', re.IGNORECASE)
-
-DEBUG_FILES = {
-    '_ctypes_test',
-    '_testbuffer',
-    '_testcapi',
-    '_testconsole',
-    '_testimportmultiple',
-    '_testmultiphase',
-    'xxlimited',
-    'python3_dstub',
-}
-
-EXCLUDE_FROM_LIBRARY = {
-    '__pycache__',
-    'idlelib',
-    'pydoc_data',
-    'site-packages',
-    'tkinter',
-    'turtledemo',
-}
-
-EXCLUDE_FROM_EMBEDDABLE_LIBRARY = {
-    'ensurepip',
-    'venv',
-}
-
-EXCLUDE_FILE_FROM_LIBRARY = {
-    'bdist_wininst.py',
-}
-
-EXCLUDE_FILE_FROM_LIBS = {
-    'liblzma',
-    'python3stub',
-}
-
-EXCLUDED_FILES = {
-    'pyshellext',
-}
-
-def is_not_debug(p):
-    if DEBUG_RE.search(p.name):
-        return False
-
-    if TKTCL_RE.search(p.name):
-        return False
-
-    return p.stem.lower() not in DEBUG_FILES and p.stem.lower() not in EXCLUDED_FILES
-
-def is_not_debug_or_python(p):
-    return is_not_debug(p) and not PYTHON_DLL_RE.search(p.name)
-
-def include_in_lib(p):
-    name = p.name.lower()
-    if p.is_dir():
-        if name in EXCLUDE_FROM_LIBRARY:
-            return False
-        if name == 'test' and p.parts[-2].lower() == 'lib':
-            return False
-        if name in {'test', 'tests'} and p.parts[-3].lower() == 'lib':
-            return False
-        return True
-
-    if name in EXCLUDE_FILE_FROM_LIBRARY:
-        return False
-
-    suffix = p.suffix.lower()
-    return suffix not in {'.pyc', '.pyo', '.exe'}
-
-def include_in_embeddable_lib(p):
-    if p.is_dir() and p.name.lower() in EXCLUDE_FROM_EMBEDDABLE_LIBRARY:
-        return False
-
-    return include_in_lib(p)
-
-def include_in_libs(p):
-    if not is_not_debug(p):
-        return False
-
-    return p.stem.lower() not in EXCLUDE_FILE_FROM_LIBS
-
-def include_in_tools(p):
-    if p.is_dir() and p.name.lower() in {'scripts', 'i18n', 'pynche', 'demo', 'parser'}:
-        return True
-
-    return p.suffix.lower() in {'.py', '.pyw', '.txt'}
-
-BASE_NAME = 'python{0.major}{0.minor}'.format(sys.version_info)
-
-FULL_LAYOUT = [
-    ('/', '$build', 'python.exe', is_not_debug),
-    ('/', '$build', 'pythonw.exe', is_not_debug),
-    ('/', '$build', 'python{}.dll'.format(sys.version_info.major), is_not_debug),
-    ('/', '$build', '{}.dll'.format(BASE_NAME), is_not_debug),
-    ('DLLs/', '$build', '*.pyd', is_not_debug),
-    ('DLLs/', '$build', '*.dll', is_not_debug_or_python),
-    ('include/', 'include', '*.h', None),
-    ('include/', 'PC', 'pyconfig.h', None),
-    ('Lib/', 'Lib', '**/*', include_in_lib),
-    ('libs/', '$build', '*.lib', include_in_libs),
-    ('Tools/', 'Tools', '**/*', include_in_tools),
-]
-
-EMBED_LAYOUT = [
-    ('/', '$build', 'python*.exe', is_not_debug),
-    ('/', '$build', '*.pyd', is_not_debug),
-    ('/', '$build', '*.dll', is_not_debug),
-    ('{}.zip'.format(BASE_NAME), 'Lib', '**/*', include_in_embeddable_lib),
-]
-
-if os.getenv('DOC_FILENAME'):
-    FULL_LAYOUT.append(('Doc/', 'Doc/build/htmlhelp', os.getenv('DOC_FILENAME'), None))
-if os.getenv('VCREDIST_PATH'):
-    FULL_LAYOUT.append(('/', os.getenv('VCREDIST_PATH'), 'vcruntime*.dll', None))
-    EMBED_LAYOUT.append(('/', os.getenv('VCREDIST_PATH'), 'vcruntime*.dll', None))
-
-def copy_to_layout(target, rel_sources):
-    count = 0
-
-    if target.suffix.lower() == '.zip':
-        if target.exists():
-            target.unlink()
-
-        with ZipFile(str(target), 'w', ZIP_DEFLATED) as f:
-            with tempfile.TemporaryDirectory() as tmpdir:
-                for s, rel in rel_sources:
-                    if rel.suffix.lower() == '.py':
-                        pyc = Path(tmpdir) / rel.with_suffix('.pyc').name
-                        try:
-                            py_compile.compile(str(s), str(pyc), str(rel), doraise=True, optimize=2)
-                        except py_compile.PyCompileError:
-                            f.write(str(s), str(rel))
-                        else:
-                            f.write(str(pyc), str(rel.with_suffix('.pyc')))
-                    else:
-                        f.write(str(s), str(rel))
-                    count += 1
-
-    else:
-        for s, rel in rel_sources:
-            dest = target / rel
-            try:
-                dest.parent.mkdir(parents=True)
-            except FileExistsError:
-                pass
-            if dest.is_file():
-                dest.chmod(stat.S_IWRITE)
-            shutil.copy(str(s), str(dest))
-            if dest.is_file():
-                dest.chmod(stat.S_IWRITE)
-            count += 1
-
-    return count
-
-def rglob(root, pattern, condition):
-    dirs = [root]
-    recurse = pattern[:3] in {'**/', '**\\'}
-    while dirs:
-        d = dirs.pop(0)
-        for f in d.glob(pattern[3:] if recurse else pattern):
-            if recurse and f.is_dir() and (not condition or condition(f)):
-                dirs.append(f)
-            elif f.is_file() and (not condition or condition(f)):
-                yield f, f.relative_to(root)
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-s', '--source', metavar='dir', help='The directory containing the repository root', type=Path)
-    parser.add_argument('-o', '--out', metavar='file', help='The name of the output archive', type=Path, default=None)
-    parser.add_argument('-t', '--temp', metavar='dir', help='A directory to temporarily extract files into', type=Path, default=None)
-    parser.add_argument('-e', '--embed', help='Create an embedding layout', action='store_true', default=False)
-    parser.add_argument('-b', '--build', help='Specify the build directory', type=Path, default=None)
-    ns = parser.parse_args()
-
-    source = ns.source or (Path(__file__).resolve().parent.parent.parent)
-    out = ns.out
-    build = ns.build or Path(sys.exec_prefix)
-    assert isinstance(source, Path)
-    assert not out or isinstance(out, Path)
-    assert isinstance(build, Path)
-
-    if ns.temp:
-        temp = ns.temp
-        delete_temp = False
-    else:
-        temp = Path(tempfile.mkdtemp())
-        delete_temp = True
-
-    if out:
-        try:
-            out.parent.mkdir(parents=True)
-        except FileExistsError:
-            pass
-    try:
-        temp.mkdir(parents=True)
-    except FileExistsError:
-        pass
-
-    layout = EMBED_LAYOUT if ns.embed else FULL_LAYOUT
-
-    try:
-        for t, s, p, c in layout:
-            if s == '$build':
-                fs = build
-            else:
-                fs = source / s
-            files = rglob(fs, p, c)
-            extra_files = []
-            if s == 'Lib' and p == '**/*':
-                extra_files.append((
-                    source / 'tools' / 'msi' / 'distutils.command.bdist_wininst.py',
-                    Path('distutils') / 'command' / 'bdist_wininst.py'
-                ))
-            copied = copy_to_layout(temp / t.rstrip('/'), chain(files, extra_files))
-            print('Copied {} files'.format(copied))
-
-        if ns.embed:
-            with open(str(temp / (BASE_NAME + '._pth')), 'w') as f:
-                print(BASE_NAME + '.zip', file=f)
-                print('.', file=f)
-                print('', file=f)
-                print('# Uncomment to run site.main() automatically', file=f)
-                print('#import site', file=f)
-
-        if out:
-            total = copy_to_layout(out, rglob(temp, '**/*', None))
-            print('Wrote {} files to {}'.format(total, out))
-    finally:
-        if delete_temp:
-            shutil.rmtree(temp, True)
-
-
-if __name__ == "__main__":
-    sys.exit(int(main() or 0))
diff --git a/Tools/msi/sdktools.psm1 b/Tools/msi/sdktools.psm1
new file mode 100644
index 0000000..81a74d3
--- /dev/null
+++ b/Tools/msi/sdktools.psm1
@@ -0,0 +1,43 @@
+function Find-Tool {
+    param([string]$toolname)
+
+    $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
+    $tool = (gci -r "$kitroot\Bin\*\x64\$toolname" | sort FullName -Desc | select -First 1)
+    if (-not $tool) {
+        throw "$toolname is not available"
+    }
+    Write-Host "Found $toolname at $($tool.FullName)"
+    return $tool.FullName
+}
+
+Set-Alias SignTool (Find-Tool "signtool.exe") -Scope Script
+
+function Sign-File {
+    param([string]$certname, [string]$certsha1, [string]$certfile, [string]$description, [string[]]$files)
+
+    if (-not $description) {
+        $description = $env:SigningDescription;
+        if (-not $description) {
+            $description = "Python";
+        }
+    }
+    if (-not $certname) {
+        $certname = $env:SigningCertificate;
+    }
+    if (-not $certfile) {
+        $certfile = $env:SigningCertificateFile;
+    }
+
+    foreach ($a in $files) {
+        if ($certsha1) {
+            SignTool sign /sha1 $certsha1 /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a
+        } elseif ($certname) {
+            SignTool sign /n $certname /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a
+        } elseif ($certfile) {
+            SignTool sign /f $certfile /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a
+        } else {
+            SignTool sign /a /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a
+        }
+    }
+}
+
diff --git a/Tools/msi/sign_build.ps1 b/Tools/msi/sign_build.ps1
new file mode 100644
index 0000000..6668eb3
--- /dev/null
+++ b/Tools/msi/sign_build.ps1
@@ -0,0 +1,34 @@
+<#
+.Synopsis
+    Recursively signs the contents of a directory.
+.Description
+    Given the file patterns, code signs the contents.
+.Parameter root
+    The root directory to sign.
+.Parameter patterns
+    The file patterns to sign
+.Parameter description
+    The description to add to the signature (optional).
+.Parameter certname
+    The name of the certificate to sign with (optional).
+.Parameter certsha1
+    The SHA1 hash of the certificate to sign with (optional).
+#>
+param(
+    [Parameter(Mandatory=$true)][string]$root,
+    [string[]]$patterns=@("*.exe", "*.dll", "*.pyd"),
+    [string]$description,
+    [string]$certname,
+    [string]$certsha1,
+    [string]$certfile
+)
+
+$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent;
+Import-Module $tools\sdktools.psm1 -WarningAction SilentlyContinue -Force
+
+pushd $root
+try {
+    Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files (gci -r $patterns)
+} finally {
+    popd
+}
\ No newline at end of file