Implement Windows release builds in Azure Pipelines (GH-14065)

Includes backported fixes from GH-14091

6 files changed, 154 insertions, 100 deletions

diff --git a/PC/layout/main.py b/PC/layout/main.py
index 624033e..c39aab2 100644
--- a/PC/layout/main.py
+++ b/PC/layout/main.py
@@ -31,6 +31,7 @@ from .support.logging import *
 from .support.options import *
 from .support.pip import *
 from .support.props import *
+from .support.nuspec import *
 
 BDIST_WININST_FILES_ONLY = FileNameSet("wininst-*", "bdist_wininst.py")
 BDIST_WININST_STUB = "PC/layout/support/distutils.command.bdist_wininst.py"
@@ -66,6 +67,7 @@ DATA_DIRS = FileNameSet("data")
 TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser")
 TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt")
 
+
 def copy_if_modified(src, dest):
     try:
         dest_stat = os.stat(dest)
@@ -73,12 +75,15 @@ def copy_if_modified(src, dest):
         do_copy = True
     else:
         src_stat = os.stat(src)
-        do_copy = (src_stat.st_mtime != dest_stat.st_mtime or
-                   src_stat.st_size != dest_stat.st_size)
+        do_copy = (
+            src_stat.st_mtime != dest_stat.st_mtime
+            or src_stat.st_size != dest_stat.st_size
+        )
 
     if do_copy:
         shutil.copy2(src, dest)
 
+
 def get_lib_layout(ns):
     def _c(f):
         if f in EXCLUDE_FROM_LIB:
@@ -119,7 +124,7 @@ def get_tcltk_lib(ns):
         except FileNotFoundError:
             pass
         if not tcl_lib or not os.path.isdir(tcl_lib):
-            warn("Failed to find TCL_LIBRARY")
+            log_warning("Failed to find TCL_LIBRARY")
             return
 
     for dest, src in rglob(Path(tcl_lib).parent, "**/*"):
@@ -168,7 +173,7 @@ def get_layout(ns):
     for dest, src in rglob(ns.build, "vcruntime*.dll"):
         yield dest, src
 
-    yield "LICENSE.txt", ns.source / "LICENSE"
+    yield "LICENSE.txt", ns.build / "LICENSE.txt"
 
     for dest, src in rglob(ns.build, ("*.pyd", "*.dll")):
         if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS:
@@ -222,15 +227,12 @@ def get_layout(ns):
         yield dest, src
 
     if ns.include_pip:
-        pip_dir = get_pip_dir(ns)
-        if not pip_dir.is_dir():
-            log_warning("Failed to find {} - pip will not be included", pip_dir)
-        else:
-            pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}"
-            for dest, src in rglob(pip_dir, "**/*"):
-                if src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB:
-                    continue
-                yield pkg_root.format(dest), src
+        for dest, src in get_pip_layout(ns):
+            if isinstance(src, tuple) or not (
+                src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB
+            ):
+                continue
+            yield dest, src
 
     if ns.include_chm:
         for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME):
@@ -244,6 +246,10 @@ def get_layout(ns):
         for dest, src in get_props_layout(ns):
             yield dest, src
 
+    if ns.include_nuspec:
+        for dest, src in get_nuspec_layout(ns):
+            yield dest, src
+
     for dest, src in get_appx_layout(ns):
         yield dest, src
 
@@ -287,7 +293,9 @@ def _py_temp_compile(src, ns, dest_dir=None, checked=True):
         return None
 
     dest = (dest_dir or ns.temp) / (src.stem + ".py")
-    return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked)
+    return _compile_one_py(
+        src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked
+    )
 
 
 def _write_to_zip(zf, dest, src, ns, checked=True):
@@ -361,28 +369,9 @@ def generate_source_files(ns):
             print("# Uncomment to run site.main() automatically", file=f)
             print("#import site", file=f)
 
-    if ns.include_appxmanifest:
-        log_info("Generating AppxManifest.xml in {}", ns.temp)
-        ns.temp.mkdir(parents=True, exist_ok=True)
-
-        with open(ns.temp / "AppxManifest.xml", "wb") as f:
-            f.write(get_appxmanifest(ns))
-
-        with open(ns.temp / "_resources.xml", "wb") as f:
-            f.write(get_resources_xml(ns))
-
     if ns.include_pip:
-        pip_dir = get_pip_dir(ns)
-        if not (pip_dir / "pip").is_dir():
-            log_info("Extracting pip to {}", pip_dir)
-            pip_dir.mkdir(parents=True, exist_ok=True)
-            extract_pip_files(ns)
-
-    if ns.include_props:
-        log_info("Generating {} in {}", PYTHON_PROPS_NAME, ns.temp)
-        ns.temp.mkdir(parents=True, exist_ok=True)
-        with open(ns.temp / PYTHON_PROPS_NAME, "wb") as f:
-            f.write(get_props(ns))
+        log_info("Extracting pip")
+        extract_pip_files(ns)
 
 
 def _create_zip_file(ns):
@@ -427,6 +416,18 @@ def copy_files(files, ns):
                     log_info("Processed {} files", count)
             log_debug("Processing {!s}", src)
 
+            if isinstance(src, tuple):
+                src, content = src
+                if ns.copy:
+                    log_debug("Copy {} -> {}", src, ns.copy / dest)
+                    (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True)
+                    with open(ns.copy / dest, "wb") as f:
+                        f.write(content)
+                if ns.zip:
+                    log_debug("Zip {} into {}", src, ns.zip)
+                    zip_file.writestr(str(dest), content)
+                continue
+
             if (
                 ns.precompile
                 and src in PY_FILES
diff --git a/PC/layout/support/appxmanifest.py b/PC/layout/support/appxmanifest.py
index 49a35fa..58fba84 100644
--- a/PC/layout/support/appxmanifest.py
+++ b/PC/layout/support/appxmanifest.py
@@ -17,12 +17,7 @@ from xml.etree import ElementTree as ET
 
 from .constants import *
 
-__all__ = []
-
-
-def public(f):
-    __all__.append(f.__name__)
-    return f
+__all__ = ["get_appx_layout"]
 
 
 APPX_DATA = dict(
@@ -166,9 +161,7 @@ REGISTRY = {
             "Help": {
                 "Main Python Documentation": {
                     "_condition": lambda ns: ns.include_chm,
-                    "": "[{{AppVPackageRoot}}]\\Doc\\{}".format(
-                        PYTHON_CHM_NAME
-                    ),
+                    "": "[{{AppVPackageRoot}}]\\Doc\\{}".format(PYTHON_CHM_NAME),
                 },
                 "Local Python Documentation": {
                     "_condition": lambda ns: ns.include_html_doc,
@@ -239,31 +232,6 @@ def _fixup_sccd(ns, sccd, new_hash=None):
     return sccd
 
 
-@public
-def get_appx_layout(ns):
-    if not ns.include_appxmanifest:
-        return
-
-    yield "AppxManifest.xml", ns.temp / "AppxManifest.xml"
-    yield "_resources.xml", ns.temp / "_resources.xml"
-    icons = ns.source / "PC" / "icons"
-    yield "_resources/pythonx44.png", icons / "pythonx44.png"
-    yield "_resources/pythonx44$targetsize-44_altform-unplated.png", icons / "pythonx44.png"
-    yield "_resources/pythonx50.png", icons / "pythonx50.png"
-    yield "_resources/pythonx50$targetsize-50_altform-unplated.png", icons / "pythonx50.png"
-    yield "_resources/pythonx150.png", icons / "pythonx150.png"
-    yield "_resources/pythonx150$targetsize-150_altform-unplated.png", icons / "pythonx150.png"
-    yield "_resources/pythonwx44.png", icons / "pythonwx44.png"
-    yield "_resources/pythonwx44$targetsize-44_altform-unplated.png", icons / "pythonwx44.png"
-    yield "_resources/pythonwx150.png", icons / "pythonwx150.png"
-    yield "_resources/pythonwx150$targetsize-150_altform-unplated.png", icons / "pythonwx150.png"
-    sccd = ns.source / SCCD_FILENAME
-    if sccd.is_file():
-        # This should only be set for side-loading purposes.
-        sccd = _fixup_sccd(ns, sccd, os.getenv("APPX_DATA_SHA256"))
-        yield sccd.name, sccd
-
-
 def find_or_add(xml, element, attr=None, always_add=False):
     if always_add:
         e = None
@@ -393,7 +361,6 @@ def disable_registry_virtualization(xml):
     e = find_or_add(e, "rescap:Capability", ("Name", "unvirtualizedResources"))
 
 
-@public
 def get_appxmanifest(ns):
     for k, v in APPXMANIFEST_NS.items():
         ET.register_namespace(k, v)
@@ -481,6 +448,29 @@ def get_appxmanifest(ns):
     return buffer.getbuffer()
 
 
-@public
 def get_resources_xml(ns):
     return RESOURCES_XML_TEMPLATE.encode("utf-8")
+
+
+def get_appx_layout(ns):
+    if not ns.include_appxmanifest:
+        return
+
+    yield "AppxManifest.xml", ("AppxManifest.xml", get_appxmanifest(ns))
+    yield "_resources.xml", ("_resources.xml", get_resources_xml(ns))
+    icons = ns.source / "PC" / "icons"
+    yield "_resources/pythonx44.png", icons / "pythonx44.png"
+    yield "_resources/pythonx44$targetsize-44_altform-unplated.png", icons / "pythonx44.png"
+    yield "_resources/pythonx50.png", icons / "pythonx50.png"
+    yield "_resources/pythonx50$targetsize-50_altform-unplated.png", icons / "pythonx50.png"
+    yield "_resources/pythonx150.png", icons / "pythonx150.png"
+    yield "_resources/pythonx150$targetsize-150_altform-unplated.png", icons / "pythonx150.png"
+    yield "_resources/pythonwx44.png", icons / "pythonwx44.png"
+    yield "_resources/pythonwx44$targetsize-44_altform-unplated.png", icons / "pythonwx44.png"
+    yield "_resources/pythonwx150.png", icons / "pythonwx150.png"
+    yield "_resources/pythonwx150$targetsize-150_altform-unplated.png", icons / "pythonwx150.png"
+    sccd = ns.source / SCCD_FILENAME
+    if sccd.is_file():
+        # This should only be set for side-loading purposes.
+        sccd = _fixup_sccd(ns, sccd, os.getenv("APPX_DATA_SHA256"))
+        yield sccd.name, sccd
diff --git a/PC/layout/support/nuspec.py b/PC/layout/support/nuspec.py
new file mode 100644
index 0000000..ba26ff3
--- /dev/null
+++ b/PC/layout/support/nuspec.py
@@ -0,0 +1,66 @@
+"""
+Provides .props file.
+"""
+
+import os
+
+from .constants import *
+
+__all__ = ["get_nuspec_layout"]
+
+PYTHON_NUSPEC_NAME = "python.nuspec"
+
+NUSPEC_DATA = {
+    "PYTHON_TAG": VER_DOT,
+    "PYTHON_VERSION": os.getenv("PYTHON_NUSPEC_VERSION"),
+    "PYTHON_BITNESS": "64-bit" if IS_X64 else "32-bit",
+    "PACKAGENAME": os.getenv("PYTHON_NUSPEC_PACKAGENAME"),
+    "PACKAGETITLE": os.getenv("PYTHON_NUSPEC_PACKAGETITLE"),
+    "FILELIST": r'    <file src="**\*" target="tools" />',
+}
+
+if not NUSPEC_DATA["PYTHON_VERSION"]:
+    if VER_NAME:
+        NUSPEC_DATA["PYTHON_VERSION"] = "{}.{}-{}{}".format(
+            VER_DOT, VER_MICRO, VER_NAME, VER_SERIAL
+        )
+    else:
+        NUSPEC_DATA["PYTHON_VERSION"] = "{}.{}".format(VER_DOT, VER_MICRO)
+
+if not NUSPEC_DATA["PACKAGETITLE"]:
+    NUSPEC_DATA["PACKAGETITLE"] = "Python" if IS_X64 else "Python (32-bit)"
+
+if not NUSPEC_DATA["PACKAGENAME"]:
+    NUSPEC_DATA["PACKAGENAME"] = "python" if IS_X64 else "pythonx86"
+
+FILELIST_WITH_PROPS = r"""    <file src="**\*" exclude="python.props" target="tools" />
+    <file src="python.props" target="build\native" />"""
+
+NUSPEC_TEMPLATE = r"""<?xml version="1.0"?>
+<package>
+  <metadata>
+    <id>{PACKAGENAME}</id>
+    <title>{PACKAGETITLE}</title>
+    <version>{PYTHON_VERSION}</version>
+    <authors>Python Software Foundation</authors>
+    <license type="file">tools\LICENSE.txt</license>
+    <projectUrl>https://www.python.org/</projectUrl>
+    <description>Installs {PYTHON_BITNESS} Python for use in build scenarios.</description>
+    <iconUrl>https://www.python.org/static/favicon.ico</iconUrl>
+    <tags>python</tags>
+  </metadata>
+  <files>
+{FILELIST}
+  </files>
+</package>
+"""
+
+
+def get_nuspec_layout(ns):
+    if ns.include_all or ns.include_nuspec:
+        data = NUSPEC_DATA
+        if ns.include_all or ns.include_props:
+            data = dict(data)
+            data["FILELIST"] = FILELIST_WITH_PROPS
+        nuspec = NUSPEC_TEMPLATE.format_map(data)
+        yield "python.nuspec", ("python.nuspec", nuspec.encode("utf-8"))
diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py
index 00f0566..c8ae4e3 100644
--- a/PC/layout/support/options.py
+++ b/PC/layout/support/options.py
@@ -30,6 +30,7 @@ OPTIONS = {
     "launchers": {"help": "specific launchers"},
     "appxmanifest": {"help": "an appxmanifest"},
     "props": {"help": "a python.props file"},
+    "nuspec": {"help": "a python.nuspec file"},
     "chm": {"help": "the CHM documentation"},
     "html-doc": {"help": "the HTML documentation"},
 }
@@ -60,13 +61,11 @@ PRESETS = {
             "stable",
             "distutils",
             "venv",
-            "props"
+            "props",
+            "nuspec",
         ],
     },
-    "iot": {
-        "help": "Windows IoT Core",
-        "options": ["stable", "pip"],
-    },
+    "iot": {"help": "Windows IoT Core", "options": ["stable", "pip"]},
     "default": {
         "help": "development kit package",
         "options": [
diff --git a/PC/layout/support/pip.py b/PC/layout/support/pip.py
index 369a923..eada456 100644
--- a/PC/layout/support/pip.py
+++ b/PC/layout/support/pip.py
@@ -11,15 +11,11 @@ import shutil
 import subprocess
 import sys
 
-__all__ = []
+from .filesets import *
 
+__all__ = ["extract_pip_files", "get_pip_layout"]
 
-def public(f):
-    __all__.append(f.__name__)
-    return f
 
-
-@public
 def get_pip_dir(ns):
     if ns.copy:
         if ns.zip_lib:
@@ -29,10 +25,23 @@ def get_pip_dir(ns):
         return ns.temp / "packages"
 
 
-@public
+def get_pip_layout(ns):
+    pip_dir = get_pip_dir(ns)
+    if not pip_dir.is_dir():
+        log_warning("Failed to find {} - pip will not be included", pip_dir)
+    else:
+        pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}"
+        for dest, src in rglob(pip_dir, "**/*"):
+            yield pkg_root.format(dest), src
+        yield "pip.ini", ("pip.ini", b"[global]\nuser=yes")
+
+
 def extract_pip_files(ns):
     dest = get_pip_dir(ns)
-    dest.mkdir(parents=True, exist_ok=True)
+    try:
+        dest.mkdir(parents=True, exist_ok=False)
+    except IOError:
+        return
 
     src = ns.source / "Lib" / "ensurepip" / "_bundled"
 
@@ -58,6 +67,7 @@ def extract_pip_files(ns):
             "--target",
             str(dest),
             "--no-index",
+            "--no-compile",
             "--no-cache-dir",
             "-f",
             str(src),
diff --git a/PC/layout/support/props.py b/PC/layout/support/props.py
index 3a047d2..4d3b061 100644
--- a/PC/layout/support/props.py
+++ b/PC/layout/support/props.py
@@ -6,13 +6,7 @@ import os
 
 from .constants import *
 
-__all__ = ["PYTHON_PROPS_NAME"]
-
-
-def public(f):
-    __all__.append(f.__name__)
-    return f
-
+__all__ = ["get_props_layout"]
 
 PYTHON_PROPS_NAME = "python.props"
 
@@ -97,14 +91,8 @@ PROPS_TEMPLATE = r"""<?xml version="1.0" encoding="utf-8"?>
 """
 
 
-@public
 def get_props_layout(ns):
     if ns.include_all or ns.include_props:
-        yield "python.props", ns.temp / "python.props"
-
-
-@public
-def get_props(ns):
-    # TODO: Filter contents of props file according to included/excluded items
-    props = PROPS_TEMPLATE.format_map(PROPS_DATA)
-    return props.encode("utf-8")
+        # TODO: Filter contents of props file according to included/excluded items
+        props = PROPS_TEMPLATE.format_map(PROPS_DATA)
+        yield "python.props", ("python.props", props.encode("utf-8"))