bpo-34977: Add Windows App Store package (GH-10245)

1 files changed, 612 insertions, 0 deletions

diff --git a/PC/layout/main.py b/PC/layout/main.py
new file mode 100644
index 0000000..82d0536
--- /dev/null
+++ b/PC/layout/main.py
@@ -0,0 +1,612 @@
+"""
+Generates a layout of Python for Windows from a build.
+
+See python make_layout.py --help for usage.
+"""
+
+__author__ = "Steve Dower <steve.dower@python.org>"
+__version__ = "3.8"
+
+import argparse
+import functools
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import zipfile
+
+from pathlib import Path
+
+if __name__ == "__main__":
+    # Started directly, so enable relative imports
+    __path__ = [str(Path(__file__).resolve().parent)]
+
+from .support.appxmanifest import *
+from .support.catalog import *
+from .support.constants import *
+from .support.filesets import *
+from .support.logging import *
+from .support.options import *
+from .support.pip import *
+from .support.props import *
+
+BDIST_WININST_FILES_ONLY = FileNameSet("wininst-*", "bdist_wininst.py")
+BDIST_WININST_STUB = "PC/layout/support/distutils.command.bdist_wininst.py"
+
+TEST_PYDS_ONLY = FileStemSet("xxlimited", "_ctypes_test", "_test*")
+TEST_DIRS_ONLY = FileNameSet("test", "tests")
+
+IDLE_DIRS_ONLY = FileNameSet("idlelib")
+
+TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter")
+TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo")
+TCLTK_FILES_ONLY = FileNameSet("turtle.py")
+
+VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip")
+
+EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext")
+EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle")
+EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt")
+EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*")
+EXCLUDE_FROM_CATALOG = FileSuffixSet(".exe", ".pyd", ".dll")
+
+REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*")
+
+LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt")
+
+PY_FILES = FileSuffixSet(".py")
+PYC_FILES = FileSuffixSet(".pyc")
+CAT_FILES = FileSuffixSet(".cat")
+CDF_FILES = FileSuffixSet(".cdf")
+
+DATA_DIRS = FileNameSet("data")
+
+TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser")
+TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt")
+
+
+def get_lib_layout(ns):
+    def _c(f):
+        if f in EXCLUDE_FROM_LIB:
+            return False
+        if f.is_dir():
+            if f in TEST_DIRS_ONLY:
+                return ns.include_tests
+            if f in TCLTK_DIRS_ONLY:
+                return ns.include_tcltk
+            if f in IDLE_DIRS_ONLY:
+                return ns.include_idle
+            if f in VENV_DIRS_ONLY:
+                return ns.include_venv
+        else:
+            if f in TCLTK_FILES_ONLY:
+                return ns.include_tcltk
+            if f in BDIST_WININST_FILES_ONLY:
+                return ns.include_bdist_wininst
+        return True
+
+    for dest, src in rglob(ns.source / "Lib", "**/*", _c):
+        yield dest, src
+
+    if not ns.include_bdist_wininst:
+        src = ns.source / BDIST_WININST_STUB
+        yield Path("distutils/command/bdist_wininst.py"), src
+
+
+def get_tcltk_lib(ns):
+    if not ns.include_tcltk:
+        return
+
+    tcl_lib = os.getenv("TCL_LIBRARY")
+    if not tcl_lib or not os.path.isdir(tcl_lib):
+        try:
+            with open(ns.build / "TCL_LIBRARY.env", "r", encoding="utf-8-sig") as f:
+                tcl_lib = f.read().strip()
+        except FileNotFoundError:
+            pass
+        if not tcl_lib or not os.path.isdir(tcl_lib):
+            warn("Failed to find TCL_LIBRARY")
+            return
+
+    for dest, src in rglob(Path(tcl_lib).parent, "**/*"):
+        yield "tcl/{}".format(dest), src
+
+
+def get_layout(ns):
+    def in_build(f, dest="", new_name=None):
+        n, _, x = f.rpartition(".")
+        n = new_name or n
+        src = ns.build / f
+        if ns.debug and src not in REQUIRED_DLLS:
+            if not src.stem.endswith("_d"):
+                src = src.parent / (src.stem + "_d" + src.suffix)
+            if not n.endswith("_d"):
+                n += "_d"
+                f = n + "." + x
+        yield dest + n + "." + x, src
+        if ns.include_symbols:
+            pdb = src.with_suffix(".pdb")
+            if pdb.is_file():
+                yield dest + n + ".pdb", pdb
+        if ns.include_dev:
+            lib = src.with_suffix(".lib")
+            if lib.is_file():
+                yield "libs/" + n + ".lib", lib
+
+    yield from in_build("python_uwp.exe", new_name="python")
+    yield from in_build("pythonw_uwp.exe", new_name="pythonw")
+
+    yield from in_build(PYTHON_DLL_NAME)
+
+    if ns.include_launchers:
+        if ns.include_pip:
+            yield from in_build("python_uwp.exe", new_name="pip")
+        if ns.include_idle:
+            yield from in_build("pythonw_uwp.exe", new_name="idle")
+
+    if ns.include_stable:
+        yield from in_build(PYTHON_STABLE_DLL_NAME)
+
+    for dest, src in rglob(ns.build, "vcruntime*.dll"):
+        yield dest, src
+
+    for dest, src in rglob(ns.build, ("*.pyd", "*.dll")):
+        if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS:
+            continue
+        if src in EXCLUDE_FROM_PYDS:
+            continue
+        if src in TEST_PYDS_ONLY and not ns.include_tests:
+            continue
+        if src in TCLTK_PYDS_ONLY and not ns.include_tcltk:
+            continue
+
+        yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/")
+
+    if ns.zip_lib:
+        zip_name = PYTHON_ZIP_NAME
+        yield zip_name, ns.temp / zip_name
+    else:
+        for dest, src in get_lib_layout(ns):
+            yield "Lib/{}".format(dest), src
+
+        if ns.include_venv:
+            yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python")
+            yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw")
+
+    if ns.include_tools:
+
+        def _c(d):
+            if d.is_dir():
+                return d in TOOLS_DIRS
+            return d in TOOLS_FILES
+
+        for dest, src in rglob(ns.source / "Tools", "**/*", _c):
+            yield "Tools/{}".format(dest), src
+
+    if ns.include_underpth:
+        yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME
+
+    if ns.include_dev:
+
+        def _c(d):
+            if d.is_dir():
+                return d.name != "internal"
+            return True
+
+        for dest, src in rglob(ns.source / "Include", "**/*.h", _c):
+            yield "include/{}".format(dest), src
+        src = ns.source / "PC" / "pyconfig.h"
+        yield "include/pyconfig.h", src
+
+    for dest, src in get_tcltk_lib(ns):
+        yield dest, src
+
+    if ns.include_pip:
+        pip_dir = get_pip_dir(ns)
+        if not pip_dir.is_dir():
+            log_warning("Failed to find {} - pip will not be included", pip_dir)
+        else:
+            pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}"
+            for dest, src in rglob(pip_dir, "**/*"):
+                if src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB:
+                    continue
+                yield pkg_root.format(dest), src
+
+    if ns.include_chm:
+        for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME):
+            yield "Doc/{}".format(dest), src
+
+    if ns.include_html_doc:
+        for dest, src in rglob(ns.doc_build / "html", "**/*"):
+            yield "Doc/html/{}".format(dest), src
+
+    if ns.include_props:
+        for dest, src in get_props_layout(ns):
+            yield dest, src
+
+    for dest, src in get_appx_layout(ns):
+        yield dest, src
+
+    if ns.include_cat:
+        if ns.flat_dlls:
+            yield ns.include_cat.name, ns.include_cat
+        else:
+            yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat
+
+
+def _compile_one_py(src, dest, name, optimize):
+    import py_compile
+
+    if dest is not None:
+        dest = str(dest)
+
+    try:
+        return Path(
+            py_compile.compile(
+                str(src),
+                dest,
+                str(name),
+                doraise=True,
+                optimize=optimize,
+                invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+            )
+        )
+    except py_compile.PyCompileError:
+        log_warning("Failed to compile {}", src)
+        return None
+
+
+def _py_temp_compile(src, ns, dest_dir=None):
+    if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS:
+        return None
+
+    dest = (dest_dir or ns.temp) / (src.stem + ".py")
+    return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2)
+
+
+def _write_to_zip(zf, dest, src, ns):
+    pyc = _py_temp_compile(src, ns)
+    if pyc:
+        try:
+            zf.write(str(pyc), dest.with_suffix(".pyc"))
+        finally:
+            try:
+                pyc.unlink()
+            except:
+                log_exception("Failed to delete {}", pyc)
+        return
+
+    if src in LIB2TO3_GRAMMAR_FILES:
+        from lib2to3.pgen2.driver import load_grammar
+
+        tmp = ns.temp / src.name
+        try:
+            shutil.copy(src, tmp)
+            load_grammar(str(tmp))
+            for f in ns.temp.glob(src.stem + "*.pickle"):
+                zf.write(str(f), str(dest.parent / f.name))
+                try:
+                    f.unlink()
+                except:
+                    log_exception("Failed to delete {}", f)
+        except:
+            log_exception("Failed to compile {}", src)
+        finally:
+            try:
+                tmp.unlink()
+            except:
+                log_exception("Failed to delete {}", tmp)
+
+    zf.write(str(src), str(dest))
+
+
+def generate_source_files(ns):
+    if ns.zip_lib:
+        zip_name = PYTHON_ZIP_NAME
+        zip_path = ns.temp / zip_name
+        if zip_path.is_file():
+            zip_path.unlink()
+        elif zip_path.is_dir():
+            log_error(
+                "Cannot create zip file because a directory exists by the same name"
+            )
+            return
+        log_info("Generating {} in {}", zip_name, ns.temp)
+        ns.temp.mkdir(parents=True, exist_ok=True)
+        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for dest, src in get_lib_layout(ns):
+                _write_to_zip(zf, dest, src, ns)
+
+    if ns.include_underpth:
+        log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp)
+        ns.temp.mkdir(parents=True, exist_ok=True)
+        with open(ns.temp / PYTHON_PTH_NAME, "w", encoding="utf-8") as f:
+            if ns.zip_lib:
+                print(PYTHON_ZIP_NAME, file=f)
+                if ns.include_pip:
+                    print("packages", file=f)
+            else:
+                print("Lib", file=f)
+                print("Lib/site-packages", file=f)
+            if not ns.flat_dlls:
+                print("DLLs", file=f)
+            print(".", file=f)
+            print(file=f)
+            print("# Uncomment to run site.main() automatically", file=f)
+            print("#import site", file=f)
+
+    if ns.include_appxmanifest:
+        log_info("Generating AppxManifest.xml in {}", ns.temp)
+        ns.temp.mkdir(parents=True, exist_ok=True)
+
+        with open(ns.temp / "AppxManifest.xml", "wb") as f:
+            f.write(get_appxmanifest(ns))
+
+        with open(ns.temp / "_resources.xml", "wb") as f:
+            f.write(get_resources_xml(ns))
+
+    if ns.include_pip:
+        pip_dir = get_pip_dir(ns)
+        if not (pip_dir / "pip").is_dir():
+            log_info("Extracting pip to {}", pip_dir)
+            pip_dir.mkdir(parents=True, exist_ok=True)
+            extract_pip_files(ns)
+
+    if ns.include_props:
+        log_info("Generating {} in {}", PYTHON_PROPS_NAME, ns.temp)
+        ns.temp.mkdir(parents=True, exist_ok=True)
+        with open(ns.temp / PYTHON_PROPS_NAME, "wb") as f:
+            f.write(get_props(ns))
+
+
+def _create_zip_file(ns):
+    if not ns.zip:
+        return None
+
+    if ns.zip.is_file():
+        try:
+            ns.zip.unlink()
+        except OSError:
+            log_exception("Unable to remove {}", ns.zip)
+            sys.exit(8)
+    elif ns.zip.is_dir():
+        log_error("Cannot create ZIP file because {} is a directory", ns.zip)
+        sys.exit(8)
+
+    ns.zip.parent.mkdir(parents=True, exist_ok=True)
+    return zipfile.ZipFile(ns.zip, "w", zipfile.ZIP_DEFLATED)
+
+
+def copy_files(files, ns):
+    if ns.copy:
+        ns.copy.mkdir(parents=True, exist_ok=True)
+
+    try:
+        total = len(files)
+    except TypeError:
+        total = None
+    count = 0
+
+    zip_file = _create_zip_file(ns)
+    try:
+        need_compile = []
+        in_catalog = []
+
+        for dest, src in files:
+            count += 1
+            if count % 10 == 0:
+                if total:
+                    log_info("Processed {:>4} of {} files", count, total)
+                else:
+                    log_info("Processed {} files", count)
+            log_debug("Processing {!s}", src)
+
+            if (
+                ns.precompile
+                and src in PY_FILES
+                and src not in EXCLUDE_FROM_COMPILE
+                and src.parent not in DATA_DIRS
+                and os.path.normcase(str(dest)).startswith(os.path.normcase("Lib"))
+            ):
+                if ns.copy:
+                    need_compile.append((dest, ns.copy / dest))
+                else:
+                    (ns.temp / "Lib" / dest).parent.mkdir(parents=True, exist_ok=True)
+                    shutil.copy2(src, ns.temp / "Lib" / dest)
+                    need_compile.append((dest, ns.temp / "Lib" / dest))
+
+            if src not in EXCLUDE_FROM_CATALOG:
+                in_catalog.append((src.name, src))
+
+            if ns.copy:
+                log_debug("Copy {} -> {}", src, ns.copy / dest)
+                (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True)
+                try:
+                    shutil.copy2(src, ns.copy / dest)
+                except shutil.SameFileError:
+                    pass
+
+            if ns.zip:
+                log_debug("Zip {} into {}", src, ns.zip)
+                zip_file.write(src, str(dest))
+
+        if need_compile:
+            for dest, src in need_compile:
+                compiled = [
+                    _compile_one_py(src, None, dest, optimize=0),
+                    _compile_one_py(src, None, dest, optimize=1),
+                    _compile_one_py(src, None, dest, optimize=2),
+                ]
+                for c in compiled:
+                    if not c:
+                        continue
+                    cdest = Path(dest).parent / Path(c).relative_to(src.parent)
+                    if ns.zip:
+                        log_debug("Zip {} into {}", c, ns.zip)
+                        zip_file.write(c, str(cdest))
+                    in_catalog.append((cdest.name, cdest))
+
+        if ns.catalog:
+            # Just write out the CDF now. Compilation and signing is
+            # an extra step
+            log_info("Generating {}", ns.catalog)
+            ns.catalog.parent.mkdir(parents=True, exist_ok=True)
+            write_catalog(ns.catalog, in_catalog)
+
+    finally:
+        if zip_file:
+            zip_file.close()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v", help="Increase verbosity", action="count")
+    parser.add_argument(
+        "-s",
+        "--source",
+        metavar="dir",
+        help="The directory containing the repository root",
+        type=Path,
+        default=None,
+    )
+    parser.add_argument(
+        "-b", "--build", metavar="dir", help="Specify the build directory", type=Path
+    )
+    parser.add_argument(
+        "--doc-build",
+        metavar="dir",
+        help="Specify the docs build directory",
+        type=Path,
+        default=None,
+    )
+    parser.add_argument(
+        "--copy",
+        metavar="directory",
+        help="The name of the directory to copy an extracted layout to",
+        type=Path,
+        default=None,
+    )
+    parser.add_argument(
+        "--zip",
+        metavar="file",
+        help="The ZIP file to write all files to",
+        type=Path,
+        default=None,
+    )
+    parser.add_argument(
+        "--catalog",
+        metavar="file",
+        help="The CDF file to write catalog entries to",
+        type=Path,
+        default=None,
+    )
+    parser.add_argument(
+        "--log",
+        metavar="file",
+        help="Write all operations to the specified file",
+        type=Path,
+        default=None,
+    )
+    parser.add_argument(
+        "-t",
+        "--temp",
+        metavar="file",
+        help="A temporary working directory",
+        type=Path,
+        default=None,
+    )
+    parser.add_argument(
+        "-d", "--debug", help="Include debug build", action="store_true"
+    )
+    parser.add_argument(
+        "-p",
+        "--precompile",
+        help="Include .pyc files instead of .py",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-z", "--zip-lib", help="Include library in a ZIP file", action="store_true"
+    )
+    parser.add_argument(
+        "--flat-dlls", help="Does not create a DLLs directory", action="store_true"
+    )
+    parser.add_argument(
+        "-a",
+        "--include-all",
+        help="Include all optional components",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--include-cat",
+        metavar="file",
+        help="Specify the catalog file to include",
+        type=Path,
+        default=None,
+    )
+    for opt, help in get_argparse_options():
+        parser.add_argument(opt, help=help, action="store_true")
+
+    ns = parser.parse_args()
+    update_presets(ns)
+
+    ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent)
+    ns.build = ns.build or Path(sys.executable).parent
+    ns.temp = ns.temp or Path(tempfile.mkdtemp())
+    ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build")
+    if not ns.source.is_absolute():
+        ns.source = (Path.cwd() / ns.source).resolve()
+    if not ns.build.is_absolute():
+        ns.build = (Path.cwd() / ns.build).resolve()
+    if not ns.temp.is_absolute():
+        ns.temp = (Path.cwd() / ns.temp).resolve()
+    if not ns.doc_build.is_absolute():
+        ns.doc_build = (Path.cwd() / ns.doc_build).resolve()
+    if ns.include_cat and not ns.include_cat.is_absolute():
+        ns.include_cat = (Path.cwd() / ns.include_cat).resolve()
+
+    if ns.copy and not ns.copy.is_absolute():
+        ns.copy = (Path.cwd() / ns.copy).resolve()
+    if ns.zip and not ns.zip.is_absolute():
+        ns.zip = (Path.cwd() / ns.zip).resolve()
+    if ns.catalog and not ns.catalog.is_absolute():
+        ns.catalog = (Path.cwd() / ns.catalog).resolve()
+
+    configure_logger(ns)
+
+    log_info(
+        """OPTIONS
+Source: {ns.source}
+Build:  {ns.build}
+Temp:   {ns.temp}
+
+Copy to: {ns.copy}
+Zip to:  {ns.zip}
+Catalog: {ns.catalog}""",
+        ns=ns,
+    )
+
+    if ns.include_idle and not ns.include_tcltk:
+        log_warning("Assuming --include-tcltk to support --include-idle")
+        ns.include_tcltk = True
+
+    try:
+        generate_source_files(ns)
+        files = list(get_layout(ns))
+        copy_files(files, ns)
+    except KeyboardInterrupt:
+        log_info("Interrupted by Ctrl+C")
+        return 3
+    except SystemExit:
+        raise
+    except:
+        log_exception("Unhandled error")
+
+    if error_was_logged():
+        log_error("Errors occurred.")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(int(main() or 0))