diff options
Diffstat (limited to 'PC/layout/main.py')
| -rw-r--r-- | PC/layout/main.py | 612 |
1 files changed, 612 insertions, 0 deletions
diff --git a/PC/layout/main.py b/PC/layout/main.py new file mode 100644 index 0000000..82d0536 --- /dev/null +++ b/PC/layout/main.py @@ -0,0 +1,612 @@ +""" +Generates a layout of Python for Windows from a build. + +See python make_layout.py --help for usage. +""" + +__author__ = "Steve Dower <steve.dower@python.org>" +__version__ = "3.8" + +import argparse +import functools +import os +import re +import shutil +import subprocess +import sys +import tempfile +import zipfile + +from pathlib import Path + +if __name__ == "__main__": + # Started directly, so enable relative imports + __path__ = [str(Path(__file__).resolve().parent)] + +from .support.appxmanifest import * +from .support.catalog import * +from .support.constants import * +from .support.filesets import * +from .support.logging import * +from .support.options import * +from .support.pip import * +from .support.props import * + +BDIST_WININST_FILES_ONLY = FileNameSet("wininst-*", "bdist_wininst.py") +BDIST_WININST_STUB = "PC/layout/support/distutils.command.bdist_wininst.py" + +TEST_PYDS_ONLY = FileStemSet("xxlimited", "_ctypes_test", "_test*") +TEST_DIRS_ONLY = FileNameSet("test", "tests") + +IDLE_DIRS_ONLY = FileNameSet("idlelib") + +TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter") +TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo") +TCLTK_FILES_ONLY = FileNameSet("turtle.py") + +VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip") + +EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext") +EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle") +EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt") +EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*") +EXCLUDE_FROM_CATALOG = FileSuffixSet(".exe", ".pyd", ".dll") + +REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*") + +LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt") + +PY_FILES = FileSuffixSet(".py") +PYC_FILES = FileSuffixSet(".pyc") +CAT_FILES = FileSuffixSet(".cat") +CDF_FILES = FileSuffixSet(".cdf") + +DATA_DIRS = FileNameSet("data") + +TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser") +TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt") + + +def get_lib_layout(ns): + def _c(f): + if f in EXCLUDE_FROM_LIB: + return False + if f.is_dir(): + if f in TEST_DIRS_ONLY: + return ns.include_tests + if f in TCLTK_DIRS_ONLY: + return ns.include_tcltk + if f in IDLE_DIRS_ONLY: + return ns.include_idle + if f in VENV_DIRS_ONLY: + return ns.include_venv + else: + if f in TCLTK_FILES_ONLY: + return ns.include_tcltk + if f in BDIST_WININST_FILES_ONLY: + return ns.include_bdist_wininst + return True + + for dest, src in rglob(ns.source / "Lib", "**/*", _c): + yield dest, src + + if not ns.include_bdist_wininst: + src = ns.source / BDIST_WININST_STUB + yield Path("distutils/command/bdist_wininst.py"), src + + +def get_tcltk_lib(ns): + if not ns.include_tcltk: + return + + tcl_lib = os.getenv("TCL_LIBRARY") + if not tcl_lib or not os.path.isdir(tcl_lib): + try: + with open(ns.build / "TCL_LIBRARY.env", "r", encoding="utf-8-sig") as f: + tcl_lib = f.read().strip() + except FileNotFoundError: + pass + if not tcl_lib or not os.path.isdir(tcl_lib): + warn("Failed to find TCL_LIBRARY") + return + + for dest, src in rglob(Path(tcl_lib).parent, "**/*"): + yield "tcl/{}".format(dest), src + + +def get_layout(ns): + def in_build(f, dest="", new_name=None): + n, _, x = f.rpartition(".") + n = new_name or n + src = ns.build / f + if ns.debug and src not in REQUIRED_DLLS: + if not src.stem.endswith("_d"): + src = src.parent / (src.stem + "_d" + src.suffix) + if not n.endswith("_d"): + n += "_d" + f = n + "." + x + yield dest + n + "." + x, src + if ns.include_symbols: + pdb = src.with_suffix(".pdb") + if pdb.is_file(): + yield dest + n + ".pdb", pdb + if ns.include_dev: + lib = src.with_suffix(".lib") + if lib.is_file(): + yield "libs/" + n + ".lib", lib + + yield from in_build("python_uwp.exe", new_name="python") + yield from in_build("pythonw_uwp.exe", new_name="pythonw") + + yield from in_build(PYTHON_DLL_NAME) + + if ns.include_launchers: + if ns.include_pip: + yield from in_build("python_uwp.exe", new_name="pip") + if ns.include_idle: + yield from in_build("pythonw_uwp.exe", new_name="idle") + + if ns.include_stable: + yield from in_build(PYTHON_STABLE_DLL_NAME) + + for dest, src in rglob(ns.build, "vcruntime*.dll"): + yield dest, src + + for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): + if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: + continue + if src in EXCLUDE_FROM_PYDS: + continue + if src in TEST_PYDS_ONLY and not ns.include_tests: + continue + if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: + continue + + yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + + if ns.zip_lib: + zip_name = PYTHON_ZIP_NAME + yield zip_name, ns.temp / zip_name + else: + for dest, src in get_lib_layout(ns): + yield "Lib/{}".format(dest), src + + if ns.include_venv: + yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") + yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") + + if ns.include_tools: + + def _c(d): + if d.is_dir(): + return d in TOOLS_DIRS + return d in TOOLS_FILES + + for dest, src in rglob(ns.source / "Tools", "**/*", _c): + yield "Tools/{}".format(dest), src + + if ns.include_underpth: + yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME + + if ns.include_dev: + + def _c(d): + if d.is_dir(): + return d.name != "internal" + return True + + for dest, src in rglob(ns.source / "Include", "**/*.h", _c): + yield "include/{}".format(dest), src + src = ns.source / "PC" / "pyconfig.h" + yield "include/pyconfig.h", src + + for dest, src in get_tcltk_lib(ns): + yield dest, src + + if ns.include_pip: + pip_dir = get_pip_dir(ns) + if not pip_dir.is_dir(): + log_warning("Failed to find {} - pip will not be included", pip_dir) + else: + pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}" + for dest, src in rglob(pip_dir, "**/*"): + if src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB: + continue + yield pkg_root.format(dest), src + + if ns.include_chm: + for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME): + yield "Doc/{}".format(dest), src + + if ns.include_html_doc: + for dest, src in rglob(ns.doc_build / "html", "**/*"): + yield "Doc/html/{}".format(dest), src + + if ns.include_props: + for dest, src in get_props_layout(ns): + yield dest, src + + for dest, src in get_appx_layout(ns): + yield dest, src + + if ns.include_cat: + if ns.flat_dlls: + yield ns.include_cat.name, ns.include_cat + else: + yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat + + +def _compile_one_py(src, dest, name, optimize): + import py_compile + + if dest is not None: + dest = str(dest) + + try: + return Path( + py_compile.compile( + str(src), + dest, + str(name), + doraise=True, + optimize=optimize, + invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, + ) + ) + except py_compile.PyCompileError: + log_warning("Failed to compile {}", src) + return None + + +def _py_temp_compile(src, ns, dest_dir=None): + if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS: + return None + + dest = (dest_dir or ns.temp) / (src.stem + ".py") + return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2) + + +def _write_to_zip(zf, dest, src, ns): + pyc = _py_temp_compile(src, ns) + if pyc: + try: + zf.write(str(pyc), dest.with_suffix(".pyc")) + finally: + try: + pyc.unlink() + except: + log_exception("Failed to delete {}", pyc) + return + + if src in LIB2TO3_GRAMMAR_FILES: + from lib2to3.pgen2.driver import load_grammar + + tmp = ns.temp / src.name + try: + shutil.copy(src, tmp) + load_grammar(str(tmp)) + for f in ns.temp.glob(src.stem + "*.pickle"): + zf.write(str(f), str(dest.parent / f.name)) + try: + f.unlink() + except: + log_exception("Failed to delete {}", f) + except: + log_exception("Failed to compile {}", src) + finally: + try: + tmp.unlink() + except: + log_exception("Failed to delete {}", tmp) + + zf.write(str(src), str(dest)) + + +def generate_source_files(ns): + if ns.zip_lib: + zip_name = PYTHON_ZIP_NAME + zip_path = ns.temp / zip_name + if zip_path.is_file(): + zip_path.unlink() + elif zip_path.is_dir(): + log_error( + "Cannot create zip file because a directory exists by the same name" + ) + return + log_info("Generating {} in {}", zip_name, ns.temp) + ns.temp.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + for dest, src in get_lib_layout(ns): + _write_to_zip(zf, dest, src, ns) + + if ns.include_underpth: + log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp) + ns.temp.mkdir(parents=True, exist_ok=True) + with open(ns.temp / PYTHON_PTH_NAME, "w", encoding="utf-8") as f: + if ns.zip_lib: + print(PYTHON_ZIP_NAME, file=f) + if ns.include_pip: + print("packages", file=f) + else: + print("Lib", file=f) + print("Lib/site-packages", file=f) + if not ns.flat_dlls: + print("DLLs", file=f) + print(".", file=f) + print(file=f) + print("# Uncomment to run site.main() automatically", file=f) + print("#import site", file=f) + + if ns.include_appxmanifest: + log_info("Generating AppxManifest.xml in {}", ns.temp) + ns.temp.mkdir(parents=True, exist_ok=True) + + with open(ns.temp / "AppxManifest.xml", "wb") as f: + f.write(get_appxmanifest(ns)) + + with open(ns.temp / "_resources.xml", "wb") as f: + f.write(get_resources_xml(ns)) + + if ns.include_pip: + pip_dir = get_pip_dir(ns) + if not (pip_dir / "pip").is_dir(): + log_info("Extracting pip to {}", pip_dir) + pip_dir.mkdir(parents=True, exist_ok=True) + extract_pip_files(ns) + + if ns.include_props: + log_info("Generating {} in {}", PYTHON_PROPS_NAME, ns.temp) + ns.temp.mkdir(parents=True, exist_ok=True) + with open(ns.temp / PYTHON_PROPS_NAME, "wb") as f: + f.write(get_props(ns)) + + +def _create_zip_file(ns): + if not ns.zip: + return None + + if ns.zip.is_file(): + try: + ns.zip.unlink() + except OSError: + log_exception("Unable to remove {}", ns.zip) + sys.exit(8) + elif ns.zip.is_dir(): + log_error("Cannot create ZIP file because {} is a directory", ns.zip) + sys.exit(8) + + ns.zip.parent.mkdir(parents=True, exist_ok=True) + return zipfile.ZipFile(ns.zip, "w", zipfile.ZIP_DEFLATED) + + +def copy_files(files, ns): + if ns.copy: + ns.copy.mkdir(parents=True, exist_ok=True) + + try: + total = len(files) + except TypeError: + total = None + count = 0 + + zip_file = _create_zip_file(ns) + try: + need_compile = [] + in_catalog = [] + + for dest, src in files: + count += 1 + if count % 10 == 0: + if total: + log_info("Processed {:>4} of {} files", count, total) + else: + log_info("Processed {} files", count) + log_debug("Processing {!s}", src) + + if ( + ns.precompile + and src in PY_FILES + and src not in EXCLUDE_FROM_COMPILE + and src.parent not in DATA_DIRS + and os.path.normcase(str(dest)).startswith(os.path.normcase("Lib")) + ): + if ns.copy: + need_compile.append((dest, ns.copy / dest)) + else: + (ns.temp / "Lib" / dest).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, ns.temp / "Lib" / dest) + need_compile.append((dest, ns.temp / "Lib" / dest)) + + if src not in EXCLUDE_FROM_CATALOG: + in_catalog.append((src.name, src)) + + if ns.copy: + log_debug("Copy {} -> {}", src, ns.copy / dest) + (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True) + try: + shutil.copy2(src, ns.copy / dest) + except shutil.SameFileError: + pass + + if ns.zip: + log_debug("Zip {} into {}", src, ns.zip) + zip_file.write(src, str(dest)) + + if need_compile: + for dest, src in need_compile: + compiled = [ + _compile_one_py(src, None, dest, optimize=0), + _compile_one_py(src, None, dest, optimize=1), + _compile_one_py(src, None, dest, optimize=2), + ] + for c in compiled: + if not c: + continue + cdest = Path(dest).parent / Path(c).relative_to(src.parent) + if ns.zip: + log_debug("Zip {} into {}", c, ns.zip) + zip_file.write(c, str(cdest)) + in_catalog.append((cdest.name, cdest)) + + if ns.catalog: + # Just write out the CDF now. Compilation and signing is + # an extra step + log_info("Generating {}", ns.catalog) + ns.catalog.parent.mkdir(parents=True, exist_ok=True) + write_catalog(ns.catalog, in_catalog) + + finally: + if zip_file: + zip_file.close() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-v", help="Increase verbosity", action="count") + parser.add_argument( + "-s", + "--source", + metavar="dir", + help="The directory containing the repository root", + type=Path, + default=None, + ) + parser.add_argument( + "-b", "--build", metavar="dir", help="Specify the build directory", type=Path + ) + parser.add_argument( + "--doc-build", + metavar="dir", + help="Specify the docs build directory", + type=Path, + default=None, + ) + parser.add_argument( + "--copy", + metavar="directory", + help="The name of the directory to copy an extracted layout to", + type=Path, + default=None, + ) + parser.add_argument( + "--zip", + metavar="file", + help="The ZIP file to write all files to", + type=Path, + default=None, + ) + parser.add_argument( + "--catalog", + metavar="file", + help="The CDF file to write catalog entries to", + type=Path, + default=None, + ) + parser.add_argument( + "--log", + metavar="file", + help="Write all operations to the specified file", + type=Path, + default=None, + ) + parser.add_argument( + "-t", + "--temp", + metavar="file", + help="A temporary working directory", + type=Path, + default=None, + ) + parser.add_argument( + "-d", "--debug", help="Include debug build", action="store_true" + ) + parser.add_argument( + "-p", + "--precompile", + help="Include .pyc files instead of .py", + action="store_true", + ) + parser.add_argument( + "-z", "--zip-lib", help="Include library in a ZIP file", action="store_true" + ) + parser.add_argument( + "--flat-dlls", help="Does not create a DLLs directory", action="store_true" + ) + parser.add_argument( + "-a", + "--include-all", + help="Include all optional components", + action="store_true", + ) + parser.add_argument( + "--include-cat", + metavar="file", + help="Specify the catalog file to include", + type=Path, + default=None, + ) + for opt, help in get_argparse_options(): + parser.add_argument(opt, help=help, action="store_true") + + ns = parser.parse_args() + update_presets(ns) + + ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) + ns.build = ns.build or Path(sys.executable).parent + ns.temp = ns.temp or Path(tempfile.mkdtemp()) + ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") + if not ns.source.is_absolute(): + ns.source = (Path.cwd() / ns.source).resolve() + if not ns.build.is_absolute(): + ns.build = (Path.cwd() / ns.build).resolve() + if not ns.temp.is_absolute(): + ns.temp = (Path.cwd() / ns.temp).resolve() + if not ns.doc_build.is_absolute(): + ns.doc_build = (Path.cwd() / ns.doc_build).resolve() + if ns.include_cat and not ns.include_cat.is_absolute(): + ns.include_cat = (Path.cwd() / ns.include_cat).resolve() + + if ns.copy and not ns.copy.is_absolute(): + ns.copy = (Path.cwd() / ns.copy).resolve() + if ns.zip and not ns.zip.is_absolute(): + ns.zip = (Path.cwd() / ns.zip).resolve() + if ns.catalog and not ns.catalog.is_absolute(): + ns.catalog = (Path.cwd() / ns.catalog).resolve() + + configure_logger(ns) + + log_info( + """OPTIONS +Source: {ns.source} +Build: {ns.build} +Temp: {ns.temp} + +Copy to: {ns.copy} +Zip to: {ns.zip} +Catalog: {ns.catalog}""", + ns=ns, + ) + + if ns.include_idle and not ns.include_tcltk: + log_warning("Assuming --include-tcltk to support --include-idle") + ns.include_tcltk = True + + try: + generate_source_files(ns) + files = list(get_layout(ns)) + copy_files(files, ns) + except KeyboardInterrupt: + log_info("Interrupted by Ctrl+C") + return 3 + except SystemExit: + raise + except: + log_exception("Unhandled error") + + if error_was_logged(): + log_error("Errors occurred.") + return 1 + + +if __name__ == "__main__": + sys.exit(int(main() or 0)) |