diff options
Diffstat (limited to 'PC/layout/main.py')
| -rw-r--r-- | PC/layout/main.py | 612 |
1 files changed, 0 insertions, 612 deletions
diff --git a/PC/layout/main.py b/PC/layout/main.py deleted file mode 100644 index 82d0536..0000000 --- a/PC/layout/main.py +++ /dev/null @@ -1,612 +0,0 @@ -""" -Generates a layout of Python for Windows from a build. - -See python make_layout.py --help for usage. -""" - -__author__ = "Steve Dower <steve.dower@python.org>" -__version__ = "3.8" - -import argparse -import functools -import os -import re -import shutil -import subprocess -import sys -import tempfile -import zipfile - -from pathlib import Path - -if __name__ == "__main__": - # Started directly, so enable relative imports - __path__ = [str(Path(__file__).resolve().parent)] - -from .support.appxmanifest import * -from .support.catalog import * -from .support.constants import * -from .support.filesets import * -from .support.logging import * -from .support.options import * -from .support.pip import * -from .support.props import * - -BDIST_WININST_FILES_ONLY = FileNameSet("wininst-*", "bdist_wininst.py") -BDIST_WININST_STUB = "PC/layout/support/distutils.command.bdist_wininst.py" - -TEST_PYDS_ONLY = FileStemSet("xxlimited", "_ctypes_test", "_test*") -TEST_DIRS_ONLY = FileNameSet("test", "tests") - -IDLE_DIRS_ONLY = FileNameSet("idlelib") - -TCLTK_PYDS_ONLY = FileStemSet("tcl*", "tk*", "_tkinter") -TCLTK_DIRS_ONLY = FileNameSet("tkinter", "turtledemo") -TCLTK_FILES_ONLY = FileNameSet("turtle.py") - -VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip") - -EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext") -EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle") -EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt") -EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*") -EXCLUDE_FROM_CATALOG = FileSuffixSet(".exe", ".pyd", ".dll") - -REQUIRED_DLLS = FileStemSet("libcrypto*", "libssl*") - -LIB2TO3_GRAMMAR_FILES = FileNameSet("Grammar.txt", "PatternGrammar.txt") - -PY_FILES = FileSuffixSet(".py") -PYC_FILES = FileSuffixSet(".pyc") -CAT_FILES = FileSuffixSet(".cat") -CDF_FILES = FileSuffixSet(".cdf") - -DATA_DIRS = FileNameSet("data") - -TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser") -TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt") - - -def get_lib_layout(ns): - def _c(f): - if f in EXCLUDE_FROM_LIB: - return False - if f.is_dir(): - if f in TEST_DIRS_ONLY: - return ns.include_tests - if f in TCLTK_DIRS_ONLY: - return ns.include_tcltk - if f in IDLE_DIRS_ONLY: - return ns.include_idle - if f in VENV_DIRS_ONLY: - return ns.include_venv - else: - if f in TCLTK_FILES_ONLY: - return ns.include_tcltk - if f in BDIST_WININST_FILES_ONLY: - return ns.include_bdist_wininst - return True - - for dest, src in rglob(ns.source / "Lib", "**/*", _c): - yield dest, src - - if not ns.include_bdist_wininst: - src = ns.source / BDIST_WININST_STUB - yield Path("distutils/command/bdist_wininst.py"), src - - -def get_tcltk_lib(ns): - if not ns.include_tcltk: - return - - tcl_lib = os.getenv("TCL_LIBRARY") - if not tcl_lib or not os.path.isdir(tcl_lib): - try: - with open(ns.build / "TCL_LIBRARY.env", "r", encoding="utf-8-sig") as f: - tcl_lib = f.read().strip() - except FileNotFoundError: - pass - if not tcl_lib or not os.path.isdir(tcl_lib): - warn("Failed to find TCL_LIBRARY") - return - - for dest, src in rglob(Path(tcl_lib).parent, "**/*"): - yield "tcl/{}".format(dest), src - - -def get_layout(ns): - def in_build(f, dest="", new_name=None): - n, _, x = f.rpartition(".") - n = new_name or n - src = ns.build / f - if ns.debug and src not in REQUIRED_DLLS: - if not src.stem.endswith("_d"): - src = src.parent / (src.stem + "_d" + src.suffix) - if not n.endswith("_d"): - n += "_d" - f = n + "." + x - yield dest + n + "." + x, src - if ns.include_symbols: - pdb = src.with_suffix(".pdb") - if pdb.is_file(): - yield dest + n + ".pdb", pdb - if ns.include_dev: - lib = src.with_suffix(".lib") - if lib.is_file(): - yield "libs/" + n + ".lib", lib - - yield from in_build("python_uwp.exe", new_name="python") - yield from in_build("pythonw_uwp.exe", new_name="pythonw") - - yield from in_build(PYTHON_DLL_NAME) - - if ns.include_launchers: - if ns.include_pip: - yield from in_build("python_uwp.exe", new_name="pip") - if ns.include_idle: - yield from in_build("pythonw_uwp.exe", new_name="idle") - - if ns.include_stable: - yield from in_build(PYTHON_STABLE_DLL_NAME) - - for dest, src in rglob(ns.build, "vcruntime*.dll"): - yield dest, src - - for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): - if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: - continue - if src in EXCLUDE_FROM_PYDS: - continue - if src in TEST_PYDS_ONLY and not ns.include_tests: - continue - if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: - continue - - yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") - - if ns.zip_lib: - zip_name = PYTHON_ZIP_NAME - yield zip_name, ns.temp / zip_name - else: - for dest, src in get_lib_layout(ns): - yield "Lib/{}".format(dest), src - - if ns.include_venv: - yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") - yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") - - if ns.include_tools: - - def _c(d): - if d.is_dir(): - return d in TOOLS_DIRS - return d in TOOLS_FILES - - for dest, src in rglob(ns.source / "Tools", "**/*", _c): - yield "Tools/{}".format(dest), src - - if ns.include_underpth: - yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME - - if ns.include_dev: - - def _c(d): - if d.is_dir(): - return d.name != "internal" - return True - - for dest, src in rglob(ns.source / "Include", "**/*.h", _c): - yield "include/{}".format(dest), src - src = ns.source / "PC" / "pyconfig.h" - yield "include/pyconfig.h", src - - for dest, src in get_tcltk_lib(ns): - yield dest, src - - if ns.include_pip: - pip_dir = get_pip_dir(ns) - if not pip_dir.is_dir(): - log_warning("Failed to find {} - pip will not be included", pip_dir) - else: - pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}" - for dest, src in rglob(pip_dir, "**/*"): - if src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB: - continue - yield pkg_root.format(dest), src - - if ns.include_chm: - for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME): - yield "Doc/{}".format(dest), src - - if ns.include_html_doc: - for dest, src in rglob(ns.doc_build / "html", "**/*"): - yield "Doc/html/{}".format(dest), src - - if ns.include_props: - for dest, src in get_props_layout(ns): - yield dest, src - - for dest, src in get_appx_layout(ns): - yield dest, src - - if ns.include_cat: - if ns.flat_dlls: - yield ns.include_cat.name, ns.include_cat - else: - yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat - - -def _compile_one_py(src, dest, name, optimize): - import py_compile - - if dest is not None: - dest = str(dest) - - try: - return Path( - py_compile.compile( - str(src), - dest, - str(name), - doraise=True, - optimize=optimize, - invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, - ) - ) - except py_compile.PyCompileError: - log_warning("Failed to compile {}", src) - return None - - -def _py_temp_compile(src, ns, dest_dir=None): - if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS: - return None - - dest = (dest_dir or ns.temp) / (src.stem + ".py") - return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2) - - -def _write_to_zip(zf, dest, src, ns): - pyc = _py_temp_compile(src, ns) - if pyc: - try: - zf.write(str(pyc), dest.with_suffix(".pyc")) - finally: - try: - pyc.unlink() - except: - log_exception("Failed to delete {}", pyc) - return - - if src in LIB2TO3_GRAMMAR_FILES: - from lib2to3.pgen2.driver import load_grammar - - tmp = ns.temp / src.name - try: - shutil.copy(src, tmp) - load_grammar(str(tmp)) - for f in ns.temp.glob(src.stem + "*.pickle"): - zf.write(str(f), str(dest.parent / f.name)) - try: - f.unlink() - except: - log_exception("Failed to delete {}", f) - except: - log_exception("Failed to compile {}", src) - finally: - try: - tmp.unlink() - except: - log_exception("Failed to delete {}", tmp) - - zf.write(str(src), str(dest)) - - -def generate_source_files(ns): - if ns.zip_lib: - zip_name = PYTHON_ZIP_NAME - zip_path = ns.temp / zip_name - if zip_path.is_file(): - zip_path.unlink() - elif zip_path.is_dir(): - log_error( - "Cannot create zip file because a directory exists by the same name" - ) - return - log_info("Generating {} in {}", zip_name, ns.temp) - ns.temp.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: - for dest, src in get_lib_layout(ns): - _write_to_zip(zf, dest, src, ns) - - if ns.include_underpth: - log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp) - ns.temp.mkdir(parents=True, exist_ok=True) - with open(ns.temp / PYTHON_PTH_NAME, "w", encoding="utf-8") as f: - if ns.zip_lib: - print(PYTHON_ZIP_NAME, file=f) - if ns.include_pip: - print("packages", file=f) - else: - print("Lib", file=f) - print("Lib/site-packages", file=f) - if not ns.flat_dlls: - print("DLLs", file=f) - print(".", file=f) - print(file=f) - print("# Uncomment to run site.main() automatically", file=f) - print("#import site", file=f) - - if ns.include_appxmanifest: - log_info("Generating AppxManifest.xml in {}", ns.temp) - ns.temp.mkdir(parents=True, exist_ok=True) - - with open(ns.temp / "AppxManifest.xml", "wb") as f: - f.write(get_appxmanifest(ns)) - - with open(ns.temp / "_resources.xml", "wb") as f: - f.write(get_resources_xml(ns)) - - if ns.include_pip: - pip_dir = get_pip_dir(ns) - if not (pip_dir / "pip").is_dir(): - log_info("Extracting pip to {}", pip_dir) - pip_dir.mkdir(parents=True, exist_ok=True) - extract_pip_files(ns) - - if ns.include_props: - log_info("Generating {} in {}", PYTHON_PROPS_NAME, ns.temp) - ns.temp.mkdir(parents=True, exist_ok=True) - with open(ns.temp / PYTHON_PROPS_NAME, "wb") as f: - f.write(get_props(ns)) - - -def _create_zip_file(ns): - if not ns.zip: - return None - - if ns.zip.is_file(): - try: - ns.zip.unlink() - except OSError: - log_exception("Unable to remove {}", ns.zip) - sys.exit(8) - elif ns.zip.is_dir(): - log_error("Cannot create ZIP file because {} is a directory", ns.zip) - sys.exit(8) - - ns.zip.parent.mkdir(parents=True, exist_ok=True) - return zipfile.ZipFile(ns.zip, "w", zipfile.ZIP_DEFLATED) - - -def copy_files(files, ns): - if ns.copy: - ns.copy.mkdir(parents=True, exist_ok=True) - - try: - total = len(files) - except TypeError: - total = None - count = 0 - - zip_file = _create_zip_file(ns) - try: - need_compile = [] - in_catalog = [] - - for dest, src in files: - count += 1 - if count % 10 == 0: - if total: - log_info("Processed {:>4} of {} files", count, total) - else: - log_info("Processed {} files", count) - log_debug("Processing {!s}", src) - - if ( - ns.precompile - and src in PY_FILES - and src not in EXCLUDE_FROM_COMPILE - and src.parent not in DATA_DIRS - and os.path.normcase(str(dest)).startswith(os.path.normcase("Lib")) - ): - if ns.copy: - need_compile.append((dest, ns.copy / dest)) - else: - (ns.temp / "Lib" / dest).parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, ns.temp / "Lib" / dest) - need_compile.append((dest, ns.temp / "Lib" / dest)) - - if src not in EXCLUDE_FROM_CATALOG: - in_catalog.append((src.name, src)) - - if ns.copy: - log_debug("Copy {} -> {}", src, ns.copy / dest) - (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True) - try: - shutil.copy2(src, ns.copy / dest) - except shutil.SameFileError: - pass - - if ns.zip: - log_debug("Zip {} into {}", src, ns.zip) - zip_file.write(src, str(dest)) - - if need_compile: - for dest, src in need_compile: - compiled = [ - _compile_one_py(src, None, dest, optimize=0), - _compile_one_py(src, None, dest, optimize=1), - _compile_one_py(src, None, dest, optimize=2), - ] - for c in compiled: - if not c: - continue - cdest = Path(dest).parent / Path(c).relative_to(src.parent) - if ns.zip: - log_debug("Zip {} into {}", c, ns.zip) - zip_file.write(c, str(cdest)) - in_catalog.append((cdest.name, cdest)) - - if ns.catalog: - # Just write out the CDF now. Compilation and signing is - # an extra step - log_info("Generating {}", ns.catalog) - ns.catalog.parent.mkdir(parents=True, exist_ok=True) - write_catalog(ns.catalog, in_catalog) - - finally: - if zip_file: - zip_file.close() - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-v", help="Increase verbosity", action="count") - parser.add_argument( - "-s", - "--source", - metavar="dir", - help="The directory containing the repository root", - type=Path, - default=None, - ) - parser.add_argument( - "-b", "--build", metavar="dir", help="Specify the build directory", type=Path - ) - parser.add_argument( - "--doc-build", - metavar="dir", - help="Specify the docs build directory", - type=Path, - default=None, - ) - parser.add_argument( - "--copy", - metavar="directory", - help="The name of the directory to copy an extracted layout to", - type=Path, - default=None, - ) - parser.add_argument( - "--zip", - metavar="file", - help="The ZIP file to write all files to", - type=Path, - default=None, - ) - parser.add_argument( - "--catalog", - metavar="file", - help="The CDF file to write catalog entries to", - type=Path, - default=None, - ) - parser.add_argument( - "--log", - metavar="file", - help="Write all operations to the specified file", - type=Path, - default=None, - ) - parser.add_argument( - "-t", - "--temp", - metavar="file", - help="A temporary working directory", - type=Path, - default=None, - ) - parser.add_argument( - "-d", "--debug", help="Include debug build", action="store_true" - ) - parser.add_argument( - "-p", - "--precompile", - help="Include .pyc files instead of .py", - action="store_true", - ) - parser.add_argument( - "-z", "--zip-lib", help="Include library in a ZIP file", action="store_true" - ) - parser.add_argument( - "--flat-dlls", help="Does not create a DLLs directory", action="store_true" - ) - parser.add_argument( - "-a", - "--include-all", - help="Include all optional components", - action="store_true", - ) - parser.add_argument( - "--include-cat", - metavar="file", - help="Specify the catalog file to include", - type=Path, - default=None, - ) - for opt, help in get_argparse_options(): - parser.add_argument(opt, help=help, action="store_true") - - ns = parser.parse_args() - update_presets(ns) - - ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) - ns.build = ns.build or Path(sys.executable).parent - ns.temp = ns.temp or Path(tempfile.mkdtemp()) - ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") - if not ns.source.is_absolute(): - ns.source = (Path.cwd() / ns.source).resolve() - if not ns.build.is_absolute(): - ns.build = (Path.cwd() / ns.build).resolve() - if not ns.temp.is_absolute(): - ns.temp = (Path.cwd() / ns.temp).resolve() - if not ns.doc_build.is_absolute(): - ns.doc_build = (Path.cwd() / ns.doc_build).resolve() - if ns.include_cat and not ns.include_cat.is_absolute(): - ns.include_cat = (Path.cwd() / ns.include_cat).resolve() - - if ns.copy and not ns.copy.is_absolute(): - ns.copy = (Path.cwd() / ns.copy).resolve() - if ns.zip and not ns.zip.is_absolute(): - ns.zip = (Path.cwd() / ns.zip).resolve() - if ns.catalog and not ns.catalog.is_absolute(): - ns.catalog = (Path.cwd() / ns.catalog).resolve() - - configure_logger(ns) - - log_info( - """OPTIONS -Source: {ns.source} -Build: {ns.build} -Temp: {ns.temp} - -Copy to: {ns.copy} -Zip to: {ns.zip} -Catalog: {ns.catalog}""", - ns=ns, - ) - - if ns.include_idle and not ns.include_tcltk: - log_warning("Assuming --include-tcltk to support --include-idle") - ns.include_tcltk = True - - try: - generate_source_files(ns) - files = list(get_layout(ns)) - copy_files(files, ns) - except KeyboardInterrupt: - log_info("Interrupted by Ctrl+C") - return 3 - except SystemExit: - raise - except: - log_exception("Unhandled error") - - if error_was_logged(): - log_error("Errors occurred.") - return 1 - - -if __name__ == "__main__": - sys.exit(int(main() or 0)) |