From 090591636c4f03ce06a039079bd7716a5b23631e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 17 Sep 2021 16:31:31 -0600 Subject: bpo-45020: Freeze os, site, and codecs. (gh-28398) https://bugs.python.org/issue45020 --- Lib/test/support/__init__.py | 18 +++++++++--------- Lib/test/test_embed.py | 12 +++++++----- Lib/test/test_import/__init__.py | 5 +++-- Lib/test/test_site.py | 6 +++--- Makefile.pre.in | 12 ++++++++++++ PCbuild/_freeze_module.vcxproj | 15 +++++++++++++++ PCbuild/_freeze_module.vcxproj.filters | 9 +++++++++ Python/frozen.c | 11 ++++++++++- Tools/scripts/freeze_modules.py | 22 +++++++++++----------- 9 files changed, 79 insertions(+), 31 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 89f5e5a..04516a5 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -99,6 +99,13 @@ SHORT_TIMEOUT = 30.0 # option. LONG_TIMEOUT = 5 * 60.0 +# TEST_HOME_DIR refers to the top level directory of the "test" package +# that contains Python's regression test suite +TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR) +STDLIB_DIR = os.path.dirname(TEST_HOME_DIR) +REPO_ROOT = os.path.dirname(STDLIB_DIR) + class Error(Exception): """Base class for regression test exceptions.""" @@ -148,9 +155,7 @@ def load_package_tests(pkg_dir, loader, standard_tests, pattern): """ if pattern is None: pattern = "test*" - top_dir = os.path.dirname( # Lib - os.path.dirname( # test - os.path.dirname(__file__))) # support + top_dir = STDLIB_DIR package_tests = loader.discover(start_dir=pkg_dir, top_level_dir=top_dir, pattern=pattern) @@ -459,11 +464,6 @@ PGO = False # PGO task. If this is True, PGO is also True. PGO_EXTENDED = False -# TEST_HOME_DIR refers to the top level directory of the "test" package -# that contains Python's regression test suite -TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR) - # TEST_DATA_DIR is used as a target download location for remote resources TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data") @@ -1390,7 +1390,7 @@ class PythonSymlink: self._env = {k.upper(): os.getenv(k) for k in os.environ} self._env["PYTHONHOME"] = os.path.dirname(self.real) if sysconfig.is_python_build(True): - self._env["PYTHONPATH"] = os.path.dirname(os.__file__) + self._env["PYTHONPATH"] = STDLIB_DIR def __enter__(self): os.symlink(self.real, self.link) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e5e7c83..941e307 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -8,6 +8,7 @@ from collections import namedtuple import contextlib import json import os +import os.path import re import shutil import subprocess @@ -52,15 +53,13 @@ def remove_python_envvars(): class EmbeddingTestsMixin: def setUp(self): - here = os.path.abspath(__file__) - basepath = os.path.dirname(os.path.dirname(os.path.dirname(here))) exename = "_testembed" if MS_WINDOWS: ext = ("_d" if debug_build(sys.executable) else "") + ".exe" exename += ext exepath = os.path.dirname(sys.executable) else: - exepath = os.path.join(basepath, "Programs") + exepath = os.path.join(support.REPO_ROOT, "Programs") self.test_exe = exe = os.path.join(exepath, exename) if not os.path.exists(exe): self.skipTest("%r doesn't exist" % exe) @@ -68,7 +67,7 @@ class EmbeddingTestsMixin: # "Py_Initialize: Unable to get the locale encoding # LookupError: no codec search functions registered: can't find encoding" self.oldcwd = os.getcwd() - os.chdir(basepath) + os.chdir(support.REPO_ROOT) def tearDown(self): os.chdir(self.oldcwd) @@ -1304,7 +1303,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): lib_dynload = os.path.join(pyvenv_home, 'lib') os.makedirs(lib_dynload) # getpathp.c uses Lib\os.py as the LANDMARK - shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py')) + shutil.copyfile( + os.path.join(support.STDLIB_DIR, 'os.py'), + os.path.join(lib_dynload, 'os.py'), + ) filename = os.path.join(tmpdir, 'pyvenv.cfg') with open(filename, "w", encoding="utf8") as fp: diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index afbc12c..8857fd5 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -19,7 +19,8 @@ import unittest from unittest import mock from test.support import os_helper -from test.support import (is_jython, swap_attr, swap_item, cpython_only) +from test.support import ( + STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) from test.support.os_helper import ( @@ -495,7 +496,7 @@ class ImportTests(unittest.TestCase): env = None env = {k.upper(): os.environ[k] for k in os.environ} - env["PYTHONPATH"] = tmp2 + ";" + os.path.dirname(os.__file__) + env["PYTHONPATH"] = tmp2 + ";" + STDLIB_DIR # Test 1: import with added DLL directory subprocess.check_call([ diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index fa74e77..5f06a0d 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -591,7 +591,7 @@ class _pthFileTests(unittest.TestCase): return sys_path def test_underpth_nosite_file(self): - libpath = os.path.dirname(os.path.dirname(encodings.__file__)) + libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) pth_lines = [ 'fake-path-name', @@ -619,7 +619,7 @@ class _pthFileTests(unittest.TestCase): ) def test_underpth_file(self): - libpath = os.path.dirname(os.path.dirname(encodings.__file__)) + libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) exe_file = self._create_underpth_exe([ 'fake-path-name', @@ -644,7 +644,7 @@ class _pthFileTests(unittest.TestCase): def test_underpth_dll_file(self): - libpath = os.path.dirname(os.path.dirname(encodings.__file__)) + libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) exe_file = self._create_underpth_exe([ 'fake-path-name', diff --git a/Makefile.pre.in b/Makefile.pre.in index d6b2179..9ea359b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -762,6 +762,9 @@ Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h +Python/frozen_modules/codecs.h: Programs/_freeze_module Lib/codecs.py + Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py $(srcdir)/Python/frozen_modules/codecs.h + Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h @@ -780,6 +783,12 @@ Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h +Python/frozen_modules/os.h: Programs/_freeze_module Lib/os.py + Programs/_freeze_module os $(srcdir)/Lib/os.py $(srcdir)/Python/frozen_modules/os.h + +Python/frozen_modules/site.h: Programs/_freeze_module Lib/site.py + Programs/_freeze_module site $(srcdir)/Lib/site.py $(srcdir)/Python/frozen_modules/site.h + Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h @@ -1023,12 +1032,15 @@ FROZEN_FILES = \ Python/frozen_modules/importlib__bootstrap_external.h \ Python/frozen_modules/zipimport.h \ Python/frozen_modules/abc.h \ + Python/frozen_modules/codecs.h \ Python/frozen_modules/io.h \ Python/frozen_modules/_collections_abc.h \ Python/frozen_modules/_sitebuiltins.h \ Python/frozen_modules/genericpath.h \ Python/frozen_modules/ntpath.h \ Python/frozen_modules/posixpath.h \ + Python/frozen_modules/os.h \ + Python/frozen_modules/site.h \ Python/frozen_modules/stat.h \ Python/frozen_modules/__hello__.h # End FROZEN_FILES diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index ef0069c..382351c 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -250,6 +250,11 @@ $(IntDir)abc.g.h $(PySourcePath)Python\frozen_modules\abc.h + + codecs + $(IntDir)codecs.g.h + $(PySourcePath)Python\frozen_modules\codecs.h + io $(IntDir)io.g.h @@ -280,6 +285,16 @@ $(IntDir)posixpath.g.h $(PySourcePath)Python\frozen_modules\posixpath.h + + os + $(IntDir)os.g.h + $(PySourcePath)Python\frozen_modules\os.h + + + site + $(IntDir)site.g.h + $(PySourcePath)Python\frozen_modules\site.h + stat $(IntDir)stat.g.h diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 71fb41c..4a1c90f 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -28,6 +28,9 @@ Python Files + + Python Files + Python Files @@ -46,6 +49,12 @@ Python Files + + Python Files + + + Python Files + Python Files diff --git a/Python/frozen.c b/Python/frozen.c index cf384d3..05b5281 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -42,12 +42,15 @@ #include "frozen_modules/importlib__bootstrap_external.h" #include "frozen_modules/zipimport.h" #include "frozen_modules/abc.h" +#include "frozen_modules/codecs.h" #include "frozen_modules/io.h" #include "frozen_modules/_collections_abc.h" #include "frozen_modules/_sitebuiltins.h" #include "frozen_modules/genericpath.h" #include "frozen_modules/ntpath.h" #include "frozen_modules/posixpath.h" +#include "frozen_modules/os.h" +#include "frozen_modules/site.h" #include "frozen_modules/stat.h" #include "frozen_modules/__hello__.h" /* End includes */ @@ -62,15 +65,21 @@ static const struct _frozen _PyImport_FrozenModules[] = { (int)sizeof(_Py_M__importlib__bootstrap_external)}, {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)}, - /* stdlib */ + /* stdlib - startup, without site (python -S) */ {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc)}, + {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs)}, {"io", _Py_M__io, (int)sizeof(_Py_M__io)}, + + /* stdlib - startup, with site */ {"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc)}, {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins)}, {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath)}, {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath)}, {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)}, + {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)}, + {"os", _Py_M__os, (int)sizeof(_Py_M__os)}, + {"site", _Py_M__site, (int)sizeof(_Py_M__site)}, {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat)}, /* Test module */ diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index bfbea5d..b3ae5c7 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -66,16 +66,16 @@ FROZEN = [ # on a builtin zip file instead of a filesystem. 'zipimport', ]), - ('stdlib', [ - # For the moment we skip codecs, encodings.*, os, and site. - # These modules have different generated files depending on - # if a debug or non-debug build. (See bpo-45186 and bpo-45188.) - # without site (python -S) + ('stdlib - startup, without site (python -S)', [ 'abc', - #'codecs', - # '', + 'codecs', + # For now we do not freeze the encodings, due # to the noise all + # those extra modules add to the text printed during the build. + # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.) + #'', 'io', - # with site + ]), + ('stdlib - startup, with site', [ '_collections_abc', '_sitebuiltins', 'genericpath', @@ -83,9 +83,9 @@ FROZEN = [ 'posixpath', # We must explicitly mark os.path as a frozen module # even though it will never be imported. - #f'{OS_PATH} : os.path', - #'os', - #'site', + f'{OS_PATH} : os.path', + 'os', + 'site', 'stat', ]), ('Test module', [ -- cgit v0.12