diff options
Diffstat (limited to 'Lib/test/test_embed.py')
| -rw-r--r-- | Lib/test/test_embed.py | 342 |
1 files changed, 322 insertions, 20 deletions
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e02acbc..ed2b96f 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -3,15 +3,19 @@ from test import support import unittest from collections import namedtuple +import contextlib import json import os import re +import shutil import subprocess import sys +import tempfile import textwrap MS_WINDOWS = (os.name == 'nt') +MACOS = (sys.platform == 'darwin') PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 @@ -25,6 +29,12 @@ API_PYTHON = 2 API_ISOLATED = 3 +def debug_build(program): + program = os.path.basename(program) + name = os.path.splitext(program)[0] + return name.endswith("_d") + + def remove_python_envvars(): env = dict(os.environ) # Remove PYTHON* environment variables to get deterministic environment @@ -40,7 +50,7 @@ class EmbeddingTestsMixin: basepath = os.path.dirname(os.path.dirname(os.path.dirname(here))) exename = "_testembed" if MS_WINDOWS: - ext = ("_d" if "_d" in sys.executable else "") + ".exe" + ext = ("_d" if debug_build(sys.executable) else "") + ".exe" exename += ext exepath = os.path.dirname(sys.executable) else: @@ -58,7 +68,8 @@ class EmbeddingTestsMixin: os.chdir(self.oldcwd) def run_embedded_interpreter(self, *args, env=None, - timeout=None, returncode=0, input=None): + timeout=None, returncode=0, input=None, + cwd=None): """Runs a test in the embedded interpreter""" cmd = [self.test_exe] cmd.extend(args) @@ -72,7 +83,8 @@ class EmbeddingTestsMixin: stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, - env=env) + env=env, + cwd=cwd) try: (out, err) = p.communicate(input=input, timeout=timeout) except: @@ -460,6 +472,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): EXPECTED_CONFIG = None + @classmethod + def tearDownClass(cls): + # clear cache + cls.EXPECTED_CONFIG = None + def main_xoptions(self, xoptions_list): xoptions = {} for opt in xoptions_list: @@ -470,7 +487,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): xoptions[opt] = True return xoptions - def _get_expected_config(self, env): + def _get_expected_config_impl(self): + env = remove_python_envvars() code = textwrap.dedent(''' import json import sys @@ -489,23 +507,37 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): args = [sys.executable, '-S', '-c', code] proc = subprocess.run(args, env=env, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.PIPE) if proc.returncode: raise Exception(f"failed to get the default config: " f"stdout={proc.stdout!r} stderr={proc.stderr!r}") stdout = proc.stdout.decode('utf-8') + # ignore stderr try: return json.loads(stdout) except json.JSONDecodeError: self.fail(f"fail to decode stdout: {stdout!r}") + def _get_expected_config(self): + cls = InitConfigTests + if cls.EXPECTED_CONFIG is None: + cls.EXPECTED_CONFIG = self._get_expected_config_impl() + + # get a copy + configs = {} + for config_key, config_value in cls.EXPECTED_CONFIG.items(): + config = {} + for key, value in config_value.items(): + if isinstance(value, list): + value = value.copy() + config[key] = value + configs[config_key] = config + return configs + def get_expected_config(self, expected_preconfig, expected, env, api, modify_path_cb=None): cls = self.__class__ - if cls.EXPECTED_CONFIG is None: - cls.EXPECTED_CONFIG = self._get_expected_config(env) - configs = {key: dict(value) - for key, value in self.EXPECTED_CONFIG.items()} + configs = self._get_expected_config() pre_config = configs['pre_config'] for key, value in expected_preconfig.items(): @@ -553,9 +585,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if value is self.GET_DEFAULT_CONFIG: expected[key] = config[key] - prepend_path = expected['pythonpath_env'] - if prepend_path is not None: - expected['module_search_paths'] = [prepend_path, *expected['module_search_paths']] + pythonpath_env = expected['pythonpath_env'] + if pythonpath_env is not None: + paths = pythonpath_env.split(os.path.pathsep) + expected['module_search_paths'] = [*paths, *expected['module_search_paths']] if modify_path_cb is not None: expected['module_search_paths'] = expected['module_search_paths'].copy() modify_path_cb(expected['module_search_paths']) @@ -603,13 +636,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.assertEqual(configs['global_config'], expected) def check_all_configs(self, testname, expected_config=None, - expected_preconfig=None, modify_path_cb=None, stderr=None, - *, api): - env = remove_python_envvars() - - if api == API_ISOLATED: + expected_preconfig=None, modify_path_cb=None, + stderr=None, *, api, preconfig_api=None, + env=None, ignore_stderr=False, cwd=None): + new_env = remove_python_envvars() + if env is not None: + new_env.update(env) + env = new_env + + if preconfig_api is None: + preconfig_api = api + if preconfig_api == API_ISOLATED: default_preconfig = self.PRE_CONFIG_ISOLATED - elif api == API_PYTHON: + elif preconfig_api == API_PYTHON: default_preconfig = self.PRE_CONFIG_PYTHON else: default_preconfig = self.PRE_CONFIG_COMPAT @@ -631,10 +670,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): expected_config, env, api, modify_path_cb) - out, err = self.run_embedded_interpreter(testname, env=env) + out, err = self.run_embedded_interpreter(testname, + env=env, cwd=cwd) if stderr is None and not expected_config['verbose']: stderr = "" - if stderr is not None: + if stderr is not None and not ignore_stderr: self.assertEqual(err.rstrip(), stderr) try: configs = json.loads(out) @@ -965,6 +1005,268 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_all_configs("test_init_dont_parse_argv", config, pre_config, api=API_PYTHON) + def default_program_name(self, config): + if MS_WINDOWS: + program_name = 'python' + executable = self.test_exe + else: + program_name = 'python3' + if MACOS: + executable = self.test_exe + else: + executable = shutil.which(program_name) or '' + config.update({ + 'program_name': program_name, + 'base_executable': executable, + 'executable': executable, + }) + + def test_init_setpath(self): + # Test Py_SetPath() + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + + config = { + 'module_search_paths': paths, + 'prefix': '', + 'base_prefix': '', + 'exec_prefix': '', + 'base_exec_prefix': '', + } + self.default_program_name(config) + env = {'TESTPATH': os.path.pathsep.join(paths)} + self.check_all_configs("test_init_setpath", config, + api=API_COMPAT, env=env, + ignore_stderr=True) + + def test_init_setpath_config(self): + # Test Py_SetPath() with PyConfig + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + + config = { + # set by Py_SetPath() + 'module_search_paths': paths, + 'prefix': '', + 'base_prefix': '', + 'exec_prefix': '', + 'base_exec_prefix': '', + # overriden by PyConfig + 'program_name': 'conf_program_name', + 'base_executable': 'conf_executable', + 'executable': 'conf_executable', + } + env = {'TESTPATH': os.path.pathsep.join(paths)} + self.check_all_configs("test_init_setpath_config", config, + api=API_PYTHON, env=env, ignore_stderr=True) + + def module_search_paths(self, prefix=None, exec_prefix=None): + config = self._get_expected_config() + if prefix is None: + prefix = config['config']['prefix'] + if exec_prefix is None: + exec_prefix = config['config']['prefix'] + if MS_WINDOWS: + return config['config']['module_search_paths'] + else: + ver = sys.version_info + return [ + os.path.join(prefix, 'lib', + f'python{ver.major}{ver.minor}.zip'), + os.path.join(prefix, 'lib', + f'python{ver.major}.{ver.minor}'), + os.path.join(exec_prefix, 'lib', + f'python{ver.major}.{ver.minor}', 'lib-dynload'), + ] + + @contextlib.contextmanager + def tmpdir_with_python(self): + # Temporary directory with a copy of the Python program + with tempfile.TemporaryDirectory() as tmpdir: + # bpo-38234: On macOS and FreeBSD, the temporary directory + # can be symbolic link. For example, /tmp can be a symbolic link + # to /var/tmp. Call realpath() to resolve all symbolic links. + tmpdir = os.path.realpath(tmpdir) + + if MS_WINDOWS: + # Copy pythonXY.dll (or pythonXY_d.dll) + ver = sys.version_info + dll = f'python{ver.major}{ver.minor}' + if debug_build(sys.executable): + dll += '_d' + dll += '.dll' + dll = os.path.join(os.path.dirname(self.test_exe), dll) + dll_copy = os.path.join(tmpdir, os.path.basename(dll)) + shutil.copyfile(dll, dll_copy) + + # Copy Python program + exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe)) + shutil.copyfile(self.test_exe, exec_copy) + shutil.copystat(self.test_exe, exec_copy) + self.test_exe = exec_copy + + yield tmpdir + + def test_init_setpythonhome(self): + # Test Py_SetPythonHome(home) with PYTHONPATH env var + config = self._get_expected_config() + paths = config['config']['module_search_paths'] + paths_str = os.path.pathsep.join(paths) + + for path in paths: + if not os.path.isdir(path): + continue + if os.path.exists(os.path.join(path, 'os.py')): + home = os.path.dirname(path) + break + else: + self.fail(f"Unable to find home in {paths!r}") + + prefix = exec_prefix = home + ver = sys.version_info + expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) + + config = { + 'home': home, + 'module_search_paths': expected_paths, + 'prefix': prefix, + 'base_prefix': prefix, + 'exec_prefix': exec_prefix, + 'base_exec_prefix': exec_prefix, + 'pythonpath_env': paths_str, + } + self.default_program_name(config) + env = {'TESTHOME': home, 'PYTHONPATH': paths_str} + self.check_all_configs("test_init_setpythonhome", config, + api=API_COMPAT, env=env) + + def copy_paths_by_env(self, config): + all_configs = self._get_expected_config() + paths = all_configs['config']['module_search_paths'] + paths_str = os.path.pathsep.join(paths) + config['pythonpath_env'] = paths_str + env = {'PYTHONPATH': paths_str} + return env + + @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt') + def test_init_pybuilddir(self): + # Test path configuration with pybuilddir.txt configuration file + + with self.tmpdir_with_python() as tmpdir: + # pybuilddir.txt is a sub-directory relative to the current + # directory (tmpdir) + subdir = 'libdir' + libdir = os.path.join(tmpdir, subdir) + os.mkdir(libdir) + + filename = os.path.join(tmpdir, 'pybuilddir.txt') + with open(filename, "w", encoding="utf8") as fp: + fp.write(subdir) + + module_search_paths = self.module_search_paths() + module_search_paths[-1] = libdir + + executable = self.test_exe + config = { + 'base_executable': executable, + 'executable': executable, + 'module_search_paths': module_search_paths, + } + env = self.copy_paths_by_env(config) + self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=True, cwd=tmpdir) + + def test_init_pyvenv_cfg(self): + # Test path configuration with pyvenv.cfg configuration file + + with self.tmpdir_with_python() as tmpdir, \ + tempfile.TemporaryDirectory() as pyvenv_home: + ver = sys.version_info + + if not MS_WINDOWS: + lib_dynload = os.path.join(pyvenv_home, + 'lib', + f'python{ver.major}.{ver.minor}', + 'lib-dynload') + os.makedirs(lib_dynload) + else: + lib_dynload = os.path.join(pyvenv_home, 'lib') + os.makedirs(lib_dynload) + # getpathp.c uses Lib\os.py as the LANDMARK + shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py')) + + filename = os.path.join(tmpdir, 'pyvenv.cfg') + with open(filename, "w", encoding="utf8") as fp: + print("home = %s" % pyvenv_home, file=fp) + print("include-system-site-packages = false", file=fp) + + paths = self.module_search_paths() + if not MS_WINDOWS: + paths[-1] = lib_dynload + else: + for index, path in enumerate(paths): + if index == 0: + paths[index] = os.path.join(tmpdir, os.path.basename(path)) + else: + paths[index] = os.path.join(pyvenv_home, os.path.basename(path)) + paths[-1] = pyvenv_home + + executable = self.test_exe + exec_prefix = pyvenv_home + config = { + 'base_exec_prefix': exec_prefix, + 'exec_prefix': exec_prefix, + 'base_executable': executable, + 'executable': executable, + 'module_search_paths': paths, + } + if MS_WINDOWS: + config['base_prefix'] = pyvenv_home + config['prefix'] = pyvenv_home + env = self.copy_paths_by_env(config) + self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=True, cwd=tmpdir) + + def test_global_pathconfig(self): + # Test C API functions getting the path configuration: + # + # - Py_GetExecPrefix() + # - Py_GetPath() + # - Py_GetPrefix() + # - Py_GetProgramFullPath() + # - Py_GetProgramName() + # - Py_GetPythonHome() + # + # The global path configuration (_Py_path_config) must be a copy + # of the path configuration of PyInterpreter.config (PyConfig). + ctypes = support.import_module('ctypes') + _testinternalcapi = support.import_module('_testinternalcapi') + + def get_func(name): + func = getattr(ctypes.pythonapi, name) + func.argtypes = () + func.restype = ctypes.c_wchar_p + return func + + Py_GetPath = get_func('Py_GetPath') + Py_GetPrefix = get_func('Py_GetPrefix') + Py_GetExecPrefix = get_func('Py_GetExecPrefix') + Py_GetProgramName = get_func('Py_GetProgramName') + Py_GetProgramFullPath = get_func('Py_GetProgramFullPath') + Py_GetPythonHome = get_func('Py_GetPythonHome') + + config = _testinternalcapi.get_configs()['config'] + + self.assertEqual(Py_GetPath().split(os.path.pathsep), + config['module_search_paths']) + self.assertEqual(Py_GetPrefix(), config['prefix']) + self.assertEqual(Py_GetExecPrefix(), config['exec_prefix']) + self.assertEqual(Py_GetProgramName(), config['program_name']) + self.assertEqual(Py_GetProgramFullPath(), config['executable']) + self.assertEqual(Py_GetPythonHome(), config['home']) + class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): def test_open_code_hook(self): |