diff options
| author | Steve Dower <steve.dower@python.org> | 2021-12-03 00:08:42 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-12-03 00:08:42 (GMT) |
| commit | 99fcf1505218464c489d419d4500f126b6d6dc28 (patch) | |
| tree | a9d607d854e943b3651248eadbe2f31f8c410021 | |
| parent | 9f2f7e42269db74a89fc8cd74d82a875787f01d7 (diff) | |
| download | cpython-99fcf1505218464c489d419d4500f126b6d6dc28.zip cpython-99fcf1505218464c489d419d4500f126b6d6dc28.tar.gz cpython-99fcf1505218464c489d419d4500f126b6d6dc28.tar.bz2 | |
bpo-45582: Port getpath[p].c to Python (GH-29041)
The getpath.py file is frozen at build time and executed as code over a namespace. It is never imported, nor is it meant to be importable or reusable. However, it should be easier to read, modify, and patch than the previous code.
This commit attempts to preserve every previously tested quirk, but these may be changed in the future to better align platforms.
40 files changed, 3516 insertions, 3668 deletions
@@ -74,6 +74,7 @@ Mac/pythonw Misc/python.pc Misc/python-embed.pc Misc/python-config.sh +Modules/getpath.h Modules/Setup.config Modules/Setup.local Modules/Setup.stdlib diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 3642a8b..7c8fb7b 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -479,6 +479,9 @@ PyConfig Fields which are already initialized are left unchanged. + Fields for :ref:`path configuration <init-path-config>` are no longer + calculated or modified when calling this function, as of Python 3.11. + The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are @@ -493,6 +496,12 @@ PyConfig parsed, and arguments are only parsed if :c:member:`PyConfig.parse_argv` equals ``1``. + .. versionchanged:: 3.11 + :c:func:`PyConfig_Read` no longer calculates all paths, and so fields + listed under :ref:`Python Path Configuration <init-path-config>` may + no longer be updated until :c:func:`Py_InitializeFromConfig` is + called. + .. c:function:: void PyConfig_Clear(PyConfig *config) Release configuration memory. @@ -848,12 +857,19 @@ PyConfig Default: value of the ``PLATLIBDIR`` macro which is set by the :option:`configure --with-platlibdir option <--with-platlibdir>` - (default: ``"lib"``). + (default: ``"lib"``, or ``"DLLs"`` on Windows). Part of the :ref:`Python Path Configuration <init-path-config>` input. .. versionadded:: 3.9 + .. versionchanged:: 3.11 + This macro is now used on Windows to locate the standard + library extension modules, typically under ``DLLs``. However, + for compatibility, note that this value is ignored for any + non-standard layouts, including in-tree builds and virtual + environments. + .. c:member:: wchar_t* pythonpath_env Module search paths (:data:`sys.path`) as a string separated by ``DELIM`` @@ -870,9 +886,9 @@ PyConfig Module search paths: :data:`sys.path`. - If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the - function calculating the :ref:`Python Path Configuration <init-path-config>` - overrides the :c:member:`~PyConfig.module_search_paths` and sets + If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, + :c:func:`Py_InitializeFromConfig` will replace + :c:member:`~PyConfig.module_search_paths` and sets :c:member:`~PyConfig.module_search_paths_set` to ``1``. Default: empty list (``module_search_paths``) and ``0`` @@ -944,16 +960,16 @@ PyConfig .. c:member:: int pathconfig_warnings - On Unix, if non-zero, calculating the :ref:`Python Path Configuration - <init-path-config>` can log warnings into ``stderr``. If equals to 0, - suppress these warnings. - - It has no effect on Windows. + If non-zero, calculation of path configuration is allowed to log + warnings into ``stderr``. If equals to 0, suppress these warnings. Default: ``1`` in Python mode, ``0`` in isolated mode. Part of the :ref:`Python Path Configuration <init-path-config>` input. + .. versionchanged:: 3.11 + Now also applies on Windows. + .. c:member:: wchar_t* prefix The site-specific directory prefix where the platform independent Python @@ -1305,10 +1321,9 @@ variables, command line arguments (:c:member:`PyConfig.argv` is not parsed) and user site directory. The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left unchanged. Signal handlers are not installed. -Configuration files are still used with this configuration. Set the -:ref:`Python Path Configuration <init-path-config>` ("output fields") to ignore these -configuration files and avoid the function computing the default path -configuration. +Configuration files are still used with this configuration to determine +paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified +to avoid computing the default path configuration. .. _init-python-config: diff --git a/Include/cpython/fileutils.h b/Include/cpython/fileutils.h index ccf37e9..7ab2290 100644 --- a/Include/cpython/fileutils.h +++ b/Include/cpython/fileutils.h @@ -135,12 +135,6 @@ PyAPI_FUNC(wchar_t*) _Py_wrealpath( size_t resolved_path_len); #endif -#ifndef MS_WINDOWS -PyAPI_FUNC(int) _Py_isabs(const wchar_t *path); -#endif - -PyAPI_FUNC(int) _Py_abspath(const wchar_t *path, wchar_t **abspath_p); - PyAPI_FUNC(wchar_t*) _Py_wgetcwd( wchar_t *buf, /* Number of characters of 'buf' buffer diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 2be4c25..2ba1224 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -213,6 +213,9 @@ typedef struct PyConfig { // If non-zero, disallow threads, subprocesses, and fork. // Default: 0. int _isolated_interpreter; + + // If non-zero, we believe we're running from a source tree. + int _is_python_build; } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index d1caf9c..38df73b 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -74,14 +74,15 @@ extern int _Py_EncodeNonUnicodeWchar_InPlace( Py_ssize_t size); #endif +extern int _Py_isabs(const wchar_t *path); +extern int _Py_abspath(const wchar_t *path, wchar_t **abspath_p); extern wchar_t * _Py_join_relfile(const wchar_t *dirname, const wchar_t *relfile); extern int _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize); extern size_t _Py_find_basename(const wchar_t *filename); -PyAPI_FUNC(int) _Py_normalize_path(const wchar_t *path, - wchar_t *buf, const size_t buf_len); +PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); // Macros to protect CRT calls against instant termination when passed an diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 9014fcd..be545f4 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -166,6 +166,10 @@ extern PyStatus _PyConfig_SetPyArgv( PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config); PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict); +extern void _Py_DumpPathConfig(PyThreadState *tstate); + +PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject(); + /* --- Function used for testing ---------------------------------- */ diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index a258aab..b8deaa0 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -8,65 +8,15 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -typedef struct _PyPathConfig { - /* Full path to the Python program */ - wchar_t *program_full_path; - wchar_t *prefix; - wchar_t *exec_prefix; - wchar_t *stdlib_dir; - /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */ - wchar_t *module_search_path; - /* Python program name */ - wchar_t *program_name; - /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ - wchar_t *home; -#ifdef MS_WINDOWS - /* isolated and site_import are used to set Py_IsolatedFlag and - Py_NoSiteFlag flags on Windows in read_pth_file(). These fields - are ignored when their value are equal to -1 (unset). */ - int isolated; - int site_import; - /* Set when a venv is detected */ - wchar_t *base_executable; -#endif -} _PyPathConfig; - -#ifdef MS_WINDOWS -# define _PyPathConfig_INIT \ - {.module_search_path = NULL, \ - .isolated = -1, \ - .site_import = -1} -#else -# define _PyPathConfig_INIT \ - {.module_search_path = NULL} -#endif -/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */ - -PyAPI_DATA(_PyPathConfig) _Py_path_config; -#ifdef MS_WINDOWS -PyAPI_DATA(wchar_t*) _Py_dll_path; -#endif - -extern void _PyPathConfig_ClearGlobal(void); +PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void); +extern PyStatus _PyPathConfig_ReadGlobal(PyConfig *config); +extern PyStatus _PyPathConfig_UpdateGlobal(const PyConfig *config); +extern const wchar_t * _PyPathConfig_GetGlobalModuleSearchPath(void); -extern PyStatus _PyPathConfig_Calculate( - _PyPathConfig *pathconfig, - const PyConfig *config); extern int _PyPathConfig_ComputeSysPath0( const PyWideStringList *argv, PyObject **path0); -extern PyStatus _Py_FindEnvConfigValue( - FILE *env_file, - const wchar_t *key, - wchar_t **value_p); - -#ifdef MS_WINDOWS -extern wchar_t* _Py_GetDLLPath(void); -#endif -extern PyStatus _PyConfig_WritePathConfig(const PyConfig *config); -extern void _Py_DumpPathConfig(PyThreadState *tstate); -extern PyObject* _PyPathConfig_AsDict(void); #ifdef __cplusplus } diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 527c7ae..58483a0 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -70,7 +70,7 @@ def isabs(s): if s.replace('/', '\\').startswith('\\\\?\\'): return True s = splitdrive(s)[1] - return len(s) > 0 and s[0] in _get_bothseps(s) + return len(s) > 0 and s[0] and s[0] in _get_bothseps(s) # Join two (or more) paths. @@ -268,11 +268,13 @@ def ismount(path): root, rest = splitdrive(path) if root and root[0] in seps: return (not rest) or (rest in seps) - if rest in seps: + if rest and rest in seps: return True if _getvolumepathname: - return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps) + x = path.rstrip(seps) + y =_getvolumepathname(path).rstrip(seps) + return x.casefold() == y.casefold() else: return False @@ -459,56 +461,68 @@ def expandvars(path): # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! +try: + from nt import _path_normpath -def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - sep = b'\\' - altsep = b'/' - curdir = b'.' - pardir = b'..' - special_prefixes = (b'\\\\.\\', b'\\\\?\\') - else: - sep = '\\' - altsep = '/' - curdir = '.' - pardir = '..' - special_prefixes = ('\\\\.\\', '\\\\?\\') - if path.startswith(special_prefixes): - # in the case of paths with these prefixes: - # \\.\ -> device names - # \\?\ -> literal paths - # do not do any normalization, but return the path - # unchanged apart from the call to os.fspath() - return path - path = path.replace(altsep, sep) - prefix, path = splitdrive(path) - - # collapse initial backslashes - if path.startswith(sep): - prefix += sep - path = path.lstrip(sep) - - comps = path.split(sep) - i = 0 - while i < len(comps): - if not comps[i] or comps[i] == curdir: - del comps[i] - elif comps[i] == pardir: - if i > 0 and comps[i-1] != pardir: - del comps[i-1:i+1] - i -= 1 - elif i == 0 and prefix.endswith(sep): +except ImportError: + def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'\\' + altsep = b'/' + curdir = b'.' + pardir = b'..' + special_prefixes = (b'\\\\.\\', b'\\\\?\\') + else: + sep = '\\' + altsep = '/' + curdir = '.' + pardir = '..' + special_prefixes = ('\\\\.\\', '\\\\?\\') + if path.startswith(special_prefixes): + # in the case of paths with these prefixes: + # \\.\ -> device names + # \\?\ -> literal paths + # do not do any normalization, but return the path + # unchanged apart from the call to os.fspath() + return path + path = path.replace(altsep, sep) + prefix, path = splitdrive(path) + + # collapse initial backslashes + if path.startswith(sep): + prefix += sep + path = path.lstrip(sep) + + comps = path.split(sep) + i = 0 + while i < len(comps): + if not comps[i] or comps[i] == curdir: del comps[i] + elif comps[i] == pardir: + if i > 0 and comps[i-1] != pardir: + del comps[i-1:i+1] + i -= 1 + elif i == 0 and prefix.endswith(sep): + del comps[i] + else: + i += 1 else: i += 1 - else: - i += 1 - # If the path is now empty, substitute '.' - if not prefix and not comps: - comps.append(curdir) - return prefix + sep.join(comps) + # If the path is now empty, substitute '.' + if not prefix and not comps: + comps.append(curdir) + return prefix + sep.join(comps) + +else: + def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." + return _path_normpath(path) or "." + def _abspath_fallback(path): """Return the absolute version of a path as a fallback function in case diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 1953746..a46c667 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -334,43 +334,55 @@ def expandvars(path): # It should be understood that this may change the meaning of the path # if it contains symbolic links! -def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - sep = b'/' - empty = b'' - dot = b'.' - dotdot = b'..' - else: - sep = '/' - empty = '' - dot = '.' - dotdot = '..' - if path == empty: - return dot - initial_slashes = path.startswith(sep) - # POSIX allows one or two initial slashes, but treats three or more - # as single slash. - # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13) - if (initial_slashes and - path.startswith(sep*2) and not path.startswith(sep*3)): - initial_slashes = 2 - comps = path.split(sep) - new_comps = [] - for comp in comps: - if comp in (empty, dot): - continue - if (comp != dotdot or (not initial_slashes and not new_comps) or - (new_comps and new_comps[-1] == dotdot)): - new_comps.append(comp) - elif new_comps: - new_comps.pop() - comps = new_comps - path = sep.join(comps) - if initial_slashes: - path = sep*initial_slashes + path - return path or dot +try: + from posix import _path_normpath + +except ImportError: + def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'/' + empty = b'' + dot = b'.' + dotdot = b'..' + else: + sep = '/' + empty = '' + dot = '.' + dotdot = '..' + if path == empty: + return dot + initial_slashes = path.startswith(sep) + # POSIX allows one or two initial slashes, but treats three or more + # as single slash. + # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13) + if (initial_slashes and + path.startswith(sep*2) and not path.startswith(sep*3)): + initial_slashes = 2 + comps = path.split(sep) + new_comps = [] + for comp in comps: + if comp in (empty, dot): + continue + if (comp != dotdot or (not initial_slashes and not new_comps) or + (new_comps and new_comps[-1] == dotdot)): + new_comps.append(comp) + elif new_comps: + new_comps.pop() + comps = new_comps + path = sep.join(comps) + if initial_slashes: + path = sep*initial_slashes + path + return path or dot + +else: + def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." + return _path_normpath(path) or "." def abspath(path): diff --git a/Lib/site.py b/Lib/site.py index e129f3b..b11cd48 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -361,11 +361,11 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) - libdirs = [sys.platlibdir] - if sys.platlibdir != "lib": - libdirs.append("lib") - if os.sep == '/': + libdirs = [sys.platlibdir] + if sys.platlibdir != "lib": + libdirs.append("lib") + for libdir in libdirs: path = os.path.join(prefix, libdir, "python%d.%d" % sys.version_info[:2], @@ -373,10 +373,7 @@ def getsitepackages(prefixes=None): sitepackages.append(path) else: sitepackages.append(prefix) - - for libdir in libdirs: - path = os.path.join(prefix, libdir, "site-packages") - sitepackages.append(path) + sitepackages.append(os.path.join(prefix, "Lib", "site-packages")) return sitepackages def addsitepackages(known_paths, prefixes=None): diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index daf9f00..da918b7 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -216,6 +216,11 @@ def _expand_vars(scheme, vars): if vars is None: vars = {} _extend_dict(vars, get_config_vars()) + if os.name == 'nt': + # On Windows we want to substitute 'lib' for schemes rather + # than the native value (without modifying vars, in case it + # was passed in) + vars = vars | {'platlibdir': 'lib'} for key, value in _INSTALL_SCHEMES[scheme].items(): if os.name in ('posix', 'nt'): diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index d05907e..e7b7106 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -20,6 +20,7 @@ class SetConfigTests(unittest.TestCase): self.sys_copy = dict(sys.__dict__) def tearDown(self): + _testinternalcapi.reset_path_config() _testinternalcapi.set_config(self.old_config) sys.__dict__.clear() sys.__dict__.update(self.sys_copy) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 4781c7f..3620a76 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -13,6 +13,7 @@ import re import shutil import subprocess import sys +import sysconfig import tempfile import textwrap @@ -445,6 +446,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '_init_main': 1, '_isolated_interpreter': 0, 'use_frozen_modules': 1, + '_is_python_build': IGNORE_CONFIG, } if MS_WINDOWS: CONFIG_COMPAT.update({ @@ -508,32 +510,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), )) - # path config - if MS_WINDOWS: - PATH_CONFIG = { - 'isolated': -1, - 'site_import': -1, - 'python3_dll': GET_DEFAULT_CONFIG, - } - else: - PATH_CONFIG = {} - # other keys are copied by COPY_PATH_CONFIG - - COPY_PATH_CONFIG = [ - # Copy core config to global config for expected values - 'prefix', - 'exec_prefix', - 'program_name', - 'home', - 'stdlib_dir', - # program_full_path and module_search_path are copied indirectly from - # the core configuration in check_path_config(). - ] - if MS_WINDOWS: - COPY_PATH_CONFIG.extend(( - 'base_executable', - )) - EXPECTED_CONFIG = None @classmethod @@ -599,8 +575,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): return configs def get_expected_config(self, expected_preconfig, expected, - expected_pathconfig, env, api, - modify_path_cb=None): + env, api, modify_path_cb=None): configs = self._get_expected_config() pre_config = configs['pre_config'] @@ -608,11 +583,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if value is self.GET_DEFAULT_CONFIG: expected_preconfig[key] = pre_config[key] - path_config = configs['path_config'] - for key, value in expected_pathconfig.items(): - if value is self.GET_DEFAULT_CONFIG: - expected_pathconfig[key] = path_config[key] - if not expected_preconfig['configure_locale'] or api == API_COMPAT: # there is no easy way to get the locale encoding before # setlocale(LC_CTYPE, "") is called: don't test encodings @@ -705,18 +675,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.assertEqual(configs['global_config'], expected) - def check_path_config(self, configs, expected): - config = configs['config'] - - for key in self.COPY_PATH_CONFIG: - expected[key] = config[key] - expected['module_search_path'] = os.path.pathsep.join(config['module_search_paths']) - expected['program_full_path'] = config['executable'] - - self.assertEqual(configs['path_config'], expected) - def check_all_configs(self, testname, expected_config=None, - expected_preconfig=None, expected_pathconfig=None, + expected_preconfig=None, modify_path_cb=None, stderr=None, *, api, preconfig_api=None, env=None, ignore_stderr=False, cwd=None): @@ -740,10 +700,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if expected_config is None: expected_config = {} - if expected_pathconfig is None: - expected_pathconfig = {} - expected_pathconfig = dict(self.PATH_CONFIG, **expected_pathconfig) - if api == API_PYTHON: default_config = self.CONFIG_PYTHON elif api == API_ISOLATED: @@ -754,7 +710,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.get_expected_config(expected_preconfig, expected_config, - expected_pathconfig, env, api, modify_path_cb) @@ -772,7 +727,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_pre_config(configs, expected_preconfig) self.check_config(configs, expected_config) self.check_global_config(configs) - self.check_path_config(configs, expected_pathconfig) return configs def test_init_default_config(self): @@ -1039,10 +993,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_all_configs("test_init_dont_configure_locale", {}, preconfig, api=API_PYTHON) + @unittest.skip('as of 3.11 this test no longer works because ' + 'path calculations do not occur on read') def test_init_read_set(self): config = { 'program_name': './init_read_set', 'executable': 'my_executable', + 'base_executable': 'my_executable', } def modify_path(path): path.insert(1, "test_path_insert1") @@ -1156,7 +1113,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', - 'use_frozen_modules': -1, } self.default_program_name(config) env = {'TESTPATH': os.path.pathsep.join(paths)} @@ -1180,7 +1136,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # The current getpath.c doesn't determine the stdlib dir # in this case. 'stdlib_dir': '', - 'use_frozen_modules': -1, + 'use_frozen_modules': 1, # overridden by PyConfig 'program_name': 'conf_program_name', 'base_executable': 'conf_executable', @@ -1210,13 +1166,16 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ] @contextlib.contextmanager - def tmpdir_with_python(self): + def tmpdir_with_python(self, subdir=None): # Temporary directory with a copy of the Python program with tempfile.TemporaryDirectory() as tmpdir: # bpo-38234: On macOS and FreeBSD, the temporary directory # can be symbolic link. For example, /tmp can be a symbolic link # to /var/tmp. Call realpath() to resolve all symbolic links. tmpdir = os.path.realpath(tmpdir) + if subdir: + tmpdir = os.path.normpath(os.path.join(tmpdir, subdir)) + os.makedirs(tmpdir) if MS_WINDOWS: # Copy pythonXY.dll (or pythonXY_d.dll) @@ -1260,11 +1219,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): prefix = exec_prefix = home if MS_WINDOWS: - stdlib = os.path.join(home, sys.platlibdir) + stdlib = os.path.join(home, "Lib") + # Because we are specifying 'home', module search paths + # are fairly static + expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')] else: version = f'{sys.version_info.major}.{sys.version_info.minor}' stdlib = os.path.join(home, sys.platlibdir, f'python{version}') - expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) + expected_paths = self.module_search_paths(prefix=home, exec_prefix=home) config = { 'home': home, @@ -1291,7 +1253,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): env = {'PYTHONPATH': paths_str} return env - @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt') + @unittest.skipIf(MS_WINDOWS, 'See test_init_pybuilddir_win32') def test_init_pybuilddir(self): # Test path configuration with pybuilddir.txt configuration file @@ -1300,6 +1262,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # directory (tmpdir) subdir = 'libdir' libdir = os.path.join(tmpdir, subdir) + # The stdlib dir is dirname(executable) + VPATH + 'Lib' + stdlibdir = os.path.join(tmpdir, 'Lib') os.mkdir(libdir) filename = os.path.join(tmpdir, 'pybuilddir.txt') @@ -1307,21 +1271,58 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): fp.write(subdir) module_search_paths = self.module_search_paths() + module_search_paths[-2] = stdlibdir module_search_paths[-1] = libdir executable = self.test_exe config = { + 'base_exec_prefix': sysconfig.get_config_var("exec_prefix"), + 'base_prefix': sysconfig.get_config_var("prefix"), 'base_executable': executable, 'executable': executable, 'module_search_paths': module_search_paths, - 'stdlib_dir': STDLIB_INSTALL, - 'use_frozen_modules': 1 if STDLIB_INSTALL else -1, + 'stdlib_dir': stdlibdir, } env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, api=API_COMPAT, env=env, ignore_stderr=True, cwd=tmpdir) + @unittest.skipUnless(MS_WINDOWS, 'See test_init_pybuilddir') + def test_init_pybuilddir_win32(self): + # Test path configuration with pybuilddir.txt configuration file + + with self.tmpdir_with_python(r'PCbuild\arch') as tmpdir: + # The prefix is dirname(executable) + VPATH + prefix = os.path.normpath(os.path.join(tmpdir, r'..\..')) + # The stdlib dir is dirname(executable) + VPATH + 'Lib' + stdlibdir = os.path.normpath(os.path.join(tmpdir, r'..\..\Lib')) + + filename = os.path.join(tmpdir, 'pybuilddir.txt') + with open(filename, "w", encoding="utf8") as fp: + fp.write(tmpdir) + + module_search_paths = self.module_search_paths() + module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3])) + module_search_paths[-2] = stdlibdir + module_search_paths[-1] = tmpdir + + executable = self.test_exe + config = { + 'base_exec_prefix': prefix, + 'base_prefix': prefix, + 'base_executable': executable, + 'executable': executable, + 'prefix': prefix, + 'exec_prefix': prefix, + 'module_search_paths': module_search_paths, + 'stdlib_dir': stdlibdir, + } + env = self.copy_paths_by_env(config) + self.check_all_configs("test_init_compat_config", config, + api=API_COMPAT, env=env, + ignore_stderr=False, cwd=tmpdir) + def test_init_pyvenv_cfg(self): # Test path configuration with pyvenv.cfg configuration file @@ -1336,12 +1337,12 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'lib-dynload') os.makedirs(lib_dynload) else: - lib_dynload = os.path.join(pyvenv_home, 'lib') - os.makedirs(lib_dynload) - # getpathp.c uses Lib\os.py as the LANDMARK + lib_folder = os.path.join(pyvenv_home, 'Lib') + os.makedirs(lib_folder) + # getpath.py uses Lib\os.py as the LANDMARK shutil.copyfile( os.path.join(support.STDLIB_DIR, 'os.py'), - os.path.join(lib_dynload, 'os.py'), + os.path.join(lib_folder, 'os.py'), ) filename = os.path.join(tmpdir, 'pyvenv.cfg') @@ -1355,43 +1356,38 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): else: for index, path in enumerate(paths): if index == 0: + # Because we copy the DLLs into tmpdir as well, the zip file + # entry in sys.path will be there. For a regular venv, it will + # usually be in the home directory. paths[index] = os.path.join(tmpdir, os.path.basename(path)) else: paths[index] = os.path.join(pyvenv_home, os.path.basename(path)) paths[-1] = pyvenv_home executable = self.test_exe + base_executable = os.path.join(pyvenv_home, os.path.basename(executable)) exec_prefix = pyvenv_home config = { + 'base_prefix': sysconfig.get_config_var("prefix"), 'base_exec_prefix': exec_prefix, 'exec_prefix': exec_prefix, - 'base_executable': executable, + 'base_executable': base_executable, 'executable': executable, 'module_search_paths': paths, } - path_config = {} if MS_WINDOWS: config['base_prefix'] = pyvenv_home config['prefix'] = pyvenv_home - config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib') + config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') config['use_frozen_modules'] = 1 - - ver = sys.version_info - dll = f'python{ver.major}' - if debug_build(executable): - dll += '_d' - dll += '.DLL' - dll = os.path.join(os.path.dirname(executable), dll) - path_config['python3_dll'] = dll else: - # The current getpath.c doesn't determine the stdlib dir - # in this case. - config['stdlib_dir'] = STDLIB_INSTALL - config['use_frozen_modules'] = 1 if STDLIB_INSTALL else -1 + # cannot reliably assume stdlib_dir here because it + # depends too much on our build. But it ought to be found + config['stdlib_dir'] = self.IGNORE_CONFIG + config['use_frozen_modules'] = 1 env = self.copy_paths_by_env(config) self.check_all_configs("test_init_compat_config", config, - expected_pathconfig=path_config, api=API_COMPAT, env=env, ignore_stderr=True, cwd=tmpdir) @@ -1502,10 +1498,14 @@ class SetConfigTests(unittest.TestCase): def test_set_config(self): # bpo-42260: Test _PyInterpreterState_SetConfig() import_helper.import_module('_testcapi') - cmd = [sys.executable, '-I', '-m', 'test._test_embed_set_config'] + cmd = [sys.executable, '-X', 'utf8', '-I', '-m', 'test._test_embed_set_config'] proc = subprocess.run(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + encoding='utf-8', errors='backslashreplace') + if proc.returncode and support.verbose: + print(proc.stdout) + print(proc.stderr) self.assertEqual(proc.returncode, 0, (proc.returncode, proc.stdout, proc.stderr)) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py new file mode 100644 index 0000000..c18689c --- /dev/null +++ b/Lib/test/test_getpath.py @@ -0,0 +1,879 @@ +import copy +import ntpath +import pathlib +import posixpath +import sys +import unittest + +from test.support import verbose + +try: + # If we are in a source tree, use the original source file for tests + SOURCE = (pathlib.Path(__file__).absolute().parent.parent.parent / "Modules/getpath.py").read_bytes() +except FileNotFoundError: + # Try from _testcapimodule instead + from _testinternalcapi import get_getpath_codeobject + SOURCE = get_getpath_codeobject() + + +class MockGetPathTests(unittest.TestCase): + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.maxDiff = None + + def test_normal_win32(self): + "Test a 'standard' install layout on Windows." + ns = MockNTNamespace( + argv0=r"C:\Python\python.exe", + real_executable=r"C:\Python\python.exe", + ) + ns.add_known_xfile(r"C:\Python\python.exe") + ns.add_known_file(r"C:\Python\Lib\os.py") + ns.add_known_dir(r"C:\Python\DLLs") + expected = dict( + executable=r"C:\Python\python.exe", + base_executable=r"C:\Python\python.exe", + prefix=r"C:\Python", + exec_prefix=r"C:\Python", + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + r"C:\Python\Lib", + r"C:\Python\DLLs", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_buildtree_win32(self): + "Test an in-build-tree layout on Windows." + ns = MockNTNamespace( + argv0=r"C:\CPython\PCbuild\amd64\python.exe", + real_executable=r"C:\CPython\PCbuild\amd64\python.exe", + ) + ns.add_known_xfile(r"C:\CPython\PCbuild\amd64\python.exe") + ns.add_known_file(r"C:\CPython\Lib\os.py") + ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""]) + expected = dict( + executable=r"C:\CPython\PCbuild\amd64\python.exe", + base_executable=r"C:\CPython\PCbuild\amd64\python.exe", + prefix=r"C:\CPython", + exec_prefix=r"C:\CPython", + build_prefix=r"C:\CPython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + r"C:\CPython\PCbuild\amd64\python98.zip", + r"C:\CPython\Lib", + r"C:\CPython\PCbuild\amd64", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_win32(self): + """Test a venv layout on Windows. + + This layout is discovered by the presence of %__PYVENV_LAUNCHER__%, + specifying the original launcher executable. site.py is responsible + for updating prefix and exec_prefix. + """ + ns = MockNTNamespace( + argv0=r"C:\Python\python.exe", + ENV___PYVENV_LAUNCHER__=r"C:\venv\Scripts\python.exe", + real_executable=r"C:\Python\python.exe", + ) + ns.add_known_xfile(r"C:\Python\python.exe") + ns.add_known_xfile(r"C:\venv\Scripts\python.exe") + ns.add_known_file(r"C:\Python\Lib\os.py") + ns.add_known_dir(r"C:\Python\DLLs") + ns.add_known_file(r"C:\venv\pyvenv.cfg", [ + r"home = C:\Python" + ]) + expected = dict( + executable=r"C:\venv\Scripts\python.exe", + prefix=r"C:\Python", + exec_prefix=r"C:\Python", + base_executable=r"C:\Python\python.exe", + base_prefix=r"C:\Python", + base_exec_prefix=r"C:\Python", + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + r"C:\Python\Lib", + r"C:\Python", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_registry_win32(self): + """Test registry lookup on Windows. + + On Windows there are registry entries that are intended for other + applications to register search paths. + """ + hkey = rf"HKLM\Software\Python\PythonCore\9.8-XY\PythonPath" + winreg = MockWinreg({ + hkey: None, + f"{hkey}\\Path1": "path1-dir", + f"{hkey}\\Path1\\Subdir": "not-subdirs", + }) + ns = MockNTNamespace( + argv0=r"C:\Python\python.exe", + real_executable=r"C:\Python\python.exe", + winreg=winreg, + ) + ns.add_known_xfile(r"C:\Python\python.exe") + ns.add_known_file(r"C:\Python\Lib\os.py") + ns.add_known_dir(r"C:\Python\DLLs") + expected = dict( + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + "path1-dir", + # should not contain not-subdirs + r"C:\Python\Lib", + r"C:\Python\DLLs", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + ns["config"]["use_environment"] = 0 + ns["config"]["module_search_paths_set"] = 0 + ns["config"]["module_search_paths"] = None + expected = dict( + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + r"C:\Python\Lib", + r"C:\Python\DLLs", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_normal_win32(self): + "Test a 'standard' install layout via symlink on Windows." + ns = MockNTNamespace( + argv0=r"C:\LinkedFrom\python.exe", + real_executable=r"C:\Python\python.exe", + ) + ns.add_known_xfile(r"C:\LinkedFrom\python.exe") + ns.add_known_xfile(r"C:\Python\python.exe") + ns.add_known_link(r"C:\LinkedFrom\python.exe", r"C:\Python\python.exe") + ns.add_known_file(r"C:\Python\Lib\os.py") + ns.add_known_dir(r"C:\Python\DLLs") + expected = dict( + executable=r"C:\LinkedFrom\python.exe", + base_executable=r"C:\LinkedFrom\python.exe", + prefix=r"C:\Python", + exec_prefix=r"C:\Python", + module_search_paths_set=1, + module_search_paths=[ + r"C:\Python\python98.zip", + r"C:\Python\Lib", + r"C:\Python\DLLs", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_buildtree_win32(self): + "Test an in-build-tree layout via symlink on Windows." + ns = MockNTNamespace( + argv0=r"C:\LinkedFrom\python.exe", + real_executable=r"C:\CPython\PCbuild\amd64\python.exe", + ) + ns.add_known_xfile(r"C:\LinkedFrom\python.exe") + ns.add_known_xfile(r"C:\CPython\PCbuild\amd64\python.exe") + ns.add_known_link(r"C:\LinkedFrom\python.exe", r"C:\CPython\PCbuild\amd64\python.exe") + ns.add_known_file(r"C:\CPython\Lib\os.py") + ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""]) + expected = dict( + executable=r"C:\LinkedFrom\python.exe", + base_executable=r"C:\LinkedFrom\python.exe", + prefix=r"C:\CPython", + exec_prefix=r"C:\CPython", + build_prefix=r"C:\CPython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + r"C:\CPython\PCbuild\amd64\python98.zip", + r"C:\CPython\Lib", + r"C:\CPython\PCbuild\amd64", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_normal_posix(self): + "Test a 'standard' install layout on *nix" + ns = MockPosixNamespace( + PREFIX="/usr", + argv0="python", + ENV_PATH="/usr/bin", + ) + ns.add_known_xfile("/usr/bin/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + expected = dict( + executable="/usr/bin/python", + base_executable="/usr/bin/python", + prefix="/usr", + exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_buildpath_posix(self): + """Test an in-build-tree layout on POSIX. + + This layout is discovered from the presence of pybuilddir.txt, which + contains the relative path from the executable's directory to the + platstdlib path. + """ + ns = MockPosixNamespace( + argv0=r"/home/cpython/python", + PREFIX="/usr/local", + ) + ns.add_known_xfile("/home/cpython/python") + ns.add_known_xfile("/usr/local/bin/python") + ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.linux-x86_64-9.8"]) + ns.add_known_file("/home/cpython/Lib/os.py") + ns.add_known_dir("/home/cpython/lib-dynload") + expected = dict( + executable="/home/cpython/python", + prefix="/usr/local", + exec_prefix="/usr/local", + base_executable="/home/cpython/python", + build_prefix="/home/cpython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + "/usr/local/lib/python98.zip", + "/home/cpython/Lib", + "/home/cpython/build/lib.linux-x86_64-9.8", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_posix(self): + "Test a venv layout on *nix." + ns = MockPosixNamespace( + argv0="python", + PREFIX="/usr", + ENV_PATH="/venv/bin:/usr/bin", + ) + ns.add_known_xfile("/usr/bin/python") + ns.add_known_xfile("/venv/bin/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + ns.add_known_file("/venv/pyvenv.cfg", [ + r"home = /usr/bin" + ]) + expected = dict( + executable="/venv/bin/python", + prefix="/usr", + exec_prefix="/usr", + base_executable="/usr/bin/python", + base_prefix="/usr", + base_exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_normal_posix(self): + "Test a 'standard' install layout via symlink on *nix" + ns = MockPosixNamespace( + PREFIX="/usr", + argv0="/linkfrom/python", + ) + ns.add_known_xfile("/linkfrom/python") + ns.add_known_xfile("/usr/bin/python") + ns.add_known_link("/linkfrom/python", "/usr/bin/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + expected = dict( + executable="/linkfrom/python", + base_executable="/linkfrom/python", + prefix="/usr", + exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_buildpath_posix(self): + """Test an in-build-tree layout on POSIX. + + This layout is discovered from the presence of pybuilddir.txt, which + contains the relative path from the executable's directory to the + platstdlib path. + """ + ns = MockPosixNamespace( + argv0=r"/linkfrom/python", + PREFIX="/usr/local", + ) + ns.add_known_xfile("/linkfrom/python") + ns.add_known_xfile("/home/cpython/python") + ns.add_known_link("/linkfrom/python", "/home/cpython/python") + ns.add_known_xfile("/usr/local/bin/python") + ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.linux-x86_64-9.8"]) + ns.add_known_file("/home/cpython/Lib/os.py") + ns.add_known_dir("/home/cpython/lib-dynload") + expected = dict( + executable="/linkfrom/python", + prefix="/usr/local", + exec_prefix="/usr/local", + base_executable="/linkfrom/python", + build_prefix="/home/cpython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + "/usr/local/lib/python98.zip", + "/home/cpython/Lib", + "/home/cpython/build/lib.linux-x86_64-9.8", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_custom_platlibdir_posix(self): + "Test an install with custom platlibdir on *nix" + ns = MockPosixNamespace( + PREFIX="/usr", + argv0="/linkfrom/python", + PLATLIBDIR="lib64", + ) + ns.add_known_xfile("/usr/bin/python") + ns.add_known_file("/usr/lib64/python9.8/os.py") + ns.add_known_dir("/usr/lib64/python9.8/lib-dynload") + expected = dict( + executable="/linkfrom/python", + base_executable="/linkfrom/python", + prefix="/usr", + exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib64/python98.zip", + "/usr/lib64/python9.8", + "/usr/lib64/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_macos(self): + """Test a venv layout on macOS. + + This layout is discovered when 'executable' and 'real_executable' match, + but $__PYVENV_LAUNCHER__ has been set to the original process. + """ + ns = MockPosixNamespace( + os_name="darwin", + argv0="/usr/bin/python", + PREFIX="/usr", + ENV___PYVENV_LAUNCHER__="/framework/Python9.8/python", + real_executable="/usr/bin/python", + ) + ns.add_known_xfile("/usr/bin/python") + ns.add_known_xfile("/framework/Python9.8/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + ns.add_known_file("/framework/Python9.8/pyvenv.cfg", [ + "home = /usr/bin" + ]) + expected = dict( + executable="/framework/Python9.8/python", + prefix="/usr", + exec_prefix="/usr", + base_executable="/usr/bin/python", + base_prefix="/usr", + base_exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_normal_macos(self): + "Test a 'standard' install layout via symlink on macOS" + ns = MockPosixNamespace( + os_name="darwin", + PREFIX="/usr", + argv0="python", + ENV_PATH="/linkfrom:/usr/bin", + # real_executable on macOS matches the invocation path + real_executable="/linkfrom/python", + ) + ns.add_known_xfile("/linkfrom/python") + ns.add_known_xfile("/usr/bin/python") + ns.add_known_link("/linkfrom/python", "/usr/bin/python") + ns.add_known_file("/usr/lib/python9.8/os.py") + ns.add_known_dir("/usr/lib/python9.8/lib-dynload") + expected = dict( + executable="/linkfrom/python", + base_executable="/linkfrom/python", + prefix="/usr", + exec_prefix="/usr", + module_search_paths_set=1, + module_search_paths=[ + "/usr/lib/python98.zip", + "/usr/lib/python9.8", + "/usr/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_symlink_buildpath_macos(self): + """Test an in-build-tree layout via symlink on macOS. + + This layout is discovered from the presence of pybuilddir.txt, which + contains the relative path from the executable's directory to the + platstdlib path. + """ + ns = MockPosixNamespace( + os_name="darwin", + argv0=r"python", + ENV_PATH="/linkfrom:/usr/bin", + PREFIX="/usr/local", + # real_executable on macOS matches the invocation path + real_executable="/linkfrom/python", + ) + ns.add_known_xfile("/linkfrom/python") + ns.add_known_xfile("/home/cpython/python") + ns.add_known_link("/linkfrom/python", "/home/cpython/python") + ns.add_known_xfile("/usr/local/bin/python") + ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.macos-9.8"]) + ns.add_known_file("/home/cpython/Lib/os.py") + ns.add_known_dir("/home/cpython/lib-dynload") + expected = dict( + executable="/linkfrom/python", + prefix="/usr/local", + exec_prefix="/usr/local", + base_executable="/linkfrom/python", + build_prefix="/home/cpython", + _is_python_build=1, + module_search_paths_set=1, + module_search_paths=[ + "/usr/local/lib/python98.zip", + "/home/cpython/Lib", + "/home/cpython/build/lib.macos-9.8", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + +# ****************************************************************************** + +DEFAULT_NAMESPACE = dict( + PREFIX="", + EXEC_PREFIX="", + PYTHONPATH="", + VPATH="", + PLATLIBDIR="", + PYDEBUGEXT="", + VERSION_MAJOR=9, # fixed version number for ease + VERSION_MINOR=8, # of testing + PYWINVER=None, + EXE_SUFFIX=None, + + ENV_PATH="", + ENV_PYTHONHOME="", + ENV_PYTHONEXECUTABLE="", + ENV___PYVENV_LAUNCHER__="", + argv0="", + py_setpath="", + real_executable="", + executable_dir="", + library="", + winreg=None, + build_prefix=None, + venv_prefix=None, +) + +DEFAULT_CONFIG = dict( + home=None, + platlibdir=None, + pythonpath=None, + program_name=None, + prefix=None, + exec_prefix=None, + base_prefix=None, + base_exec_prefix=None, + executable=None, + base_executable="", + stdlib_dir=None, + platstdlib_dir=None, + module_search_paths=None, + module_search_paths_set=0, + pythonpath_env=None, + argv=None, + orig_argv=None, + + isolated=0, + use_environment=1, + use_site=1, +) + +class MockNTNamespace(dict): + def __init__(self, *a, argv0=None, config=None, **kw): + self.update(DEFAULT_NAMESPACE) + self["config"] = DEFAULT_CONFIG.copy() + self["os_name"] = "nt" + self["PLATLIBDIR"] = "DLLs" + self["PYWINVER"] = "9.8-XY" + self["VPATH"] = r"..\.." + super().__init__(*a, **kw) + if argv0: + self["config"]["orig_argv"] = [argv0] + if config: + self["config"].update(config) + self._files = {} + self._links = {} + self._dirs = set() + self._warnings = [] + + def add_known_file(self, path, lines=None): + self._files[path.casefold()] = list(lines or ()) + self.add_known_dir(path.rpartition("\\")[0]) + + def add_known_xfile(self, path): + self.add_known_file(path) + + def add_known_link(self, path, target): + self._links[path.casefold()] = target + + def add_known_dir(self, path): + p = path.rstrip("\\").casefold() + while p: + self._dirs.add(p) + p = p.rpartition("\\")[0] + + def __missing__(self, key): + try: + return getattr(self, key) + except AttributeError: + raise KeyError(key) from None + + def abspath(self, path): + if self.isabs(path): + return path + return self.joinpath("C:\\Absolute", path) + + def basename(self, path): + return path.rpartition("\\")[2] + + def dirname(self, path): + name = path.rstrip("\\").rpartition("\\")[0] + if name[1:] == ":": + return name + "\\" + return name + + def hassuffix(self, path, suffix): + return path.casefold().endswith(suffix.casefold()) + + def isabs(self, path): + return path[1:3] == ":\\" + + def isdir(self, path): + if verbose: + print("Check if", path, "is a dir") + return path.casefold() in self._dirs + + def isfile(self, path): + if verbose: + print("Check if", path, "is a file") + return path.casefold() in self._files + + def ismodule(self, path): + if verbose: + print("Check if", path, "is a module") + path = path.casefold() + return path in self._files and path.rpartition(".")[2] == "py".casefold() + + def isxfile(self, path): + if verbose: + print("Check if", path, "is a executable") + path = path.casefold() + return path in self._files and path.rpartition(".")[2] == "exe".casefold() + + def joinpath(self, *path): + return ntpath.normpath(ntpath.join(*path)) + + def readlines(self, path): + try: + return self._files[path.casefold()] + except KeyError: + raise FileNotFoundError(path) from None + + def realpath(self, path, _trail=None): + if verbose: + print("Read link from", path) + try: + link = self._links[path.casefold()] + except KeyError: + return path + if _trail is None: + _trail = set() + elif link.casefold() in _trail: + raise OSError("circular link") + _trail.add(link.casefold()) + return self.realpath(link, _trail) + + def warn(self, message): + self._warnings.append(message) + if verbose: + print(message) + + +class MockWinreg: + HKEY_LOCAL_MACHINE = "HKLM" + HKEY_CURRENT_USER = "HKCU" + + def __init__(self, keys): + self.keys = {k.casefold(): v for k, v in keys.items()} + self.open = {} + + def __repr__(self): + return "<MockWinreg>" + + def __eq__(self, other): + return isinstance(other, type(self)) + + def open_keys(self): + return list(self.open) + + def OpenKeyEx(self, hkey, subkey): + if verbose: + print(f"OpenKeyEx({hkey}, {subkey})") + key = f"{hkey}\\{subkey}".casefold() + if key in self.keys: + self.open[key] = self.open.get(key, 0) + 1 + return key + raise FileNotFoundError() + + def CloseKey(self, hkey): + if verbose: + print(f"CloseKey({hkey})") + hkey = hkey.casefold() + if hkey not in self.open: + raise RuntimeError("key is not open") + self.open[hkey] -= 1 + if not self.open[hkey]: + del self.open[hkey] + + def EnumKey(self, hkey, i): + if verbose: + print(f"EnumKey({hkey}, {i})") + hkey = hkey.casefold() + if hkey not in self.open: + raise RuntimeError("key is not open") + prefix = f'{hkey}\\' + subkeys = [k[len(prefix):] for k in sorted(self.keys) if k.startswith(prefix)] + subkeys[:] = [k for k in subkeys if '\\' not in k] + for j, n in enumerate(subkeys): + if j == i: + return n.removeprefix(prefix) + raise OSError("end of enumeration") + + def QueryValue(self, hkey): + if verbose: + print(f"QueryValue({hkey})") + hkey = hkey.casefold() + if hkey not in self.open: + raise RuntimeError("key is not open") + try: + return self.keys[hkey] + except KeyError: + raise OSError() + + +class MockPosixNamespace(dict): + def __init__(self, *a, argv0=None, config=None, **kw): + self.update(DEFAULT_NAMESPACE) + self["config"] = DEFAULT_CONFIG.copy() + self["os_name"] = "posix" + self["PLATLIBDIR"] = "lib" + super().__init__(*a, **kw) + if argv0: + self["config"]["orig_argv"] = [argv0] + if config: + self["config"].update(config) + self._files = {} + self._xfiles = set() + self._links = {} + self._dirs = set() + self._warnings = [] + + def add_known_file(self, path, lines=None): + self._files[path] = list(lines or ()) + self.add_known_dir(path.rpartition("/")[0]) + + def add_known_xfile(self, path): + self.add_known_file(path) + self._xfiles.add(path) + + def add_known_link(self, path, target): + self._links[path] = target + + def add_known_dir(self, path): + p = path.rstrip("/") + while p: + self._dirs.add(p) + p = p.rpartition("/")[0] + + def __missing__(self, key): + try: + return getattr(self, key) + except AttributeError: + raise KeyError(key) from None + + def abspath(self, path): + if self.isabs(path): + return path + return self.joinpath("/Absolute", path) + + def basename(self, path): + return path.rpartition("/")[2] + + def dirname(self, path): + return path.rstrip("/").rpartition("/")[0] + + def hassuffix(self, path, suffix): + return path.endswith(suffix) + + def isabs(self, path): + return path[0:1] == "/" + + def isdir(self, path): + if verbose: + print("Check if", path, "is a dir") + return path in self._dirs + + def isfile(self, path): + if verbose: + print("Check if", path, "is a file") + return path in self._files + + def ismodule(self, path): + if verbose: + print("Check if", path, "is a module") + return path in self._files and path.rpartition(".")[2] == "py" + + def isxfile(self, path): + if verbose: + print("Check if", path, "is an xfile") + return path in self._xfiles + + def joinpath(self, *path): + return posixpath.normpath(posixpath.join(*path)) + + def readlines(self, path): + try: + return self._files[path] + except KeyError: + raise FileNotFoundError(path) from None + + def realpath(self, path, _trail=None): + if verbose: + print("Read link from", path) + try: + link = self._links[path] + except KeyError: + return path + if _trail is None: + _trail = set() + elif link in _trail: + raise OSError("circular link") + _trail.add(link) + return self.realpath(link, _trail) + + def warn(self, message): + self._warnings.append(message) + if verbose: + print(message) + + +def diff_dict(before, after, prefix="global"): + diff = [] + for k in sorted(before): + if k[:2] == "__": + continue + if k == "config": + diff_dict(before[k], after[k], prefix="config") + continue + if k in after and after[k] != before[k]: + diff.append((k, before[k], after[k])) + if not diff: + return + max_k = max(len(k) for k, _, _ in diff) + indent = " " * (len(prefix) + 1 + max_k) + if verbose: + for k, b, a in diff: + if b: + print("{}.{} -{!r}\n{} +{!r}".format(prefix, k.ljust(max_k), b, indent, a)) + else: + print("{}.{} +{!r}".format(prefix, k.ljust(max_k), a)) + + +def dump_dict(before, after, prefix="global"): + if not verbose or not after: + return + max_k = max(len(k) for k in after) + for k, v in sorted(after.items(), key=lambda i: i[0]): + if k[:2] == "__": + continue + if k == "config": + dump_dict(before[k], after[k], prefix="config") + continue + try: + if v != before[k]: + print("{}.{} {!r} (was {!r})".format(prefix, k.ljust(max_k), v, before[k])) + continue + except KeyError: + pass + print("{}.{} {!r}".format(prefix, k.ljust(max_k), v)) + + +def getpath(ns, keys): + before = copy.deepcopy(ns) + failed = True + try: + exec(SOURCE, ns) + failed = False + finally: + if failed: + dump_dict(before, ns) + else: + diff_dict(before, ns) + return { + k: ns['config'].get(k, ns.get(k, ...)) + for k in keys + } diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 661c59d..a8d87e5 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -741,8 +741,9 @@ class TestNtpath(NtpathTestCase): # (or any other volume root). The drive-relative # locations below cannot then refer to mount points # - drive, path = ntpath.splitdrive(sys.executable) - with os_helper.change_cwd(ntpath.dirname(sys.executable)): + test_cwd = os.getenv("SystemRoot") + drive, path = ntpath.splitdrive(test_cwd) + with os_helper.change_cwd(test_cwd): self.assertFalse(ntpath.ismount(drive.lower())) self.assertFalse(ntpath.ismount(drive.upper())) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 5f06a0d..76d35da 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -11,6 +11,7 @@ from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr from test.support.os_helper import TESTFN, EnvironmentVarGuard, change_cwd +import ast import builtins import encodings import glob @@ -300,7 +301,8 @@ class HelperFunctionsTests(unittest.TestCase): self.assertEqual(len(dirs), 2) self.assertEqual(dirs[0], 'xoxo') wanted = os.path.join('xoxo', 'lib', 'site-packages') - self.assertEqual(dirs[1], wanted) + self.assertEqual(os.path.normcase(dirs[1]), + os.path.normcase(wanted)) @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_no_home_directory(self): @@ -497,13 +499,14 @@ class StartupImportTests(unittest.TestCase): def test_startup_imports(self): # Get sys.path in isolated mode (python3 -I) - popen = subprocess.Popen([sys.executable, '-I', '-c', - 'import sys; print(repr(sys.path))'], + popen = subprocess.Popen([sys.executable, '-X', 'utf8', '-I', + '-c', 'import sys; print(repr(sys.path))'], stdout=subprocess.PIPE, - encoding='utf-8') + encoding='utf-8', + errors='surrogateescape') stdout = popen.communicate()[0] self.assertEqual(popen.returncode, 0, repr(stdout)) - isolated_paths = eval(stdout) + isolated_paths = ast.literal_eval(stdout) # bpo-27807: Even with -I, the site module executes all .pth files # found in sys.path (see site.addpackage()). Skip the test if at least @@ -515,14 +518,15 @@ class StartupImportTests(unittest.TestCase): # This tests checks which modules are loaded by Python when it # initially starts upon startup. - popen = subprocess.Popen([sys.executable, '-I', '-v', '-c', - 'import sys; print(set(sys.modules))'], + popen = subprocess.Popen([sys.executable, '-X', 'utf8', '-I', '-v', + '-c', 'import sys; print(set(sys.modules))'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8') + encoding='utf-8', + errors='surrogateescape') stdout, stderr = popen.communicate() self.assertEqual(popen.returncode, 0, (stdout, stderr)) - modules = eval(stdout) + modules = ast.literal_eval(stdout) self.assertIn('site', modules) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 9408657..506266d 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -102,6 +102,10 @@ class TestSysConfig(unittest.TestCase): def test_get_path(self): config_vars = get_config_vars() + if os.name == 'nt': + # On Windows, we replace the native platlibdir name with the + # default so that POSIX schemes resolve correctly + config_vars = config_vars | {'platlibdir': 'lib'} for scheme in _INSTALL_SCHEMES: for name in _INSTALL_SCHEMES[scheme]: expected = _INSTALL_SCHEMES[scheme][name].format(**config_vars) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 94d6265..de714de 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -15,7 +15,7 @@ import subprocess import sys import tempfile from test.support import (captured_stdout, captured_stderr, requires_zlib, - skip_if_broken_multiprocessing_synchronize) + skip_if_broken_multiprocessing_synchronize, verbose) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv @@ -40,6 +40,8 @@ def check_output(cmd, encoding=None): encoding=encoding) out, err = p.communicate() if p.returncode: + if verbose and err: + print(err.decode('utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) return out, err @@ -194,7 +196,7 @@ class BasicTest(BaseTest): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode()) + self.assertEqual(out.strip(), expected.encode(), prefix) if sys.platform == 'win32': ENV_SUBDIRS = ( diff --git a/Makefile.pre.in b/Makefile.pre.in index d5be6ed..3dc131b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -308,7 +308,6 @@ COVERAGE_REPORT_OPTIONS=--no-branch-coverage --title "CPython lcov report" # Modules MODULE_OBJS= \ Modules/config.o \ - Modules/getpath.o \ Modules/main.o \ Modules/gcmodule.o @@ -505,6 +504,7 @@ LIBRARY_OBJS_OMIT_FROZEN= \ LIBRARY_OBJS= \ $(LIBRARY_OBJS_OMIT_FROZEN) \ $(DEEPFREEZE_OBJS) \ + Modules/getpath.o \ Python/frozen.o ########################################################################## @@ -1061,8 +1061,11 @@ FROZEN_FILES_OUT = \ Programs/_freeze_module.o: Programs/_freeze_module.c Makefile -Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) - $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) +Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile + $(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Modules/getpath_noop.c + +Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o + $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o $(LIBS) $(MODLIBS) $(SYSLIBS) # BEGIN: freezing modules @@ -1130,6 +1133,10 @@ Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py Tools/scripts/freeze_modules.py: $(FREEZE_MODULE) +# We manually freeze getpath.py rather than through freeze_modules +Modules/getpath.h: Programs/_freeze_module Modules/getpath.py + Programs/_freeze_module getpath $(srcdir)/Modules/getpath.py $(srcdir)/Modules/getpath.h + .PHONY: regen-frozen regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py @@ -1170,12 +1177,13 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \ -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \ -o $@ $(srcdir)/Modules/getbuildinfo.c -Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile +Modules/getpath.o: $(srcdir)/Modules/getpath.c Modules/getpath.h Makefile $(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \ -DPREFIX='"$(prefix)"' \ -DEXEC_PREFIX='"$(exec_prefix)"' \ -DVERSION='"$(VERSION)"' \ -DVPATH='"$(VPATH)"' \ + -DPLATLIBDIR='"$(PLATLIBDIR)"' \ -o $@ $(srcdir)/Modules/getpath.c Programs/python.o: $(srcdir)/Programs/python.c @@ -1210,11 +1218,6 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydt $(MULTIARCH_CPPFLAGS) \ -o $@ $(srcdir)/Python/sysmodule.c -Python/initconfig.o: $(srcdir)/Python/initconfig.c - $(CC) -c $(PY_CORE_CFLAGS) \ - -DPLATLIBDIR='"$(PLATLIBDIR)"' \ - -o $@ $(srcdir)/Python/initconfig.c - $(IO_OBJS): $(IO_H) .PHONY: regen-pegen-metaparser diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst new file mode 100644 index 0000000..45fa75e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst @@ -0,0 +1,3 @@ +Path calculation (known as ``getpath``) has been reimplemented as a frozen +Python module. This should have no visible impact, but may affect +calculation of all paths referenced in :mod:`sys` and :mod:`sysconfig`. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index c230ba2..19babb0 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -14,10 +14,11 @@ #include "Python.h" #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_fileutils.h" // _Py_normalize_path +#include "pycore_fileutils.h" // _Py_normpath #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() +#include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -273,6 +274,14 @@ error: } +static PyObject * +test_reset_path_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(arg)) +{ + _PyPathConfig_ClearGlobal(); + Py_RETURN_NONE; +} + + static PyObject* test_atomic_funcs(PyObject *self, PyObject *Py_UNUSED(args)) { @@ -378,15 +387,15 @@ normalize_path(PyObject *self, PyObject *filename) return NULL; } - wchar_t buf[MAXPATHLEN + 1]; - int res = _Py_normalize_path(encoded, buf, Py_ARRAY_LENGTH(buf)); + PyObject *result = PyUnicode_FromWideChar(_Py_normpath(encoded, size), -1); PyMem_Free(encoded); - if (res != 0) { - PyErr_SetString(PyExc_ValueError, "string too long"); - return NULL; - } - return PyUnicode_FromWideChar(buf, -1); + return result; +} + +static PyObject * +get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) { + return _Py_Get_Getpath_CodeObject(); } @@ -399,9 +408,11 @@ static PyMethodDef TestMethods[] = { {"test_hashtable", test_hashtable, METH_NOARGS}, {"get_config", test_get_config, METH_NOARGS}, {"set_config", test_set_config, METH_O}, + {"reset_path_config", test_reset_path_config, METH_NOARGS}, {"test_atomic_funcs", test_atomic_funcs, METH_NOARGS}, {"test_edit_cost", test_edit_cost, METH_NOARGS}, {"normalize_path", normalize_path, METH_O, NULL}, + {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 7921c22..86da087 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1317,6 +1317,38 @@ exit: #endif /* defined(MS_WINDOWS) */ +PyDoc_STRVAR(os__path_normpath__doc__, +"_path_normpath($module, /, path)\n" +"--\n" +"\n" +"Basic path normalization."); + +#define OS__PATH_NORMPATH_METHODDEF \ + {"_path_normpath", (PyCFunction)(void(*)(void))os__path_normpath, METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__}, + +static PyObject * +os__path_normpath_impl(PyObject *module, PyObject *path); + +static PyObject * +os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "_path_normpath", 0}; + PyObject *argsbuf[1]; + PyObject *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + path = args[0]; + return_value = os__path_normpath_impl(module, path); + +exit: + return return_value; +} + PyDoc_STRVAR(os_mkdir__doc__, "mkdir($module, /, path, mode=511, *, dir_fd=None)\n" "--\n" @@ -9263,4 +9295,4 @@ exit: #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=65a85d7d3f2c487e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=05505f171cdcff72 input=a9049054013a1b77]*/ diff --git a/Modules/getpath.c b/Modules/getpath.c index 4dbd502..32d5db9 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -1,1617 +1,935 @@ /* Return the initial module search path. */ #include "Python.h" -#include "pycore_fileutils.h" +#include "marshal.h" // PyMarshal_ReadObjectFromString +#include "osdefs.h" // DELIM #include "pycore_initconfig.h" +#include "pycore_fileutils.h" #include "pycore_pathconfig.h" -#include "osdefs.h" // DELIM +#include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() +#include <wchar.h> -#include <stdlib.h> // getenv() -#include <string.h> -#include <sys/types.h> +#ifdef MS_WINDOWS +# include <windows.h> // GetFullPathNameW(), MAX_PATH +# include <pathcch.h> +#endif #ifdef __APPLE__ # include <mach-o/dyld.h> #endif -/* Search in some common locations for the associated Python libraries. - * - * Two directories must be found, the platform independent directory - * (prefix), containing the common .py and .pyc files, and the platform - * dependent directory (exec_prefix), containing the shared library - * modules. Note that prefix and exec_prefix can be the same directory, - * but for some installations, they are different. - * - * Py_GetPath() carries out separate searches for prefix and exec_prefix. - * Each search tries a number of different locations until a ``landmark'' - * file or directory is found. If no prefix or exec_prefix is found, a - * warning message is issued and the preprocessor defined PREFIX and - * EXEC_PREFIX are used (even though they will not work); python carries on - * as best as is possible, but most imports will fail. - * - * Before any searches are done, the location of the executable is - * determined. If argv[0] has one or more slashes in it, it is used - * unchanged. Otherwise, it must have been invoked from the shell's path, - * so we search $PATH for the named executable and use that. If the - * executable was not found on $PATH (or there was no $PATH environment - * variable), the original argv[0] string is used. - * - * Next, the executable location is examined to see if it is a symbolic - * link. If so, the link is chased (correctly interpreting a relative - * pathname if one is found) and the directory of the link target is used. - * - * Finally, argv0_path is set to the directory containing the executable - * (i.e. the last component is stripped). - * - * With argv0_path in hand, we perform a number of steps. The same steps - * are performed for prefix and for exec_prefix, but with a different - * landmark. - * - * Step 1. Are we running python out of the build directory? This is - * checked by looking for a different kind of landmark relative to - * argv0_path. For prefix, the landmark's path is derived from the VPATH - * preprocessor variable (taking into account that its value is almost, but - * not quite, what we need). For exec_prefix, the landmark is - * pybuilddir.txt. If the landmark is found, we're done. - * - * For the remaining steps, the prefix landmark will always be - * lib/python$VERSION/os.py and the exec_prefix will always be - * lib/python$VERSION/lib-dynload, where $VERSION is Python's version - * number as supplied by the Makefile. Note that this means that no more - * build directory checking is performed; if the first step did not find - * the landmarks, the assumption is that python is running from an - * installed setup. - * - * Step 2. See if the $PYTHONHOME environment variable points to the - * installed location of the Python libraries. If $PYTHONHOME is set, then - * it points to prefix and exec_prefix. $PYTHONHOME can be a single - * directory, which is used for both, or the prefix and exec_prefix - * directories separated by a colon. - * - * Step 3. Try to find prefix and exec_prefix relative to argv0_path, - * backtracking up the path until it is exhausted. This is the most common - * step to succeed. Note that if prefix and exec_prefix are different, - * exec_prefix is more likely to be found; however if exec_prefix is a - * subdirectory of prefix, both will be found. - * - * Step 4. Search the directories pointed to by the preprocessor variables - * PREFIX and EXEC_PREFIX. These are supplied by the Makefile but can be - * passed in as options to the configure script. - * - * That's it! - * - * Well, almost. Once we have determined prefix and exec_prefix, the - * preprocessor variable PYTHONPATH is used to construct a path. Each - * relative path on PYTHONPATH is prefixed with prefix. Then the directory - * containing the shared library modules is appended. The environment - * variable $PYTHONPATH is inserted in front of it all. Finally, the - * prefix and exec_prefix globals are tweaked so they reflect the values - * expected by other code, by stripping the "lib/python$VERSION/..." stuff - * off. If either points to the build directory, the globals are reset to - * the corresponding preprocessor variables (so sys.prefix will reflect the - * installation location, even though sys.path points into the build - * directory). This seems to make more sense given that currently the only - * known use of sys.prefix and sys.exec_prefix is for the ILU installation - * process to find the installed Python tree. - * - * An embedding application can use Py_SetPath() to override all of - * these automatic path computations. - * - * NOTE: Windows MSVC builds use PC/getpathp.c instead! - */ - -#ifdef __cplusplus -extern "C" { -#endif - +/* Reference the precompiled getpath.py */ +#include "getpath.h" #if (!defined(PREFIX) || !defined(EXEC_PREFIX) \ - || !defined(VERSION) || !defined(VPATH)) -#error "PREFIX, EXEC_PREFIX, VERSION and VPATH macros must be defined" + || !defined(VERSION) || !defined(VPATH) \ + || !defined(PLATLIBDIR)) +#error "PREFIX, EXEC_PREFIX, VERSION, VPATH and PLATLIBDIR macros must be defined" #endif -#ifndef LANDMARK -#define LANDMARK L"os.py" +#if !defined(PYTHONPATH) +#define PYTHONPATH NULL #endif -#define BUILD_LANDMARK L"Modules/Setup.local" - -#define PATHLEN_ERR() _PyStatus_ERR("path configuration: path too long") - -typedef struct { - wchar_t *path_env; /* PATH environment variable */ - - wchar_t *pythonpath_macro; /* PYTHONPATH macro */ - wchar_t *prefix_macro; /* PREFIX macro */ - wchar_t *exec_prefix_macro; /* EXEC_PREFIX macro */ - wchar_t *vpath_macro; /* VPATH macro */ - - wchar_t *lib_python; /* <platlibdir> / "pythonX.Y" */ - - int prefix_found; /* found platform independent libraries? */ - int exec_prefix_found; /* found the platform dependent libraries? */ - - int warnings; - const wchar_t *pythonpath_env; - const wchar_t *platlibdir; - - wchar_t *argv0_path; - wchar_t *zip_path; - wchar_t *prefix; - wchar_t *exec_prefix; -} PyCalculatePath; - -static const wchar_t delimiter[2] = {DELIM, '\0'}; -static const wchar_t separator[2] = {SEP, '\0'}; - - -static void -reduce(wchar_t *dir) -{ - size_t i = wcslen(dir); - while (i > 0 && dir[i] != SEP) { - --i; - } - dir[i] = '\0'; -} - +#if !defined(PYDEBUGEXT) +#define PYDEBUGEXT NULL +#endif -/* Is file, not directory */ -static int -isfile(const wchar_t *filename) -{ - struct stat buf; - if (_Py_wstat(filename, &buf) != 0) { - return 0; - } - if (!S_ISREG(buf.st_mode)) { - return 0; - } - return 1; -} +#if !defined(PYWINVER) +#ifdef MS_DLL_ID +#define PYWINVER MS_DLL_ID +#else +#define PYWINVER NULL +#endif +#endif +#if !defined(EXE_SUFFIX) +#if defined(MS_WINDOWS) || defined(__CYGWIN__) || defined(__MINGW32__) +#define EXE_SUFFIX L".exe" +#else +#define EXE_SUFFIX NULL +#endif +#endif -/* Is executable file */ -static int -isxfile(const wchar_t *filename) -{ - struct stat buf; - if (_Py_wstat(filename, &buf) != 0) { - return 0; - } - if (!S_ISREG(buf.st_mode)) { - return 0; - } - if ((buf.st_mode & 0111) == 0) { - return 0; - } - return 1; -} +/* HELPER FUNCTIONS for getpath.py */ -/* Is directory */ -static int -isdir(const wchar_t *filename) +static PyObject * +getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) { - struct stat buf; - if (_Py_wstat(filename, &buf) != 0) { - return 0; + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; } - if (!S_ISDIR(buf.st_mode)) { - return 0; + Py_ssize_t len; + path = PyUnicode_AsWideCharString(pathobj, &len); + if (path) { + wchar_t *abs; + if (_Py_abspath(path, &abs) == 0 && abs) { + r = PyUnicode_FromWideChar(_Py_normpath(abs, -1), -1); + PyMem_RawFree((void *)abs); + } else { + PyErr_SetString(PyExc_OSError, "failed to make path absolute"); + } + PyMem_Free((void *)path); } - return 1; + return r; } -/* Add a path component, by appending stuff to buffer. - buflen: 'buffer' length in characters including trailing NUL. - - If path2 is empty: - - - if path doesn't end with SEP and is not empty, add SEP to path - - otherwise, do nothing. */ -static PyStatus -joinpath(wchar_t *path, const wchar_t *path2, size_t path_len) +static PyObject * +getpath_basename(PyObject *Py_UNUSED(self), PyObject *args) { - if (_Py_isabs(path2)) { - if (wcslen(path2) >= path_len) { - return PATHLEN_ERR(); - } - wcscpy(path, path2); - } - else { - if (_Py_add_relfile(path, path2, path_len) < 0) { - return PATHLEN_ERR(); - } - return _PyStatus_OK(); + const char *path; + if (!PyArg_ParseTuple(args, "s", &path)) { + return NULL; } - return _PyStatus_OK(); + const char *name = strrchr(path, SEP); + return PyUnicode_FromString(name ? name + 1 : path); } -static wchar_t* -substring(const wchar_t *str, size_t len) +static PyObject * +getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args) { - wchar_t *substr = PyMem_RawMalloc((len + 1) * sizeof(wchar_t)); - if (substr == NULL) { + const char *path; + if (!PyArg_ParseTuple(args, "s", &path)) { return NULL; } - - if (len) { - memcpy(substr, str, len * sizeof(wchar_t)); + const char *name = strrchr(path, SEP); + if (!name) { + return PyUnicode_FromStringAndSize(NULL, 0); } - substr[len] = L'\0'; - return substr; + return PyUnicode_FromStringAndSize(path, (name - path)); } -static wchar_t* -joinpath2(const wchar_t *path, const wchar_t *path2) +static PyObject * +getpath_isabs(PyObject *Py_UNUSED(self), PyObject *args) { - if (_Py_isabs(path2)) { - return _PyMem_RawWcsdup(path2); + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; } - return _Py_join_relfile(path, path2); -} - - -static inline int -safe_wcscpy(wchar_t *dst, const wchar_t *src, size_t n) -{ - size_t srclen = wcslen(src); - if (n <= srclen) { - dst[0] = L'\0'; - return -1; + path = PyUnicode_AsWideCharString(pathobj, NULL); + if (path) { + r = _Py_isabs(path) ? Py_True : Py_False; + PyMem_Free((void *)path); } - memcpy(dst, src, (srclen + 1) * sizeof(wchar_t)); - return 0; + Py_XINCREF(r); + return r; } -/* copy_absolute requires that path be allocated at least - 'abs_path_len' characters (including trailing NUL). */ -static PyStatus -copy_absolute(wchar_t *abs_path, const wchar_t *path, size_t abs_path_len) +static PyObject * +getpath_hassuffix(PyObject *Py_UNUSED(self), PyObject *args) { - if (_Py_isabs(path)) { - if (safe_wcscpy(abs_path, path, abs_path_len) < 0) { - return PATHLEN_ERR(); - } + PyObject *r = NULL; + PyObject *pathobj; + PyObject *suffixobj; + const wchar_t *path; + const wchar_t *suffix; + if (!PyArg_ParseTuple(args, "UU", &pathobj, &suffixobj)) { + return NULL; } - else { - if (!_Py_wgetcwd(abs_path, abs_path_len)) { - /* unable to get the current directory */ - if (safe_wcscpy(abs_path, path, abs_path_len) < 0) { - return PATHLEN_ERR(); + Py_ssize_t len, suffixLen; + path = PyUnicode_AsWideCharString(pathobj, &len); + if (path) { + suffix = PyUnicode_AsWideCharString(suffixobj, &suffixLen); + if (suffix) { + if (suffixLen < len || +#ifdef MS_WINDOWS + wcsicmp(&path[len - suffixLen], suffix) != 0 +#else + wcscmp(&path[len - suffixLen], suffix) != 0 +#endif + ) { + r = Py_False; + } else { + r = Py_True; } - return _PyStatus_OK(); - } - if (path[0] == '.' && path[1] == SEP) { - path += 2; - } - PyStatus status = joinpath(abs_path, path, abs_path_len); - if (_PyStatus_EXCEPTION(status)) { - return status; + Py_INCREF(r); + PyMem_Free((void *)suffix); } + PyMem_Free((void *)path); } - return _PyStatus_OK(); -} - - -/* path_len: path length in characters including trailing NUL */ -static PyStatus -absolutize(wchar_t **path_p) -{ - assert(!_Py_isabs(*path_p)); - - wchar_t abs_path[MAXPATHLEN+1]; - wchar_t *path = *path_p; - - PyStatus status = copy_absolute(abs_path, path, Py_ARRAY_LENGTH(abs_path)); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - PyMem_RawFree(*path_p); - *path_p = _PyMem_RawWcsdup(abs_path); - if (*path_p == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); + return r; } -/* Is module -- check for .pyc too */ -static PyStatus -ismodule(const wchar_t *path, int *result) +static PyObject * +getpath_isdir(PyObject *Py_UNUSED(self), PyObject *args) { - wchar_t *filename = joinpath2(path, LANDMARK); - if (filename == NULL) { - return _PyStatus_NO_MEMORY(); - } - - if (isfile(filename)) { - PyMem_RawFree(filename); - *result = 1; - return _PyStatus_OK(); - } - - /* Check for the compiled version of prefix. */ - size_t len = wcslen(filename); - wchar_t *pyc = PyMem_RawMalloc((len + 2) * sizeof(wchar_t)); - if (pyc == NULL) { - PyMem_RawFree(filename); - return _PyStatus_NO_MEMORY(); + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; } - - memcpy(pyc, filename, len * sizeof(wchar_t)); - pyc[len] = L'c'; - pyc[len + 1] = L'\0'; - *result = isfile(pyc); - - PyMem_RawFree(filename); - PyMem_RawFree(pyc); - - return _PyStatus_OK(); -} - - -#if defined(__CYGWIN__) || defined(__MINGW32__) -#ifndef EXE_SUFFIX -#define EXE_SUFFIX L".exe" + path = PyUnicode_AsWideCharString(pathobj, NULL); + if (path) { +#ifdef MS_WINDOWS + r = (GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False; +#else + struct stat st; + r = (_Py_wstat(path, &st) == 0) && S_ISDIR(st.st_mode) ? Py_True : Py_False; #endif - -/* pathlen: 'path' length in characters including trailing NUL */ -static PyStatus -add_exe_suffix(wchar_t **progpath_p) -{ - wchar_t *progpath = *progpath_p; - - /* Check for already have an executable suffix */ - size_t n = wcslen(progpath); - size_t s = wcslen(EXE_SUFFIX); - if (wcsncasecmp(EXE_SUFFIX, progpath + n - s, s) == 0) { - return _PyStatus_OK(); - } - - wchar_t *progpath2 = PyMem_RawMalloc((n + s + 1) * sizeof(wchar_t)); - if (progpath2 == NULL) { - return _PyStatus_NO_MEMORY(); - } - - memcpy(progpath2, progpath, n * sizeof(wchar_t)); - memcpy(progpath2 + n, EXE_SUFFIX, s * sizeof(wchar_t)); - progpath2[n+s] = L'\0'; - - if (isxfile(progpath2)) { - PyMem_RawFree(*progpath_p); - *progpath_p = progpath2; - } - else { - PyMem_RawFree(progpath2); + PyMem_Free((void *)path); } - return _PyStatus_OK(); + Py_XINCREF(r); + return r; } -#endif -/* search_for_prefix requires that argv0_path be no more than MAXPATHLEN - bytes long. -*/ -static PyStatus -search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, - wchar_t *prefix, size_t prefix_len, int *found) +static PyObject * +getpath_isfile(PyObject *Py_UNUSED(self), PyObject *args) { - PyStatus status; - - /* If PYTHONHOME is set, we believe it unconditionally */ - if (pathconfig->home) { - /* Path: <home> / <lib_python> */ - if (safe_wcscpy(prefix, pathconfig->home, prefix_len) < 0) { - return PATHLEN_ERR(); - } - wchar_t *delim = wcschr(prefix, DELIM); - if (delim) { - *delim = L'\0'; - } - status = joinpath(prefix, calculate->lib_python, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - *found = 1; - return _PyStatus_OK(); - } - - /* Check to see if argv0_path is in the build directory - - Path: <argv0_path> / <BUILD_LANDMARK define> */ - wchar_t *path = joinpath2(calculate->argv0_path, BUILD_LANDMARK); - if (path == NULL) { - return _PyStatus_NO_MEMORY(); - } - - int is_build_dir = isfile(path); - PyMem_RawFree(path); - - if (is_build_dir) { - /* argv0_path is the build directory (BUILD_LANDMARK exists), - now also check LANDMARK using ismodule(). */ - - /* Path: <argv0_path> / <VPATH macro> / Lib */ - /* or if VPATH is empty: <argv0_path> / Lib */ - if (safe_wcscpy(prefix, calculate->argv0_path, prefix_len) < 0) { - return PATHLEN_ERR(); - } - - status = joinpath(prefix, calculate->vpath_macro, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - status = joinpath(prefix, L"Lib", prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - int module; - status = ismodule(prefix, &module); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (module) { - /* BUILD_LANDMARK and LANDMARK found */ - *found = -1; - return _PyStatus_OK(); - } - } - - /* Search from argv0_path, until root is found */ - status = copy_absolute(prefix, calculate->argv0_path, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - do { - /* Path: <argv0_path or substring> / <lib_python> / LANDMARK */ - size_t n = wcslen(prefix); - status = joinpath(prefix, calculate->lib_python, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - int module; - status = ismodule(prefix, &module); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (module) { - *found = 1; - return _PyStatus_OK(); - } - prefix[n] = L'\0'; - reduce(prefix); - } while (prefix[0]); - - /* Look at configure's PREFIX. - Path: <PREFIX macro> / <lib_python> / LANDMARK */ - if (safe_wcscpy(prefix, calculate->prefix_macro, prefix_len) < 0) { - return PATHLEN_ERR(); - } - status = joinpath(prefix, calculate->lib_python, prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - int module; - status = ismodule(prefix, &module); - if (_PyStatus_EXCEPTION(status)) { - return status; + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; } - if (module) { - *found = 1; - return _PyStatus_OK(); + path = PyUnicode_AsWideCharString(pathobj, NULL); + if (path) { +#ifdef MS_WINDOWS + r = !(GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False; +#else + struct stat st; + r = (_Py_wstat(path, &st) == 0) && S_ISREG(st.st_mode) ? Py_True : Py_False; +#endif + PyMem_Free((void *)path); } - - /* Fail */ - *found = 0; - return _PyStatus_OK(); + Py_XINCREF(r); + return r; } -static PyStatus -calculate_set_stdlib_dir(PyCalculatePath *calculate, _PyPathConfig *pathconfig) +static PyObject * +getpath_isxfile(PyObject *Py_UNUSED(self), PyObject *args) { - // Note that, unlike calculate_set_prefix(), here we allow a negative - // prefix_found. That means the source tree Lib dir gets used. - if (!calculate->prefix_found) { - return _PyStatus_OK(); - } - PyStatus status; - wchar_t *prefix = calculate->prefix; - if (!_Py_isabs(prefix)) { - prefix = _PyMem_RawWcsdup(prefix); - if (prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - status = absolutize(&prefix); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - wchar_t buf[MAXPATHLEN + 1]; - int res = _Py_normalize_path(prefix, buf, Py_ARRAY_LENGTH(buf)); - if (prefix != calculate->prefix) { - PyMem_RawFree(prefix); - } - if (res < 0) { - return PATHLEN_ERR(); + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + Py_ssize_t cchPath; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; } - pathconfig->stdlib_dir = _PyMem_RawWcsdup(buf); - if (pathconfig->stdlib_dir == NULL) { - return _PyStatus_NO_MEMORY(); + path = PyUnicode_AsWideCharString(pathobj, &cchPath); + if (path) { +#ifdef MS_WINDOWS + const wchar_t *ext; + r = (GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) && + SUCCEEDED(PathCchFindExtension(path, cchPath, &ext)) && + (CompareStringOrdinal(ext, -1, L".exe", -1, 1 /* ignore case */) == CSTR_EQUAL) + ? Py_True : Py_False; +#else + struct stat st; + r = (_Py_wstat(path, &st) == 0) && + S_ISREG(st.st_mode) && + (st.st_mode & 0111) + ? Py_True : Py_False; +#endif + PyMem_Free((void *)path); } - return _PyStatus_OK(); + Py_XINCREF(r); + return r; } -static PyStatus -calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig) +static PyObject * +getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args) { - wchar_t prefix[MAXPATHLEN+1]; - memset(prefix, 0, sizeof(prefix)); - size_t prefix_len = Py_ARRAY_LENGTH(prefix); - - PyStatus status; - status = search_for_prefix(calculate, pathconfig, - prefix, prefix_len, - &calculate->prefix_found); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - if (!calculate->prefix_found) { - if (calculate->warnings) { - fprintf(stderr, - "Could not find platform independent libraries <prefix>\n"); - } - - calculate->prefix = joinpath2(calculate->prefix_macro, - calculate->lib_python); - } - else { - calculate->prefix = _PyMem_RawWcsdup(prefix); - } - - if (calculate->prefix == NULL) { - return _PyStatus_NO_MEMORY(); + if (!PyTuple_Check(args)) { + PyErr_SetString(PyExc_TypeError, "requires tuple of arguments"); + return NULL; } - return _PyStatus_OK(); -} - - -static PyStatus -calculate_set_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - /* Reduce prefix and exec_prefix to their essence, - * e.g. /usr/local/lib/python1.5 is reduced to /usr/local. - * If we're loading relative to the build directory, - * return the compiled-in defaults instead. - */ - if (calculate->prefix_found > 0) { - wchar_t *prefix = _PyMem_RawWcsdup(calculate->prefix); - if (prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - - reduce(prefix); - reduce(prefix); - if (prefix[0]) { - pathconfig->prefix = prefix; - } - else { - PyMem_RawFree(prefix); - - /* The prefix is the root directory, but reduce() chopped - off the "/". */ - pathconfig->prefix = _PyMem_RawWcsdup(separator); - if (pathconfig->prefix == NULL) { - return _PyStatus_NO_MEMORY(); + Py_ssize_t n = PyTuple_GET_SIZE(args); + if (n == 0) { + return PyUnicode_FromString(NULL); + } + /* Convert all parts to wchar and accumulate max final length */ + wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *)); + memset(parts, 0, n * sizeof(wchar_t *)); + Py_ssize_t cchFinal = 0; + Py_ssize_t first = 0; + + for (Py_ssize_t i = 0; i < n; ++i) { + PyObject *s = PyTuple_GET_ITEM(args, i); + Py_ssize_t cch; + if (s == Py_None) { + cch = 0; + } else if (PyUnicode_Check(s)) { + parts[i] = PyUnicode_AsWideCharString(s, &cch); + if (!parts[i]) { + cchFinal = -1; + break; } + if (_Py_isabs(parts[i])) { + first = i; + } + } else { + PyErr_SetString(PyExc_TypeError, "all arguments to joinpath() must be str or None"); + cchFinal = -1; + break; } + cchFinal += cch + 1; } - else { - pathconfig->prefix = _PyMem_RawWcsdup(calculate->prefix_macro); - if (pathconfig->prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - return _PyStatus_OK(); -} - - -static PyStatus -calculate_pybuilddir(const wchar_t *argv0_path, - wchar_t *exec_prefix, size_t exec_prefix_len, - int *found) -{ - PyStatus status; - - /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" - is written by setup.py and contains the relative path to the location - of shared library modules. - - Filename: <argv0_path> / "pybuilddir.txt" */ - wchar_t *filename = joinpath2(argv0_path, L"pybuilddir.txt"); - if (filename == NULL) { - return _PyStatus_NO_MEMORY(); - } - - FILE *fp = _Py_wfopen(filename, L"rb"); - PyMem_RawFree(filename); - if (fp == NULL) { - errno = 0; - return _PyStatus_OK(); - } - - char buf[MAXPATHLEN + 1]; - size_t n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, fp); - buf[n] = '\0'; - fclose(fp); - - size_t dec_len; - wchar_t *pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len); - if (!pybuilddir) { - return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len); - } - - /* Path: <argv0_path> / <pybuilddir content> */ - if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) { - PyMem_RawFree(pybuilddir); - return PATHLEN_ERR(); - } - status = joinpath(exec_prefix, pybuilddir, exec_prefix_len); - PyMem_RawFree(pybuilddir); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - *found = -1; - return _PyStatus_OK(); -} - -/* search_for_exec_prefix requires that argv0_path be no more than - MAXPATHLEN bytes long. -*/ -static PyStatus -search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig, - wchar_t *exec_prefix, size_t exec_prefix_len, - int *found) -{ - PyStatus status; - - /* If PYTHONHOME is set, we believe it unconditionally */ - if (pathconfig->home) { - /* Path: <home> / <lib_python> / "lib-dynload" */ - wchar_t *delim = wcschr(pathconfig->home, DELIM); - if (delim) { - if (safe_wcscpy(exec_prefix, delim+1, exec_prefix_len) < 0) { - return PATHLEN_ERR(); - } + wchar_t *final = cchFinal > 0 ? (wchar_t *)PyMem_Malloc(cchFinal * sizeof(wchar_t)) : NULL; + if (!final) { + for (Py_ssize_t i = 0; i < n; ++i) { + PyMem_Free(parts[i]); } - else { - if (safe_wcscpy(exec_prefix, pathconfig->home, exec_prefix_len) < 0) { - return PATHLEN_ERR(); - } - } - status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; + PyMem_Free(parts); + if (cchFinal) { + PyErr_NoMemory(); + return NULL; } - status = joinpath(exec_prefix, L"lib-dynload", exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - *found = 1; - return _PyStatus_OK(); - } - - /* Check for pybuilddir.txt */ - assert(*found == 0); - status = calculate_pybuilddir(calculate->argv0_path, - exec_prefix, exec_prefix_len, found); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (*found) { - return _PyStatus_OK(); + return PyUnicode_FromStringAndSize(NULL, 0); } - /* Search from argv0_path, until root is found */ - status = copy_absolute(exec_prefix, calculate->argv0_path, exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - do { - /* Path: <argv0_path or substring> / <lib_python> / "lib-dynload" */ - size_t n = wcslen(exec_prefix); - status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - status = joinpath(exec_prefix, L"lib-dynload", exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; + final[0] = '\0'; + /* Now join all the paths. The final result should be shorter than the buffer */ + for (Py_ssize_t i = 0; i < n; ++i) { + if (!parts[i]) { + continue; } - if (isdir(exec_prefix)) { - *found = 1; - return _PyStatus_OK(); + if (i >= first && final) { + if (!final[0]) { + /* final is definitely long enough to fit any individual part */ + wcscpy(final, parts[i]); + } else if (_Py_add_relfile(final, parts[i], cchFinal) < 0) { + /* if we fail, keep iterating to free memory, but stop adding parts */ + PyMem_Free(final); + final = NULL; + } } - exec_prefix[n] = L'\0'; - reduce(exec_prefix); - } while (exec_prefix[0]); - - /* Look at configure's EXEC_PREFIX. - - Path: <EXEC_PREFIX macro> / <lib_python> / "lib-dynload" */ - if (safe_wcscpy(exec_prefix, calculate->exec_prefix_macro, exec_prefix_len) < 0) { - return PATHLEN_ERR(); + PyMem_Free(parts[i]); } - status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - status = joinpath(exec_prefix, L"lib-dynload", exec_prefix_len); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (isdir(exec_prefix)) { - *found = 1; - return _PyStatus_OK(); + PyMem_Free(parts); + if (!final) { + PyErr_SetString(PyExc_SystemError, "failed to join paths"); + return NULL; } - - /* Fail */ - *found = 0; - return _PyStatus_OK(); + PyObject *r = PyUnicode_FromWideChar(_Py_normpath(final, -1), -1); + PyMem_Free(final); + return r; } -static PyStatus -calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig) +static PyObject * +getpath_readlines(PyObject *Py_UNUSED(self), PyObject *args) { - PyStatus status; - wchar_t exec_prefix[MAXPATHLEN+1]; - memset(exec_prefix, 0, sizeof(exec_prefix)); - size_t exec_prefix_len = Py_ARRAY_LENGTH(exec_prefix); - - status = search_for_exec_prefix(calculate, pathconfig, - exec_prefix, exec_prefix_len, - &calculate->exec_prefix_found); - if (_PyStatus_EXCEPTION(status)) { - return status; + PyObject *r = NULL; + PyObject *pathobj; + const wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; } - - if (!calculate->exec_prefix_found) { - if (calculate->warnings) { - fprintf(stderr, - "Could not find platform dependent libraries <exec_prefix>\n"); - } - - /* <platlibdir> / "lib-dynload" */ - wchar_t *lib_dynload = joinpath2(calculate->platlibdir, - L"lib-dynload"); - if (lib_dynload == NULL) { - return _PyStatus_NO_MEMORY(); - } - - calculate->exec_prefix = joinpath2(calculate->exec_prefix_macro, - lib_dynload); - PyMem_RawFree(lib_dynload); - - if (calculate->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } + path = PyUnicode_AsWideCharString(pathobj, NULL); + if (!path) { + return NULL; } - else { - /* If we found EXEC_PREFIX do *not* reduce it! (Yet.) */ - calculate->exec_prefix = _PyMem_RawWcsdup(exec_prefix); - if (calculate->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } + FILE *fp = _Py_wfopen(path, L"rb"); + PyMem_Free((void *)path); + if (!fp) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; } - return _PyStatus_OK(); -} - - -static PyStatus -calculate_set_exec_prefix(PyCalculatePath *calculate, - _PyPathConfig *pathconfig) -{ - if (calculate->exec_prefix_found > 0) { - wchar_t *exec_prefix = _PyMem_RawWcsdup(calculate->exec_prefix); - if (exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - reduce(exec_prefix); - reduce(exec_prefix); - reduce(exec_prefix); - - if (exec_prefix[0]) { - pathconfig->exec_prefix = exec_prefix; - } - else { - /* empty string: use SEP instead */ - PyMem_RawFree(exec_prefix); - - /* The exec_prefix is the root directory, but reduce() chopped - off the "/". */ - pathconfig->exec_prefix = _PyMem_RawWcsdup(separator); - if (pathconfig->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } + r = PyList_New(0); + if (!r) { + fclose(fp); + return NULL; } - else { - pathconfig->exec_prefix = _PyMem_RawWcsdup(calculate->exec_prefix_macro); - if (pathconfig->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } + const size_t MAX_FILE = 32 * 1024; + char *buffer = (char *)PyMem_Malloc(MAX_FILE); + if (!buffer) { + Py_DECREF(r); + fclose(fp); + return NULL; } - return _PyStatus_OK(); -} - - -/* Similar to shutil.which(). - If found, write the path into *abs_path_p. */ -static PyStatus -calculate_which(const wchar_t *path_env, wchar_t *program_name, - wchar_t **abs_path_p) -{ - while (1) { - wchar_t *delim = wcschr(path_env, DELIM); - wchar_t *abs_path; - - if (delim) { - wchar_t *path = substring(path_env, delim - path_env); - if (path == NULL) { - return _PyStatus_NO_MEMORY(); - } - abs_path = joinpath2(path, program_name); - PyMem_RawFree(path); - } - else { - abs_path = joinpath2(path_env, program_name); - } - - if (abs_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - - if (isxfile(abs_path)) { - *abs_path_p = abs_path; - return _PyStatus_OK(); - } - PyMem_RawFree(abs_path); - if (!delim) { - break; - } - path_env = delim + 1; + size_t cb = fread(buffer, 1, MAX_FILE, fp); + fclose(fp); + if (!cb) { + return r; } - - /* not found */ - return _PyStatus_OK(); -} - - -#ifdef __APPLE__ -static PyStatus -calculate_program_macos(wchar_t **abs_path_p) -{ - char execpath[MAXPATHLEN + 1]; - uint32_t nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; - - /* On Mac OS X, if a script uses an interpreter of the form - "#!/opt/python2.3/bin/python", the kernel only passes "python" - as argv[0], which falls through to the $PATH search below. - If /opt/python2.3/bin isn't in your path, or is near the end, - this algorithm may incorrectly find /usr/bin/python. To work - around this, we can use _NSGetExecutablePath to get a better - hint of what the intended interpreter was, although this - will fail if a relative path was used. but in that case, - absolutize() should help us out below - */ - if (_NSGetExecutablePath(execpath, &nsexeclength) != 0 - || (wchar_t)execpath[0] != SEP) - { - /* _NSGetExecutablePath() failed or the path is relative */ - return _PyStatus_OK(); + if (cb >= MAX_FILE) { + Py_DECREF(r); + PyErr_SetString(PyExc_MemoryError, + "cannot read file larger than 32KB during initialization"); + return NULL; } + buffer[cb] = '\0'; size_t len; - *abs_path_p = Py_DecodeLocale(execpath, &len); - if (*abs_path_p == NULL) { - return DECODE_LOCALE_ERR("executable path", len); - } - return _PyStatus_OK(); -} -#endif /* __APPLE__ */ - - -static PyStatus -calculate_program_impl(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - assert(pathconfig->program_full_path == NULL); - - PyStatus status; - - /* If there is no slash in the argv0 path, then we have to - * assume python is on the user's $PATH, since there's no - * other way to find a directory to start the search from. If - * $PATH isn't exported, you lose. - */ - if (wcschr(pathconfig->program_name, SEP)) { - pathconfig->program_full_path = _PyMem_RawWcsdup(pathconfig->program_name); - if (pathconfig->program_full_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); - } - -#ifdef __APPLE__ - wchar_t *abs_path = NULL; - status = calculate_program_macos(&abs_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (abs_path) { - pathconfig->program_full_path = abs_path; - return _PyStatus_OK(); + wchar_t *wbuffer = _Py_DecodeUTF8_surrogateescape(buffer, cb, &len); + PyMem_Free((void *)buffer); + if (!wbuffer) { + Py_DECREF(r); + PyErr_NoMemory(); + return NULL; } -#endif /* __APPLE__ */ - if (calculate->path_env) { - wchar_t *abs_path = NULL; - status = calculate_which(calculate->path_env, pathconfig->program_name, - &abs_path); - if (_PyStatus_EXCEPTION(status)) { - return status; + wchar_t *p1 = wbuffer; + wchar_t *p2 = p1; + while ((p2 = wcschr(p1, L'\n')) != NULL) { + size_t cb = p2 - p1; + while (cb && (p1[cb] == L'\n' || p1[cb] == L'\r')) { + --cb; } - if (abs_path) { - pathconfig->program_full_path = abs_path; - return _PyStatus_OK(); + PyObject *u = PyUnicode_FromWideChar(p1, cb + 1); + if (!u || PyList_Append(r, u) < 0) { + Py_XDECREF(u); + Py_CLEAR(r); + break; } + Py_DECREF(u); + p1 = p2 + 1; } - - /* In the last resort, use an empty string */ - pathconfig->program_full_path = _PyMem_RawWcsdup(L""); - if (pathconfig->program_full_path == NULL) { - return _PyStatus_NO_MEMORY(); + if (r && p1 && *p1) { + PyObject *u = PyUnicode_FromWideChar(p1, -1); + if (!u || PyList_Append(r, u) < 0) { + Py_CLEAR(r); + } + Py_XDECREF(u); } - return _PyStatus_OK(); + PyMem_RawFree(wbuffer); + return r; } -/* Calculate pathconfig->program_full_path */ -static PyStatus -calculate_program(PyCalculatePath *calculate, _PyPathConfig *pathconfig) +static PyObject * +getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) { - PyStatus status; - - status = calculate_program_impl(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; + PyObject *pathobj; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; } - - if (pathconfig->program_full_path[0] != '\0') { - /* program_full_path is not empty */ - - /* Make sure that program_full_path is an absolute path */ - if (!_Py_isabs(pathconfig->program_full_path)) { - status = absolutize(&pathconfig->program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - -#if defined(__CYGWIN__) || defined(__MINGW32__) - /* For these platforms it is necessary to ensure that the .exe suffix - * is appended to the filename, otherwise there is potential for - * sys.executable to return the name of a directory under the same - * path (bpo-28441). - */ - status = add_exe_suffix(&pathconfig->program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } -#endif +#if defined(HAVE_READLINK) + /* This readlink calculation only resolves a symlinked file, and + does not resolve any path segments. This is consistent with + prior releases, however, the realpath implementation below is + potentially correct in more cases. */ + PyObject *r = NULL; + int nlink = 0; + wchar_t *path = PyUnicode_AsWideCharString(pathobj, NULL); + if (!path) { + goto done; } - return _PyStatus_OK(); -} - - -#if HAVE_READLINK -static PyStatus -resolve_symlinks(wchar_t **path_p) -{ - wchar_t new_path[MAXPATHLEN + 1]; - const size_t new_path_len = Py_ARRAY_LENGTH(new_path); - unsigned int nlink = 0; - - while (1) { - int linklen = _Py_wreadlink(*path_p, new_path, new_path_len); + wchar_t *path2 = _PyMem_RawWcsdup(path); + PyMem_Free((void *)path); + path = path2; + while (path) { + wchar_t resolved[MAXPATHLEN + 1]; + int linklen = _Py_wreadlink(path, resolved, Py_ARRAY_LENGTH(resolved)); if (linklen == -1) { - /* not a symbolic link: we are done */ + r = PyUnicode_FromWideChar(path, -1); break; } - - if (_Py_isabs(new_path)) { - PyMem_RawFree(*path_p); - *path_p = _PyMem_RawWcsdup(new_path); - if (*path_p == NULL) { - return _PyStatus_NO_MEMORY(); + if (_Py_isabs(resolved)) { + PyMem_RawFree((void *)path); + path = _PyMem_RawWcsdup(resolved); + } else { + wchar_t *s = wcsrchr(path, SEP); + if (s) { + *s = L'\0'; } + path2 = _Py_normpath(_Py_join_relfile(path, resolved), -1); + PyMem_RawFree((void *)path); + path = path2; } - else { - /* new_path is relative to path */ - reduce(*path_p); - - wchar_t *abs_path = joinpath2(*path_p, new_path); - if (abs_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - - PyMem_RawFree(*path_p); - *path_p = abs_path; - } - nlink++; /* 40 is the Linux kernel 4.2 limit */ if (nlink >= 40) { - return _PyStatus_ERR("maximum number of symbolic links reached"); + PyErr_SetString(PyExc_OSError, "maximum number of symbolic links reached"); + break; } } - return _PyStatus_OK(); -} -#endif /* HAVE_READLINK */ - - -#ifdef WITH_NEXT_FRAMEWORK -static PyStatus -calculate_argv0_path_framework(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - NSModule pythonModule; - - /* On Mac OS X we have a special case if we're running from a framework. - This is because the python home should be set relative to the library, - which is in the framework, not relative to the executable, which may - be outside of the framework. Except when we're in the build - directory... */ - pythonModule = NSModuleForSymbol(NSLookupAndBindSymbol("_Py_Initialize")); - - /* Use dylib functions to find out where the framework was loaded from */ - const char* modPath = NSLibraryNameForModule(pythonModule); - if (modPath == NULL) { - return _PyStatus_OK(); - } - - /* We're in a framework. - See if we might be in the build directory. The framework in the - build directory is incomplete, it only has the .dylib and a few - needed symlinks, it doesn't have the Lib directories and such. - If we're running with the framework from the build directory we must - be running the interpreter in the build directory, so we use the - build-directory-specific logic to find Lib and such. */ - size_t len; - wchar_t* wbuf = Py_DecodeLocale(modPath, &len); - if (wbuf == NULL) { - return DECODE_LOCALE_ERR("framework location", len); + if (!path) { + PyErr_NoMemory(); } - - /* Path: reduce(modPath) / lib_python / LANDMARK */ - PyStatus status; - - wchar_t *parent = _PyMem_RawWcsdup(wbuf); - if (parent == NULL) { - status = _PyStatus_NO_MEMORY(); +done: + PyMem_RawFree((void *)path); + return r; + +#elif defined(HAVE_REALPATH) + PyObject *r = NULL; + struct stat st; + const char *narrow = NULL; + wchar_t *path = PyUnicode_AsWideCharString(pathobj, NULL); + if (!path) { goto done; } - - reduce(parent); - wchar_t *lib_python = joinpath2(parent, calculate->lib_python); - PyMem_RawFree(parent); - - if (lib_python == NULL) { - status = _PyStatus_NO_MEMORY(); + narrow = Py_EncodeLocale(path, NULL); + if (!narrow) { + PyErr_NoMemory(); goto done; } - - int module; - status = ismodule(lib_python, &module); - PyMem_RawFree(lib_python); - - if (_PyStatus_EXCEPTION(status)) { + if (lstat(narrow, &st)) { + PyErr_SetFromErrno(PyExc_OSError); goto done; } - if (!module) { - /* We are in the build directory so use the name of the - executable - we know that the absolute path is passed */ - PyMem_RawFree(calculate->argv0_path); - calculate->argv0_path = _PyMem_RawWcsdup(pathconfig->program_full_path); - if (calculate->argv0_path == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - - status = _PyStatus_OK(); + if (!S_ISLNK(st.st_mode)) { + Py_INCREF(pathobj); + r = pathobj; goto done; } - - /* Use the location of the library as argv0_path */ - PyMem_RawFree(calculate->argv0_path); - calculate->argv0_path = wbuf; - return _PyStatus_OK(); - + wchar_t resolved[MAXPATHLEN+1]; + if (_Py_wrealpath(path, resolved, MAXPATHLEN) == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + } else { + r = PyUnicode_FromWideChar(resolved, -1); + } done: - PyMem_RawFree(wbuf); - return status; -} + PyMem_Free((void *)path); + PyMem_Free((void *)narrow); + return r; #endif + Py_INCREF(pathobj); + return pathobj; +} -static PyStatus -calculate_argv0_path(PyCalculatePath *calculate, - _PyPathConfig *pathconfig) -{ - PyStatus status; - - calculate->argv0_path = _PyMem_RawWcsdup(pathconfig->program_full_path); - if (calculate->argv0_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - -#ifdef WITH_NEXT_FRAMEWORK - status = calculate_argv0_path_framework(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } -#endif - - status = resolve_symlinks(&calculate->argv0_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - reduce(calculate->argv0_path); +static PyMethodDef getpath_methods[] = { + {"abspath", getpath_abspath, METH_VARARGS, NULL}, + {"basename", getpath_basename, METH_VARARGS, NULL}, + {"dirname", getpath_dirname, METH_VARARGS, NULL}, + {"hassuffix", getpath_hassuffix, METH_VARARGS, NULL}, + {"isabs", getpath_isabs, METH_VARARGS, NULL}, + {"isdir", getpath_isdir, METH_VARARGS, NULL}, + {"isfile", getpath_isfile, METH_VARARGS, NULL}, + {"isxfile", getpath_isxfile, METH_VARARGS, NULL}, + {"joinpath", getpath_joinpath, METH_VARARGS, NULL}, + {"readlines", getpath_readlines, METH_VARARGS, NULL}, + {"realpath", getpath_realpath, METH_VARARGS, NULL}, + {NULL, NULL, 0, NULL} +}; - return _PyStatus_OK(); -} +/* Two implementations of warn() to use depending on whether warnings + are enabled or not. */ -static PyStatus -calculate_open_pyenv(PyCalculatePath *calculate, FILE **env_file_p) +static PyObject * +getpath_warn(PyObject *Py_UNUSED(self), PyObject *args) { - *env_file_p = NULL; - - const wchar_t *env_cfg = L"pyvenv.cfg"; - - /* Filename: <argv0_path> / "pyvenv.cfg" */ - wchar_t *filename = joinpath2(calculate->argv0_path, env_cfg); - if (filename == NULL) { - return _PyStatus_NO_MEMORY(); + PyObject *msgobj; + if (!PyArg_ParseTuple(args, "U", &msgobj)) { + return NULL; } + fprintf(stderr, "%s\n", PyUnicode_AsUTF8(msgobj)); + Py_RETURN_NONE; +} - *env_file_p = _Py_wfopen(filename, L"r"); - PyMem_RawFree(filename); - if (*env_file_p != NULL) { - return _PyStatus_OK(); +static PyObject * +getpath_nowarn(PyObject *Py_UNUSED(self), PyObject *args) +{ + Py_RETURN_NONE; +} - } - /* fopen() failed: reset errno */ - errno = 0; +static PyMethodDef getpath_warn_method = {"warn", getpath_warn, METH_VARARGS, NULL}; +static PyMethodDef getpath_nowarn_method = {"warn", getpath_nowarn, METH_VARARGS, NULL}; - /* Path: <basename(argv0_path)> / "pyvenv.cfg" */ - wchar_t *parent = _PyMem_RawWcsdup(calculate->argv0_path); - if (parent == NULL) { - return _PyStatus_NO_MEMORY(); +/* Add the helper functions to the dict */ +static int +funcs_to_dict(PyObject *dict, int warnings) +{ + for (PyMethodDef *m = getpath_methods; m->ml_name; ++m) { + PyObject *f = PyCFunction_NewEx(m, NULL, NULL); + if (!f) { + return 0; + } + if (PyDict_SetItemString(dict, m->ml_name, f) < 0) { + Py_DECREF(f); + return 0; + } + Py_DECREF(f); } - reduce(parent); - - filename = joinpath2(parent, env_cfg); - PyMem_RawFree(parent); - if (filename == NULL) { - return _PyStatus_NO_MEMORY(); + PyMethodDef *m2 = warnings ? &getpath_warn_method : &getpath_nowarn_method; + PyObject *f = PyCFunction_NewEx(m2, NULL, NULL); + if (!f) { + return 0; } - - *env_file_p = _Py_wfopen(filename, L"r"); - PyMem_RawFree(filename); - - if (*env_file_p == NULL) { - /* fopen() failed: reset errno */ - errno = 0; + if (PyDict_SetItemString(dict, m2->ml_name, f) < 0) { + Py_DECREF(f); + return 0; } - return _PyStatus_OK(); + Py_DECREF(f); + return 1; } -/* Search for an "pyvenv.cfg" environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. - - Write the 'home' variable of pyvenv.cfg into calculate->argv0_path. */ -static PyStatus -calculate_read_pyenv(PyCalculatePath *calculate) +/* Add a wide-character string constant to the dict */ +static int +wchar_to_dict(PyObject *dict, const char *key, const wchar_t *s) { - PyStatus status; - FILE *env_file = NULL; - - status = calculate_open_pyenv(calculate, &env_file); - if (_PyStatus_EXCEPTION(status)) { - assert(env_file == NULL); - return status; - } - if (env_file == NULL) { - /* pyvenv.cfg not found */ - return _PyStatus_OK(); - } - - /* Look for a 'home' variable and set argv0_path to it, if found */ - wchar_t *home = NULL; - status = _Py_FindEnvConfigValue(env_file, L"home", &home); - if (_PyStatus_EXCEPTION(status)) { - fclose(env_file); - return status; - } - - if (home) { - PyMem_RawFree(calculate->argv0_path); - calculate->argv0_path = home; + PyObject *u; + int r; + if (s && s[0]) { + u = PyUnicode_FromWideChar(s, -1); + if (!u) { + return 0; + } + } else { + u = Py_None; + Py_INCREF(u); } - fclose(env_file); - return _PyStatus_OK(); + r = PyDict_SetItemString(dict, key, u) == 0; + Py_DECREF(u); + return r; } -static PyStatus -calculate_zip_path(PyCalculatePath *calculate) +/* Add a narrow string constant to the dict, using default locale decoding */ +static int +decode_to_dict(PyObject *dict, const char *key, const char *s) { - PyStatus res; - - /* Path: <platlibdir> / "pythonXY.zip" */ - wchar_t *path = joinpath2(calculate->platlibdir, - L"python" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) - L".zip"); - if (path == NULL) { - return _PyStatus_NO_MEMORY(); + PyObject *u = NULL; + int r; + if (s && s[0]) { + size_t len; + const wchar_t *w = Py_DecodeLocale(s, &len); + if (w) { + u = PyUnicode_FromWideChar(w, len); + PyMem_RawFree((void *)w); + } + if (!u) { + return 0; + } + } else { + u = Py_None; + Py_INCREF(u); } + r = PyDict_SetItemString(dict, key, u) == 0; + Py_DECREF(u); + return r; +} - if (calculate->prefix_found > 0) { - /* Use the reduced prefix returned by Py_GetPrefix() - - Path: <basename(basename(prefix))> / <platlibdir> / "pythonXY.zip" */ - wchar_t *parent = _PyMem_RawWcsdup(calculate->prefix); - if (parent == NULL) { - res = _PyStatus_NO_MEMORY(); - goto done; +/* Add an environment variable to the dict, optionally clearing it afterwards */ +static int +env_to_dict(PyObject *dict, const char *key, int and_clear) +{ + PyObject *u = NULL; + int r = 0; + assert(strncmp(key, "ENV_", 4) == 0); + assert(strlen(key) < 64); +#ifdef MS_WINDOWS + wchar_t wkey[64]; + // Quick convert to wchar_t, since we know key is ASCII + wchar_t *wp = wkey; + for (const char *p = &key[4]; *p; ++p) { + assert(*p < 128); + *wp++ = *p; + } + *wp = L'\0'; + const wchar_t *v = _wgetenv(wkey); + if (v) { + u = PyUnicode_FromWideChar(v, -1); + if (!u) { + PyErr_Clear(); + } + } +#else + const char *v = getenv(&key[4]); + if (v) { + size_t len; + const wchar_t *w = Py_DecodeLocale(v, &len); + if (w) { + u = PyUnicode_FromWideChar(w, len); + if (!u) { + PyErr_Clear(); + } + PyMem_RawFree((void *)w); } - reduce(parent); - reduce(parent); - calculate->zip_path = joinpath2(parent, path); - PyMem_RawFree(parent); } - else { - calculate->zip_path = joinpath2(calculate->prefix_macro, path); - } - - if (calculate->zip_path == NULL) { - res = _PyStatus_NO_MEMORY(); - goto done; +#endif + if (u) { + r = PyDict_SetItemString(dict, key, u) == 0; + Py_DECREF(u); + } else { + r = PyDict_SetItemString(dict, key, Py_None) == 0; + } + if (r && and_clear) { +#ifdef MS_WINDOWS + _wputenv_s(wkey, L""); +#else + unsetenv(&key[4]); +#endif } - - res = _PyStatus_OK(); - -done: - PyMem_RawFree(path); - return res; + return r; } -static PyStatus -calculate_module_search_path(PyCalculatePath *calculate, - _PyPathConfig *pathconfig) +/* Add an integer constant to the dict */ +static int +int_to_dict(PyObject *dict, const char *key, int v) { - /* Calculate size of return buffer */ - size_t bufsz = 0; - if (calculate->pythonpath_env != NULL) { - bufsz += wcslen(calculate->pythonpath_env) + 1; + PyObject *o; + int r; + o = PyLong_FromLong(v); + if (!o) { + return 0; } + r = PyDict_SetItemString(dict, key, o) == 0; + Py_DECREF(o); + return r; +} - wchar_t *defpath = calculate->pythonpath_macro; - size_t prefixsz = wcslen(calculate->prefix) + 1; - while (1) { - wchar_t *delim = wcschr(defpath, DELIM); - - if (!_Py_isabs(defpath)) { - /* Paths are relative to prefix */ - bufsz += prefixsz; - } - if (delim) { - bufsz += delim - defpath + 1; - } - else { - bufsz += wcslen(defpath) + 1; - break; +#ifdef MS_WINDOWS +static int +winmodule_to_dict(PyObject *dict, const char *key, HMODULE mod) +{ + wchar_t *buffer = NULL; + for (DWORD cch = 256; buffer == NULL && cch < (1024 * 1024); cch *= 2) { + buffer = (wchar_t*)PyMem_RawMalloc(cch * sizeof(wchar_t)); + if (buffer) { + if (GetModuleFileNameW(mod, buffer, cch) == cch) { + PyMem_RawFree(buffer); + buffer = NULL; + } } - defpath = delim + 1; - } - - bufsz += wcslen(calculate->zip_path) + 1; - bufsz += wcslen(calculate->exec_prefix) + 1; - - /* Allocate the buffer */ - wchar_t *buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); - if (buf == NULL) { - return _PyStatus_NO_MEMORY(); } - buf[0] = '\0'; - - /* Run-time value of $PYTHONPATH goes first */ - if (calculate->pythonpath_env) { - wcscpy(buf, calculate->pythonpath_env); - wcscat(buf, delimiter); - } - - /* Next is the default zip path */ - wcscat(buf, calculate->zip_path); - wcscat(buf, delimiter); - - /* Next goes merge of compile-time $PYTHONPATH with - * dynamically located prefix. - */ - defpath = calculate->pythonpath_macro; - while (1) { - wchar_t *delim = wcschr(defpath, DELIM); + int r = wchar_to_dict(dict, key, buffer); + PyMem_RawFree(buffer); + return r; +} +#endif - if (!_Py_isabs(defpath)) { - wcscat(buf, calculate->prefix); - if (prefixsz >= 2 && calculate->prefix[prefixsz - 2] != SEP && - defpath[0] != (delim ? DELIM : L'\0')) - { - /* not empty */ - wcscat(buf, separator); - } - } - if (delim) { - size_t len = delim - defpath + 1; - size_t end = wcslen(buf) + len; - wcsncat(buf, defpath, len); - buf[end] = '\0'; +/* Add the current executable's path to the dict */ +static int +progname_to_dict(PyObject *dict, const char *key) +{ +#ifdef MS_WINDOWS + return winmodule_to_dict(dict, key, NULL); +#elif defined(__APPLE__) + char *path; + uint32_t pathLen = 256; + while (pathLen) { + path = PyMem_RawMalloc((pathLen + 1) * sizeof(char)); + if (!path) { + return 0; + } + if (_NSGetExecutablePath(path, &pathLen) != 0) { + PyMem_RawFree(path); + continue; } - else { - wcscat(buf, defpath); - break; + // Only keep if the path is absolute + if (path[0] == SEP) { + int r = decode_to_dict(dict, key, path); + PyMem_RawFree(path); + return r; } - defpath = delim + 1; + // Fall back and store None + PyMem_RawFree(path); + break; } - wcscat(buf, delimiter); - - /* Finally, on goes the directory for dynamic-load modules */ - wcscat(buf, calculate->exec_prefix); - - pathconfig->module_search_path = buf; - return _PyStatus_OK(); +#endif + return PyDict_SetItemString(dict, key, Py_None) == 0; } -static PyStatus -calculate_init(PyCalculatePath *calculate, const PyConfig *config) +/* Add the runtime library's path to the dict */ +static int +library_to_dict(PyObject *dict, const char *key) { - size_t len; - - calculate->warnings = config->pathconfig_warnings; - calculate->pythonpath_env = config->pythonpath_env; - calculate->platlibdir = config->platlibdir; - - const char *path = getenv("PATH"); - if (path) { - calculate->path_env = Py_DecodeLocale(path, &len); - if (!calculate->path_env) { - return DECODE_LOCALE_ERR("PATH environment variable", len); - } +#ifdef MS_WINDOWS + extern HMODULE PyWin_DLLhModule; + if (PyWin_DLLhModule) { + return winmodule_to_dict(dict, key, PyWin_DLLhModule); } +#elif defined(WITH_NEXT_FRAMEWORK) + static const char modPath[MAXPATHLEN + 1]; + static int modPathInitialized = -1; + if (modPathInitialized < 0) { + NSModule pythonModule; + modPathInitialized = 0; - /* Decode macros */ - calculate->pythonpath_macro = Py_DecodeLocale(PYTHONPATH, &len); - if (!calculate->pythonpath_macro) { - return DECODE_LOCALE_ERR("PYTHONPATH macro", len); - } - calculate->prefix_macro = Py_DecodeLocale(PREFIX, &len); - if (!calculate->prefix_macro) { - return DECODE_LOCALE_ERR("PREFIX macro", len); - } - calculate->exec_prefix_macro = Py_DecodeLocale(EXEC_PREFIX, &len); - if (!calculate->exec_prefix_macro) { - return DECODE_LOCALE_ERR("EXEC_PREFIX macro", len); - } - calculate->vpath_macro = Py_DecodeLocale(VPATH, &len); - if (!calculate->vpath_macro) { - return DECODE_LOCALE_ERR("VPATH macro", len); - } + /* On Mac OS X we have a special case if we're running from a framework. + This is because the python home should be set relative to the library, + which is in the framework, not relative to the executable, which may + be outside of the framework. Except when we're in the build + directory... */ + pythonModule = NSModuleForSymbol(NSLookupAndBindSymbol("_Py_Initialize")); - // <platlibdir> / "pythonX.Y" - wchar_t *pyversion = Py_DecodeLocale("python" VERSION, &len); - if (!pyversion) { - return DECODE_LOCALE_ERR("VERSION macro", len); + /* Use dylib functions to find out where the framework was loaded from */ + const char *path = NSLibraryNameForModule(pythonModule); + if (path) { + strncpy(modPath, path, MAXPATHLEN); + } } - calculate->lib_python = joinpath2(config->platlibdir, pyversion); - PyMem_RawFree(pyversion); - if (calculate->lib_python == NULL) { - return _PyStatus_NO_MEMORY(); + if (modPathInitialized > 0) { + return decode_to_dict(dict, key, modPath); } - - return _PyStatus_OK(); +#endif + return PyDict_SetItemString(dict, key, Py_None) == 0; } -static void -calculate_free(PyCalculatePath *calculate) +PyObject * +_Py_Get_Getpath_CodeObject() { - PyMem_RawFree(calculate->pythonpath_macro); - PyMem_RawFree(calculate->prefix_macro); - PyMem_RawFree(calculate->exec_prefix_macro); - PyMem_RawFree(calculate->vpath_macro); - PyMem_RawFree(calculate->lib_python); - PyMem_RawFree(calculate->path_env); - PyMem_RawFree(calculate->zip_path); - PyMem_RawFree(calculate->argv0_path); - PyMem_RawFree(calculate->prefix); - PyMem_RawFree(calculate->exec_prefix); + return PyMarshal_ReadObjectFromString( + (const char*)_Py_M__getpath, sizeof(_Py_M__getpath)); } -static PyStatus -calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - PyStatus status; - - if (pathconfig->program_full_path == NULL) { - status = calculate_program(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } +/* Perform the actual path calculation. - status = calculate_argv0_path(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + When compute_path_config is 0, this only reads any initialised path + config values into the PyConfig struct. For example, Py_SetHome() or + Py_SetPath(). The only error should be due to failed memory allocation. - /* If a pyvenv.cfg configure file is found, - argv0_path is overridden with its 'home' variable. */ - status = calculate_read_pyenv(calculate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + When compute_path_config is 1, full path calculation is performed. + The GIL must be held, and there may be filesystem access, side + effects, and potential unraisable errors that are reported directly + to stderr. - status = calculate_prefix(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + Calling this function multiple times on the same PyConfig is only + safe because already-configured values are not recalculated. To + actually recalculate paths, you need a clean PyConfig. +*/ +PyStatus +_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) +{ + PyStatus status = _PyPathConfig_ReadGlobal(config); - status = calculate_zip_path(calculate); - if (_PyStatus_EXCEPTION(status)) { + if (_PyStatus_EXCEPTION(status) || !compute_path_config) { return status; } - status = calculate_exec_prefix(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; + if (!_PyThreadState_UncheckedGet()) { + return PyStatus_Error("cannot calculate path configuration without GIL"); } - if ((!calculate->prefix_found || !calculate->exec_prefix_found) - && calculate->warnings) - { - fprintf(stderr, - "Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]\n"); + PyObject *configDict = _PyConfig_AsDict(config); + if (!configDict) { + PyErr_Clear(); + return PyStatus_NoMemory(); } - if (pathconfig->module_search_path == NULL) { - status = calculate_module_search_path(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + PyObject *dict = PyDict_New(); + if (!dict) { + PyErr_Clear(); + Py_DECREF(configDict); + return PyStatus_NoMemory(); } - if (pathconfig->stdlib_dir == NULL) { - /* This must be done *before* calculate_set_prefix() is called. */ - status = calculate_set_stdlib_dir(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + if (PyDict_SetItemString(dict, "config", configDict) < 0) { + PyErr_Clear(); + Py_DECREF(configDict); + Py_DECREF(dict); + return PyStatus_NoMemory(); } + /* reference now held by dict */ + Py_DECREF(configDict); - if (pathconfig->prefix == NULL) { - status = calculate_set_prefix(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } + PyObject *co = _Py_Get_Getpath_CodeObject(); + if (!co || !PyCode_Check(co)) { + PyErr_Clear(); + Py_XDECREF(co); + Py_DECREF(dict); + return PyStatus_Error("error reading frozen getpath.py"); } - if (pathconfig->exec_prefix == NULL) { - status = calculate_set_exec_prefix(calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; +#ifdef MS_WINDOWS + PyObject *winreg = PyImport_ImportModule("winreg"); + if (!winreg || PyDict_SetItemString(dict, "winreg", winreg) < 0) { + PyErr_Clear(); + Py_XDECREF(winreg); + if (PyDict_SetItemString(dict, "winreg", Py_None) < 0) { + PyErr_Clear(); + Py_DECREF(co); + Py_DECREF(dict); + return PyStatus_Error("error importing winreg module"); } + } else { + Py_DECREF(winreg); } - return _PyStatus_OK(); -} - - -/* Calculate the Python path configuration. - - Inputs: - - - PATH environment variable - - Macros: PYTHONPATH, PREFIX, EXEC_PREFIX, VERSION (ex: "3.9"). - PREFIX and EXEC_PREFIX are generated by the configure script. - PYTHONPATH macro is the default search path. - - pybuilddir.txt file - - pyvenv.cfg configuration file - - PyConfig fields ('config' function argument): - - - pathconfig_warnings - - pythonpath_env (PYTHONPATH environment variable) - - - _PyPathConfig fields ('pathconfig' function argument): - - - program_name: see config_init_program_name() - - home: Py_SetPythonHome() or PYTHONHOME environment variable - - - current working directory: see copy_absolute() - - Outputs, 'pathconfig' fields: - - - program_full_path - - module_search_path - - prefix - - exec_prefix - - If a field is already set (non NULL), it is left unchanged. */ -PyStatus -_PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config) -{ - PyStatus status; - PyCalculatePath calculate; - memset(&calculate, 0, sizeof(calculate)); +#endif - status = calculate_init(&calculate, config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } + if ( +#ifdef MS_WINDOWS + !decode_to_dict(dict, "os_name", "nt") || +#elif defined(__APPLE__) + !decode_to_dict(dict, "os_name", "darwin") || +#else + !decode_to_dict(dict, "os_name", "posix") || +#endif + !decode_to_dict(dict, "PREFIX", PREFIX) || + !decode_to_dict(dict, "EXEC_PREFIX", EXEC_PREFIX) || + !decode_to_dict(dict, "PYTHONPATH", PYTHONPATH) || + !decode_to_dict(dict, "VPATH", VPATH) || + !decode_to_dict(dict, "PLATLIBDIR", PLATLIBDIR) || + !decode_to_dict(dict, "PYDEBUGEXT", PYDEBUGEXT) || + !int_to_dict(dict, "VERSION_MAJOR", PY_MAJOR_VERSION) || + !int_to_dict(dict, "VERSION_MINOR", PY_MINOR_VERSION) || + !decode_to_dict(dict, "PYWINVER", PYWINVER) || + !wchar_to_dict(dict, "EXE_SUFFIX", EXE_SUFFIX) || + !env_to_dict(dict, "ENV_PATH", 0) || + !env_to_dict(dict, "ENV_PYTHONHOME", 0) || + !env_to_dict(dict, "ENV_PYTHONEXECUTABLE", 0) || + !env_to_dict(dict, "ENV___PYVENV_LAUNCHER__", 1) || + !progname_to_dict(dict, "real_executable") || + !library_to_dict(dict, "library") || + !wchar_to_dict(dict, "executable_dir", NULL) || + !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) || + !funcs_to_dict(dict, config->pathconfig_warnings) || +#ifndef MS_WINDOWS + PyDict_SetItemString(dict, "winreg", Py_None) < 0 || +#endif + PyDict_SetItemString(dict, "__builtins__", PyEval_GetBuiltins()) < 0 + ) { + Py_DECREF(co); + Py_DECREF(dict); + _PyErr_WriteUnraisableMsg("error evaluating initial values", NULL); + return PyStatus_Error("error evaluating initial values"); + } + + PyObject *r = PyEval_EvalCode(co, dict, dict); + Py_DECREF(co); + + if (!r) { + Py_DECREF(dict); + _PyErr_WriteUnraisableMsg("error evaluating path", NULL); + return PyStatus_Error("error evaluating path"); + } + Py_DECREF(r); + +#if 0 + PyObject *it = PyObject_GetIter(configDict); + for (PyObject *k = PyIter_Next(it); k; k = PyIter_Next(it)) { + if (!strcmp("__builtins__", PyUnicode_AsUTF8(k))) { + Py_DECREF(k); + continue; + } + fprintf(stderr, "%s = ", PyUnicode_AsUTF8(k)); + PyObject *o = PyDict_GetItem(configDict, k); + o = PyObject_Repr(o); + fprintf(stderr, "%s\n", PyUnicode_AsUTF8(o)); + Py_DECREF(o); + Py_DECREF(k); + } + Py_DECREF(it); +#endif - status = calculate_path(&calculate, pathconfig); - if (_PyStatus_EXCEPTION(status)) { - goto done; + if (_PyConfig_FromDict(config, configDict) < 0) { + _PyErr_WriteUnraisableMsg("reading getpath results", NULL); + Py_DECREF(dict); + return PyStatus_Error("error getting getpath results"); } - /* program_full_path must an either an empty string or an absolute path */ - assert(wcslen(pathconfig->program_full_path) == 0 - || _Py_isabs(pathconfig->program_full_path)); - - status = _PyStatus_OK(); + Py_DECREF(dict); -done: - calculate_free(&calculate); - return status; -} - -#ifdef __cplusplus + return _PyStatus_OK(); } -#endif diff --git a/Modules/getpath.py b/Modules/getpath.py new file mode 100644 index 0000000..2eadfba --- /dev/null +++ b/Modules/getpath.py @@ -0,0 +1,727 @@ +# ****************************************************************************** +# getpath.py +# ****************************************************************************** + +# This script is designed to be precompiled to bytecode, frozen into the +# main binary, and then directly evaluated. It is not an importable module, +# and does not import any other modules (besides winreg on Windows). +# Rather, the values listed below must be specified in the globals dict +# used when evaluating the bytecode. + +# See _PyConfig_InitPathConfig in Modules/getpath.c for the execution. + +# ****************************************************************************** +# REQUIRED GLOBALS +# ****************************************************************************** + +# ** Helper functions ** +# abspath(path) -- make relative paths absolute against CWD +# basename(path) -- the filename of path +# dirname(path) -- the directory name of path +# hassuffix(path, suffix) -- returns True if path has suffix +# isabs(path) -- path is absolute or not +# isdir(path) -- path exists and is a directory +# isfile(path) -- path exists and is a file +# isxfile(path) -- path exists and is an executable file +# joinpath(*paths) -- combine the paths +# readlines(path) -- a list of each line of text in the UTF-8 encoded file +# realpath(path) -- resolves symlinks in path +# warn(message) -- print a warning (if enabled) + +# ** Values known at compile time ** +# os_name -- [in] one of 'nt', 'posix', 'darwin' +# PREFIX -- [in] sysconfig.get_config_var(...) +# EXEC_PREFIX -- [in] sysconfig.get_config_var(...) +# PYTHONPATH -- [in] sysconfig.get_config_var(...) +# VPATH -- [in] sysconfig.get_config_var(...) +# PLATLIBDIR -- [in] sysconfig.get_config_var(...) +# PYDEBUGEXT -- [in, opt] '_d' on Windows for debug builds +# EXE_SUFFIX -- [in, opt] '.exe' on Windows/Cygwin/similar +# VERSION_MAJOR -- [in] sys.version_info.major +# VERSION_MINOR -- [in] sys.version_info.minor +# PYWINVER -- [in] the Windows platform-specific version (e.g. 3.8-32) + +# ** Values read from the environment ** +# There is no need to check the use_environment flag before reading +# these, as the flag will be tested in this script. +# Also note that ENV_PYTHONPATH is read from config['pythonpath_env'] +# to allow for embedders who choose to specify it via that struct. +# ENV_PATH -- [in] getenv(...) +# ENV_PYTHONHOME -- [in] getenv(...) +# ENV_PYTHONEXECUTABLE -- [in] getenv(...) +# ENV___PYVENV_LAUNCHER__ -- [in] getenv(...) + +# ** Values calculated at runtime ** +# config -- [in/out] dict of the PyConfig structure +# real_executable -- [in, optional] resolved path to main process +# On Windows and macOS, read directly from the running process +# Otherwise, leave None and it will be calculated from executable +# executable_dir -- [in, optional] real directory containing binary +# If None, will be calculated from real_executable or executable +# py_setpath -- [in] argument provided to Py_SetPath +# If None, 'prefix' and 'exec_prefix' may be updated in config +# library -- [in, optional] path of dylib/DLL/so +# Only used for locating ._pth files +# winreg -- [in, optional] the winreg module (only on Windows) + +# ****************************************************************************** +# HIGH-LEVEL ALGORITHM +# ****************************************************************************** + +# IMPORTANT: The code is the actual specification at time of writing. +# This prose description is based on the original comment from the old +# getpath.c to help capture the intent, but should not be considered +# a specification. + +# Search in some common locations for the associated Python libraries. + +# Two directories must be found, the platform independent directory +# (prefix), containing the common .py and .pyc files, and the platform +# dependent directory (exec_prefix), containing the shared library +# modules. Note that prefix and exec_prefix can be the same directory, +# but for some installations, they are different. + +# This script carries out separate searches for prefix and exec_prefix. +# Each search tries a number of different locations until a ``landmark'' +# file or directory is found. If no prefix or exec_prefix is found, a +# warning message is issued and the preprocessor defined PREFIX and +# EXEC_PREFIX are used (even though they will not work); python carries on +# as best as is possible, but most imports will fail. + +# Before any searches are done, the location of the executable is +# determined. If Py_SetPath() was called, or if we are running on +# Windows, the 'real_executable' path is used (if known). Otherwise, +# we use the config-specified program name or default to argv[0]. +# If this has one or more slashes in it, it is made absolute against +# the current working directory. If it only contains a name, it must +# have been invoked from the shell's path, so we search $PATH for the +# named executable and use that. If the executable was not found on +# $PATH (or there was no $PATH environment variable), the original +# argv[0] string is used. + +# At this point, provided Py_SetPath was not used, the +# __PYVENV_LAUNCHER__ variable may override the executable (on macOS, +# the PYTHON_EXECUTABLE variable may also override). This allows +# certain launchers that run Python as a subprocess to properly +# specify the executable path. They are not intended for users. + +# Next, the executable location is examined to see if it is a symbolic +# link. If so, the link is realpath-ed and the directory of the link +# target is used for the remaining searches. The same steps are +# performed for prefix and for exec_prefix, but with different landmarks. + +# Step 1. Are we running in a virtual environment? Unless 'home' has +# been specified another way, check for a pyvenv.cfg and use its 'home' +# property to override the executable dir used later for prefix searches. +# We do not activate the venv here - that is performed later by site.py. + +# Step 2. Is there a ._pth file? A ._pth file lives adjacent to the +# runtime library (if any) or the actual executable (not the symlink), +# and contains precisely the intended contents of sys.path as relative +# paths (to its own location). Its presence also enables isolated mode +# and suppresses other environment variable usage. Unless already +# specified by Py_SetHome(), the directory containing the ._pth file is +# set as 'home'. + +# Step 3. Are we running python out of the build directory? This is +# checked by looking for the BUILDDIR_TXT file, which contains the +# relative path to the platlib dir. The executable_dir value is +# derived from joining the VPATH preprocessor variable to the +# directory containing pybuilddir.txt. If it is not found, the +# BUILD_LANDMARK file is found, which is part of the source tree. +# prefix is then found by searching up for a file that should only +# exist in the source tree, and the stdlib dir is set to prefix/Lib. + +# Step 4. If 'home' is set, either by Py_SetHome(), ENV_PYTHONHOME, +# a pyvenv.cfg file, ._pth file, or by detecting a build directory, it +# is assumed to point to prefix and exec_prefix. $PYTHONHOME can be a +# single directory, which is used for both, or the prefix and exec_prefix +# directories separated by DELIM (colon on POSIX; semicolon on Windows). + +# Step 5. Try to find prefix and exec_prefix relative to executable_dir, +# backtracking up the path until it is exhausted. This is the most common +# step to succeed. Note that if prefix and exec_prefix are different, +# exec_prefix is more likely to be found; however if exec_prefix is a +# subdirectory of prefix, both will be found. + +# Step 6. Search the directories pointed to by the preprocessor variables +# PREFIX and EXEC_PREFIX. These are supplied by the Makefile but can be +# passed in as options to the configure script. + +# That's it! + +# Well, almost. Once we have determined prefix and exec_prefix, the +# preprocessor variable PYTHONPATH is used to construct a path. Each +# relative path on PYTHONPATH is prefixed with prefix. Then the directory +# containing the shared library modules is appended. The environment +# variable $PYTHONPATH is inserted in front of it all. On POSIX, if we are +# in a build directory, both prefix and exec_prefix are reset to the +# corresponding preprocessor variables (so sys.prefix will reflect the +# installation location, even though sys.path points into the build +# directory). This seems to make more sense given that currently the only +# known use of sys.prefix and sys.exec_prefix is for the ILU installation +# process to find the installed Python tree. + +# An embedding application can use Py_SetPath() to override all of +# these automatic path computations. + + +# ****************************************************************************** +# PLATFORM CONSTANTS +# ****************************************************************************** + +platlibdir = config.get('platlibdir') or PLATLIBDIR + +if os_name == 'posix' or os_name == 'darwin': + BUILDDIR_TXT = 'pybuilddir.txt' + BUILD_LANDMARK = 'Modules/Setup.local' + DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] + VENV_LANDMARK = 'pyvenv.cfg' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + DELIM = ':' + SEP = '/' + +elif os_name == 'nt': + BUILDDIR_TXT = 'pybuilddir.txt' + BUILD_LANDMARK = r'..\..\Modules\Setup.local' + DEFAULT_PROGRAM_NAME = f'python' + STDLIB_SUBDIR = 'Lib' + STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}\\os.py', f'{STDLIB_SUBDIR}\\os.pyc'] + PLATSTDLIB_LANDMARK = f'{platlibdir}' + BUILDSTDLIB_LANDMARKS = ['Lib\\os.py'] + VENV_LANDMARK = 'pyvenv.cfg' + ZIP_LANDMARK = f'python{VERSION_MAJOR}{VERSION_MINOR}{PYDEBUGEXT or ""}.zip' + WINREG_KEY = f'SOFTWARE\\Python\\PythonCore\\{PYWINVER}\\PythonPath' + DELIM = ';' + SEP = '\\' + + +# ****************************************************************************** +# HELPER FUNCTIONS (note that we prefer C functions for performance) +# ****************************************************************************** + +def search_up(prefix, *landmarks, test=isfile): + while prefix: + if any(test(joinpath(prefix, f)) for f in landmarks): + return prefix + prefix = dirname(prefix) + + +# ****************************************************************************** +# READ VARIABLES FROM config +# ****************************************************************************** + +program_name = config.get('program_name') +home = config.get('home') +executable = config.get('executable') +base_executable = config.get('base_executable') +prefix = config.get('prefix') +exec_prefix = config.get('exec_prefix') +base_prefix = config.get('base_prefix') +base_exec_prefix = config.get('base_exec_prefix') +ENV_PYTHONPATH = config['pythonpath_env'] +use_environment = config.get('use_environment', 1) + +pythonpath = config.get('module_search_paths') + +real_executable_dir = None +stdlib_dir = None +platstdlib_dir = None + +# ****************************************************************************** +# CALCULATE program_name +# ****************************************************************************** + +program_name_was_set = bool(program_name) + +if not program_name: + try: + program_name = config.get('orig_argv', [])[0] + except IndexError: + pass + +if not program_name: + program_name = DEFAULT_PROGRAM_NAME + +if EXE_SUFFIX and not hassuffix(program_name, EXE_SUFFIX) and isxfile(program_name + EXE_SUFFIX): + program_name = program_name + EXE_SUFFIX + + +# ****************************************************************************** +# CALCULATE executable +# ****************************************************************************** + +if py_setpath: + # When Py_SetPath has been called, executable defaults to + # the real executable path. + if not executable: + executable = real_executable + +if not executable and SEP in program_name: + # Resolve partial path program_name against current directory + executable = abspath(program_name) + +if not executable: + # All platforms default to real_executable if known at this + # stage. POSIX does not set this value. + executable = real_executable +elif os_name == 'darwin': + # QUIRK: On macOS we may know the real executable path, but + # if our caller has lied to us about it (e.g. most of + # test_embed), we need to use their path in order to detect + # whether we are in a build tree. This is true even if the + # executable path was provided in the config. + real_executable = executable + +if not executable and program_name: + # Resolve names against PATH. + # NOTE: The use_environment value is ignored for this lookup. + # To properly isolate, launch Python with a full path. + for p in ENV_PATH.split(DELIM): + p = joinpath(p, program_name) + if isxfile(p): + executable = p + break + +if not executable: + executable = '' + # When we cannot calculate the executable, subsequent searches + # look in the current working directory. Here, we emulate that + # (the former getpath.c would do it apparently by accident). + executable_dir = abspath('.') + # Also need to set this fallback in case we are running from a + # build directory with an invalid argv0 (i.e. test_sys.test_executable) + real_executable_dir = executable_dir + +if ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__: + # If set, these variables imply that we should be using them as + # sys.executable and when searching for venvs. However, we should + # use the argv0 path for prefix calculation + base_executable = executable + if not real_executable: + real_executable = executable + executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__ + executable_dir = dirname(executable) + + +# ****************************************************************************** +# CALCULATE (default) home +# ****************************************************************************** + +# Used later to distinguish between Py_SetPythonHome and other +# ways that it may have been set +home_was_set = False + +if home: + home_was_set = True +elif use_environment and ENV_PYTHONHOME and not py_setpath: + home = ENV_PYTHONHOME + + +# ****************************************************************************** +# READ pyvenv.cfg +# ****************************************************************************** + +venv_prefix = None + +# Calling Py_SetPythonHome(), Py_SetPath() or +# setting $PYTHONHOME will override venv detection. +if not home and not py_setpath: + try: + # prefix2 is just to avoid calculating dirname again later, + # as the path in venv_prefix is the more common case. + venv_prefix2 = executable_dir or dirname(executable) + venv_prefix = dirname(venv_prefix2) + try: + # Read pyvenv.cfg from one level above executable + pyvenvcfg = readlines(joinpath(venv_prefix, VENV_LANDMARK)) + except FileNotFoundError: + # Try the same directory as executable + pyvenvcfg = readlines(joinpath(venv_prefix2, VENV_LANDMARK)) + venv_prefix = venv_prefix2 + except FileNotFoundError: + venv_prefix = None + pyvenvcfg = [] + + for line in pyvenvcfg: + key, had_equ, value = line.partition('=') + if had_equ and key.strip().lower() == 'home': + executable_dir = real_executable_dir = value.strip() + base_executable = joinpath(executable_dir, basename(executable)) + break + else: + venv_prefix = None + + +# ****************************************************************************** +# CALCULATE base_executable, real_executable AND executable_dir +# ****************************************************************************** + +if not base_executable: + base_executable = executable or real_executable or '' + +if not real_executable: + real_executable = base_executable + +try: + real_executable = realpath(real_executable) +except OSError as ex: + # Only warn if the file actually exists and was unresolvable + # Otherwise users who specify a fake executable may get spurious warnings. + if isfile(real_executable): + warn(f'Failed to find real location of {base_executable}') + +if not executable_dir and os_name == 'darwin' and library: + # QUIRK: macOS checks adjacent to its library early + library_dir = dirname(library) + if any(isfile(joinpath(library_dir, p)) for p in STDLIB_LANDMARKS): + # Exceptions here should abort the whole process (to match + # previous behavior) + executable_dir = realpath(library_dir) + real_executable_dir = executable_dir + +# If we do not have the executable's directory, we can calculate it. +# This is the directory used to find prefix/exec_prefix if necessary. +if not executable_dir: + executable_dir = real_executable_dir = dirname(real_executable) + +# If we do not have the real executable's directory, we calculate it. +# This is the directory used to detect build layouts. +if not real_executable_dir: + real_executable_dir = dirname(real_executable) + +# ****************************************************************************** +# DETECT _pth FILE +# ****************************************************************************** + +# The contents of an optional ._pth file are used to totally override +# sys.path calcualation. Its presence also implies isolated mode and +# no-site (unless explicitly requested) +pth = None +pth_dir = None + +# Calling Py_SetPythonHome() or Py_SetPath() will override ._pth search, +# but environment variables and command-line options cannot. +if not py_setpath and not home_was_set: + # Check adjacent to the main DLL/dylib/so + if library: + try: + pth = readlines(library.rpartition('.')[0] + '._pth') + pth_dir = dirname(library) + except FileNotFoundError: + pass + + # Check adjacent to the original executable, even if we + # redirected to actually launch Python. This may allow a + # venv to override the base_executable's ._pth file, but + # it cannot override the library's one. + if not pth_dir: + try: + pth = readlines(executable.rpartition('.')[0] + '._pth') + pth_dir = dirname(executable) + except FileNotFoundError: + pass + + # If we found a ._pth file, disable environment and home + # detection now. Later, we will do the rest. + if pth_dir: + use_environment = 0 + home = pth_dir + pythonpath = [] + + +# ****************************************************************************** +# CHECK FOR BUILD DIRECTORY +# ****************************************************************************** + +build_prefix = None + +if not home_was_set and real_executable_dir and not py_setpath: + # Detect a build marker and use it to infer prefix, exec_prefix, + # stdlib_dir and the platstdlib_dir directories. + try: + platstdlib_dir = joinpath( + real_executable_dir, + readlines(joinpath(real_executable_dir, BUILDDIR_TXT))[0], + ) + build_prefix = joinpath(real_executable_dir, VPATH) + except FileNotFoundError: + if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)): + build_prefix = joinpath(real_executable_dir, VPATH) + if os_name == 'nt': + # QUIRK: Windows builds need platstdlib_dir to be the executable + # dir. Normally the builddir marker handles this, but in this + # case we need to correct manually. + platstdlib_dir = real_executable_dir + + if build_prefix: + if os_name == 'nt': + # QUIRK: No searching for more landmarks on Windows + build_stdlib_prefix = build_prefix + else: + build_stdlib_prefix = search_up(build_prefix, *BUILDSTDLIB_LANDMARKS) + # Always use the build prefix for stdlib + if build_stdlib_prefix: + stdlib_dir = joinpath(build_stdlib_prefix, 'Lib') + else: + stdlib_dir = joinpath(build_prefix, 'Lib') + # Only use the build prefix for prefix if it hasn't already been set + if not prefix: + prefix = build_stdlib_prefix + # Do not warn, because 'prefix' never equals 'build_prefix' on POSIX + #elif not venv_prefix and prefix != build_prefix: + # warn('Detected development environment but prefix is already set') + if not exec_prefix: + exec_prefix = build_prefix + # Do not warn, because 'exec_prefix' never equals 'build_prefix' on POSIX + #elif not venv_prefix and exec_prefix != build_prefix: + # warn('Detected development environment but exec_prefix is already set') + config['_is_python_build'] = 1 + + +# ****************************************************************************** +# CALCULATE prefix AND exec_prefix +# ****************************************************************************** + +if py_setpath: + # As documented, calling Py_SetPath will force both prefix + # and exec_prefix to the empty string. + prefix = exec_prefix = '' + +else: + # Read prefix and exec_prefix from explicitly set home + if home: + # When multiple paths are listed with ':' or ';' delimiters, + # split into prefix:exec_prefix + prefix, had_delim, exec_prefix = home.partition(DELIM) + if not had_delim: + exec_prefix = prefix + + + # First try to detect prefix by looking alongside our runtime library, if known + if library and not prefix: + library_dir = dirname(library) + if ZIP_LANDMARK: + if os_name == 'nt': + # QUIRK: Windows does not search up for ZIP file + if isfile(joinpath(library_dir, ZIP_LANDMARK)): + prefix = library_dir + else: + prefix = search_up(library_dir, ZIP_LANDMARK) + if STDLIB_SUBDIR and STDLIB_LANDMARKS and not prefix: + if any(isfile(joinpath(library_dir, f)) for f in STDLIB_LANDMARKS): + prefix = library_dir + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + + + # Detect prefix by looking for zip file + if ZIP_LANDMARK and executable_dir and not prefix: + if os_name == 'nt': + # QUIRK: Windows does not search up for ZIP file + if isfile(joinpath(executable_dir, ZIP_LANDMARK)): + prefix = executable_dir + else: + prefix = search_up(executable_dir, ZIP_LANDMARK) + if prefix: + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + if not isdir(stdlib_dir): + stdlib_dir = None + + + # Detect prefix by searching from our executable location for the stdlib_dir + if STDLIB_SUBDIR and STDLIB_LANDMARKS and executable_dir and not prefix: + prefix = search_up(executable_dir, *STDLIB_LANDMARKS) + if prefix: + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + + if PREFIX and not prefix: + prefix = PREFIX + if not any(isfile(joinpath(prefix, f)) for f in STDLIB_LANDMARKS): + warn('Could not find platform independent libraries <prefix>') + + if not prefix: + prefix = abspath('') + warn('Could not find platform independent libraries <prefix>') + + + # Detect exec_prefix by searching from executable for the platstdlib_dir + if PLATSTDLIB_LANDMARK and not exec_prefix: + if executable_dir: + exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir) + if not exec_prefix: + if EXEC_PREFIX: + exec_prefix = EXEC_PREFIX + if not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): + warn('Could not find platform dependent libraries <exec_prefix>') + else: + warn('Could not find platform dependent libraries <exec_prefix>') + + # Fallback: assume exec_prefix == prefix + if not exec_prefix: + exec_prefix = prefix + + + if not prefix or not exec_prefix: + warn('Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]') + + +# If we haven't set [plat]stdlib_dir already, set them now +if not stdlib_dir: + if prefix: + stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) + else: + stdlib_dir = '' + +if not platstdlib_dir: + if exec_prefix: + platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK) + else: + platstdlib_dir = '' + + +# For a venv, update the main prefix/exec_prefix but leave the base ones unchanged +# XXX: We currently do not update prefix here, but it happens in site.py +#if venv_prefix: +# base_prefix = prefix +# base_exec_prefix = exec_prefix +# prefix = exec_prefix = venv_prefix + + +# ****************************************************************************** +# UPDATE pythonpath (sys.path) +# ****************************************************************************** + +if py_setpath: + # If Py_SetPath was called then it overrides any existing search path + config['module_search_paths'] = py_setpath.split(DELIM) + config['module_search_paths_set'] = 1 + +elif not pythonpath: + # If pythonpath was already set, we leave it alone. + # This won't matter in normal use, but if an embedded host is trying to + # recalculate paths while running then we do not want to change it. + pythonpath = [] + + # First add entries from the process environment + if use_environment and ENV_PYTHONPATH: + for p in ENV_PYTHONPATH.split(DELIM): + pythonpath.append(abspath(p)) + + # Then add the default zip file + if os_name == 'nt': + # QUIRK: Windows uses the library directory rather than the prefix + if library: + library_dir = dirname(library) + else: + library_dir = executable_dir + pythonpath.append(joinpath(library_dir, ZIP_LANDMARK)) + elif build_prefix or venv_prefix: + # QUIRK: POSIX uses the default prefix when in the build directory + # or a venv + pythonpath.append(joinpath(PREFIX, ZIP_LANDMARK)) + else: + pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) + + if os_name == 'nt' and use_environment and winreg: + # QUIRK: Windows also lists paths in the registry. Paths are stored + # as the default value of each subkey of + # {HKCU,HKLM}\Software\Python\PythonCore\{winver}\PythonPath + # where winver is sys.winver (typically '3.x' or '3.x-32') + for hk in (winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE): + try: + key = winreg.OpenKeyEx(hk, WINREG_KEY) + try: + i = 0 + while True: + try: + keyname = winreg.EnumKey(key, i) + subkey = winreg.OpenKeyEx(key, keyname) + if not subkey: + continue + try: + v = winreg.QueryValue(subkey) + finally: + winreg.CloseKey(subkey) + if isinstance(v, str): + pythonpath.append(v) + i += 1 + except OSError: + break + finally: + winreg.CloseKey(key) + except OSError: + pass + + # Then add any entries compiled into the PYTHONPATH macro. + if PYTHONPATH: + for p in PYTHONPATH.split(DELIM): + pythonpath.append(joinpath(prefix, p)) + + # Then add stdlib_dir and platstdlib_dir + if stdlib_dir: + pythonpath.append(stdlib_dir) + if platstdlib_dir: + if os_name == 'nt' and venv_prefix: + # QUIRK: Windows appends executable_dir instead of platstdlib_dir + # when in a venv + pythonpath.append(executable_dir) + else: + pythonpath.append(platstdlib_dir) + + config['module_search_paths'] = pythonpath + config['module_search_paths_set'] = 1 + + +# ****************************************************************************** +# POSIX prefix/exec_prefix QUIRKS +# ****************************************************************************** + +# QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running +# in build directory. This happens after pythonpath calculation. +if os_name != 'nt' and build_prefix: + prefix = config.get('prefix') or PREFIX + exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix + + +# ****************************************************************************** +# SET pythonpath FROM _PTH FILE +# ****************************************************************************** + +if pth: + config['isolated'] = 1 + config['use_environment'] = 0 + config['site_import'] = 0 + pythonpath = [] + for line in pth: + line = line.partition('#')[0].strip() + if not line: + pass + elif line == 'import site': + config['site_import'] = 1 + elif line.startswith('import '): + warn("unsupported 'import' line in ._pth file") + else: + pythonpath.append(joinpath(pth_dir, line)) + config['module_search_paths'] = pythonpath + config['module_search_paths_set'] = 1 + +# ****************************************************************************** +# UPDATE config FROM CALCULATED VALUES +# ****************************************************************************** + +config['program_name'] = program_name +config['home'] = home +config['executable'] = executable +config['base_executable'] = base_executable +config['prefix'] = prefix +config['exec_prefix'] = exec_prefix +config['base_prefix'] = base_prefix or prefix +config['base_exec_prefix'] = base_exec_prefix or exec_prefix + +config['platlibdir'] = platlibdir +config['stdlib_dir'] = stdlib_dir +config['platstdlib_dir'] = platstdlib_dir diff --git a/Modules/getpath_noop.c b/Modules/getpath_noop.c new file mode 100644 index 0000000..c10e41d --- /dev/null +++ b/Modules/getpath_noop.c @@ -0,0 +1,10 @@ +/* Implements the getpath API for compiling with no functionality */ + +#include "Python.h" +#include "pycore_pathconfig.h" + +PyStatus +_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) +{ + return PyStatus_Error("path configuration is unsupported"); +} diff --git a/Modules/main.c b/Modules/main.c index c537e6b..b9bcea39 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -534,11 +534,16 @@ pymain_repl(PyConfig *config, int *exitcode) static void pymain_run_python(int *exitcode) { + PyObject *main_importer_path = NULL; PyInterpreterState *interp = _PyInterpreterState_GET(); /* pymain_run_stdin() modify the config */ PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); - PyObject *main_importer_path = NULL; + /* ensure path config is written into global variables */ + if (_PyStatus_EXCEPTION(_PyPathConfig_UpdateGlobal(config))) { + goto error; + } + if (config->run_filename != NULL) { /* If filename is a package (ex: directory or ZIP file) which contains __main__.py, main_importer_path is set to filename and will be diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a89cff7..b1c2914 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4445,6 +4445,33 @@ os__path_splitroot_impl(PyObject *module, path_t *path) /*[clinic input] +os._path_normpath + + path: object + +Basic path normalization. +[clinic start generated code]*/ + +static PyObject * +os__path_normpath_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/ +{ + if (!PyUnicode_Check(path)) { + PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", + Py_TYPE(path)->tp_name); + return NULL; + } + Py_ssize_t len; + wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); + if (!buffer) { + return NULL; + } + PyObject *result = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + PyMem_Free(buffer); + return result; +} + +/*[clinic input] os.mkdir path : path_t @@ -14826,6 +14853,7 @@ static PyMethodDef posix_methods[] = { OS__GETFINALPATHNAME_METHODDEF OS__GETVOLUMEPATHNAME_METHODDEF OS__PATH_SPLITROOT_METHODDEF + OS__PATH_NORMPATH_METHODDEF OS_GETLOADAVG_METHODDEF OS_URANDOM_METHODDEF OS_SETRESUID_METHODDEF diff --git a/PC/config_minimal.c b/PC/config_minimal.c index adb1c44..928a4ef 100644 --- a/PC/config_minimal.c +++ b/PC/config_minimal.c @@ -5,6 +5,10 @@ #include "Python.h" +/* Define extern variables omitted from minimal builds */ +void *PyWin_DLLhModule = NULL; + + extern PyObject* PyInit_faulthandler(void); extern PyObject* PyInit__tracemalloc(void); extern PyObject* PyInit_gc(void); diff --git a/PC/getpathp.c b/PC/getpathp.c deleted file mode 100644 index a273480..0000000 --- a/PC/getpathp.c +++ /dev/null @@ -1,1174 +0,0 @@ - -/* Return the initial module search path. */ -/* Used by DOS, Windows 3.1, Windows 95/98, Windows NT. */ - -/* ---------------------------------------------------------------- - PATH RULES FOR WINDOWS: - This describes how sys.path is formed on Windows. It describes the - functionality, not the implementation (ie, the order in which these - are actually fetched is different). The presence of a python._pth or - pythonXY._pth file alongside the program overrides these rules - see - below. - - * Python always adds an empty entry at the start, which corresponds - to the current directory. - - * If the PYTHONPATH env. var. exists, its entries are added next. - - * We look in the registry for "application paths" - that is, sub-keys - under the main PythonPath registry key. These are added next (the - order of sub-key processing is undefined). - HKEY_CURRENT_USER is searched and added first. - HKEY_LOCAL_MACHINE is searched and added next. - (Note that all known installers only use HKLM, so HKCU is typically - empty) - - * We attempt to locate the "Python Home" - if the PYTHONHOME env var - is set, we believe it. Otherwise, we use the path of our host .EXE's - to try and locate one of our "landmarks" and deduce our home. - - If we DO have a Python Home: The relevant sub-directories (Lib, - DLLs, etc) are based on the Python Home - - If we DO NOT have a Python Home, the core Python Path is - loaded from the registry. (This is the main PythonPath key, - and both HKLM and HKCU are combined to form the path) - - * Iff - we can not locate the Python Home, have not had a PYTHONPATH - specified, and can't locate any Registry entries (ie, we have _nothing_ - we can assume is a good path), a default path with relative entries is - used (eg. .\Lib;.\DLLs, etc) - - - If a '._pth' file exists adjacent to the executable with the same base name - (e.g. python._pth adjacent to python.exe) or adjacent to the shared library - (e.g. python36._pth adjacent to python36.dll), it is used in preference to - the above process. The shared library file takes precedence over the - executable. The path file must contain a list of paths to add to sys.path, - one per line. Each path is relative to the directory containing the file. - Blank lines and comments beginning with '#' are permitted. - - In the presence of this ._pth file, no other paths are added to the search - path, the registry finder is not enabled, site.py is not imported and - isolated mode is enabled. The site package can be enabled by including a - line reading "import site"; no other imports are recognized. Any invalid - entry (other than directories that do not exist) will result in immediate - termination of the program. - - - The end result of all this is: - * When running python.exe, or any other .exe in the main Python directory - (either an installed version, or directly from the PCbuild directory), - the core path is deduced, and the core paths in the registry are - ignored. Other "application paths" in the registry are always read. - - * When Python is hosted in another exe (different directory, embedded via - COM, etc), the Python Home will not be deduced, so the core path from - the registry is used. Other "application paths" in the registry are - always read. - - * If Python can't find its home and there is no registry (eg, frozen - exe, some very strange installation setup) you get a path with - some default, but relative, paths. - - * An embedding application can use Py_SetPath() to override all of - these automatic path computations. - - * An install of Python can fully specify the contents of sys.path using - either a 'EXENAME._pth' or 'DLLNAME._pth' file, optionally including - "import site" to enable the site module. - - ---------------------------------------------------------------- */ - - -#include "Python.h" -#include "pycore_fileutils.h" // _Py_add_relfile() -#include "pycore_initconfig.h" // PyStatus -#include "pycore_pathconfig.h" // _PyPathConfig -#include "osdefs.h" // SEP, ALTSEP -#include <wchar.h> - -#ifndef MS_WINDOWS -#error getpathp.c should only be built on Windows -#endif - -#include <windows.h> -#include <pathcch.h> - -#ifdef HAVE_SYS_TYPES_H -#include <sys/types.h> -#endif /* HAVE_SYS_TYPES_H */ - -#ifdef HAVE_SYS_STAT_H -#include <sys/stat.h> -#endif /* HAVE_SYS_STAT_H */ - -#include <string.h> - -/* Search in some common locations for the associated Python libraries. - * - * Py_GetPath() tries to return a sensible Python module search path. - * - * The approach is an adaptation for Windows of the strategy used in - * ../Modules/getpath.c; it uses the Windows Registry as one of its - * information sources. - * - * Py_SetPath() can be used to override this mechanism. Call Py_SetPath - * with a semicolon separated path prior to calling Py_Initialize. - */ - -#define STDLIB_SUBDIR L"lib" - -#define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow") - - -typedef struct { - const wchar_t *path_env; /* PATH environment variable */ - const wchar_t *home; /* PYTHONHOME environment variable */ - - /* Registry key "Software\Python\PythonCore\X.Y\PythonPath" - where X.Y is the Python version (major.minor) */ - wchar_t *machine_path; /* from HKEY_LOCAL_MACHINE */ - wchar_t *user_path; /* from HKEY_CURRENT_USER */ - - const wchar_t *pythonpath_env; -} PyCalculatePath; - - -/* determine if "ch" is a separator character */ -static int -is_sep(wchar_t ch) -{ -#ifdef ALTSEP - return ch == SEP || ch == ALTSEP; -#else - return ch == SEP; -#endif -} - - -/* assumes 'dir' null terminated in bounds. Never writes - beyond existing terminator. */ -static void -reduce(wchar_t *dir) -{ - size_t i = wcsnlen_s(dir, MAXPATHLEN+1); - if (i >= MAXPATHLEN+1) { - Py_FatalError("buffer overflow in getpathp.c's reduce()"); - } - - while (i > 0 && !is_sep(dir[i])) - --i; - dir[i] = '\0'; -} - - -static int -change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) -{ - if (src && src != dest) { - size_t src_len = wcsnlen_s(src, MAXPATHLEN+1); - size_t i = src_len; - if (i >= MAXPATHLEN+1) { - Py_FatalError("buffer overflow in getpathp.c's reduce()"); - } - - while (i > 0 && src[i] != '.' && !is_sep(src[i])) - --i; - - if (i == 0) { - dest[0] = '\0'; - return -1; - } - - if (is_sep(src[i])) { - i = src_len; - } - - if (wcsncpy_s(dest, MAXPATHLEN+1, src, i)) { - dest[0] = '\0'; - return -1; - } - } else { - wchar_t *s = wcsrchr(dest, L'.'); - if (s) { - s[0] = '\0'; - } - } - - if (wcscat_s(dest, MAXPATHLEN+1, ext)) { - dest[0] = '\0'; - return -1; - } - - return 0; -} - - -static int -exists(const wchar_t *filename) -{ - return GetFileAttributesW(filename) != 0xFFFFFFFF; -} - - -/* Is module -- check for .pyc too. - Assumes 'filename' MAXPATHLEN+1 bytes long - - may extend 'filename' by one character. */ -static int -ismodule(wchar_t *filename) -{ - size_t n; - - if (exists(filename)) { - return 1; - } - - /* Check for the compiled version of prefix. */ - n = wcsnlen_s(filename, MAXPATHLEN+1); - if (n < MAXPATHLEN) { - int exist = 0; - filename[n] = L'c'; - filename[n + 1] = L'\0'; - exist = exists(filename); - // Drop the 'c' we just added. - filename[n] = L'\0'; - return exist; - } - return 0; -} - - -/* Add a path component, by appending stuff to buffer. - buffer must have at least MAXPATHLEN + 1 bytes allocated, and contain a - NUL-terminated string with no more than MAXPATHLEN characters (not counting - the trailing NUL). It's a fatal error if it contains a string longer than - that (callers must be careful!). If these requirements are met, it's - guaranteed that buffer will still be a NUL-terminated string with no more - than MAXPATHLEN characters at exit. If stuff is too long, only as much of - stuff as fits will be appended. -*/ - -static void -join(wchar_t *buffer, const wchar_t *stuff) -{ - if (_Py_add_relfile(buffer, stuff, MAXPATHLEN+1) < 0) { - Py_FatalError("buffer overflow in getpathp.c's join()"); - } -} - -/* Call PathCchCanonicalizeEx(path): remove navigation elements such as "." - and ".." to produce a direct, well-formed path. */ -static PyStatus -canonicalize(wchar_t *buffer, const wchar_t *path) -{ - if (buffer == NULL) { - return _PyStatus_NO_MEMORY(); - } - - const wchar_t *pathTail; - if (FAILED(PathCchSkipRoot(path, &pathTail)) || path == pathTail) { - wchar_t buff[MAXPATHLEN + 1]; - if (!GetCurrentDirectoryW(MAXPATHLEN, buff)) { - return _PyStatus_ERR("unable to find current working directory"); - } - if (FAILED(PathCchCombineEx(buff, MAXPATHLEN + 1, buff, path, PATHCCH_ALLOW_LONG_PATHS))) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - if (FAILED(PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, buff, PATHCCH_ALLOW_LONG_PATHS))) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - return _PyStatus_OK(); - } - - if (FAILED(PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, path, PATHCCH_ALLOW_LONG_PATHS))) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - return _PyStatus_OK(); -} - -static int -is_stdlibdir(wchar_t *stdlibdir) -{ - wchar_t *filename = stdlibdir; -#ifndef LANDMARK -# define LANDMARK L"os.py" -#endif - /* join() ensures 'landmark' can not overflow prefix if too long. */ - join(filename, LANDMARK); - return ismodule(filename); -} - -/* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd. - assumption provided by only caller, calculate_path() */ -static int -search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path) -{ - /* Search from argv0_path, until LANDMARK is found. - We guarantee 'prefix' is null terminated in bounds. */ - wcscpy_s(prefix, MAXPATHLEN+1, argv0_path); - if (!prefix[0]) { - return 0; - } - wchar_t stdlibdir[MAXPATHLEN+1]; - wcscpy_s(stdlibdir, Py_ARRAY_LENGTH(stdlibdir), prefix); - /* We initialize with the longest possible path, in case it doesn't fit. - This also gives us an initial SEP at stdlibdir[wcslen(prefix)]. */ - join(stdlibdir, STDLIB_SUBDIR); - do { - assert(stdlibdir[wcslen(prefix)] == SEP); - /* Due to reduce() and our initial value, this result - is guaranteed to fit. */ - wcscpy(&stdlibdir[wcslen(prefix) + 1], STDLIB_SUBDIR); - if (is_stdlibdir(stdlibdir)) { - return 1; - } - reduce(prefix); - } while (prefix[0]); - return 0; -} - - -static int -get_dllpath(wchar_t *dllpath) -{ -#ifdef Py_ENABLE_SHARED - extern HANDLE PyWin_DLLhModule; - if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, dllpath, MAXPATHLEN)) { - return 0; - } -#endif - return -1; -} - - -#ifdef Py_ENABLE_SHARED - -/* a string loaded from the DLL at startup.*/ -extern const char *PyWin_DLLVersionString; - -/* Load a PYTHONPATH value from the registry. - Load from either HKEY_LOCAL_MACHINE or HKEY_CURRENT_USER. - - Works in both Unicode and 8bit environments. Only uses the - Ex family of functions so it also works with Windows CE. - - Returns NULL, or a pointer that should be freed. - - XXX - this code is pretty strange, as it used to also - work on Win16, where the buffer sizes were not available - in advance. It could be simplied now Win16/Win32s is dead! -*/ -static wchar_t * -getpythonregpath(HKEY keyBase, int skipcore) -{ - HKEY newKey = 0; - DWORD dataSize = 0; - DWORD numKeys = 0; - LONG rc; - wchar_t *retval = NULL; - WCHAR *dataBuf = NULL; - static const WCHAR keyPrefix[] = L"Software\\Python\\PythonCore\\"; - static const WCHAR keySuffix[] = L"\\PythonPath"; - size_t versionLen, keyBufLen; - DWORD index; - WCHAR *keyBuf = NULL; - WCHAR *keyBufPtr; - WCHAR **ppPaths = NULL; - - /* Tried to use sysget("winver") but here is too early :-( */ - versionLen = strlen(PyWin_DLLVersionString); - /* Space for all the chars, plus one \0 */ - keyBufLen = sizeof(keyPrefix) + - sizeof(WCHAR)*(versionLen-1) + - sizeof(keySuffix); - keyBuf = keyBufPtr = PyMem_RawMalloc(keyBufLen); - if (keyBuf==NULL) { - goto done; - } - - memcpy_s(keyBufPtr, keyBufLen, keyPrefix, sizeof(keyPrefix)-sizeof(WCHAR)); - keyBufPtr += Py_ARRAY_LENGTH(keyPrefix) - 1; - mbstowcs(keyBufPtr, PyWin_DLLVersionString, versionLen); - keyBufPtr += versionLen; - /* NULL comes with this one! */ - memcpy(keyBufPtr, keySuffix, sizeof(keySuffix)); - /* Open the root Python key */ - rc=RegOpenKeyExW(keyBase, - keyBuf, /* subkey */ - 0, /* reserved */ - KEY_READ, - &newKey); - if (rc!=ERROR_SUCCESS) { - goto done; - } - /* Find out how big our core buffer is, and how many subkeys we have */ - rc = RegQueryInfoKeyW(newKey, NULL, NULL, NULL, &numKeys, NULL, NULL, - NULL, NULL, &dataSize, NULL, NULL); - if (rc!=ERROR_SUCCESS) { - goto done; - } - if (skipcore) { - dataSize = 0; /* Only count core ones if we want them! */ - } - /* Allocate a temp array of char buffers, so we only need to loop - reading the registry once - */ - ppPaths = PyMem_RawCalloc(numKeys, sizeof(WCHAR *)); - if (ppPaths==NULL) { - goto done; - } - /* Loop over all subkeys, allocating a temp sub-buffer. */ - for(index=0;index<numKeys;index++) { - WCHAR keyBuf[MAX_PATH+1]; - HKEY subKey = 0; - DWORD reqdSize = MAX_PATH+1; - /* Get the sub-key name */ - DWORD rc = RegEnumKeyExW(newKey, index, keyBuf, &reqdSize, - NULL, NULL, NULL, NULL ); - if (rc!=ERROR_SUCCESS) { - goto done; - } - /* Open the sub-key */ - rc=RegOpenKeyExW(newKey, - keyBuf, /* subkey */ - 0, /* reserved */ - KEY_READ, - &subKey); - if (rc!=ERROR_SUCCESS) { - goto done; - } - /* Find the value of the buffer size, malloc, then read it */ - RegQueryValueExW(subKey, NULL, 0, NULL, NULL, &reqdSize); - if (reqdSize) { - ppPaths[index] = PyMem_RawMalloc(reqdSize); - if (ppPaths[index]) { - RegQueryValueExW(subKey, NULL, 0, NULL, - (LPBYTE)ppPaths[index], - &reqdSize); - dataSize += reqdSize + 1; /* 1 for the ";" */ - } - } - RegCloseKey(subKey); - } - - /* return null if no path to return */ - if (dataSize == 0) { - goto done; - } - - /* original datasize from RegQueryInfo doesn't include the \0 */ - dataBuf = PyMem_RawMalloc((dataSize+1) * sizeof(WCHAR)); - if (dataBuf) { - WCHAR *szCur = dataBuf; - /* Copy our collected strings */ - for (index=0;index<numKeys;index++) { - if (index > 0) { - *(szCur++) = L';'; - dataSize--; - } - if (ppPaths[index]) { - Py_ssize_t len = wcslen(ppPaths[index]); - wcsncpy(szCur, ppPaths[index], len); - szCur += len; - assert(dataSize > (DWORD)len); - dataSize -= (DWORD)len; - } - } - if (skipcore) { - *szCur = '\0'; - } - else { - /* If we have no values, we don't need a ';' */ - if (numKeys) { - *(szCur++) = L';'; - dataSize--; - } - /* Now append the core path entries - - this will include the NULL - */ - rc = RegQueryValueExW(newKey, NULL, 0, NULL, - (LPBYTE)szCur, &dataSize); - if (rc != ERROR_SUCCESS) { - PyMem_RawFree(dataBuf); - goto done; - } - } - /* And set the result - caller must free */ - retval = dataBuf; - } -done: - /* Loop freeing my temp buffers */ - if (ppPaths) { - for(index=0; index<numKeys; index++) - PyMem_RawFree(ppPaths[index]); - PyMem_RawFree(ppPaths); - } - if (newKey) { - RegCloseKey(newKey); - } - PyMem_RawFree(keyBuf); - return retval; -} -#endif /* Py_ENABLE_SHARED */ - - -static PyStatus -get_program_full_path(_PyPathConfig *pathconfig) -{ - PyStatus status; - const wchar_t *pyvenv_launcher; - wchar_t program_full_path[MAXPATHLEN+1]; - memset(program_full_path, 0, sizeof(program_full_path)); - - if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) { - /* GetModuleFileName should never fail when passed NULL */ - return _PyStatus_ERR("Cannot determine program path"); - } - - /* The launcher may need to force the executable path to a - * different environment, so override it here. */ - pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__"); - if (pyvenv_launcher && pyvenv_launcher[0]) { - /* If overridden, preserve the original full path */ - if (pathconfig->base_executable == NULL) { - pathconfig->base_executable = PyMem_RawMalloc( - sizeof(wchar_t) * (MAXPATHLEN + 1)); - if (pathconfig->base_executable == NULL) { - return _PyStatus_NO_MEMORY(); - } - - status = canonicalize(pathconfig->base_executable, - program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher); - /* bpo-35873: Clear the environment variable to avoid it being - * inherited by child processes. */ - _wputenv_s(L"__PYVENV_LAUNCHER__", L""); - } - - if (pathconfig->program_full_path == NULL) { - pathconfig->program_full_path = PyMem_RawMalloc( - sizeof(wchar_t) * (MAXPATHLEN + 1)); - if (pathconfig->program_full_path == NULL) { - return _PyStatus_NO_MEMORY(); - } - - status = canonicalize(pathconfig->program_full_path, - program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - return _PyStatus_OK(); -} - - -static PyStatus -read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path, - int *found) -{ - PyStatus status; - wchar_t *buf = NULL; - wchar_t *wline = NULL; - FILE *sp_file; - - sp_file = _Py_wfopen(path, L"r"); - if (sp_file == NULL) { - return _PyStatus_OK(); - } - - wcscpy_s(prefix, MAXPATHLEN+1, path); - reduce(prefix); - pathconfig->isolated = 1; - pathconfig->site_import = 0; - - size_t bufsiz = MAXPATHLEN; - size_t prefixlen = wcslen(prefix); - - buf = (wchar_t*)PyMem_RawMalloc(bufsiz * sizeof(wchar_t)); - if (buf == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - buf[0] = '\0'; - - while (!feof(sp_file)) { - char line[MAXPATHLEN + 1]; - char *p = fgets(line, Py_ARRAY_LENGTH(line), sp_file); - if (!p) { - break; - } - if (*p == '\0' || *p == '\r' || *p == '\n' || *p == '#') { - continue; - } - while (*++p) { - if (*p == '\r' || *p == '\n') { - *p = '\0'; - break; - } - } - - if (strcmp(line, "import site") == 0) { - pathconfig->site_import = 1; - continue; - } - else if (strncmp(line, "import ", 7) == 0) { - status = _PyStatus_ERR("only 'import site' is supported " - "in ._pth file"); - goto done; - } - - DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0); - wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t)); - if (wline == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1); - wline[wn] = '\0'; - - size_t usedsiz = wcslen(buf); - while (usedsiz + wn + prefixlen + 4 > bufsiz) { - bufsiz += MAXPATHLEN; - wchar_t *tmp = (wchar_t*)PyMem_RawRealloc(buf, (bufsiz + 1) * - sizeof(wchar_t)); - if (tmp == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - buf = tmp; - } - - if (usedsiz) { - wcscat_s(buf, bufsiz, L";"); - usedsiz += 1; - } - - errno_t result; - _Py_BEGIN_SUPPRESS_IPH - result = wcscat_s(buf, bufsiz, prefix); - _Py_END_SUPPRESS_IPH - - if (result == EINVAL) { - status = _PyStatus_ERR("invalid argument during ._pth processing"); - goto done; - } else if (result == ERANGE) { - status = _PyStatus_ERR("buffer overflow during ._pth processing"); - goto done; - } - - wchar_t *b = &buf[usedsiz]; - join(b, wline); - - PyMem_RawFree(wline); - wline = NULL; - } - - if (pathconfig->module_search_path == NULL) { - pathconfig->module_search_path = _PyMem_RawWcsdup(buf); - if (pathconfig->module_search_path == NULL) { - status = _PyStatus_NO_MEMORY(); - goto done; - } - } - - *found = 1; - status = _PyStatus_OK(); - goto done; - -done: - PyMem_RawFree(buf); - PyMem_RawFree(wline); - fclose(sp_file); - return status; -} - - -static int -get_pth_filename(PyCalculatePath *calculate, wchar_t *filename, - const _PyPathConfig *pathconfig) -{ - if (!get_dllpath(filename) && - !change_ext(filename, filename, L"._pth") && - exists(filename)) - { - return 1; - } - if (pathconfig->program_full_path[0] && - !change_ext(filename, pathconfig->program_full_path, L"._pth") && - exists(filename)) - { - return 1; - } - return 0; -} - - -static PyStatus -calculate_pth_file(PyCalculatePath *calculate, _PyPathConfig *pathconfig, - wchar_t *prefix, int *found) -{ - wchar_t filename[MAXPATHLEN+1]; - - if (!get_pth_filename(calculate, filename, pathconfig)) { - return _PyStatus_OK(); - } - - return read_pth_file(pathconfig, prefix, filename, found); -} - - -/* Search for an environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. -*/ -static PyStatus -calculate_pyvenv_file(PyCalculatePath *calculate, - wchar_t *argv0_path, size_t argv0_path_len) -{ - wchar_t filename[MAXPATHLEN+1]; - const wchar_t *env_cfg = L"pyvenv.cfg"; - - /* Filename: <argv0_path_len> / "pyvenv.cfg" */ - wcscpy_s(filename, MAXPATHLEN+1, argv0_path); - join(filename, env_cfg); - - FILE *env_file = _Py_wfopen(filename, L"r"); - if (env_file == NULL) { - errno = 0; - - /* Filename: <basename(basename(argv0_path_len))> / "pyvenv.cfg" */ - reduce(filename); - reduce(filename); - join(filename, env_cfg); - - env_file = _Py_wfopen(filename, L"r"); - if (env_file == NULL) { - errno = 0; - return _PyStatus_OK(); - } - } - - /* Look for a 'home' variable and set argv0_path to it, if found */ - wchar_t *home = NULL; - PyStatus status = _Py_FindEnvConfigValue(env_file, L"home", &home); - if (_PyStatus_EXCEPTION(status)) { - fclose(env_file); - return status; - } - if (home) { - wcscpy_s(argv0_path, argv0_path_len, home); - PyMem_RawFree(home); - } - fclose(env_file); - return _PyStatus_OK(); -} - - -static void -calculate_home_prefix(PyCalculatePath *calculate, - const wchar_t *argv0_path, - const wchar_t *zip_path, - wchar_t *prefix) -{ - if (calculate->home == NULL || *calculate->home == '\0') { - if (zip_path[0] && exists(zip_path)) { - wcscpy_s(prefix, MAXPATHLEN+1, zip_path); - reduce(prefix); - calculate->home = prefix; - } - else if (search_for_prefix(prefix, argv0_path)) { - calculate->home = prefix; - } - else { - calculate->home = NULL; - } - } - else { - wcscpy_s(prefix, MAXPATHLEN+1, calculate->home); - } -} - - -static PyStatus -calculate_module_search_path(PyCalculatePath *calculate, - _PyPathConfig *pathconfig, - const wchar_t *argv0_path, - wchar_t *prefix, - const wchar_t *zip_path) -{ - int skiphome = calculate->home==NULL ? 0 : 1; -#ifdef Py_ENABLE_SHARED - if (!Py_IgnoreEnvironmentFlag) { - calculate->machine_path = getpythonregpath(HKEY_LOCAL_MACHINE, - skiphome); - calculate->user_path = getpythonregpath(HKEY_CURRENT_USER, skiphome); - } -#endif - /* We only use the default relative PYTHONPATH if we haven't - anything better to use! */ - int skipdefault = (calculate->pythonpath_env != NULL || - calculate->home != NULL || - calculate->machine_path != NULL || - calculate->user_path != NULL); - - /* We need to construct a path from the following parts. - (1) the PYTHONPATH environment variable, if set; - (2) for Win32, the zip archive file path; - (3) for Win32, the machine_path and user_path, if set; - (4) the PYTHONPATH config macro, with the leading "." - of each component replaced with home, if set; - (5) the directory containing the executable (argv0_path). - The length calculation calculates #4 first. - Extra rules: - - If PYTHONHOME is set (in any way) item (3) is ignored. - - If registry values are used, (4) and (5) are ignored. - */ - - /* Calculate size of return buffer */ - size_t bufsz = 0; - if (calculate->home != NULL) { - const wchar_t *p; - bufsz = 1; - for (p = PYTHONPATH; *p; p++) { - if (*p == DELIM) { - bufsz++; /* number of DELIM plus one */ - } - } - bufsz *= wcslen(calculate->home); - } - bufsz += wcslen(PYTHONPATH) + 1; - bufsz += wcslen(argv0_path) + 1; - if (calculate->user_path) { - bufsz += wcslen(calculate->user_path) + 1; - } - if (calculate->machine_path) { - bufsz += wcslen(calculate->machine_path) + 1; - } - bufsz += wcslen(zip_path) + 1; - if (calculate->pythonpath_env != NULL) { - bufsz += wcslen(calculate->pythonpath_env) + 1; - } - - wchar_t *buf, *start_buf; - buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); - if (buf == NULL) { - return _PyStatus_NO_MEMORY(); - } - start_buf = buf; - - if (calculate->pythonpath_env) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), - calculate->pythonpath_env)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - if (zip_path[0]) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), zip_path)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - if (calculate->user_path) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->user_path)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - if (calculate->machine_path) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->machine_path)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - if (calculate->home == NULL) { - if (!skipdefault) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), PYTHONPATH)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - } else { - const wchar_t *p = PYTHONPATH; - const wchar_t *q; - size_t n; - for (;;) { - q = wcschr(p, DELIM); - if (q == NULL) { - n = wcslen(p); - } - else { - n = q-p; - } - if (p[0] == '.' && is_sep(p[1])) { - if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->home)) { - return INIT_ERR_BUFFER_OVERFLOW(); - } - buf = wcschr(buf, L'\0'); - p++; - n--; - } - wcsncpy(buf, p, n); - buf += n; - *buf++ = DELIM; - if (q == NULL) { - break; - } - p = q+1; - } - } - if (argv0_path) { - wcscpy(buf, argv0_path); - buf = wcschr(buf, L'\0'); - *buf++ = DELIM; - } - *(buf - 1) = L'\0'; - - /* Now to pull one last hack/trick. If sys.prefix is - empty, then try and find it somewhere on the paths - we calculated. We scan backwards, as our general policy - is that Python core directories are at the *end* of - sys.path. We assume that our "lib" directory is - on the path, and that our 'prefix' directory is - the parent of that. - */ - if (prefix[0] == L'\0') { - PyStatus status; - wchar_t lookBuf[MAXPATHLEN+1]; - const wchar_t *look = buf - 1; /* 'buf' is at the end of the buffer */ - while (1) { - Py_ssize_t nchars; - const wchar_t *lookEnd = look; - /* 'look' will end up one character before the - start of the path in question - even if this - is one character before the start of the buffer - */ - while (look >= start_buf && *look != DELIM) - look--; - nchars = lookEnd-look; - wcsncpy(lookBuf, look+1, nchars); - lookBuf[nchars] = L'\0'; - status = canonicalize(lookBuf, lookBuf); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - /* Up one level to the parent */ - reduce(lookBuf); - if (search_for_prefix(prefix, lookBuf)) { - break; - } - /* If we are out of paths to search - give up */ - if (look < start_buf) { - break; - } - look--; - } - } - - pathconfig->module_search_path = start_buf; - return _PyStatus_OK(); -} - - -static PyStatus -calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig) -{ - PyStatus status; - - status = get_program_full_path(pathconfig); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - /* program_full_path guaranteed \0 terminated in MAXPATH+1 bytes. */ - wchar_t argv0_path[MAXPATHLEN+1]; - memset(argv0_path, 0, sizeof(argv0_path)); - - wcscpy_s(argv0_path, MAXPATHLEN+1, pathconfig->program_full_path); - reduce(argv0_path); - - wchar_t prefix[MAXPATHLEN+1]; - memset(prefix, 0, sizeof(prefix)); - - /* Search for a sys.path file */ - int pth_found = 0; - status = calculate_pth_file(calculate, pathconfig, prefix, &pth_found); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - if (pth_found) { - goto done; - } - - status = calculate_pyvenv_file(calculate, - argv0_path, Py_ARRAY_LENGTH(argv0_path)); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - /* Calculate zip archive path from DLL or exe path */ - wchar_t zip_path[MAXPATHLEN+1]; - memset(zip_path, 0, sizeof(zip_path)); - - if (get_dllpath(zip_path) || change_ext(zip_path, zip_path, L".zip")) - { - if (change_ext(zip_path, pathconfig->program_full_path, L".zip")) { - zip_path[0] = L'\0'; - } - } - - calculate_home_prefix(calculate, argv0_path, zip_path, prefix); - - if (pathconfig->module_search_path == NULL) { - status = calculate_module_search_path(calculate, pathconfig, - argv0_path, prefix, zip_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - -done: - if (pathconfig->stdlib_dir == NULL) { - pathconfig->stdlib_dir = _Py_join_relfile(prefix, STDLIB_SUBDIR); - if (pathconfig->stdlib_dir == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - if (pathconfig->prefix == NULL) { - pathconfig->prefix = _PyMem_RawWcsdup(prefix); - if (pathconfig->prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - if (pathconfig->exec_prefix == NULL) { - pathconfig->exec_prefix = _PyMem_RawWcsdup(prefix); - if (pathconfig->exec_prefix == NULL) { - return _PyStatus_NO_MEMORY(); - } - } - - return _PyStatus_OK(); -} - - -static PyStatus -calculate_init(PyCalculatePath *calculate, _PyPathConfig *pathconfig, - const PyConfig *config) -{ - calculate->home = pathconfig->home; - calculate->path_env = _wgetenv(L"PATH"); - - calculate->pythonpath_env = config->pythonpath_env; - - return _PyStatus_OK(); -} - - -static void -calculate_free(PyCalculatePath *calculate) -{ - PyMem_RawFree(calculate->machine_path); - PyMem_RawFree(calculate->user_path); -} - - -/* Calculate the Python path configuration. - - Inputs: - - - PyConfig.pythonpath_env: PYTHONPATH environment variable - - _PyPathConfig.home: Py_SetPythonHome() or PYTHONHOME environment variable - - PATH environment variable - - __PYVENV_LAUNCHER__ environment variable - - GetModuleFileNameW(NULL): fully qualified path of the executable file of - the current process - - ._pth configuration file - - pyvenv.cfg configuration file - - Registry key "Software\Python\PythonCore\X.Y\PythonPath" - of HKEY_CURRENT_USER and HKEY_LOCAL_MACHINE where X.Y is the Python - version. - - Outputs, 'pathconfig' fields: - - - base_executable - - program_full_path - - module_search_path - - prefix - - exec_prefix - - isolated - - site_import - - If a field is already set (non NULL), it is left unchanged. */ -PyStatus -_PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config) -{ - PyStatus status; - PyCalculatePath calculate; - memset(&calculate, 0, sizeof(calculate)); - - status = calculate_init(&calculate, pathconfig, config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - status = calculate_path(&calculate, pathconfig); - -done: - calculate_free(&calculate); - return status; -} - - -/* Load python3.dll before loading any extension module that might refer - to it. That way, we can be sure that always the python3.dll corresponding - to this python DLL is loaded, not a python3.dll that might be on the path - by chance. - Return whether the DLL was found. -*/ -static int python3_checked = 0; -static HANDLE hPython3; -int -_Py_CheckPython3(void) -{ - wchar_t py3path[MAXPATHLEN+1]; - if (python3_checked) { - return hPython3 != NULL; - } - python3_checked = 1; - - /* If there is a python3.dll next to the python3y.dll, - use that DLL */ - if (!get_dllpath(py3path)) { - reduce(py3path); - join(py3path, PY3_DLLNAME); - hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); - if (hPython3 != NULL) { - return 1; - } - } - - /* If we can locate python3.dll in our application dir, - use that DLL */ - hPython3 = LoadLibraryExW(PY3_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR); - if (hPython3 != NULL) { - return 1; - } - - /* For back-compat, also search {sys.prefix}\DLLs, though - that has not been a normal install layout for a while */ - wcscpy(py3path, Py_GetPrefix()); - if (py3path[0]) { - join(py3path, L"DLLs\\" PY3_DLLNAME); - hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); - } - return hPython3 != NULL; -} diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 5d8d9f3..e0d875a 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -66,9 +66,6 @@ WIN32 is still required for the locale module. #define MS_WIN32 /* only support win32 and greater. */ #define MS_WINDOWS -#ifndef PYTHONPATH -# define PYTHONPATH L".\\DLLs;.\\lib" -#endif #define NT_THREADS #define WITH_THREAD #ifndef NETSCAPE_PI @@ -671,6 +668,4 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */ #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1 -#define PLATLIBDIR "lib" - #endif /* !Py_CONFIG_H */ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 54fef9c..7b2df4b 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -104,6 +104,7 @@ <ClCompile Include="..\Modules\faulthandler.c" /> <ClCompile Include="..\Modules\gcmodule.c" /> <ClCompile Include="..\Modules\getbuildinfo.c" /> + <ClCompile Include="..\Modules\getpath_noop.c" /> <ClCompile Include="..\Modules\posixmodule.c" /> <ClCompile Include="..\Modules\signalmodule.c" /> <ClCompile Include="..\Modules\_tracemalloc.c" /> @@ -168,7 +169,6 @@ <ClCompile Include="..\Parser\string_parser.c" /> <ClCompile Include="..\Parser\token.c" /> <ClCompile Include="..\Parser\tokenizer.c" /> - <ClCompile Include="..\PC\getpathp.c" /> <ClCompile Include="..\PC\invalid_parameter_handler.c" /> <ClCompile Include="..\PC\msvcrtmodule.c" /> <ClCompile Include="..\PC\winreg.c" /> @@ -374,9 +374,29 @@ </None> <!-- END frozen modules --> </ItemGroup> + <ItemGroup> + <!-- We manually freeze getpath.py rather than through freeze_modules --> + <GetPath Include="..\Modules\getpath.py"> + <ModName>getpath</ModName> + <IntFile>$(IntDir)getpath.g.h</IntFile> + <OutFile>$(PySourcePath)Modules\getpath.h</OutFile> + </GetPath> + </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> + <Target Name="_RebuildGetPath" AfterTargets="_RebuildFrozen" Condition="$(Configuration) != 'PGUpdate'"> + <Exec Command='"$(TargetPath)" "%(GetPath.ModName)" "%(GetPath.FullPath)" "%(GetPath.IntFile)"' /> + + <Copy SourceFiles="%(GetPath.IntFile)" + DestinationFiles="%(GetPath.OutFile)" + Condition="!Exists(%(GetPath.OutFile)) or (Exists(%(GetPath.IntFile)) and '$([System.IO.File]::ReadAllText(%(GetPath.OutFile)).Replace(`
`, `
`))' != '$([System.IO.File]::ReadAllText(%(GetPath.IntFile)).Replace(`
`, `
`))')"> + <Output TaskParameter="CopiedFiles" ItemName="_UpdatedGetPath" /> + </Copy> + + <Message Text="Updated files: @(_UpdatedGetPath->'%(Filename)%(Extension)',', ')" + Condition="'@(_UpdatedGetPath)' != ''" Importance="high" /> + </Target> <Target Name="_RebuildFrozen" AfterTargets="AfterBuild" Condition="$(Configuration) != 'PGUpdate'"> <Exec Command='"$(TargetPath)" "%(None.ModName)" "%(None.FullPath)" "%(None.DeepIntFile)"' /> diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 5894909..1c8f1b0 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -13,6 +13,395 @@ <ClCompile Include="..\Programs\_freeze_module.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\_io\_iomodule.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_tracemalloc.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\_warnings.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\abstract.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\accu.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\asdl.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\ast.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\ast_opt.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\ast_unparse.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\atexitmodule.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\bltinmodule.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\boolobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\bootstrap_hash.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_io\bufferedio.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\bytearrayobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\bytes_methods.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_io\bytesio.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\bytesobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\call.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\capsule.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\cellobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\ceval.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\classobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\codecs.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\codeobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\compile.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\complexobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\PC\config_minimal.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\context.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\descrobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\dictobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\dtoa.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\dynamic_annotations.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\dynload_win.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\enumobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\errors.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\exceptions.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\faulthandler.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_io\fileio.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\fileobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\fileutils.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\floatobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\formatter_unicode.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\frame.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\frameobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\funcobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\future.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\gcmodule.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\genericaliasobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\genobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\getargs.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\getbuildinfo.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\getcompiler.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\getcopyright.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\getopt.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\getpath_noop.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\getplatform.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\getversion.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\hamt.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\hashtable.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\import.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\importdl.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\initconfig.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\interpreteridobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\PC\invalid_parameter_handler.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_io\iobase.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\iterobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\listobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\longobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\marshal.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\memoryobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\methodobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\modsupport.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\moduleobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\PC\msvcrtmodule.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Parser\myreadline.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\mysnprintf.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\mystrtoul.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\namespaceobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\object.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\obmalloc.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\odictobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Parser\parser.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pathconfig.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Parser\peg_api.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Parser\pegen.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\picklebufobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\posixmodule.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\preconfig.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pyarena.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pyctype.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pyfpe.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pyhash.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pylifecycle.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pymath.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pystate.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pystrcmp.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pystrhex.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pystrtod.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\Python-ast.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pythonrun.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\Python-tokenize.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\pytime.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\rangeobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\setobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\signalmodule.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\sliceobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\specialize.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Parser\string_parser.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_io\stringio.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\structmember.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\structseq.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\suggestions.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\symtable.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\sysmodule.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_io\textio.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\thread.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Parser\token.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Parser\tokenizer.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\traceback.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\tupleobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\typeobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\unicodectype.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\unicodeobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\unionobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Objects\weakrefobject.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Modules\_io\winconsoleio.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\PC\winreg.c"> + <Filter>Source Files</Filter> + </ClCompile> + </ItemGroup> + <ItemGroup> + <None Include="..\Modules\getpath.py"> + <Filter>Python Files</Filter> + </None> </ItemGroup> <ItemGroup> <!-- BEGIN frozen modules --> @@ -78,4 +467,4 @@ </None> <!-- END frozen modules --> </ItemGroup> -</Project> +</Project>
\ No newline at end of file diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index b58945a..0b4329d 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -147,4 +147,11 @@ $(_PGOPath) </PropertyGroup> <WriteLinesToFile File="$(PySourcePath)python.bat" Lines="$(_Content)" Overwrite="true" Condition="'$(_Content)' != '$(_ExistingContent)'" /> </Target> + <Target Name="GeneratePyBuildDirTxt" BeforeTargets="AfterBuild"> + <PropertyGroup> + <_Content>$(OutDir)</_Content> + <_ExistingContent Condition="Exists('$(OutDir)pybuilddir.txt')">$([System.IO.File]::ReadAllText('$(OutDir)pybuilddir.txt'))</_ExistingContent> + </PropertyGroup> + <WriteLinesToFile File="$(OutDir)pybuilddir.txt" Lines="$(_Content)" Overwrite="true" Condition="'$(_Content)' != '$(_ExistingContent)'" /> + </Target> </Project> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 9d96f4b..b446e09 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -110,6 +110,19 @@ </Link> </ItemDefinitionGroup> <ItemGroup> + <ClCompile Include="..\Modules\getpath.c"> + <PreprocessorDefinitions> + PREFIX=NULL; + EXEC_PREFIX=NULL; + VERSION=NULL; + VPATH="..\\.."; + PYDEBUGEXT="$(PyDebugExt)"; + PLATLIBDIR="DLLs"; + %(PreprocessorDefinitions) + </PreprocessorDefinitions> + </ClCompile> + </ItemGroup> + <ItemGroup> <ClInclude Include="..\Include\Python.h" /> <ClInclude Include="..\Include\abstract.h" /> <ClInclude Include="..\Include\boolobject.h" /> @@ -349,6 +362,7 @@ <ClCompile Include="..\Modules\errnomodule.c" /> <ClCompile Include="..\Modules\faulthandler.c" /> <ClCompile Include="..\Modules\gcmodule.c" /> + <ClCompile Include="..\Modules\getbuildinfo.c" /> <ClCompile Include="..\Modules\itertoolsmodule.c" /> <ClCompile Include="..\Modules\main.c" /> <ClCompile Include="..\Modules\mathmodule.c" /> @@ -442,7 +456,6 @@ <ClCompile Include="..\PC\invalid_parameter_handler.c" /> <ClCompile Include="..\PC\winreg.c" /> <ClCompile Include="..\PC\config.c" /> - <ClCompile Include="..\PC\getpathp.c" /> <ClCompile Include="..\PC\msvcrtmodule.c" /> <ClCompile Include="..\Python\pyhash.c" /> <ClCompile Include="..\Python\_warnings.c" /> @@ -570,7 +583,7 @@ </PropertyGroup> <Message Text="Building $(GitTag):$(GitVersion) $(GitBranch)" Importance="high" /> <ItemGroup> - <ClCompile Include="..\Modules\getbuildinfo.c"> + <ClCompile Condition="$(Filename) == 'getbuildinfo'"> <PreprocessorDefinitions>GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions)</PreprocessorDefinitions> </ClCompile> </ItemGroup> @@ -590,4 +603,8 @@ <Target Name="_CleanVCRuntime" AfterTargets="Clean"> <Delete Files="@(VCRuntimeDLL->'$(OutDir)%(Filename)%(Extension)')" /> </Target> + + <Target Name="_DeletePyBuildDirTxt" BeforeTargets="PrepareForBuild"> + <Delete Files="$(OutDir)pybuilddir.txt" /> + </Target> </Project> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index b19f027..c1667e3 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -648,6 +648,9 @@ <ClInclude Include="$(zlibDir)\zutil.h"> <Filter>Modules\zlib</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_structseq.h"> + <Filter>Include\internal</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\_abc.c"> @@ -971,9 +974,6 @@ <ClCompile Include="..\PC\dl_nt.c"> <Filter>PC</Filter> </ClCompile> - <ClCompile Include="..\PC\getpathp.c"> - <Filter>PC</Filter> - </ClCompile> <ClCompile Include="..\PC\msvcrtmodule.c"> <Filter>PC</Filter> </ClCompile> @@ -1229,10 +1229,25 @@ <ClCompile Include="..\Objects\unionobject.c"> <Filter>Objects</Filter> </ClCompile> + <ClCompile Include="..\Python\frame.c"> + <Filter>Python</Filter> + </ClCompile> + <ClCompile Include="..\Modules\getpath.c"> + <Filter>Modules</Filter> + </ClCompile> + <ClCompile Include="..\Python\suggestions.c"> + <Filter>Python</Filter> + </ClCompile> + <ClCompile Include="..\Python\Python-tokenize.c"> + <Filter>Python</Filter> + </ClCompile> + <ClCompile Include="..\Modules\getbuildinfo.c"> + <Filter>Modules</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc"> <Filter>Resource Files</Filter> </ResourceCompile> </ItemGroup> -</Project> +</Project>
\ No newline at end of file diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 5702ab2..854b1e6 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -2,6 +2,9 @@ /* Support for dynamic loading of extension modules */ #include "Python.h" +#include "pycore_fileutils.h" // _Py_add_relfile() +#include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() +#include "pycore_pystate.h" // _PyInterpreterState_GET() #ifdef HAVE_DIRECT_H #include <direct.h> @@ -160,6 +163,60 @@ static char *GetPythonImport (HINSTANCE hModule) return NULL; } +/* Load python3.dll before loading any extension module that might refer + to it. That way, we can be sure that always the python3.dll corresponding + to this python DLL is loaded, not a python3.dll that might be on the path + by chance. + Return whether the DLL was found. +*/ +extern HMODULE PyWin_DLLhModule; +static int +_Py_CheckPython3(void) +{ + static int python3_checked = 0; + static HANDLE hPython3; + #define MAXPATHLEN 512 + wchar_t py3path[MAXPATHLEN+1]; + if (python3_checked) { + return hPython3 != NULL; + } + python3_checked = 1; + + /* If there is a python3.dll next to the python3y.dll, + use that DLL */ + if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, py3path, MAXPATHLEN)) { + wchar_t *p = wcsrchr(py3path, L'\\'); + if (p) { + wcscpy(p + 1, PY3_DLLNAME); + hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + if (hPython3 != NULL) { + return 1; + } + } + } + + /* If we can locate python3.dll in our application dir, + use that DLL */ + hPython3 = LoadLibraryExW(PY3_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR); + if (hPython3 != NULL) { + return 1; + } + + /* For back-compat, also search {sys.prefix}\DLLs, though + that has not been a normal install layout for a while */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); + assert(config->prefix); + if (config->prefix) { + wcscpy_s(py3path, MAXPATHLEN, config->prefix); + if (py3path[0] && _Py_add_relfile(py3path, L"DLLs\\" PY3_DLLNAME, MAXPATHLEN) >= 0) { + hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + } + } + return hPython3 != NULL; + #undef MAXPATHLEN +} + dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, const char *shortname, PyObject *pathname, FILE *fp) diff --git a/Python/fileutils.c b/Python/fileutils.c index ac0046c..cae6b75 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2000,13 +2000,28 @@ _Py_wrealpath(const wchar_t *path, #endif -#ifndef MS_WINDOWS int _Py_isabs(const wchar_t *path) { +#ifdef MS_WINDOWS + const wchar_t *tail; + HRESULT hr = PathCchSkipRoot(path, &tail); + if (FAILED(hr) || path == tail) { + return 0; + } + if (tail == &path[1] && (path[0] == SEP || path[0] == ALTSEP)) { + // Exclude paths with leading SEP + return 0; + } + if (tail == &path[2] && path[1] == L':') { + // Exclude drive-relative paths (e.g. C:filename.ext) + return 0; + } + return 1; +#else return (path[0] == SEP); -} #endif +} /* Get an absolute path. @@ -2017,6 +2032,22 @@ _Py_isabs(const wchar_t *path) int _Py_abspath(const wchar_t *path, wchar_t **abspath_p) { + if (path[0] == '\0' || !wcscmp(path, L".")) { + wchar_t cwd[MAXPATHLEN + 1]; + cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; + if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) { + /* unable to get the current directory */ + return -1; + } + *abspath_p = _PyMem_RawWcsdup(cwd); + return 0; + } + + if (_Py_isabs(path)) { + *abspath_p = _PyMem_RawWcsdup(path); + return 0; + } + #ifdef MS_WINDOWS wchar_t woutbuf[MAX_PATH], *woutbufp = woutbuf; DWORD result; @@ -2028,7 +2059,7 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) return -1; } - if (result > Py_ARRAY_LENGTH(woutbuf)) { + if (result >= Py_ARRAY_LENGTH(woutbuf)) { if ((size_t)result <= (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { woutbufp = PyMem_RawMalloc((size_t)result * sizeof(wchar_t)); } @@ -2055,11 +2086,6 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) *abspath_p = _PyMem_RawWcsdup(woutbufp); return 0; #else - if (_Py_isabs(path)) { - *abspath_p = _PyMem_RawWcsdup(path); - return 0; - } - wchar_t cwd[MAXPATHLEN + 1]; cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) { @@ -2102,7 +2128,8 @@ join_relfile(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, const wchar_t *relfile) { #ifdef MS_WINDOWS - if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, 0))) { + if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, + PATHCCH_ALLOW_LONG_PATHS | PATHCCH_FORCE_ENABLE_LONG_NAME_PROCESS))) { return -1; } #else @@ -2180,99 +2207,125 @@ _Py_find_basename(const wchar_t *filename) return 0; } - -/* Remove navigation elements such as "." and "..". - - This is mostly a C implementation of posixpath.normpath(). - Return 0 on success. Return -1 if "orig" is too big for the buffer. */ -int -_Py_normalize_path(const wchar_t *path, wchar_t *buf, const size_t buf_len) +/* In-place path normalisation. Returns the start of the normalized + path, which will be within the original buffer. Guaranteed to not + make the path longer, and will not fail. 'size' is the length of + the path, if known. If -1, the first null character will be assumed + to be the end of the path. */ +wchar_t * +_Py_normpath(wchar_t *path, Py_ssize_t size) { - assert(path && *path != L'\0'); - assert(*path == SEP); // an absolute path - if (wcslen(path) + 1 >= buf_len) { - return -1; + if (!path[0] || size == 0) { + return path; } + wchar_t lastC = L'\0'; + wchar_t *p1 = path; + wchar_t *pEnd = size >= 0 ? &path[size] : NULL; + wchar_t *p2 = path; + wchar_t *minP2 = path; - int dots = -1; - int check_leading = 1; - const wchar_t *buf_start = buf; - wchar_t *buf_next = buf; - // The resulting filename will never be longer than path. - for (const wchar_t *remainder = path; *remainder != L'\0'; remainder++) { - wchar_t c = *remainder; - buf_next[0] = c; - buf_next++; - if (c == SEP) { - assert(dots <= 2); - if (dots == 2) { - // Turn "/x/y/../z" into "/x/z". - buf_next -= 4; // "/../" - assert(*buf_next == SEP); - // We cap it off at the root, so "/../spam" becomes "/spam". - if (buf_next == buf_start) { - buf_next++; - } - else { - // Move to the previous SEP in the buffer. - while (*(buf_next - 1) != SEP) { - assert(buf_next != buf_start); - buf_next--; - } - } - } - else if (dots == 1) { - // Turn "/./" into "/". - buf_next -= 2; // "./" - assert(*(buf_next - 1) == SEP); - } - else if (dots == 0) { - // Turn "//" into "/". - buf_next--; - assert(*(buf_next - 1) == SEP); - if (check_leading) { - if (buf_next - 1 == buf && *(remainder + 1) != SEP) { - // Leave a leading "//" alone, unless "///...". - buf_next++; - buf_start++; - } - check_leading = 0; - } - } - dots = 0; +#define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) +#ifdef ALTSEP +#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP) +#else +#define IS_SEP(x) (*(x) == SEP) +#endif +#define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) + + // Skip leading '.\' + if (p1[0] == L'.' && IS_SEP(&p1[1])) { + path = &path[2]; + while (IS_SEP(path) && !IS_END(path)) { + path++; } - else { - check_leading = 0; - if (dots >= 0) { - if (c == L'.' && dots < 2) { - dots++; - } - else { - dots = -1; - } + p1 = p2 = minP2 = path; + lastC = SEP; + } +#ifdef MS_WINDOWS + // Skip past drive segment and update minP2 + else if (p1[0] && p1[1] == L':') { + *p2++ = *p1++; + *p2++ = *p1++; + minP2 = p2; + lastC = L':'; + } + // Skip past all \\-prefixed paths, including \\?\, \\.\, + // and network paths, including the first segment. + else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) { + int sepCount = 2; + *p2++ = SEP; + *p2++ = SEP; + p1 += 2; + for (; !IS_END(p1) && sepCount; ++p1) { + if (IS_SEP(p1)) { + --sepCount; + *p2++ = lastC = SEP; + } else { + *p2++ = lastC = *p1; } } + minP2 = p2; } - if (dots >= 0) { - // Strip any trailing dots and trailing slash. - buf_next -= dots + 1; // "/" or "/." or "/.." - assert(*buf_next == SEP); - if (buf_next == buf_start) { - // Leave the leading slash for root. - buf_next++; +#else + // Skip past two leading SEPs + else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1]) && !IS_SEP(&p1[2])) { + *p2++ = *p1++; + *p2++ = *p1++; + minP2 = p2; + lastC = SEP; + } +#endif /* MS_WINDOWS */ + + /* if pEnd is specified, check that. Else, check for null terminator */ + for (; !IS_END(p1); ++p1) { + wchar_t c = *p1; +#ifdef ALTSEP + if (c == ALTSEP) { + c = SEP; } - else { - if (dots == 2) { - // Move to the previous SEP in the buffer. - do { - assert(buf_next != buf_start); - buf_next--; - } while (*(buf_next) != SEP); +#endif + if (lastC == SEP) { + if (c == L'.') { + int sep_at_1 = SEP_OR_END(&p1[1]); + int sep_at_2 = !sep_at_1 && SEP_OR_END(&p1[2]); + if (sep_at_2 && p1[1] == L'.') { + wchar_t *p3 = p2; + while (p3 != minP2 && *--p3 == SEP) { } + while (p3 != minP2 && *(p3 - 1) != SEP) { --p3; } + if (p3[0] == L'.' && p3[1] == L'.' && IS_SEP(&p3[2])) { + // Previous segment is also ../, so append instead + *p2++ = L'.'; + *p2++ = L'.'; + lastC = L'.'; + } else if (p3[0] == SEP) { + // Absolute path, so absorb segment + p2 = p3 + 1; + } else { + p2 = p3; + } + p1 += 1; + } else if (sep_at_1) { + } else { + *p2++ = lastC = c; + } + } else if (c == SEP) { + } else { + *p2++ = lastC = c; } + } else { + *p2++ = lastC = c; + } + } + *p2 = L'\0'; + if (p2 != minP2) { + while (--p2 != minP2 && *p2 == SEP) { + *p2 = L'\0'; } } - *buf_next = L'\0'; - return 0; +#undef SEP_OR_END +#undef IS_SEP +#undef IS_END + return path; } diff --git a/Python/initconfig.c b/Python/initconfig.c index 53b624f..47ebc64 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -22,11 +22,6 @@ # endif #endif -#ifndef PLATLIBDIR -# error "PLATLIBDIR macro must be defined" -#endif - - /* --- Command line options --------------------------------------- */ /* Short usage message (with %s for argv0) */ @@ -632,7 +627,6 @@ config_check_consistency(const PyConfig *config) assert(config->parse_argv >= 0); assert(config->configure_c_stdio >= 0); assert(config->buffered_stdio >= 0); - assert(config->program_name != NULL); assert(_PyWideStringList_CheckConsistency(&config->orig_argv)); assert(_PyWideStringList_CheckConsistency(&config->argv)); /* sys.argv must be non-empty: empty argv is replaced with [''] */ @@ -641,7 +635,6 @@ config_check_consistency(const PyConfig *config) assert(_PyWideStringList_CheckConsistency(&config->warnoptions)); assert(_PyWideStringList_CheckConsistency(&config->module_search_paths)); assert(config->module_search_paths_set >= 0); - assert(config->platlibdir != NULL); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); assert(config->stdio_encoding != NULL); @@ -740,6 +733,7 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->legacy_windows_stdio = -1; #endif config->use_frozen_modules = -1; + config->_is_python_build = 0; config->code_debug_ranges = 1; } @@ -962,6 +956,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_ATTR(_isolated_interpreter); COPY_ATTR(use_frozen_modules); COPY_WSTRLIST(orig_argv); + COPY_ATTR(_is_python_build); #undef COPY_ATTR #undef COPY_WSTR_ATTR @@ -1066,6 +1061,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_INT(_isolated_interpreter); SET_ITEM_WSTRLIST(orig_argv); SET_ITEM_INT(use_frozen_modules); + SET_ITEM_INT(_is_python_build); return dict; @@ -1350,6 +1346,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_UINT(_init_main); GET_UINT(_isolated_interpreter); GET_UINT(use_frozen_modules); + GET_UINT(_is_python_build); #undef CHECK_VALUE #undef GET_UINT @@ -1489,117 +1486,6 @@ config_set_global_vars(const PyConfig *config) } -/* Get the program name: use PYTHONEXECUTABLE and __PYVENV_LAUNCHER__ - environment variables on macOS if available. */ -static PyStatus -config_init_program_name(PyConfig *config) -{ - PyStatus status; - - /* If Py_SetProgramName() was called, use its value */ - const wchar_t *program_name = _Py_path_config.program_name; - if (program_name != NULL) { - config->program_name = _PyMem_RawWcsdup(program_name); - if (config->program_name == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); - } - -#ifdef __APPLE__ - /* On MacOS X, when the Python interpreter is embedded in an - application bundle, it gets executed by a bootstrapping script - that does os.execve() with an argv[0] that's different from the - actual Python executable. This is needed to keep the Finder happy, - or rather, to work around Apple's overly strict requirements of - the process name. However, we still need a usable sys.executable, - so the actual executable path is passed in an environment variable. - See Lib/plat-mac/bundlebuilder.py for details about the bootstrap - script. */ - const char *p = config_get_env(config, "PYTHONEXECUTABLE"); - if (p != NULL) { - status = CONFIG_SET_BYTES_STR(config, &config->program_name, p, - "PYTHONEXECUTABLE environment variable"); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); - } -#ifdef WITH_NEXT_FRAMEWORK - else { - const char* pyvenv_launcher = getenv("__PYVENV_LAUNCHER__"); - if (pyvenv_launcher && *pyvenv_launcher) { - /* Used by Mac/Tools/pythonw.c to forward - * the argv0 of the stub executable - */ - status = CONFIG_SET_BYTES_STR(config, - &config->program_name, - pyvenv_launcher, - "__PYVENV_LAUNCHER__ environment variable"); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - /* - * This environment variable is used to communicate between - * the stub launcher and the real interpreter and isn't needed - * beyond this point. - * - * Clean up to avoid problems when launching other programs - * later on. - */ - (void)unsetenv("__PYVENV_LAUNCHER__"); - - return _PyStatus_OK(); - } - } -#endif /* WITH_NEXT_FRAMEWORK */ -#endif /* __APPLE__ */ - - /* Use argv[0] if available and non-empty */ - const PyWideStringList *argv = &config->argv; - if (argv->length >= 1 && argv->items[0][0] != L'\0') { - config->program_name = _PyMem_RawWcsdup(argv->items[0]); - if (config->program_name == NULL) { - return _PyStatus_NO_MEMORY(); - } - return _PyStatus_OK(); - } - - /* Last fall back: hardcoded name */ -#ifdef MS_WINDOWS - const wchar_t *default_program_name = L"python"; -#else - const wchar_t *default_program_name = L"python3"; -#endif - status = PyConfig_SetString(config, &config->program_name, - default_program_name); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); -} - -static PyStatus -config_init_executable(PyConfig *config) -{ - assert(config->executable == NULL); - - /* If Py_SetProgramFullPath() was called, use its value */ - const wchar_t *program_full_path = _Py_path_config.program_full_path; - if (program_full_path != NULL) { - PyStatus status = PyConfig_SetString(config, - &config->executable, - program_full_path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); - } - return _PyStatus_OK(); -} - - static const wchar_t* config_get_xoption(const PyConfig *config, wchar_t *name) { @@ -1619,25 +1505,6 @@ config_get_xoption_value(const PyConfig *config, wchar_t *name) static PyStatus -config_init_home(PyConfig *config) -{ - assert(config->home == NULL); - - /* If Py_SetPythonHome() was called, use its value */ - wchar_t *home = _Py_path_config.home; - if (home) { - PyStatus status = PyConfig_SetString(config, &config->home, home); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - return _PyStatus_OK(); - } - - return CONFIG_GET_ENV_DUP(config, &config->home, - L"PYTHONHOME", "PYTHONHOME"); -} - -static PyStatus config_init_hash_seed(PyConfig *config) { const char *seed_text = config_get_env(config, "PYTHONHASHSEED"); @@ -2092,44 +1959,6 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) } -/* Determine if the current build is a "development" build (e.g. running - out of the source tree) or not. - - A return value of -1 indicates that we do not know. - */ -static int -is_dev_env(PyConfig *config) -{ - // This should only ever get called early in runtime initialization, - // before the global path config is written. Otherwise we would - // use Py_GetProgramFullPath() and _Py_GetStdlibDir(). - assert(config != NULL); - - const wchar_t *executable = config->executable; - const wchar_t *stdlib = config->stdlib_dir; - if (executable == NULL || *executable == L'\0' || - stdlib == NULL || *stdlib == L'\0') { - // _PyPathConfig_Calculate() hasn't run yet. - return -1; - } - size_t len = _Py_find_basename(executable); - if (wcscmp(executable + len, L"python") != 0 && - wcscmp(executable + len, L"python.exe") != 0) { - return 0; - } - /* If dirname() is the same for both then it is a dev build. */ - if (len != _Py_find_basename(stdlib)) { - return 0; - } - // We do not bother normalizing the two filenames first since - // for config_init_import() is does the right thing as-is. - if (wcsncmp(stdlib, executable, len) != 0) { - return 0; - } - return 1; -} - - static PyStatus config_init_import(PyConfig *config, int compute_path_config) { @@ -2144,10 +1973,7 @@ config_init_import(PyConfig *config, int compute_path_config) if (config->use_frozen_modules < 0) { const wchar_t *value = config_get_xoption_value(config, L"frozen_modules"); if (value == NULL) { - int isdev = is_dev_env(config); - if (isdev >= 0) { - config->use_frozen_modules = !isdev; - } + config->use_frozen_modules = !config->_is_python_build; } else if (wcscmp(value, L"on") == 0) { config->use_frozen_modules = 1; @@ -2246,28 +2072,6 @@ config_read(PyConfig *config, int compute_path_config) return status; } - if (config->home == NULL) { - status = config_init_home(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - if (config->executable == NULL) { - status = config_init_executable(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - - if(config->platlibdir == NULL) { - status = CONFIG_SET_BYTES_STR(config, &config->platlibdir, PLATLIBDIR, - "PLATLIBDIR macro"); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - if (config->_install_importlib) { status = config_init_import(config, compute_path_config); if (_PyStatus_EXCEPTION(status)) { @@ -2890,13 +2694,6 @@ config_read_cmdline(PyConfig *config) config->parse_argv = 1; } - if (config->program_name == NULL) { - status = config_init_program_name(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - if (config->parse_argv == 1) { Py_ssize_t opt_index; status = config_parse_cmdline(config, &cmdline_warnoptions, &opt_index); @@ -3076,7 +2873,7 @@ done: PyStatus PyConfig_Read(PyConfig *config) { - return _PyConfig_Read(config, 1); + return _PyConfig_Read(config, 0); } @@ -3124,16 +2921,6 @@ _Py_GetConfigsAsDict(void) } Py_CLEAR(dict); - /* path config */ - dict = _PyPathConfig_AsDict(); - if (dict == NULL) { - goto error; - } - if (PyDict_SetItemString(result, "path_config", dict) < 0) { - goto error; - } - Py_CLEAR(dict); - return result; error: @@ -3199,6 +2986,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) PySys_WriteStderr(" environment = %i\n", config->use_environment); PySys_WriteStderr(" user site = %i\n", config->user_site_directory); PySys_WriteStderr(" import site = %i\n", config->site_import); + PySys_WriteStderr(" is in build tree = %i\n", config->_is_python_build); DUMP_CONFIG("stdlib dir", stdlib_dir); #undef DUMP_CONFIG diff --git a/Python/pathconfig.c b/Python/pathconfig.c index ad22222..4271928 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -1,6 +1,7 @@ /* Path configuration like module_search_path (sys.path) */ #include "Python.h" +#include "marshal.h" // PyMarshal_ReadObjectFromString #include "osdefs.h" // DELIM #include "pycore_initconfig.h" #include "pycore_fileutils.h" @@ -9,6 +10,8 @@ #include <wchar.h> #ifdef MS_WINDOWS # include <windows.h> // GetFullPathNameW(), MAX_PATH +# include <pathcch.h> +# include <shlwapi.h> #endif #ifdef __cplusplus @@ -16,462 +19,170 @@ extern "C" { #endif +/* External interface */ + +/* Stored values set by C API functions */ +typedef struct _PyPathConfig { + /* Full path to the Python program */ + wchar_t *program_full_path; + wchar_t *prefix; + wchar_t *exec_prefix; + wchar_t *stdlib_dir; + /* Set by Py_SetPath */ + wchar_t *module_search_path; + /* Set by _PyPathConfig_UpdateGlobal */ + wchar_t *calculated_module_search_path; + /* Python program name */ + wchar_t *program_name; + /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ + wchar_t *home; +} _PyPathConfig; + +# define _PyPathConfig_INIT \ + {.module_search_path = NULL} + + _PyPathConfig _Py_path_config = _PyPathConfig_INIT; -static int -copy_wstr(wchar_t **dst, const wchar_t *src) +const wchar_t * +_PyPathConfig_GetGlobalModuleSearchPath(void) { - assert(*dst == NULL); - if (src != NULL) { - *dst = _PyMem_RawWcsdup(src); - if (*dst == NULL) { - return -1; - } - } - else { - *dst = NULL; - } - return 0; + return _Py_path_config.module_search_path; } -static void -pathconfig_clear(_PyPathConfig *config) +void +_PyPathConfig_ClearGlobal(void) { - /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator, - since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be - called before Py_Initialize() which can changes the memory allocator. */ PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); #define CLEAR(ATTR) \ do { \ - PyMem_RawFree(ATTR); \ - ATTR = NULL; \ + PyMem_RawFree(_Py_path_config.ATTR); \ + _Py_path_config.ATTR = NULL; \ } while (0) - CLEAR(config->program_full_path); - CLEAR(config->prefix); - CLEAR(config->exec_prefix); - CLEAR(config->stdlib_dir); - CLEAR(config->module_search_path); - CLEAR(config->program_name); - CLEAR(config->home); -#ifdef MS_WINDOWS - CLEAR(config->base_executable); -#endif + CLEAR(program_full_path); + CLEAR(prefix); + CLEAR(exec_prefix); + CLEAR(stdlib_dir); + CLEAR(module_search_path); + CLEAR(calculated_module_search_path); + CLEAR(program_name); + CLEAR(home); #undef CLEAR PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } - -static PyStatus -pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2) +PyStatus +_PyPathConfig_ReadGlobal(PyConfig *config) { - pathconfig_clear(config); + PyStatus status = _PyStatus_OK(); -#define COPY_ATTR(ATTR) \ +#define COPY(ATTR) \ do { \ - if (copy_wstr(&config->ATTR, config2->ATTR) < 0) { \ - return _PyStatus_NO_MEMORY(); \ + if (_Py_path_config.ATTR && !config->ATTR) { \ + status = PyConfig_SetString(config, &config->ATTR, _Py_path_config.ATTR); \ + if (_PyStatus_EXCEPTION(status)) goto done; \ } \ } while (0) - COPY_ATTR(program_full_path); - COPY_ATTR(prefix); - COPY_ATTR(exec_prefix); - COPY_ATTR(module_search_path); - COPY_ATTR(stdlib_dir); - COPY_ATTR(program_name); - COPY_ATTR(home); -#ifdef MS_WINDOWS - config->isolated = config2->isolated; - config->site_import = config2->site_import; - COPY_ATTR(base_executable); -#endif - -#undef COPY_ATTR - - return _PyStatus_OK(); -} - - -void -_PyPathConfig_ClearGlobal(void) -{ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - pathconfig_clear(&_Py_path_config); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - -static wchar_t* -_PyWideStringList_Join(const PyWideStringList *list, wchar_t sep) -{ - size_t len = 1; /* NUL terminator */ - for (Py_ssize_t i=0; i < list->length; i++) { - if (i != 0) { - len++; - } - len += wcslen(list->items[i]); - } - - wchar_t *text = PyMem_RawMalloc(len * sizeof(wchar_t)); - if (text == NULL) { - return NULL; - } - wchar_t *str = text; - for (Py_ssize_t i=0; i < list->length; i++) { - wchar_t *path = list->items[i]; - if (i != 0) { - *str++ = sep; - } - len = wcslen(path); - memcpy(str, path, len * sizeof(wchar_t)); - str += len; - } - *str = L'\0'; - - return text; -} - - -static PyStatus -pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config) -{ - PyStatus status; - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - if (config->module_search_paths_set) { - PyMem_RawFree(pathconfig->module_search_path); - pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM); - if (pathconfig->module_search_path == NULL) { - goto no_memory; - } - } - -#define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \ - if (config->CONFIG_ATTR) { \ - PyMem_RawFree(pathconfig->PATH_ATTR); \ - pathconfig->PATH_ATTR = NULL; \ - if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \ - goto no_memory; \ - } \ - } - - COPY_CONFIG(program_full_path, executable); - COPY_CONFIG(prefix, prefix); - COPY_CONFIG(exec_prefix, exec_prefix); - COPY_CONFIG(stdlib_dir, stdlib_dir); - COPY_CONFIG(program_name, program_name); - COPY_CONFIG(home, home); -#ifdef MS_WINDOWS - COPY_CONFIG(base_executable, base_executable); -#endif - -#undef COPY_CONFIG - - status = _PyStatus_OK(); - goto done; +#define COPY2(ATTR, SRCATTR) \ + do { \ + if (_Py_path_config.SRCATTR && !config->ATTR) { \ + status = PyConfig_SetString(config, &config->ATTR, _Py_path_config.SRCATTR); \ + if (_PyStatus_EXCEPTION(status)) goto done; \ + } \ + } while (0) -no_memory: - status = _PyStatus_NO_MEMORY(); + COPY(prefix); + COPY(exec_prefix); + COPY(stdlib_dir); + COPY(program_name); + COPY(home); + COPY2(executable, program_full_path); + // module_search_path must be initialised - not read +#undef COPY +#undef COPY2 done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return status; } -PyObject * -_PyPathConfig_AsDict(void) -{ - PyObject *dict = PyDict_New(); - if (dict == NULL) { - return NULL; - } - -#define SET_ITEM(KEY, EXPR) \ - do { \ - PyObject *obj = (EXPR); \ - if (obj == NULL) { \ - goto fail; \ - } \ - int res = PyDict_SetItemString(dict, KEY, obj); \ - Py_DECREF(obj); \ - if (res < 0) { \ - goto fail; \ - } \ - } while (0) -#define SET_ITEM_STR(KEY) \ - SET_ITEM(#KEY, \ - (_Py_path_config.KEY \ - ? PyUnicode_FromWideChar(_Py_path_config.KEY, -1) \ - : (Py_INCREF(Py_None), Py_None))) -#define SET_ITEM_INT(KEY) \ - SET_ITEM(#KEY, PyLong_FromLong(_Py_path_config.KEY)) - - SET_ITEM_STR(program_full_path); - SET_ITEM_STR(prefix); - SET_ITEM_STR(exec_prefix); - SET_ITEM_STR(module_search_path); - SET_ITEM_STR(stdlib_dir); - SET_ITEM_STR(program_name); - SET_ITEM_STR(home); -#ifdef MS_WINDOWS - SET_ITEM_INT(isolated); - SET_ITEM_INT(site_import); - SET_ITEM_STR(base_executable); - - { - wchar_t py3path[MAX_PATH]; - HMODULE hPython3 = GetModuleHandleW(PY3_DLLNAME); - PyObject *obj; - if (hPython3 - && GetModuleFileNameW(hPython3, py3path, Py_ARRAY_LENGTH(py3path))) - { - obj = PyUnicode_FromWideChar(py3path, -1); - if (obj == NULL) { - goto fail; - } - } - else { - obj = Py_None; - Py_INCREF(obj); - } - if (PyDict_SetItemString(dict, "python3_dll", obj) < 0) { - Py_DECREF(obj); - goto fail; - } - Py_DECREF(obj); - } -#endif - -#undef SET_ITEM -#undef SET_ITEM_STR -#undef SET_ITEM_INT - - return dict; - -fail: - Py_DECREF(dict); - return NULL; -} - - PyStatus -_PyConfig_WritePathConfig(const PyConfig *config) +_PyPathConfig_UpdateGlobal(const PyConfig *config) { - return pathconfig_set_from_config(&_Py_path_config, config); -} - - -static PyStatus -config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig) -{ - assert(!config->module_search_paths_set); - - _PyWideStringList_Clear(&config->module_search_paths); - - const wchar_t *sys_path = pathconfig->module_search_path; - const wchar_t delim = DELIM; - while (1) { - const wchar_t *p = wcschr(sys_path, delim); - if (p == NULL) { - p = sys_path + wcslen(sys_path); /* End of string */ - } - - size_t path_len = (p - sys_path); - wchar_t *path = PyMem_RawMalloc((path_len + 1) * sizeof(wchar_t)); - if (path == NULL) { - return _PyStatus_NO_MEMORY(); - } - memcpy(path, sys_path, path_len * sizeof(wchar_t)); - path[path_len] = L'\0'; - - PyStatus status = PyWideStringList_Append(&config->module_search_paths, path); - PyMem_RawFree(path); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - if (*p == '\0') { - break; - } - sys_path = p + 1; - } - config->module_search_paths_set = 1; - return _PyStatus_OK(); -} - - -/* Calculate the path configuration: - - - exec_prefix - - module_search_path - - stdlib_dir - - prefix - - program_full_path - - On Windows, more fields are calculated: - - - base_executable - - isolated - - site_import - - On other platforms, isolated and site_import are left unchanged, and - _PyConfig_InitPathConfig() copies executable to base_executable (if it's not - set). - - Priority, highest to lowest: - - - PyConfig - - _Py_path_config: set by Py_SetPath(), Py_SetPythonHome() - and Py_SetProgramName() - - _PyPathConfig_Calculate() -*/ -static PyStatus -pathconfig_init(_PyPathConfig *pathconfig, const PyConfig *config, - int compute_path_config) -{ - PyStatus status; - PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - status = pathconfig_copy(pathconfig, &_Py_path_config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - status = pathconfig_set_from_config(pathconfig, config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - if (compute_path_config) { - status = _PyPathConfig_Calculate(pathconfig, config); - } - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return status; -} - - -static PyStatus -config_init_pathconfig(PyConfig *config, int compute_path_config) -{ - _PyPathConfig pathconfig = _PyPathConfig_INIT; - PyStatus status; - - status = pathconfig_init(&pathconfig, config, compute_path_config); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - - if (!config->module_search_paths_set - && pathconfig.module_search_path != NULL) - { - status = config_init_module_search_paths(config, &pathconfig); - if (_PyStatus_EXCEPTION(status)) { - goto done; - } - } - -#define COPY_ATTR(PATH_ATTR, CONFIG_ATTR) \ - if (config->CONFIG_ATTR == NULL && pathconfig.PATH_ATTR != NULL) { \ - if (copy_wstr(&config->CONFIG_ATTR, pathconfig.PATH_ATTR) < 0) { \ - goto no_memory; \ - } \ - } - -#ifdef MS_WINDOWS - if (config->executable != NULL && config->base_executable == NULL) { - /* If executable is set explicitly in the configuration, - ignore calculated base_executable: _PyConfig_InitPathConfig() - will copy executable to base_executable */ - } - else { - COPY_ATTR(base_executable, base_executable); - } -#endif - - COPY_ATTR(program_full_path, executable); - COPY_ATTR(prefix, prefix); - COPY_ATTR(exec_prefix, exec_prefix); - COPY_ATTR(stdlib_dir, stdlib_dir); - -#undef COPY_ATTR - -#ifdef MS_WINDOWS - /* If a ._pth file is found: isolated and site_import are overridden */ - if (pathconfig.isolated != -1) { - config->isolated = pathconfig.isolated; - } - if (pathconfig.site_import != -1) { - config->site_import = pathconfig.site_import; - } -#endif - - status = _PyStatus_OK(); - goto done; - -no_memory: - status = _PyStatus_NO_MEMORY(); +#define COPY(ATTR) \ + do { \ + if (config->ATTR) { \ + PyMem_RawFree(_Py_path_config.ATTR); \ + _Py_path_config.ATTR = _PyMem_RawWcsdup(config->ATTR); \ + if (!_Py_path_config.ATTR) goto error; \ + } \ + } while (0) -done: - pathconfig_clear(&pathconfig); - return status; -} +#define COPY2(ATTR, SRCATTR) \ + do { \ + if (config->SRCATTR) { \ + PyMem_RawFree(_Py_path_config.ATTR); \ + _Py_path_config.ATTR = _PyMem_RawWcsdup(config->SRCATTR); \ + if (!_Py_path_config.ATTR) goto error; \ + } \ + } while (0) + COPY(prefix); + COPY(exec_prefix); + COPY(stdlib_dir); + COPY(program_name); + COPY(home); + COPY2(program_full_path, executable); +#undef COPY +#undef COPY2 -PyStatus -_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) -{ - /* Do we need to calculate the path? */ - if (!config->module_search_paths_set - || config->executable == NULL - || config->prefix == NULL - || config->exec_prefix == NULL) - { - PyStatus status = config_init_pathconfig(config, compute_path_config); - if (_PyStatus_EXCEPTION(status)) { - return status; + PyMem_RawFree(_Py_path_config.module_search_path); + _Py_path_config.module_search_path = NULL; + PyMem_RawFree(_Py_path_config.calculated_module_search_path); + _Py_path_config.calculated_module_search_path = NULL; + + do { + size_t cch = 1; + for (Py_ssize_t i = 0; i < config->module_search_paths.length; ++i) { + cch += 1 + wcslen(config->module_search_paths.items[i]); } - } - if (config->base_prefix == NULL && config->prefix != NULL) { - if (copy_wstr(&config->base_prefix, config->prefix) < 0) { - return _PyStatus_NO_MEMORY(); + wchar_t *path = (wchar_t*)PyMem_RawMalloc(sizeof(wchar_t) * cch); + if (!path) { + goto error; } - } - - if (config->base_exec_prefix == NULL && config->exec_prefix != NULL) { - if (copy_wstr(&config->base_exec_prefix, - config->exec_prefix) < 0) { - return _PyStatus_NO_MEMORY(); + wchar_t *p = path; + for (Py_ssize_t i = 0; i < config->module_search_paths.length; ++i) { + wcscpy(p, config->module_search_paths.items[i]); + p = wcschr(p, L'\0'); + *p++ = DELIM; + *p = L'\0'; } - } - if (config->base_executable == NULL && config->executable != NULL) { - if (copy_wstr(&config->base_executable, - config->executable) < 0) { - return _PyStatus_NO_MEMORY(); - } - } + do { + *p = L'\0'; + } while (p != path && *--p == DELIM); + _Py_path_config.calculated_module_search_path = path; + } while (0); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return _PyStatus_OK(); -} +error: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return _PyStatus_NO_MEMORY(); +} -/* External interface */ static void _Py_NO_RETURN path_out_of_memory(const char *func) @@ -483,7 +194,7 @@ void Py_SetPath(const wchar_t *path) { if (path == NULL) { - pathconfig_clear(&_Py_path_config); + _PyPathConfig_ClearGlobal(); return; } @@ -494,6 +205,7 @@ Py_SetPath(const wchar_t *path) PyMem_RawFree(_Py_path_config.exec_prefix); PyMem_RawFree(_Py_path_config.stdlib_dir); PyMem_RawFree(_Py_path_config.module_search_path); + PyMem_RawFree(_Py_path_config.calculated_module_search_path); _Py_path_config.prefix = _PyMem_RawWcsdup(L""); _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L""); @@ -505,6 +217,7 @@ Py_SetPath(const wchar_t *path) _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L""); } _Py_path_config.module_search_path = _PyMem_RawWcsdup(path); + _Py_path_config.calculated_module_search_path = NULL; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -521,19 +234,19 @@ Py_SetPath(const wchar_t *path) void Py_SetPythonHome(const wchar_t *home) { - if (home == NULL) { - return; - } + int has_value = home && home[0]; PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); PyMem_RawFree(_Py_path_config.home); - _Py_path_config.home = _PyMem_RawWcsdup(home); + if (has_value) { + _Py_path_config.home = _PyMem_RawWcsdup(home); + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.home == NULL) { + if (has_value && _Py_path_config.home == NULL) { path_out_of_memory(__func__); } } @@ -542,19 +255,19 @@ Py_SetPythonHome(const wchar_t *home) void Py_SetProgramName(const wchar_t *program_name) { - if (program_name == NULL || program_name[0] == L'\0') { - return; - } + int has_value = program_name && program_name[0]; PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); PyMem_RawFree(_Py_path_config.program_name); - _Py_path_config.program_name = _PyMem_RawWcsdup(program_name); + if (has_value) { + _Py_path_config.program_name = _PyMem_RawWcsdup(program_name); + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.program_name == NULL) { + if (has_value && _Py_path_config.program_name == NULL) { path_out_of_memory(__func__); } } @@ -562,19 +275,19 @@ Py_SetProgramName(const wchar_t *program_name) void _Py_SetProgramFullPath(const wchar_t *program_full_path) { - if (program_full_path == NULL || program_full_path[0] == L'\0') { - return; - } + int has_value = program_full_path && program_full_path[0]; PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); PyMem_RawFree(_Py_path_config.program_full_path); - _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path); + if (has_value) { + _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path); + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.program_full_path == NULL) { + if (has_value && _Py_path_config.program_full_path == NULL) { path_out_of_memory(__func__); } } @@ -583,7 +296,12 @@ _Py_SetProgramFullPath(const wchar_t *program_full_path) wchar_t * Py_GetPath(void) { - return _Py_path_config.module_search_path; + /* If the user has provided a path, return that */ + if (_Py_path_config.module_search_path) { + return _Py_path_config.module_search_path; + } + /* If we have already done calculations, return the calculated path */ + return _Py_path_config.calculated_module_search_path; } @@ -632,6 +350,8 @@ Py_GetProgramName(void) return _Py_path_config.program_name; } + + /* Compute module search path from argv[0] or the current working directory ("-m module" case) which will be prepended to sys.argv: sys.path[0]. @@ -772,73 +492,6 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p) } -#ifdef MS_WINDOWS -#define WCSTOK wcstok_s -#else -#define WCSTOK wcstok -#endif - -/* Search for a prefix value in an environment file (pyvenv.cfg). - - - If found, copy it into *value_p: string which must be freed by - PyMem_RawFree(). - - If not found, *value_p is set to NULL. -*/ -PyStatus -_Py_FindEnvConfigValue(FILE *env_file, const wchar_t *key, - wchar_t **value_p) -{ - *value_p = NULL; - - char buffer[MAXPATHLEN * 2 + 1]; /* allow extra for key, '=', etc. */ - buffer[Py_ARRAY_LENGTH(buffer)-1] = '\0'; - - while (!feof(env_file)) { - char * p = fgets(buffer, Py_ARRAY_LENGTH(buffer) - 1, env_file); - - if (p == NULL) { - break; - } - - size_t n = strlen(p); - if (p[n - 1] != '\n') { - /* line has overflowed - bail */ - break; - } - if (p[0] == '#') { - /* Comment - skip */ - continue; - } - - wchar_t *tmpbuffer = _Py_DecodeUTF8_surrogateescape(buffer, n, NULL); - if (tmpbuffer) { - wchar_t * state; - wchar_t * tok = WCSTOK(tmpbuffer, L" \t\r\n", &state); - if ((tok != NULL) && !wcscmp(tok, key)) { - tok = WCSTOK(NULL, L" \t", &state); - if ((tok != NULL) && !wcscmp(tok, L"=")) { - tok = WCSTOK(NULL, L"\r\n", &state); - if (tok != NULL) { - *value_p = _PyMem_RawWcsdup(tok); - PyMem_RawFree(tmpbuffer); - - if (*value_p == NULL) { - return _PyStatus_NO_MEMORY(); - } - - /* found */ - return _PyStatus_OK(); - } - } - } - PyMem_RawFree(tmpbuffer); - } - } - - /* not found */ - return _PyStatus_OK(); -} - #ifdef __cplusplus } #endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c6b2a1e..84b76ea 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -459,7 +459,7 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config) } if (_Py_IsMainInterpreter(tstate->interp)) { - PyStatus status = _PyConfig_WritePathConfig(config); + PyStatus status = _PyPathConfig_UpdateGlobal(config); if (_PyStatus_EXCEPTION(status)) { _PyErr_SetFromPyStatus(status); return -1; @@ -488,7 +488,7 @@ _PyInterpreterState_SetConfig(const PyConfig *src_config) goto done; } - status = PyConfig_Read(&config); + status = _PyConfig_Read(&config, 1); if (_PyStatus_EXCEPTION(status)) { _PyErr_SetFromPyStatus(status); goto done; @@ -549,7 +549,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, config = _PyInterpreterState_GetConfig(interp); if (config->_install_importlib) { - status = _PyConfig_WritePathConfig(config); + status = _PyPathConfig_UpdateGlobal(config); if (_PyStatus_EXCEPTION(status)) { return status; } |