[3.8] bpo-38234: Backport init path config changes from master (GH-16423)

* bpo-38234: Py_SetPath() uses the program full path (GH-16357)

Py_SetPath() now sets sys.executable to the program full path
(Py_GetProgramFullPath()), rather than to the program name
(Py_GetProgramName()).

Fix also memory leaks in pathconfig_set_from_config().

(cherry picked from commit 1ce152a42eaa917d7763bce93f1e1ca72530d7ca)

* bpo-38234: Add tests for Python init path config (GH-16358)


(cherry picked from commit bb6bf7d342b4503a6227fd209fac934905b6a1aa)

* bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)

Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed
to test pyvenv.cfg and pybuilddir.txt configuration files.

Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8
encoding, not ASCII.

(cherry picked from commit 52ad33abbfb6637d74932617c7013bae0ccf6e32)

* bpo-38234: Cleanup getpath.c (GH-16367)

* search_for_prefix() directly calls reduce() if found is greater
  than 0.
* Add calculate_pybuilddir() subfunction.
* search_for_prefix(): add path string buffer for readability.
* Fix some error handling code paths: release resources on error.
* calculate_read_pyenv(): rename tmpbuffer to filename.
* test.pythoninfo now also logs windows.dll_path

(cherry picked from commit 221fd84703c545408bbb4a6e0b58459651331f5c)

* bpo-38234: Fix test_embed pathconfig tests (GH-16390)

bpo-38234: On macOS and FreeBSD, the temporary directory can be
symbolic link. For example, /tmp can be a symbolic link to /var/tmp.
Call realpath() to resolve all symbolic links.

(cherry picked from commit 00508a7407d7d300b487532e2271534b20e378a7)

* bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402)

* Add test_embed.test_init_setpath_config(): test Py_SetPath()
  with PyConfig.
* test_init_setpath() and test_init_setpythonhome() no longer call
  Py_SetProgramName(), but use the default program name.
* _PyPathConfig: isolated, site_import  and base_executable
  fields are now only available on Windows.
* If executable is set explicitly in the configuration, ignore
  calculated base_executable: _PyConfig_InitPathConfig() copies
  executable to base_executable.
* Complete path config documentation.

(cherry picked from commit 8bf39b606ef7b02c0279a80789f3c4824b0da5e9)

* bpo-38234: Complete init config documentation (GH-16404)


(cherry picked from commit 88feaecd46a8f427e30ef7ad8cfcddfe392a2402)

* bpo-38234: Fix test_embed.test_init_setpath_config() on FreeBSD (GH-16406)

Explicitly preinitializes with a Python preconfiguration to avoid
Py_SetPath() implicit preinitialization with a compat
preconfiguration.

Fix also test_init_setpath() and test_init_setpythonhome() on macOS:
use self.test_exe as the executable (and base_executable), rather
than shutil.which('python3').

(cherry picked from commit 49d99f01e6e51acec5ca57a02e857f0796bc418b)

* bpo-38234: Py_Initialize() sets global path configuration (GH-16421)

* Py_InitializeFromConfig() now writes PyConfig path configuration to
  the global path configuration (_Py_path_config).
* Add test_embed.test_get_pathconfig().
* Fix typo in _PyWideStringList_Join().

(cherry picked from commit 12f2f177fc483723406d7917194e7f655a20631b)

1 files changed, 322 insertions, 20 deletions

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index e02acbc..ed2b96f 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -3,15 +3,19 @@ from test import support
 import unittest
 
 from collections import namedtuple
+import contextlib
 import json
 import os
 import re
+import shutil
 import subprocess
 import sys
+import tempfile
 import textwrap
 
 
 MS_WINDOWS = (os.name == 'nt')
+MACOS = (sys.platform == 'darwin')
 
 PYMEM_ALLOCATOR_NOT_SET = 0
 PYMEM_ALLOCATOR_DEBUG = 2
@@ -25,6 +29,12 @@ API_PYTHON = 2
 API_ISOLATED = 3
 
 
+def debug_build(program):
+    program = os.path.basename(program)
+    name = os.path.splitext(program)[0]
+    return name.endswith("_d")
+
+
 def remove_python_envvars():
     env = dict(os.environ)
     # Remove PYTHON* environment variables to get deterministic environment
@@ -40,7 +50,7 @@ class EmbeddingTestsMixin:
         basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
         exename = "_testembed"
         if MS_WINDOWS:
-            ext = ("_d" if "_d" in sys.executable else "") + ".exe"
+            ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
             exename += ext
             exepath = os.path.dirname(sys.executable)
         else:
@@ -58,7 +68,8 @@ class EmbeddingTestsMixin:
         os.chdir(self.oldcwd)
 
     def run_embedded_interpreter(self, *args, env=None,
-                                 timeout=None, returncode=0, input=None):
+                                 timeout=None, returncode=0, input=None,
+                                 cwd=None):
         """Runs a test in the embedded interpreter"""
         cmd = [self.test_exe]
         cmd.extend(args)
@@ -72,7 +83,8 @@ class EmbeddingTestsMixin:
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
                              universal_newlines=True,
-                             env=env)
+                             env=env,
+                             cwd=cwd)
         try:
             (out, err) = p.communicate(input=input, timeout=timeout)
         except:
@@ -460,6 +472,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
     EXPECTED_CONFIG = None
 
+    @classmethod
+    def tearDownClass(cls):
+        # clear cache
+        cls.EXPECTED_CONFIG = None
+
     def main_xoptions(self, xoptions_list):
         xoptions = {}
         for opt in xoptions_list:
@@ -470,7 +487,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 xoptions[opt] = True
         return xoptions
 
-    def _get_expected_config(self, env):
+    def _get_expected_config_impl(self):
+        env = remove_python_envvars()
         code = textwrap.dedent('''
             import json
             import sys
@@ -489,23 +507,37 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         args = [sys.executable, '-S', '-c', code]
         proc = subprocess.run(args, env=env,
                               stdout=subprocess.PIPE,
-                              stderr=subprocess.STDOUT)
+                              stderr=subprocess.PIPE)
         if proc.returncode:
             raise Exception(f"failed to get the default config: "
                             f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
         stdout = proc.stdout.decode('utf-8')
+        # ignore stderr
         try:
             return json.loads(stdout)
         except json.JSONDecodeError:
             self.fail(f"fail to decode stdout: {stdout!r}")
 
+    def _get_expected_config(self):
+        cls = InitConfigTests
+        if cls.EXPECTED_CONFIG is None:
+            cls.EXPECTED_CONFIG = self._get_expected_config_impl()
+
+        # get a copy
+        configs = {}
+        for config_key, config_value in cls.EXPECTED_CONFIG.items():
+            config = {}
+            for key, value in config_value.items():
+                if isinstance(value, list):
+                    value = value.copy()
+                config[key] = value
+            configs[config_key] = config
+        return configs
+
     def get_expected_config(self, expected_preconfig, expected, env, api,
                             modify_path_cb=None):
         cls = self.__class__
-        if cls.EXPECTED_CONFIG is None:
-            cls.EXPECTED_CONFIG = self._get_expected_config(env)
-        configs = {key: dict(value)
-                   for key, value in self.EXPECTED_CONFIG.items()}
+        configs = self._get_expected_config()
 
         pre_config = configs['pre_config']
         for key, value in expected_preconfig.items():
@@ -553,9 +585,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             if value is self.GET_DEFAULT_CONFIG:
                 expected[key] = config[key]
 
-        prepend_path = expected['pythonpath_env']
-        if prepend_path is not None:
-            expected['module_search_paths'] = [prepend_path, *expected['module_search_paths']]
+        pythonpath_env = expected['pythonpath_env']
+        if pythonpath_env is not None:
+            paths = pythonpath_env.split(os.path.pathsep)
+            expected['module_search_paths'] = [*paths, *expected['module_search_paths']]
         if modify_path_cb is not None:
             expected['module_search_paths'] = expected['module_search_paths'].copy()
             modify_path_cb(expected['module_search_paths'])
@@ -603,13 +636,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         self.assertEqual(configs['global_config'], expected)
 
     def check_all_configs(self, testname, expected_config=None,
-                     expected_preconfig=None, modify_path_cb=None, stderr=None,
-                     *, api):
-        env = remove_python_envvars()
-
-        if api == API_ISOLATED:
+                          expected_preconfig=None, modify_path_cb=None,
+                          stderr=None, *, api, preconfig_api=None,
+                          env=None, ignore_stderr=False, cwd=None):
+        new_env = remove_python_envvars()
+        if env is not None:
+            new_env.update(env)
+        env = new_env
+
+        if preconfig_api is None:
+            preconfig_api = api
+        if preconfig_api == API_ISOLATED:
             default_preconfig = self.PRE_CONFIG_ISOLATED
-        elif api == API_PYTHON:
+        elif preconfig_api == API_PYTHON:
             default_preconfig = self.PRE_CONFIG_PYTHON
         else:
             default_preconfig = self.PRE_CONFIG_COMPAT
@@ -631,10 +670,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                                  expected_config, env,
                                  api, modify_path_cb)
 
-        out, err = self.run_embedded_interpreter(testname, env=env)
+        out, err = self.run_embedded_interpreter(testname,
+                                                 env=env, cwd=cwd)
         if stderr is None and not expected_config['verbose']:
             stderr = ""
-        if stderr is not None:
+        if stderr is not None and not ignore_stderr:
             self.assertEqual(err.rstrip(), stderr)
         try:
             configs = json.loads(out)
@@ -965,6 +1005,268 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         self.check_all_configs("test_init_dont_parse_argv", config, pre_config,
                                api=API_PYTHON)
 
+    def default_program_name(self, config):
+        if MS_WINDOWS:
+            program_name = 'python'
+            executable = self.test_exe
+        else:
+            program_name = 'python3'
+            if MACOS:
+                executable = self.test_exe
+            else:
+                executable = shutil.which(program_name) or ''
+        config.update({
+            'program_name': program_name,
+            'base_executable': executable,
+            'executable': executable,
+        })
+
+    def test_init_setpath(self):
+        # Test Py_SetPath()
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+
+        config = {
+            'module_search_paths': paths,
+            'prefix': '',
+            'base_prefix': '',
+            'exec_prefix': '',
+            'base_exec_prefix': '',
+        }
+        self.default_program_name(config)
+        env = {'TESTPATH': os.path.pathsep.join(paths)}
+        self.check_all_configs("test_init_setpath", config,
+                               api=API_COMPAT, env=env,
+                               ignore_stderr=True)
+
+    def test_init_setpath_config(self):
+        # Test Py_SetPath() with PyConfig
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+
+        config = {
+            # set by Py_SetPath()
+            'module_search_paths': paths,
+            'prefix': '',
+            'base_prefix': '',
+            'exec_prefix': '',
+            'base_exec_prefix': '',
+            # overriden by PyConfig
+            'program_name': 'conf_program_name',
+            'base_executable': 'conf_executable',
+            'executable': 'conf_executable',
+        }
+        env = {'TESTPATH': os.path.pathsep.join(paths)}
+        self.check_all_configs("test_init_setpath_config", config,
+                               api=API_PYTHON, env=env, ignore_stderr=True)
+
+    def module_search_paths(self, prefix=None, exec_prefix=None):
+        config = self._get_expected_config()
+        if prefix is None:
+            prefix = config['config']['prefix']
+        if exec_prefix is None:
+            exec_prefix = config['config']['prefix']
+        if MS_WINDOWS:
+            return config['config']['module_search_paths']
+        else:
+            ver = sys.version_info
+            return [
+                os.path.join(prefix, 'lib',
+                             f'python{ver.major}{ver.minor}.zip'),
+                os.path.join(prefix, 'lib',
+                             f'python{ver.major}.{ver.minor}'),
+                os.path.join(exec_prefix, 'lib',
+                             f'python{ver.major}.{ver.minor}', 'lib-dynload'),
+            ]
+
+    @contextlib.contextmanager
+    def tmpdir_with_python(self):
+        # Temporary directory with a copy of the Python program
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # bpo-38234: On macOS and FreeBSD, the temporary directory
+            # can be symbolic link. For example, /tmp can be a symbolic link
+            # to /var/tmp. Call realpath() to resolve all symbolic links.
+            tmpdir = os.path.realpath(tmpdir)
+
+            if MS_WINDOWS:
+                # Copy pythonXY.dll (or pythonXY_d.dll)
+                ver = sys.version_info
+                dll = f'python{ver.major}{ver.minor}'
+                if debug_build(sys.executable):
+                    dll += '_d'
+                dll += '.dll'
+                dll = os.path.join(os.path.dirname(self.test_exe), dll)
+                dll_copy = os.path.join(tmpdir, os.path.basename(dll))
+                shutil.copyfile(dll, dll_copy)
+
+            # Copy Python program
+            exec_copy = os.path.join(tmpdir, os.path.basename(self.test_exe))
+            shutil.copyfile(self.test_exe, exec_copy)
+            shutil.copystat(self.test_exe, exec_copy)
+            self.test_exe = exec_copy
+
+            yield tmpdir
+
+    def test_init_setpythonhome(self):
+        # Test Py_SetPythonHome(home) with PYTHONPATH env var
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+        paths_str = os.path.pathsep.join(paths)
+
+        for path in paths:
+            if not os.path.isdir(path):
+                continue
+            if os.path.exists(os.path.join(path, 'os.py')):
+                home = os.path.dirname(path)
+                break
+        else:
+            self.fail(f"Unable to find home in {paths!r}")
+
+        prefix = exec_prefix = home
+        ver = sys.version_info
+        expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
+
+        config = {
+            'home': home,
+            'module_search_paths': expected_paths,
+            'prefix': prefix,
+            'base_prefix': prefix,
+            'exec_prefix': exec_prefix,
+            'base_exec_prefix': exec_prefix,
+            'pythonpath_env': paths_str,
+        }
+        self.default_program_name(config)
+        env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
+        self.check_all_configs("test_init_setpythonhome", config,
+                               api=API_COMPAT, env=env)
+
+    def copy_paths_by_env(self, config):
+        all_configs = self._get_expected_config()
+        paths = all_configs['config']['module_search_paths']
+        paths_str = os.path.pathsep.join(paths)
+        config['pythonpath_env'] = paths_str
+        env = {'PYTHONPATH': paths_str}
+        return env
+
+    @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt')
+    def test_init_pybuilddir(self):
+        # Test path configuration with pybuilddir.txt configuration file
+
+        with self.tmpdir_with_python() as tmpdir:
+            # pybuilddir.txt is a sub-directory relative to the current
+            # directory (tmpdir)
+            subdir = 'libdir'
+            libdir = os.path.join(tmpdir, subdir)
+            os.mkdir(libdir)
+
+            filename = os.path.join(tmpdir, 'pybuilddir.txt')
+            with open(filename, "w", encoding="utf8") as fp:
+                fp.write(subdir)
+
+            module_search_paths = self.module_search_paths()
+            module_search_paths[-1] = libdir
+
+            executable = self.test_exe
+            config = {
+                'base_executable': executable,
+                'executable': executable,
+                'module_search_paths': module_search_paths,
+            }
+            env = self.copy_paths_by_env(config)
+            self.check_all_configs("test_init_compat_config", config,
+                                   api=API_COMPAT, env=env,
+                                   ignore_stderr=True, cwd=tmpdir)
+
+    def test_init_pyvenv_cfg(self):
+        # Test path configuration with pyvenv.cfg configuration file
+
+        with self.tmpdir_with_python() as tmpdir, \
+             tempfile.TemporaryDirectory() as pyvenv_home:
+            ver = sys.version_info
+
+            if not MS_WINDOWS:
+                lib_dynload = os.path.join(pyvenv_home,
+                                           'lib',
+                                           f'python{ver.major}.{ver.minor}',
+                                           'lib-dynload')
+                os.makedirs(lib_dynload)
+            else:
+                lib_dynload = os.path.join(pyvenv_home, 'lib')
+                os.makedirs(lib_dynload)
+                # getpathp.c uses Lib\os.py as the LANDMARK
+                shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py'))
+
+            filename = os.path.join(tmpdir, 'pyvenv.cfg')
+            with open(filename, "w", encoding="utf8") as fp:
+                print("home = %s" % pyvenv_home, file=fp)
+                print("include-system-site-packages = false", file=fp)
+
+            paths = self.module_search_paths()
+            if not MS_WINDOWS:
+                paths[-1] = lib_dynload
+            else:
+                for index, path in enumerate(paths):
+                    if index == 0:
+                        paths[index] = os.path.join(tmpdir, os.path.basename(path))
+                    else:
+                        paths[index] = os.path.join(pyvenv_home, os.path.basename(path))
+                paths[-1] = pyvenv_home
+
+            executable = self.test_exe
+            exec_prefix = pyvenv_home
+            config = {
+                'base_exec_prefix': exec_prefix,
+                'exec_prefix': exec_prefix,
+                'base_executable': executable,
+                'executable': executable,
+                'module_search_paths': paths,
+            }
+            if MS_WINDOWS:
+                config['base_prefix'] = pyvenv_home
+                config['prefix'] = pyvenv_home
+            env = self.copy_paths_by_env(config)
+            self.check_all_configs("test_init_compat_config", config,
+                                   api=API_COMPAT, env=env,
+                                   ignore_stderr=True, cwd=tmpdir)
+
+    def test_global_pathconfig(self):
+        # Test C API functions getting the path configuration:
+        #
+        # - Py_GetExecPrefix()
+        # - Py_GetPath()
+        # - Py_GetPrefix()
+        # - Py_GetProgramFullPath()
+        # - Py_GetProgramName()
+        # - Py_GetPythonHome()
+        #
+        # The global path configuration (_Py_path_config) must be a copy
+        # of the path configuration of PyInterpreter.config (PyConfig).
+        ctypes = support.import_module('ctypes')
+        _testinternalcapi = support.import_module('_testinternalcapi')
+
+        def get_func(name):
+            func = getattr(ctypes.pythonapi, name)
+            func.argtypes = ()
+            func.restype = ctypes.c_wchar_p
+            return func
+
+        Py_GetPath = get_func('Py_GetPath')
+        Py_GetPrefix = get_func('Py_GetPrefix')
+        Py_GetExecPrefix = get_func('Py_GetExecPrefix')
+        Py_GetProgramName = get_func('Py_GetProgramName')
+        Py_GetProgramFullPath = get_func('Py_GetProgramFullPath')
+        Py_GetPythonHome = get_func('Py_GetPythonHome')
+
+        config = _testinternalcapi.get_configs()['config']
+
+        self.assertEqual(Py_GetPath().split(os.path.pathsep),
+                         config['module_search_paths'])
+        self.assertEqual(Py_GetPrefix(), config['prefix'])
+        self.assertEqual(Py_GetExecPrefix(), config['exec_prefix'])
+        self.assertEqual(Py_GetProgramName(), config['program_name'])
+        self.assertEqual(Py_GetProgramFullPath(), config['executable'])
+        self.assertEqual(Py_GetPythonHome(), config['home'])
+
 
 class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_open_code_hook(self):