[3.8] bpo-38234: Backport init path config changes from master (GH-16423)

* bpo-38234: Py_SetPath() uses the program full path (GH-16357)

Py_SetPath() now sets sys.executable to the program full path
(Py_GetProgramFullPath()), rather than to the program name
(Py_GetProgramName()).

Fix also memory leaks in pathconfig_set_from_config().

(cherry picked from commit 1ce152a42eaa917d7763bce93f1e1ca72530d7ca)

* bpo-38234: Add tests for Python init path config (GH-16358)


(cherry picked from commit bb6bf7d342b4503a6227fd209fac934905b6a1aa)

* bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)

Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed
to test pyvenv.cfg and pybuilddir.txt configuration files.

Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8
encoding, not ASCII.

(cherry picked from commit 52ad33abbfb6637d74932617c7013bae0ccf6e32)

* bpo-38234: Cleanup getpath.c (GH-16367)

* search_for_prefix() directly calls reduce() if found is greater
  than 0.
* Add calculate_pybuilddir() subfunction.
* search_for_prefix(): add path string buffer for readability.
* Fix some error handling code paths: release resources on error.
* calculate_read_pyenv(): rename tmpbuffer to filename.
* test.pythoninfo now also logs windows.dll_path

(cherry picked from commit 221fd84703c545408bbb4a6e0b58459651331f5c)

* bpo-38234: Fix test_embed pathconfig tests (GH-16390)

bpo-38234: On macOS and FreeBSD, the temporary directory can be
symbolic link. For example, /tmp can be a symbolic link to /var/tmp.
Call realpath() to resolve all symbolic links.

(cherry picked from commit 00508a7407d7d300b487532e2271534b20e378a7)

* bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402)

* Add test_embed.test_init_setpath_config(): test Py_SetPath()
  with PyConfig.
* test_init_setpath() and test_init_setpythonhome() no longer call
  Py_SetProgramName(), but use the default program name.
* _PyPathConfig: isolated, site_import  and base_executable
  fields are now only available on Windows.
* If executable is set explicitly in the configuration, ignore
  calculated base_executable: _PyConfig_InitPathConfig() copies
  executable to base_executable.
* Complete path config documentation.

(cherry picked from commit 8bf39b606ef7b02c0279a80789f3c4824b0da5e9)

* bpo-38234: Complete init config documentation (GH-16404)


(cherry picked from commit 88feaecd46a8f427e30ef7ad8cfcddfe392a2402)

* bpo-38234: Fix test_embed.test_init_setpath_config() on FreeBSD (GH-16406)

Explicitly preinitializes with a Python preconfiguration to avoid
Py_SetPath() implicit preinitialization with a compat
preconfiguration.

Fix also test_init_setpath() and test_init_setpythonhome() on macOS:
use self.test_exe as the executable (and base_executable), rather
than shutil.which('python3').

(cherry picked from commit 49d99f01e6e51acec5ca57a02e857f0796bc418b)

* bpo-38234: Py_Initialize() sets global path configuration (GH-16421)

* Py_InitializeFromConfig() now writes PyConfig path configuration to
  the global path configuration (_Py_path_config).
* Add test_embed.test_get_pathconfig().
* Fix typo in _PyWideStringList_Join().

(cherry picked from commit 12f2f177fc483723406d7917194e7f655a20631b)

2 files changed, 59 insertions, 18 deletions

diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index 0953310..d5b8b1a 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -23,6 +23,7 @@ wchar_t *_Py_dll_path = NULL;
 static int
 copy_wstr(wchar_t **dst, const wchar_t *src)
 {
+    assert(*dst == NULL);
     if (src != NULL) {
         *dst = _PyMem_RawWcsdup(src);
         if (*dst == NULL) {
@@ -57,7 +58,10 @@ pathconfig_clear(_PyPathConfig *config)
     CLEAR(config->module_search_path);
     CLEAR(config->program_name);
     CLEAR(config->home);
+#ifdef MS_WINDOWS
     CLEAR(config->base_executable);
+#endif
+
 #undef CLEAR
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
@@ -82,9 +86,11 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
     COPY_ATTR(module_search_path);
     COPY_ATTR(program_name);
     COPY_ATTR(home);
+#ifdef MS_WINDOWS
     config->isolated = config2->isolated;
     config->site_import = config2->site_import;
     COPY_ATTR(base_executable);
+#endif
 
 #undef COPY_ATTR
 
@@ -127,7 +133,7 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
     for (Py_ssize_t i=0; i < list->length; i++) {
         wchar_t *path = list->items[i];
         if (i != 0) {
-            *str++ = SEP;
+            *str++ = sep;
         }
         len = wcslen(path);
         memcpy(str, path, len * sizeof(wchar_t));
@@ -139,11 +145,11 @@ _PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
 }
 
 
+#ifdef MS_WINDOWS
 /* Initialize _Py_dll_path on Windows. Do nothing on other platforms. */
-PyStatus
-_PyPathConfig_Init(void)
+static PyStatus
+_PyPathConfig_InitDLLPath(void)
 {
-#ifdef MS_WINDOWS
     if (_Py_dll_path == NULL) {
         /* Already set: nothing to do */
         return _PyStatus_OK();
@@ -159,9 +165,9 @@ _PyPathConfig_Init(void)
     if (_Py_dll_path == NULL) {
         return _PyStatus_NO_MEMORY();
     }
-#endif
     return _PyStatus_OK();
 }
+#endif
 
 
 static PyStatus
@@ -172,6 +178,7 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     if (config->module_search_paths_set) {
+        PyMem_RawFree(pathconfig->module_search_path);
         pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
         if (pathconfig->module_search_path == NULL) {
             goto no_memory;
@@ -180,17 +187,21 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
 
 #define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \
         if (config->CONFIG_ATTR) { \
+            PyMem_RawFree(pathconfig->PATH_ATTR); \
+            pathconfig->PATH_ATTR = NULL; \
             if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \
                 goto no_memory; \
             } \
         }
 
-    COPY_CONFIG(base_executable, base_executable);
     COPY_CONFIG(program_full_path, executable);
     COPY_CONFIG(prefix, prefix);
     COPY_CONFIG(exec_prefix, exec_prefix);
     COPY_CONFIG(program_name, program_name);
     COPY_CONFIG(home, home);
+#ifdef MS_WINDOWS
+    COPY_CONFIG(base_executable, base_executable);
+#endif
 
 #undef COPY_CONFIG
 
@@ -206,6 +217,20 @@ done:
 }
 
 
+PyStatus
+_PyConfig_WritePathConfig(const PyConfig *config)
+{
+#ifdef MS_WINDOWS
+    PyStatus status = _PyPathConfig_InitDLLPath();
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+#endif
+
+    return pathconfig_set_from_config(&_Py_path_config, config);
+}
+
+
 static PyStatus
 config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig)
 {
@@ -326,18 +351,32 @@ config_calculate_pathconfig(PyConfig *config)
             } \
         }
 
+#ifdef MS_WINDOWS
+    if (config->executable != NULL && config->base_executable == NULL) {
+        /* If executable is set explicitly in the configuration,
+           ignore calculated base_executable: _PyConfig_InitPathConfig()
+           will copy executable to base_executable */
+    }
+    else {
+        COPY_ATTR(base_executable, base_executable);
+    }
+#endif
+
     COPY_ATTR(program_full_path, executable);
     COPY_ATTR(prefix, prefix);
     COPY_ATTR(exec_prefix, exec_prefix);
-    COPY_ATTR(base_executable, base_executable);
+
 #undef COPY_ATTR
 
+#ifdef MS_WINDOWS
+    /* If a ._pth file is found: isolated and site_import are overriden */
     if (pathconfig.isolated != -1) {
         config->isolated = pathconfig.isolated;
     }
     if (pathconfig.site_import != -1) {
         config->site_import = pathconfig.site_import;
     }
+#endif
 
     status = _PyStatus_OK();
     goto done;
@@ -356,9 +395,9 @@ _PyConfig_InitPathConfig(PyConfig *config)
 {
     /* Do we need to calculate the path? */
     if (!config->module_search_paths_set
-        || (config->executable == NULL)
-        || (config->prefix == NULL)
-        || (config->exec_prefix == NULL))
+        || config->executable == NULL
+        || config->prefix == NULL
+        || config->exec_prefix == NULL)
     {
         PyStatus status = config_calculate_pathconfig(config);
         if (_PyStatus_EXCEPTION(status)) {
@@ -416,11 +455,12 @@ pathconfig_global_init(void)
 {
     PyStatus status;
 
-    /* Initialize _Py_dll_path if needed */
-    status = _PyPathConfig_Init();
+#ifdef MS_WINDOWS
+    status = _PyPathConfig_InitDLLPath();
     if (_PyStatus_EXCEPTION(status)) {
         Py_ExitStatusException(status);
     }
+#endif
 
     if (_Py_path_config.module_search_path == NULL) {
         status = pathconfig_global_read(&_Py_path_config);
@@ -438,7 +478,9 @@ pathconfig_global_init(void)
     assert(_Py_path_config.module_search_path != NULL);
     assert(_Py_path_config.program_name != NULL);
     /* home can be NULL */
+#ifdef MS_WINDOWS
     assert(_Py_path_config.base_executable != NULL);
+#endif
 }
 
 
@@ -455,16 +497,15 @@ Py_SetPath(const wchar_t *path)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    /* Getting the program name calls pathconfig_global_init() */
-    wchar_t *program_name = _PyMem_RawWcsdup(Py_GetProgramName());
+    /* Getting the program full path calls pathconfig_global_init() */
+    wchar_t *program_full_path = _PyMem_RawWcsdup(Py_GetProgramFullPath());
 
     PyMem_RawFree(_Py_path_config.program_full_path);
     PyMem_RawFree(_Py_path_config.prefix);
     PyMem_RawFree(_Py_path_config.exec_prefix);
     PyMem_RawFree(_Py_path_config.module_search_path);
 
-    /* Copy program_name to program_full_path  */
-    _Py_path_config.program_full_path = program_name;
+    _Py_path_config.program_full_path = program_full_path;
     _Py_path_config.prefix = _PyMem_RawWcsdup(L"");
     _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L"");
     _Py_path_config.module_search_path = _PyMem_RawWcsdup(path);
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 4d7873f..42a062e 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -472,7 +472,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
     config = &interp->config;
 
     if (config->_install_importlib) {
-        status = _PyPathConfig_Init();
+        status = _PyConfig_WritePathConfig(config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }
@@ -641,7 +641,7 @@ pycore_init_import_warnings(PyInterpreterState *interp, PyObject *sysmod)
     }
 
     if (config->_install_importlib) {
-        status = _PyPathConfig_Init();
+        status = _PyConfig_WritePathConfig(config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }