[3.8] bpo-38234: Backport init path config changes from master (GH-16423)

* bpo-38234: Py_SetPath() uses the program full path (GH-16357)

Py_SetPath() now sets sys.executable to the program full path
(Py_GetProgramFullPath()), rather than to the program name
(Py_GetProgramName()).

Fix also memory leaks in pathconfig_set_from_config().

(cherry picked from commit 1ce152a42eaa917d7763bce93f1e1ca72530d7ca)

* bpo-38234: Add tests for Python init path config (GH-16358)


(cherry picked from commit bb6bf7d342b4503a6227fd209fac934905b6a1aa)

* bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)

Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed
to test pyvenv.cfg and pybuilddir.txt configuration files.

Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8
encoding, not ASCII.

(cherry picked from commit 52ad33abbfb6637d74932617c7013bae0ccf6e32)

* bpo-38234: Cleanup getpath.c (GH-16367)

* search_for_prefix() directly calls reduce() if found is greater
  than 0.
* Add calculate_pybuilddir() subfunction.
* search_for_prefix(): add path string buffer for readability.
* Fix some error handling code paths: release resources on error.
* calculate_read_pyenv(): rename tmpbuffer to filename.
* test.pythoninfo now also logs windows.dll_path

(cherry picked from commit 221fd84703c545408bbb4a6e0b58459651331f5c)

* bpo-38234: Fix test_embed pathconfig tests (GH-16390)

bpo-38234: On macOS and FreeBSD, the temporary directory can be
symbolic link. For example, /tmp can be a symbolic link to /var/tmp.
Call realpath() to resolve all symbolic links.

(cherry picked from commit 00508a7407d7d300b487532e2271534b20e378a7)

* bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402)

* Add test_embed.test_init_setpath_config(): test Py_SetPath()
  with PyConfig.
* test_init_setpath() and test_init_setpythonhome() no longer call
  Py_SetProgramName(), but use the default program name.
* _PyPathConfig: isolated, site_import  and base_executable
  fields are now only available on Windows.
* If executable is set explicitly in the configuration, ignore
  calculated base_executable: _PyConfig_InitPathConfig() copies
  executable to base_executable.
* Complete path config documentation.

(cherry picked from commit 8bf39b606ef7b02c0279a80789f3c4824b0da5e9)

* bpo-38234: Complete init config documentation (GH-16404)


(cherry picked from commit 88feaecd46a8f427e30ef7ad8cfcddfe392a2402)

* bpo-38234: Fix test_embed.test_init_setpath_config() on FreeBSD (GH-16406)

Explicitly preinitializes with a Python preconfiguration to avoid
Py_SetPath() implicit preinitialization with a compat
preconfiguration.

Fix also test_init_setpath() and test_init_setpythonhome() on macOS:
use self.test_exe as the executable (and base_executable), rather
than shutil.which('python3').

(cherry picked from commit 49d99f01e6e51acec5ca57a02e857f0796bc418b)

* bpo-38234: Py_Initialize() sets global path configuration (GH-16421)

* Py_InitializeFromConfig() now writes PyConfig path configuration to
  the global path configuration (_Py_path_config).
* Add test_embed.test_get_pathconfig().
* Fix typo in _PyWideStringList_Join().

(cherry picked from commit 12f2f177fc483723406d7917194e7f655a20631b)

1 files changed, 103 insertions, 10 deletions

diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index c3ccc0e..83c266b 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -20,11 +20,12 @@
  * Executed via 'EmbeddingTests' in Lib/test/test_capi.py
  *********************************************************/
 
+/* Use path starting with "./" avoids a search along the PATH */
+#define PROGRAM_NAME L"./_testembed"
+
 static void _testembed_Py_Initialize(void)
 {
-    /* HACK: the "./" at front avoids a search along the PATH in
-       Modules/getpath.c */
-    Py_SetProgramName(L"./_testembed");
+    Py_SetProgramName(PROGRAM_NAME);
     Py_Initialize();
 }
 
@@ -363,8 +364,7 @@ config_set_wide_string_list(PyConfig *config, PyWideStringList *list,
 
 static void config_set_program_name(PyConfig *config)
 {
-    /* Use path starting with "./" avoids a search along the PATH */
-    const wchar_t *program_name = L"./_testembed";
+    const wchar_t *program_name = PROGRAM_NAME;
     config_set_string(config, &config->program_name, program_name);
 }
 
@@ -1263,7 +1263,7 @@ static int _audit_hook_run(const char *eventName, PyObject *args, void *userData
 static int test_audit_run_command(void)
 {
     AuditRunCommandTest test = {"cpython.run_command"};
-    wchar_t *argv[] = {L"./_testembed", L"-c", L"pass"};
+    wchar_t *argv[] = {PROGRAM_NAME, L"-c", L"pass"};
 
     Py_IgnoreEnvironmentFlag = 0;
     PySys_AddAuditHook(_audit_hook_run, (void*)&test);
@@ -1274,7 +1274,7 @@ static int test_audit_run_command(void)
 static int test_audit_run_file(void)
 {
     AuditRunCommandTest test = {"cpython.run_file"};
-    wchar_t *argv[] = {L"./_testembed", L"filename.py"};
+    wchar_t *argv[] = {PROGRAM_NAME, L"filename.py"};
 
     Py_IgnoreEnvironmentFlag = 0;
     PySys_AddAuditHook(_audit_hook_run, (void*)&test);
@@ -1312,21 +1312,21 @@ static int run_audit_run_test(int argc, wchar_t **argv, void *test)
 static int test_audit_run_interactivehook(void)
 {
     AuditRunCommandTest test = {"cpython.run_interactivehook", 10};
-    wchar_t *argv[] = {L"./_testembed"};
+    wchar_t *argv[] = {PROGRAM_NAME};
     return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
 }
 
 static int test_audit_run_startup(void)
 {
     AuditRunCommandTest test = {"cpython.run_startup", 10};
-    wchar_t *argv[] = {L"./_testembed"};
+    wchar_t *argv[] = {PROGRAM_NAME};
     return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
 }
 
 static int test_audit_run_stdin(void)
 {
     AuditRunCommandTest test = {"cpython.run_stdin"};
-    wchar_t *argv[] = {L"./_testembed"};
+    wchar_t *argv[] = {PROGRAM_NAME};
     return run_audit_run_test(Py_ARRAY_LENGTH(argv), argv, &test);
 }
 
@@ -1423,6 +1423,95 @@ fail:
 }
 
 
+static int test_init_setpath(void)
+{
+    char *env = getenv("TESTPATH");
+    if (!env) {
+        fprintf(stderr, "missing TESTPATH env var\n");
+        return 1;
+    }
+    wchar_t *path = Py_DecodeLocale(env, NULL);
+    if (path == NULL) {
+        fprintf(stderr, "failed to decode TESTPATH\n");
+        return 1;
+    }
+    Py_SetPath(path);
+    PyMem_RawFree(path);
+    putenv("TESTPATH=");
+
+    Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static int test_init_setpath_config(void)
+{
+    PyStatus status;
+    PyPreConfig preconfig;
+    PyPreConfig_InitPythonConfig(&preconfig);
+
+    /* Explicitly preinitializes with Python preconfiguration to avoid
+      Py_SetPath() implicit preinitialization with compat preconfiguration. */
+    status = Py_PreInitialize(&preconfig);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+
+    char *env = getenv("TESTPATH");
+    if (!env) {
+        fprintf(stderr, "missing TESTPATH env var\n");
+        return 1;
+    }
+    wchar_t *path = Py_DecodeLocale(env, NULL);
+    if (path == NULL) {
+        fprintf(stderr, "failed to decode TESTPATH\n");
+        return 1;
+    }
+    Py_SetPath(path);
+    PyMem_RawFree(path);
+    putenv("TESTPATH=");
+
+    PyConfig config;
+
+    status = PyConfig_InitPythonConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+    config_set_string(&config, &config.program_name, L"conf_program_name");
+    config_set_string(&config, &config.executable, L"conf_executable");
+    init_from_config_clear(&config);
+
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
+static int test_init_setpythonhome(void)
+{
+    char *env = getenv("TESTHOME");
+    if (!env) {
+        fprintf(stderr, "missing TESTHOME env var\n");
+        return 1;
+    }
+    wchar_t *home = Py_DecodeLocale(env, NULL);
+    if (home == NULL) {
+        fprintf(stderr, "failed to decode TESTHOME\n");
+        return 1;
+    }
+    Py_SetPythonHome(home);
+    PyMem_RawFree(home);
+    putenv("TESTHOME=");
+
+    Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static void configure_init_main(PyConfig *config)
 {
     wchar_t* argv[] = {
@@ -1559,7 +1648,11 @@ static struct TestCase TestCases[] = {
     {"test_init_run_main", test_init_run_main},
     {"test_init_main", test_init_main},
     {"test_init_sys_add", test_init_sys_add},
+    {"test_init_setpath", test_init_setpath},
+    {"test_init_setpath_config", test_init_setpath_config},
+    {"test_init_setpythonhome", test_init_setpythonhome},
     {"test_run_main", test_run_main},
+
     {"test_open_code_hook", test_open_code_hook},
     {"test_audit", test_audit},
     {"test_audit_subinterpreter", test_audit_subinterpreter},