[3.8] bpo-38234: Backport init path config changes from master (GH-16423)

* bpo-38234: Py_SetPath() uses the program full path (GH-16357)

Py_SetPath() now sets sys.executable to the program full path
(Py_GetProgramFullPath()), rather than to the program name
(Py_GetProgramName()).

Fix also memory leaks in pathconfig_set_from_config().

(cherry picked from commit 1ce152a42eaa917d7763bce93f1e1ca72530d7ca)

* bpo-38234: Add tests for Python init path config (GH-16358)


(cherry picked from commit bb6bf7d342b4503a6227fd209fac934905b6a1aa)

* bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)

Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed
to test pyvenv.cfg and pybuilddir.txt configuration files.

Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8
encoding, not ASCII.

(cherry picked from commit 52ad33abbfb6637d74932617c7013bae0ccf6e32)

* bpo-38234: Cleanup getpath.c (GH-16367)

* search_for_prefix() directly calls reduce() if found is greater
  than 0.
* Add calculate_pybuilddir() subfunction.
* search_for_prefix(): add path string buffer for readability.
* Fix some error handling code paths: release resources on error.
* calculate_read_pyenv(): rename tmpbuffer to filename.
* test.pythoninfo now also logs windows.dll_path

(cherry picked from commit 221fd84703c545408bbb4a6e0b58459651331f5c)

* bpo-38234: Fix test_embed pathconfig tests (GH-16390)

bpo-38234: On macOS and FreeBSD, the temporary directory can be
symbolic link. For example, /tmp can be a symbolic link to /var/tmp.
Call realpath() to resolve all symbolic links.

(cherry picked from commit 00508a7407d7d300b487532e2271534b20e378a7)

* bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402)

* Add test_embed.test_init_setpath_config(): test Py_SetPath()
  with PyConfig.
* test_init_setpath() and test_init_setpythonhome() no longer call
  Py_SetProgramName(), but use the default program name.
* _PyPathConfig: isolated, site_import  and base_executable
  fields are now only available on Windows.
* If executable is set explicitly in the configuration, ignore
  calculated base_executable: _PyConfig_InitPathConfig() copies
  executable to base_executable.
* Complete path config documentation.

(cherry picked from commit 8bf39b606ef7b02c0279a80789f3c4824b0da5e9)

* bpo-38234: Complete init config documentation (GH-16404)


(cherry picked from commit 88feaecd46a8f427e30ef7ad8cfcddfe392a2402)

* bpo-38234: Fix test_embed.test_init_setpath_config() on FreeBSD (GH-16406)

Explicitly preinitializes with a Python preconfiguration to avoid
Py_SetPath() implicit preinitialization with a compat
preconfiguration.

Fix also test_init_setpath() and test_init_setpythonhome() on macOS:
use self.test_exe as the executable (and base_executable), rather
than shutil.which('python3').

(cherry picked from commit 49d99f01e6e51acec5ca57a02e857f0796bc418b)

* bpo-38234: Py_Initialize() sets global path configuration (GH-16421)

* Py_InitializeFromConfig() now writes PyConfig path configuration to
  the global path configuration (_Py_path_config).
* Add test_embed.test_get_pathconfig().
* Fix typo in _PyWideStringList_Join().

(cherry picked from commit 12f2f177fc483723406d7917194e7f655a20631b)

1 files changed, 96 insertions, 30 deletions

diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py
index e9edf67..d2fa6c5 100644
--- a/Lib/test/pythoninfo.py
+++ b/Lib/test/pythoninfo.py
@@ -161,6 +161,25 @@ def collect_builtins(info_add):
     info_add('builtins.float.double_format', float.__getformat__("double"))
 
 
+def collect_urandom(info_add):
+    import os
+
+    if hasattr(os, 'getrandom'):
+        # PEP 524: Check if system urandom is initialized
+        try:
+            try:
+                os.getrandom(1, os.GRND_NONBLOCK)
+                state = 'ready (initialized)'
+            except BlockingIOError as exc:
+                state = 'not seeded yet (%s)' % exc
+            info_add('os.getrandom', state)
+        except OSError as exc:
+            # Python was compiled on a more recent Linux version
+            # than the current Linux kernel: ignore OSError(ENOSYS)
+            if exc.errno != errno.ENOSYS:
+                raise
+
+
 def collect_os(info_add):
     import os
 
@@ -180,16 +199,16 @@ def collect_os(info_add):
     )
     copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr)
 
-    call_func(info_add, 'os.cwd', os, 'getcwd')
+    call_func(info_add, 'os.getcwd', os, 'getcwd')
 
-    call_func(info_add, 'os.uid', os, 'getuid')
-    call_func(info_add, 'os.gid', os, 'getgid')
+    call_func(info_add, 'os.getuid', os, 'getuid')
+    call_func(info_add, 'os.getgid', os, 'getgid')
     call_func(info_add, 'os.uname', os, 'uname')
 
     def format_groups(groups):
         return ', '.join(map(str, groups))
 
-    call_func(info_add, 'os.groups', os, 'getgroups', formatter=format_groups)
+    call_func(info_add, 'os.getgroups', os, 'getgroups', formatter=format_groups)
 
     if hasattr(os, 'getlogin'):
         try:
@@ -202,7 +221,7 @@ def collect_os(info_add):
             info_add("os.login", login)
 
     call_func(info_add, 'os.cpu_count', os, 'cpu_count')
-    call_func(info_add, 'os.loadavg', os, 'getloadavg')
+    call_func(info_add, 'os.getloadavg', os, 'getloadavg')
 
     # Environment variables used by the stdlib and tests. Don't log the full
     # environment: filter to list to not leak sensitive information.
@@ -286,20 +305,32 @@ def collect_os(info_add):
         os.umask(mask)
         info_add("os.umask", '%03o' % mask)
 
-    if hasattr(os, 'getrandom'):
-        # PEP 524: Check if system urandom is initialized
-        try:
-            try:
-                os.getrandom(1, os.GRND_NONBLOCK)
-                state = 'ready (initialized)'
-            except BlockingIOError as exc:
-                state = 'not seeded yet (%s)' % exc
-            info_add('os.getrandom', state)
-        except OSError as exc:
-            # Python was compiled on a more recent Linux version
-            # than the current Linux kernel: ignore OSError(ENOSYS)
-            if exc.errno != errno.ENOSYS:
-                raise
+
+def collect_pwd(info_add):
+    try:
+        import pwd
+    except ImportError:
+        return
+    import os
+
+    uid = os.getuid()
+    try:
+        entry = pwd.getpwuid(uid)
+    except KeyError:
+        entry = None
+
+    info_add('pwd.getpwuid(%s)'% uid,
+             entry if entry is not None else '<KeyError>')
+
+    if entry is None:
+        # there is nothing interesting to read if the current user identifier
+        # is not the password database
+        return
+
+    if hasattr(os, 'getgrouplist'):
+        groups = os.getgrouplist(entry.pw_name, entry.pw_gid)
+        groups = ', '.join(map(str, groups))
+        info_add('os.getgrouplist', groups)
 
 
 def collect_readline(info_add):
@@ -620,36 +651,71 @@ def collect_subprocess(info_add):
     copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',))
 
 
+def collect_windows(info_add):
+    try:
+        import ctypes
+    except ImportError:
+        return
+
+    if not hasattr(ctypes, 'WinDLL'):
+        return
+
+    ntdll = ctypes.WinDLL('ntdll')
+    BOOLEAN = ctypes.c_ubyte
+
+    try:
+        RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled
+    except AttributeError:
+        res = '<function not available>'
+    else:
+        RtlAreLongPathsEnabled.restype = BOOLEAN
+        RtlAreLongPathsEnabled.argtypes = ()
+        res = bool(RtlAreLongPathsEnabled())
+    info_add('windows.RtlAreLongPathsEnabled', res)
+
+    try:
+        import _winapi
+        dll_path = _winapi.GetModuleFileName(sys.dllhandle)
+        info_add('windows.dll_path', dll_path)
+    except (ImportError, AttributeError):
+        pass
+
+
 def collect_info(info):
     error = False
     info_add = info.add
 
     for collect_func in (
-        # collect_os() should be the first, to check the getrandom() status
-        collect_os,
+        # collect_urandom() must be the first, to check the getrandom() status.
+        # Other functions may block on os.urandom() indirectly and so change
+        # its state.
+        collect_urandom,
 
         collect_builtins,
+        collect_cc,
+        collect_datetime,
+        collect_decimal,
+        collect_expat,
         collect_gdb,
+        collect_gdbm,
+        collect_get_config,
         collect_locale,
+        collect_os,
         collect_platform,
+        collect_pwd,
         collect_readline,
+        collect_resource,
         collect_socket,
         collect_sqlite,
         collect_ssl,
+        collect_subprocess,
         collect_sys,
         collect_sysconfig,
+        collect_testcapi,
         collect_time,
-        collect_datetime,
         collect_tkinter,
+        collect_windows,
         collect_zlib,
-        collect_expat,
-        collect_decimal,
-        collect_testcapi,
-        collect_resource,
-        collect_cc,
-        collect_gdbm,
-        collect_get_config,
-        collect_subprocess,
 
         # Collecting from tests should be last as they have side effects.
         collect_test_socket,