Merge with 3.6

8 files changed, 130 insertions, 64 deletions

diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst
index 5c2d864..12bdd9d 100644
--- a/Doc/using/windows.rst
+++ b/Doc/using/windows.rst
@@ -720,15 +720,24 @@ installation directory.  So, if you had installed Python to
 :file:`C:\\Python\\Lib\\` and third-party modules should be stored in
 :file:`C:\\Python\\Lib\\site-packages\\`.
 
-To completely override :data:`sys.path`, create a text file named ``'sys.path'``
-containing a list of paths alongside the Python executable. This will ignore all
-registry settings and environment variables, enable isolated mode, disable
-importing :mod:`site`, and fill :data:`sys.path` with exactly the paths listed
-in the file. Paths may be absolute or relative to the directory containing the
-file.
+To completely override :data:`sys.path`, create a ``._pth`` file with the same
+name as the DLL (``python36._pth``) or the executable (``python._pth``) and
+specify one line for each path to add to :data:`sys.path`. The file based on the
+DLL name overrides the one based on the executable, which allows paths to be
+restricted for any program loading the runtime if desired.
 
-When the ``'sys.path'`` file is missing, this is how :data:`sys.path` is
-populated on Windows:
+When the file exists, all registry and environment variables are ignored,
+isolated mode is enabled, and :mod:`site` is not imported unless one line in the
+file specifies ``import site``. Blank paths and lines starting with ``#`` are
+ignored. Each path may be absolute or relative to the location of the file.
+Import statements other than to ``site`` are not permitted, and arbitrary code
+cannot be specified.
+
+Note that ``.pth`` files (without leading underscore) will be processed normally
+by the :mod:`site` module.
+
+When no ``._pth`` file is found, this is how :data:`sys.path` is populated on
+Windows:
 
 * An empty entry is added at the start, which corresponds to the current
   directory.
@@ -782,9 +791,10 @@ The end result of all this is:
 For those who want to bundle Python into their application or distribution, the
 following advice will prevent conflicts with other installations:
 
-* Include a ``sys.path`` file alongside your executable containing the
-  directories to include. This will ignore user site-packages and other paths
-  listed in the registry or in environment variables.
+* Include a ``._pth`` file alongside your executable containing the
+  directories to include. This will ignore paths listed in the registry and
+  environment variables, and also ignore :mod:`site` unless ``import site`` is
+  listed.
 
 * If you are loading :file:`python3.dll` or :file:`python36.dll` in your own
   executable, explicitly call :c:func:`Py_SetPath` or (at least)
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
index 941a5eb..cabff60 100644
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -108,7 +108,7 @@ Windows improvements:
   which means that when the 260 character path limit may no longer apply.
   See :ref:`removing the MAX_PATH limitation <max-path>` for details.
 
-* A ``sys.path`` file can be added to force isolated mode and fully specify
+* A ``._pth`` file can be added to force isolated mode and fully specify
   all search paths to avoid registry and environment lookup. See
   :ref:`the documentation <finding_modules>` for more information.
 
diff --git a/Lib/site.py b/Lib/site.py
index 5250266..b637635 100644
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -60,7 +60,8 @@ omitted because it is not mentioned in either path configuration file.
 
 The readline module is also automatically configured to enable
 completion for systems that support it.  This can be overridden in
-sitecustomize, usercustomize or PYTHONSTARTUP.
+sitecustomize, usercustomize or PYTHONSTARTUP.  Starting Python in
+isolated mode (-I) disables automatic readline configuration.
 
 After these operations, an attempt is made to import a module
 named sitecustomize, which can perform arbitrary additional
@@ -491,7 +492,7 @@ def execsitecustomize():
             else:
                 raise
     except Exception as err:
-        if os.environ.get("PYTHONVERBOSE"):
+        if sys.flags.verbose:
             sys.excepthook(*sys.exc_info())
         else:
             sys.stderr.write(
@@ -511,7 +512,7 @@ def execusercustomize():
             else:
                 raise
     except Exception as err:
-        if os.environ.get("PYTHONVERBOSE"):
+        if sys.flags.verbose:
             sys.excepthook(*sys.exc_info())
         else:
             sys.stderr.write(
@@ -538,12 +539,13 @@ def main():
     setquit()
     setcopyright()
     sethelper()
-    enablerlcompleter()
+    if not sys.flags.isolated:
+        enablerlcompleter()
     execsitecustomize()
     if ENABLE_USER_SITE:
         execusercustomize()
 
-# Prevent edition of sys.path when python was started with -S and
+# Prevent extending of sys.path when python was started with -S and
 # site is imported later.
 if not sys.flags.no_site:
     main()
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
index 9afa56e..b048648 100644
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -140,8 +140,6 @@ class HelperFunctionsTests(unittest.TestCase):
         self.assertRegex(err_out.getvalue(), 'Traceback')
         self.assertRegex(err_out.getvalue(), 'ModuleNotFoundError')
 
-    @unittest.skipIf(sys.platform == "win32", "Windows does not raise an "
-                      "error for file paths containing null characters")
     def test_addpackage_import_bad_pth_file(self):
         # Issue 5258
         pth_dir, pth_fn = self.make_pth("abc\x00def\n")
@@ -447,10 +445,9 @@ class StartupImportTests(unittest.TestCase):
         popen = subprocess.Popen([sys.executable, '-I', '-v', '-c',
                                   'import sys; print(set(sys.modules))'],
                                  stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
+                                 stderr=subprocess.PIPE,
+                                 encoding='utf-8')
         stdout, stderr = popen.communicate()
-        stdout = stdout.decode('utf-8')
-        stderr = stderr.decode('utf-8')
         modules = eval(stdout)
 
         self.assertIn('site', modules)
@@ -474,6 +471,5 @@ class StartupImportTests(unittest.TestCase):
         if sys.platform != 'darwin':
             self.assertFalse(modules.intersection(collection_mods), stderr)
 
-
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS b/Misc/NEWS
index 088dfa9..90dfc5f 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,8 @@ What's New in Python 3.7.0 alpha 1
 Core and Builtins
 -----------------
 
+- Issue #28192: Don't import readline in isolated mode.
+
 - Issue #27441: Remove some redundant assignments to ob_size in longobject.c.
   Thanks Oren Milman.
 
@@ -70,6 +72,13 @@ Library
 - Issue #27759: Fix selectors incorrectly retain invalid file descriptors.
   Patch by Mark Williams.
 
+Windows
+-------
+
+- Issue #28137: Renames Windows path file to ._pth
+
+- Issue #28138: Windows ._pth file should allow import site
+
 Build
 -----
 
diff --git a/Modules/main.c b/Modules/main.c
index 0b82f48..6986d94 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -703,7 +703,8 @@ Py_Main(int argc, wchar_t **argv)
     PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind);
 
     if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) &&
-        isatty(fileno(stdin))) {
+        isatty(fileno(stdin)) &&
+        !Py_IsolatedFlag) {
         PyObject *v;
         v = PyImport_ImportModule("readline");
         if (v == NULL)
diff --git a/PC/getpathp.c b/PC/getpathp.c
index c05754a..31f973e 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -6,8 +6,9 @@
    PATH RULES FOR WINDOWS:
    This describes how sys.path is formed on Windows.  It describes the
    functionality, not the implementation (ie, the order in which these
-   are actually fetched is different). The presence of a sys.path file
-   alongside the program overrides these rules - see below.
+   are actually fetched is different). The presence of a python._pth or
+   pythonXY._pth file alongside the program overrides these rules - see
+   below.
 
    * Python always adds an empty entry at the start, which corresponds
      to the current directory.
@@ -37,11 +38,21 @@
      used (eg. .\Lib;.\DLLs, etc)
 
 
-   If a sys.path file exists adjacent to python.exe, it must contain a
-   list of paths to add to sys.path, one per line (like a .pth file but without
-   the ability to execute arbitrary code). Each path is relative to the
-   directory containing the file. No other paths are added to the search path,
-   and the registry finder is not enabled.
+   If a '._pth' file exists adjacent to the executable with the same base name
+   (e.g. python._pth adjacent to python.exe) or adjacent to the shared library
+   (e.g. python36._pth adjacent to python36.dll), it is used in preference to
+   the above process. The shared library file takes precedence over the
+   executable. The path file must contain a list of paths to add to sys.path,
+   one per line. Each path is relative to the directory containing the file.
+   Blank lines and comments beginning with '#' are permitted.
+
+   In the presence of this ._pth file, no other paths are added to the search
+   path, the registry finder is not enabled, site.py is not imported and
+   isolated mode is enabled. The site package can be enabled by including a
+   line reading "import site"; no other imports are recognized. Any invalid
+   entry (other than directories that do not exist) will result in immediate
+   termination of the program.
+
 
   The end result of all this is:
   * When running python.exe, or any other .exe in the main Python directory
@@ -61,8 +72,9 @@
   * An embedding application can use Py_SetPath() to override all of
     these automatic path computations.
 
-  * An isolation install of Python can disable all implicit paths by
-    providing a sys.path file.
+  * An install of Python can fully specify the contents of sys.path using
+    either a 'EXENAME._pth' or 'DLLNAME._pth' file, optionally including
+    "import site" to enable the site module.
 
    ---------------------------------------------------------------- */
 
@@ -135,6 +147,33 @@ reduce(wchar_t *dir)
     dir[i] = '\0';
 }
 
+static int
+change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext)
+{
+    size_t src_len = wcsnlen_s(src, MAXPATHLEN+1);
+    size_t i = src_len;
+    if (i >= MAXPATHLEN+1)
+        Py_FatalError("buffer overflow in getpathp.c's reduce()");
+
+    while (i > 0 && src[i] != '.' && !is_sep(src[i]))
+        --i;
+
+    if (i == 0) {
+        dest[0] = '\0';
+        return -1;
+    }
+
+    if (is_sep(src[i]))
+        i = src_len;
+
+    if (wcsncpy_s(dest, MAXPATHLEN+1, src, i) ||
+        wcscat_s(dest, MAXPATHLEN+1, ext)) {
+        dest[0] = '\0';
+        return -1;
+    }
+
+    return 0;
+}
 
 static int
 exists(wchar_t *filename)
@@ -499,12 +538,17 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value)
 }
 
 static int
-read_sys_path_file(const wchar_t *path, const wchar_t *prefix)
+read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite)
 {
     FILE *sp_file = _Py_wfopen(path, L"r");
     if (sp_file == NULL)
         return -1;
 
+    wcscpy_s(prefix, MAXPATHLEN+1, path);
+    reduce(prefix);
+    *isolated = 1;
+    *nosite = 1;
+
     size_t bufsiz = MAXPATHLEN;
     size_t prefixlen = wcslen(prefix);
 
@@ -516,16 +560,25 @@ read_sys_path_file(const wchar_t *path, const wchar_t *prefix)
         char *p = fgets(line, MAXPATHLEN + 1, sp_file);
         if (!p)
             break;
+        if (*p == '\0' || *p == '#')
+            continue;
+        while (*++p) {
+            if (*p == '\r' || *p == '\n') {
+                *p = '\0';
+                break;
+            }
+        }
 
-        DWORD n = strlen(line);
-        if (n == 0 || p[n - 1] != '\n')
-            break;
-        if (n > 2 && p[n - 1] == '\r')
-            --n;
+        if (strcmp(line, "import site") == 0) {
+            *nosite = 0;
+            continue;
+        } else if (strncmp(line, "import ", 7) == 0) {
+            Py_FatalError("only 'import site' is supported in ._pth file");
+        }
 
-        DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, n - 1, NULL, 0);
+        DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0);
         wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t));
-        wn = MultiByteToWideChar(CP_UTF8, 0, line, n - 1, wline, wn);
+        wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1);
         wline[wn] = '\0';
 
         while (wn + prefixlen + 4 > bufsiz) {
@@ -539,8 +592,8 @@ read_sys_path_file(const wchar_t *path, const wchar_t *prefix)
 
         if (buf[0])
             wcscat_s(buf, bufsiz, L";");
+
         wchar_t *b = &buf[wcslen(buf)];
-        
         wcscat_s(buf, bufsiz, prefix);
         join(b, wline);
 
@@ -586,13 +639,12 @@ calculate_path(void)
     {
         wchar_t spbuffer[MAXPATHLEN+1];
 
-        wcscpy_s(spbuffer, MAXPATHLEN+1, argv0_path);
-        join(spbuffer, L"sys.path");
-        if (exists(spbuffer) && read_sys_path_file(spbuffer, argv0_path) == 0) {
-            wcscpy_s(prefix, MAXPATHLEN + 1, argv0_path);
-            Py_IsolatedFlag = 1;
-            Py_NoSiteFlag = 1;
-            return;
+        if ((dllpath[0] && !change_ext(spbuffer, dllpath, L"._pth") && exists(spbuffer)) ||
+            (progpath[0] && !change_ext(spbuffer, progpath, L"._pth") && exists(spbuffer))) {
+
+            if (!read_pth_file(spbuffer, prefix, &Py_IsolatedFlag, &Py_NoSiteFlag)) {
+                return;
+            }
         }
     }
 
@@ -631,16 +683,7 @@ calculate_path(void)
     }
 
     /* Calculate zip archive path from DLL or exe path */
-    if (wcscpy_s(zip_path, MAXPATHLEN + 1, dllpath[0] ? dllpath : progpath)) {
-        /* exceeded buffer length - ignore zip_path */
-        zip_path[0] = '\0';
-    } else {
-        wchar_t *dot = wcsrchr(zip_path, '.');
-        if (!dot || wcscpy_s(dot, MAXPATHLEN + 1 - (dot - zip_path), L".zip")) {
-            /* exceeded buffer length - ignore zip_path */
-            zip_path[0] = L'\0';
-        }
-    }
+    change_ext(zip_path, dllpath[0] ? dllpath : progpath, L".zip");
 
     if (pythonhome == NULL || *pythonhome == '\0') {
         if (zip_path[0] && exists(zip_path)) {
diff --git a/Tools/msi/make_zip.py b/Tools/msi/make_zip.py
index e9d6dbc..ebb1766 100644
--- a/Tools/msi/make_zip.py
+++ b/Tools/msi/make_zip.py
@@ -91,11 +91,13 @@ def include_in_tools(p):
 
     return p.suffix.lower() in {'.py', '.pyw', '.txt'}
 
+BASE_NAME = 'python{0.major}{0.minor}'.format(sys.version_info)
+
 FULL_LAYOUT = [
     ('/', 'PCBuild/$arch', 'python.exe', is_not_debug),
     ('/', 'PCBuild/$arch', 'pythonw.exe', is_not_debug),
-    ('/', 'PCBuild/$arch', 'python{0.major}.dll'.format(sys.version_info), is_not_debug),
-    ('/', 'PCBuild/$arch', 'python{0.major}{0.minor}.dll'.format(sys.version_info), is_not_debug),
+    ('/', 'PCBuild/$arch', 'python{}.dll'.format(sys.version_info.major), is_not_debug),
+    ('/', 'PCBuild/$arch', '{}.dll'.format(BASE_NAME), is_not_debug),
     ('DLLs/', 'PCBuild/$arch', '*.pyd', is_not_debug),
     ('DLLs/', 'PCBuild/$arch', '*.dll', is_not_debug_or_python),
     ('include/', 'include', '*.h', None),
@@ -109,7 +111,7 @@ EMBED_LAYOUT = [
     ('/', 'PCBuild/$arch', 'python*.exe', is_not_debug),
     ('/', 'PCBuild/$arch', '*.pyd', is_not_debug),
     ('/', 'PCBuild/$arch', '*.dll', is_not_debug),
-    ('python{0.major}{0.minor}.zip'.format(sys.version_info), 'Lib', '**/*', include_in_lib),
+    ('{}.zip'.format(BASE_NAME), 'Lib', '**/*', include_in_lib),
 ]
 
 if os.getenv('DOC_FILENAME'):
@@ -209,9 +211,12 @@ def main():
             print('Copied {} files'.format(copied))
 
         if ns.embed:
-            with open(str(temp / 'sys.path'), 'w') as f:
-                print('python{0.major}{0.minor}.zip'.format(sys.version_info), file=f)
+            with open(str(temp / (BASE_NAME + '._pth')), 'w') as f:
+                print(BASE_NAME + '.zip', file=f)
                 print('.', file=f)
+                print('', file=f)
+                print('# Uncomment to run site.main() automatically', file=f)
+                print('#import site', file=f)
 
         if out:
             total = copy_to_layout(out, rglob(temp, '**/*', None))