From 1c3de541e64f75046b20cdd27bada1557e550bcd Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 10 Dec 2018 08:11:21 -0800 Subject: bpo-34977: Use venv redirector instead of original python.exe on Windows (GH-11029) --- Doc/library/venv.rst | 17 +- Doc/whatsnew/3.7.rst | 10 + Lib/test/test_venv.py | 1 + Lib/venv/__init__.py | 49 ++--- .../2018-12-07-10-00-38.bpo-34977.agQJbD.rst | 2 + PC/getpathp.c | 8 +- PC/launcher.c | 223 +++++++++++++++++++-- PCbuild/pcbuild.proj | 2 + PCbuild/venvlauncher.vcxproj | 85 ++++++++ PCbuild/venvwlauncher.vcxproj | 85 ++++++++ Tools/msi/lib/lib_files.wxs | 23 +++ 11 files changed, 452 insertions(+), 53 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2018-12-07-10-00-38.bpo-34977.agQJbD.rst create mode 100644 PCbuild/venvlauncher.vcxproj create mode 100644 PCbuild/venvwlauncher.vcxproj diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 7058d3a..efa51e2 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -130,7 +130,6 @@ creation according to their needs, the :class:`EnvBuilder` class. .. versionadded:: 3.6 Added the ``prompt`` parameter - Creators of third-party virtual environment tools will be free to use the provided ``EnvBuilder`` class as a base class. @@ -177,16 +176,15 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: setup_python(context) - Creates a copy of the Python executable (and, under Windows, DLLs) in - the environment. On a POSIX system, if a specific executable - ``python3.x`` was used, symlinks to ``python`` and ``python3`` will be - created pointing to that executable, unless files with those names - already exist. + Creates a copy of the Python executable in the environment on POSIX + systems. If a specific executable ``python3.x`` was used, symlinks to + ``python`` and ``python3`` will be created pointing to that executable, + unless files with those names already exist. .. method:: setup_scripts(context) Installs activation scripts appropriate to the platform into the virtual - environment. + environment. On Windows, also installs the ``python[w].exe`` scripts. .. method:: post_setup(context) @@ -194,6 +192,11 @@ creation according to their needs, the :class:`EnvBuilder` class. implementations to pre-install packages in the virtual environment or perform other post-creation steps. + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries, and so :meth:`setup_python` does nothing + unless running from a build in the source tree. + In addition, :class:`EnvBuilder` provides this utility method that can be called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to assist in installing custom scripts into the virtual environment. diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 8b9b64f..3e94a61 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2512,3 +2512,13 @@ In 3.7.1 the :mod:`tokenize` module now implicitly emits a ``NEWLINE`` token when provided with input that does not have a trailing new line. This behavior now matches what the C tokenizer does internally. (Contributed by Ammar Askar in :issue:`33899`.) + +Notable changes in Python 3.7.2 +=============================== + +In 3.7.2, :mod:`venv` on Windows no longer copies the original binaries, but +creates redirector scripts named ``python.exe`` and ``pythonw.exe`` instead. +This resolves a long standing issue where all virtual environments would have +to be upgraded or recreated with each Python update. However, note that this +release will still require recreation of virtual environments in order to get +the new scripts. diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 461fe7af..22a3b78 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -243,6 +243,7 @@ class BasicTest(BaseTest): self.assertIn('include-system-site-packages = %s\n' % s, data) @unittest.skipUnless(can_symlink(), 'Needs symlinks') + @unittest.skipIf(os.name == 'nt', 'Symlinks are never used on Windows') def test_symlinking(self): """ Test symlinking works as expected diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 0434208..5438b0d 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -64,10 +64,11 @@ class EnvBuilder: self.system_site_packages = False self.create_configuration(context) self.setup_python(context) + if not self.upgrade: + self.setup_scripts(context) if self.with_pip: self._setup_pip(context) if not self.upgrade: - self.setup_scripts(context) self.post_setup(context) if true_system_site_packages: # We had set it to False before, now @@ -158,14 +159,6 @@ class EnvBuilder: f.write('include-system-site-packages = %s\n' % incl) f.write('version = %d.%d.%d\n' % sys.version_info[:3]) - if os.name == 'nt': - def include_binary(self, f): - if f.endswith(('.pyd', '.dll')): - result = True - else: - result = f.startswith('python') and f.endswith('.exe') - return result - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): """ Try symlinking a file, and if that fails, fall back to copying. @@ -195,9 +188,9 @@ class EnvBuilder: binpath = context.bin_path path = context.env_exe copier = self.symlink_or_copy - copier(context.executable, path) dirname = context.python_dir if os.name != 'nt': + copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) for suffix in ('python', 'python3'): @@ -209,26 +202,22 @@ class EnvBuilder: if not os.path.islink(path): os.chmod(path, 0o755) else: - # See bpo-34011. When using a proper install, we should only need to - # copy the top-level of DLLs. - include = self.include_binary - files = [f for f in os.listdir(dirname) if include(f)] - for f in files: - src = os.path.join(dirname, f) - dst = os.path.join(binpath, f) - if dst != context.env_exe: # already done, above - copier(src, dst) - - # When creating from a build directory, we continue to copy all files. + # For normal cases, the venvlauncher will be copied from + # our scripts folder. For builds, we need to copy it + # manually. if sysconfig.is_python_build(True): - subdir = 'DLLs' - dirname = os.path.join(dirname, subdir) - if os.path.isdir(dirname): - files = [f for f in os.listdir(dirname) if include(f)] - for f in files: - src = os.path.join(dirname, f) - dst = os.path.join(binpath, f) - copier(src, dst) + suffix = '.exe' + if context.python_exe.lower().endswith('_d.exe'): + suffix = '_d.exe' + + src = os.path.join(dirname, "venvlauncher" + suffix) + dst = os.path.join(binpath, context.python_exe) + copier(src, dst) + + src = os.path.join(dirname, "venvwlauncher" + suffix) + dst = os.path.join(binpath, "pythonw" + suffix) + copier(src, dst) + # copy init.tcl over for root, dirs, files in os.walk(context.python_dir): if 'init.tcl' in files: @@ -326,7 +315,7 @@ class EnvBuilder: dstfile = os.path.join(dstdir, f) with open(srcfile, 'rb') as f: data = f.read() - if not srcfile.endswith('.exe'): + if not srcfile.endswith(('.exe', '.pdb')): try: data = data.decode('utf-8') data = self.replace_variables(data, context) diff --git a/Misc/NEWS.d/next/Windows/2018-12-07-10-00-38.bpo-34977.agQJbD.rst b/Misc/NEWS.d/next/Windows/2018-12-07-10-00-38.bpo-34977.agQJbD.rst new file mode 100644 index 0000000..12d8db2 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2018-12-07-10-00-38.bpo-34977.agQJbD.rst @@ -0,0 +1,2 @@ +venv on Windows will now use a python.exe redirector rather than copying the +actual binaries from the base environment. diff --git a/PC/getpathp.c b/PC/getpathp.c index 35c31bf..76d3a08 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -536,10 +536,16 @@ static _PyInitError get_program_full_path(const _PyCoreConfig *core_config, PyCalculatePath *calculate, _PyPathConfig *config) { + const wchar_t *pyvenv_launcher; wchar_t program_full_path[MAXPATHLEN+1]; memset(program_full_path, 0, sizeof(program_full_path)); - if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) { + /* The launcher may need to force the executable path to a + * different environment, so override it here. */ + pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__"); + if (pyvenv_launcher && pyvenv_launcher[0]) { + wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher); + } else if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) { /* GetModuleFileName should never fail when passed NULL */ return _Py_INIT_ERR("Cannot determine program path"); } diff --git a/PC/launcher.c b/PC/launcher.c index 2c2da76..4c620da 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -28,7 +28,7 @@ #define RC_NO_PYTHON 103 #define RC_NO_MEMORY 104 /* - * SCRIPT_WRAPPER is used to choose between two variants of an executable built + * SCRIPT_WRAPPER is used to choose one of the variants of an executable built * from this source file. If not defined, the PEP 397 Python launcher is built; * if defined, a script launcher of the type used by setuptools is built, which * looks for a script name related to the executable name and runs that script @@ -40,6 +40,15 @@ #if defined(SCRIPT_WRAPPER) #define RC_NO_SCRIPT 105 #endif +/* + * VENV_REDIRECT is used to choose the variant that looks for an adjacent or + * one-level-higher pyvenv.cfg, and uses its "home" property to locate and + * launch the original python.exe. + */ +#if defined(VENV_REDIRECT) +#define RC_NO_VENV_CFG 106 +#define RC_BAD_VENV_CFG 107 +#endif /* Just for now - static definition */ @@ -97,7 +106,7 @@ error(int rc, wchar_t * format, ... ) #if !defined(_WINDOWS) fwprintf(stderr, L"%ls\n", message); #else - MessageBox(NULL, message, TEXT("Python Launcher is sorry to say ..."), + MessageBoxW(NULL, message, L"Python Launcher is sorry to say ...", MB_OK); #endif exit(rc); @@ -131,6 +140,17 @@ static wchar_t * get_env(wchar_t * key) return buf; } +#if defined(_DEBUG) +#if defined(_WINDOWS) + +#define PYTHON_EXECUTABLE L"pythonw_d.exe" + +#else + +#define PYTHON_EXECUTABLE L"python_d.exe" + +#endif +#else #if defined(_WINDOWS) #define PYTHON_EXECUTABLE L"pythonw.exe" @@ -140,6 +160,7 @@ static wchar_t * get_env(wchar_t * key) #define PYTHON_EXECUTABLE L"python.exe" #endif +#endif #define MAX_VERSION_SIZE 4 @@ -1457,6 +1478,87 @@ show_python_list(wchar_t ** argv) return FALSE; /* If this has been called we cannot continue */ } +#if defined(VENV_REDIRECT) + +static int +find_home_value(const char *buffer, const char **start, DWORD *length) +{ + for (const char *s = strstr(buffer, "home"); s; s = strstr(s + 1, "\nhome")) { + if (*s == '\n') { + ++s; + } + for (int i = 4; i > 0 && *s; --i, ++s); + + while (*s && iswspace(*s)) { + ++s; + } + if (*s != L'=') { + continue; + } + + do { + ++s; + } while (*s && iswspace(*s)); + + *start = s; + char *nl = strchr(s, '\n'); + if (nl) { + *length = (DWORD)((ptrdiff_t)nl - (ptrdiff_t)s); + } else { + *length = (DWORD)strlen(s); + } + return 1; + } + return 0; +} +#endif + +static wchar_t * +wcsdup_pad(const wchar_t *s, int padding, int *newlen) +{ + size_t len = wcslen(s); + len += 1 + padding; + wchar_t *r = (wchar_t *)malloc(len * sizeof(wchar_t)); + if (!r) { + return NULL; + } + if (wcscpy_s(r, len, s)) { + free(r); + return NULL; + } + *newlen = len < MAXINT ? (int)len : MAXINT; + return r; +} + +static wchar_t * +get_process_name() +{ + DWORD bufferLen = MAX_PATH; + DWORD len = bufferLen; + wchar_t *r = NULL; + + while (!r) { + r = (wchar_t *)malloc(bufferLen * sizeof(wchar_t)); + if (!r) { + error(RC_NO_MEMORY, L"out of memory"); + return NULL; + } + len = GetModuleFileNameW(NULL, r, bufferLen); + if (len == 0) { + free(r); + error(0, L"Failed to get module name"); + return NULL; + } else if (len == bufferLen && + GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + free(r); + r = NULL; + bufferLen *= 2; + } + } + + return r; +} + static int process(int argc, wchar_t ** argv) { @@ -1464,21 +1566,27 @@ process(int argc, wchar_t ** argv) wchar_t * command; wchar_t * executable; wchar_t * p; + wchar_t * argv0; int rc = 0; - size_t plen; INSTALLED_PYTHON * ip; BOOL valid; DWORD size, attrs; - HRESULT hr; wchar_t message[MSGSIZE]; void * version_data; VS_FIXEDFILEINFO * file_info; UINT block_size; - int index; -#if defined(SCRIPT_WRAPPER) +#if defined(VENV_REDIRECT) + wchar_t * venv_cfg_path; int newlen; +#elif defined(SCRIPT_WRAPPER) wchar_t * newcommand; wchar_t * av[2]; + int newlen; + HRESULT hr; + int index; +#else + HRESULT hr; + int index; #endif setvbuf(stderr, (char *)NULL, _IONBF, 0); @@ -1496,6 +1604,7 @@ process(int argc, wchar_t ** argv) #else debug(L"launcher executable: Console\n"); #endif +#if !defined(VENV_REDIRECT) /* Get the local appdata folder (non-roaming) */ hr = SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, appdata_ini_path); @@ -1504,9 +1613,7 @@ process(int argc, wchar_t ** argv) appdata_ini_path[0] = L'\0'; } else { - plen = wcslen(appdata_ini_path); - p = &appdata_ini_path[plen]; - wcsncpy_s(p, MAX_PATH - plen, L"\\py.ini", _TRUNCATE); + wcsncat_s(appdata_ini_path, MAX_PATH, L"\\py.ini", _TRUNCATE); attrs = GetFileAttributesW(appdata_ini_path); if (attrs == INVALID_FILE_ATTRIBUTES) { debug(L"File '%ls' non-existent\n", appdata_ini_path); @@ -1515,8 +1622,9 @@ process(int argc, wchar_t ** argv) debug(L"Using local configuration file '%ls'\n", appdata_ini_path); } } - plen = GetModuleFileNameW(NULL, launcher_ini_path, MAX_PATH); - size = GetFileVersionInfoSizeW(launcher_ini_path, &size); +#endif + argv0 = get_process_name(); + size = GetFileVersionInfoSizeW(argv0, &size); if (size == 0) { winerror(GetLastError(), message, MSGSIZE); debug(L"GetFileVersionInfoSize failed: %ls\n", message); @@ -1524,7 +1632,7 @@ process(int argc, wchar_t ** argv) else { version_data = malloc(size); if (version_data) { - valid = GetFileVersionInfoW(launcher_ini_path, 0, size, + valid = GetFileVersionInfoW(argv0, 0, size, version_data); if (!valid) debug(L"GetFileVersionInfo failed: %X\n", GetLastError()); @@ -1541,15 +1649,51 @@ process(int argc, wchar_t ** argv) free(version_data); } } + +#if defined(VENV_REDIRECT) + /* Allocate some extra space for new filenames */ + venv_cfg_path = wcsdup_pad(argv0, 32, &newlen); + if (!venv_cfg_path) { + error(RC_NO_MEMORY, L"Failed to copy module name"); + } + p = wcsrchr(venv_cfg_path, L'\\'); + + if (p == NULL) { + error(RC_NO_VENV_CFG, L"No pyvenv.cfg file"); + } + p[0] = L'\0'; + wcscat_s(venv_cfg_path, newlen, L"\\pyvenv.cfg"); + attrs = GetFileAttributesW(venv_cfg_path); + if (attrs == INVALID_FILE_ATTRIBUTES) { + debug(L"File '%ls' non-existent\n", venv_cfg_path); + p[0] = '\0'; + p = wcsrchr(venv_cfg_path, L'\\'); + if (p != NULL) { + p[0] = '\0'; + wcscat_s(venv_cfg_path, newlen, L"\\pyvenv.cfg"); + attrs = GetFileAttributesW(venv_cfg_path); + if (attrs == INVALID_FILE_ATTRIBUTES) { + debug(L"File '%ls' non-existent\n", venv_cfg_path); + error(RC_NO_VENV_CFG, L"No pyvenv.cfg file"); + } + } + } + debug(L"Using venv configuration file '%ls'\n", venv_cfg_path); +#else + /* Allocate some extra space for new filenames */ + if (wcscpy_s(launcher_ini_path, MAX_PATH, argv0)) { + error(RC_NO_MEMORY, L"Failed to copy module name"); + } p = wcsrchr(launcher_ini_path, L'\\'); + if (p == NULL) { debug(L"GetModuleFileNameW returned value has no backslash: %ls\n", launcher_ini_path); launcher_ini_path[0] = L'\0'; } else { - wcsncpy_s(p, MAX_PATH - (p - launcher_ini_path), L"\\py.ini", - _TRUNCATE); + p[0] = L'\0'; + wcscat_s(launcher_ini_path, MAX_PATH, L"\\py.ini"); attrs = GetFileAttributesW(launcher_ini_path); if (attrs == INVALID_FILE_ATTRIBUTES) { debug(L"File '%ls' non-existent\n", launcher_ini_path); @@ -1558,6 +1702,7 @@ process(int argc, wchar_t ** argv) debug(L"Using global configuration file '%ls'\n", launcher_ini_path); } } +#endif command = skip_me(GetCommandLineW()); debug(L"Called with command line: %ls\n", command); @@ -1593,6 +1738,55 @@ process(int argc, wchar_t ** argv) command = newcommand; valid = FALSE; } +#elif defined(VENV_REDIRECT) + { + FILE *f; + char buffer[4096]; /* 4KB should be enough for anybody */ + char *start; + DWORD len, cch, cch_actual; + size_t cb; + if (_wfopen_s(&f, venv_cfg_path, L"r")) { + error(RC_BAD_VENV_CFG, L"Cannot read '%ls'", venv_cfg_path); + } + cb = fread_s(buffer, sizeof(buffer), sizeof(buffer[0]), + sizeof(buffer) / sizeof(buffer[0]), f); + fclose(f); + + if (!find_home_value(buffer, &start, &len)) { + error(RC_BAD_VENV_CFG, L"Cannot find home in '%ls'", + venv_cfg_path); + } + + cch = MultiByteToWideChar(CP_UTF8, 0, start, len, NULL, 0); + if (!cch) { + error(0, L"Cannot determine memory for home path"); + } + cch += (DWORD)wcslen(PYTHON_EXECUTABLE) + 1 + 1; /* include sep and null */ + executable = (wchar_t *)malloc(cch * sizeof(wchar_t)); + if (executable == NULL) { + error(RC_NO_MEMORY, L"A memory allocation failed"); + } + cch_actual = MultiByteToWideChar(CP_UTF8, 0, start, len, executable, cch); + if (!cch_actual) { + error(RC_BAD_VENV_CFG, L"Cannot decode home path in '%ls'", + venv_cfg_path); + } + if (executable[cch_actual - 1] != L'\\') { + executable[cch_actual++] = L'\\'; + executable[cch_actual] = L'\0'; + } + if (wcscat_s(executable, cch, PYTHON_EXECUTABLE)) { + error(RC_BAD_VENV_CFG, L"Cannot create executable path from '%ls'", + venv_cfg_path); + } + if (GetFileAttributesW(executable) == INVALID_FILE_ATTRIBUTES) { + error(RC_NO_PYTHON, L"No Python at '%ls'", executable); + } + if (!SetEnvironmentVariableW(L"__PYVENV_LAUNCHER__", argv0)) { + error(0, L"Failed to set launcher environment"); + } + valid = 1; + } #else if (argc <= 1) { valid = FALSE; @@ -1600,7 +1794,6 @@ process(int argc, wchar_t ** argv) } else { p = argv[1]; - plen = wcslen(p); if ((argc == 2) && // list version args (!wcsncmp(p, L"-0", wcslen(L"-0")) || !wcsncmp(p, L"--list", wcslen(L"--list")))) diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 9e103e1..77341b4 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -70,6 +70,8 @@ + + diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj new file mode 100644 index 0000000..295b363 --- /dev/null +++ b/PCbuild/venvlauncher.vcxproj @@ -0,0 +1,85 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} + venvlauncher + venvlauncher + false + + + + + Application + MultiByte + + + + + + ClCompile + + + + + + + + + _CONSOLE;VENV_REDIRECT;%(PreprocessorDefinitions) + MultiThreaded + + + PY_ICON;%(PreprocessorDefinitions) + + + version.lib;%(AdditionalDependencies) + Console + + + + + + + + + + + + + + + diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj new file mode 100644 index 0000000..e7ba25d --- /dev/null +++ b/PCbuild/venvwlauncher.vcxproj @@ -0,0 +1,85 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {FDB84CBB-2FB6-47C8-A2D6-091E0833239D} + venvwlauncher + venvwlauncher + false + + + + + Application + MultiByte + + + + + + ClCompile + + + + + + + + + _WINDOWS;VENV_REDIRECT;%(PreprocessorDefinitions) + MultiThreaded + + + PYW_ICON;%(PreprocessorDefinitions) + + + version.lib;%(AdditionalDependencies) + Windows + + + + + + + + + + + + + + + diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index da9d1c9..4bd0c57 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -2,6 +2,8 @@ + + @@ -20,10 +22,25 @@ + + + + + + + + + + + + + + + @@ -42,6 +59,12 @@ + + + + + + -- cgit v0.12