diff options
| -rw-r--r-- | Doc/library/ensurepip.rst | 6 | ||||
| -rw-r--r-- | Doc/whatsnew/3.10.rst | 12 | ||||
| -rw-r--r-- | Lib/ensurepip/__init__.py | 114 | ||||
| -rw-r--r-- | Lib/test/test_ensurepip.py | 69 | ||||
| -rw-r--r-- | Makefile.pre.in | 2 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst | 9 | ||||
| -rwxr-xr-x | configure | 28 | ||||
| -rw-r--r-- | configure.ac | 16 |
8 files changed, 220 insertions, 36 deletions
diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index a522125..fa1b42c 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -48,7 +48,7 @@ The simplest possible invocation is:: This invocation will install ``pip`` if it is not already installed, but otherwise does nothing. To ensure the installed version of ``pip`` -is at least as recent as the one bundled with ``ensurepip``, pass the +is at least as recent as the one available in ``ensurepip``, pass the ``--upgrade`` option:: python -m ensurepip --upgrade @@ -86,7 +86,7 @@ Module API .. function:: version() - Returns a string specifying the bundled version of pip that will be + Returns a string specifying the available version of pip that will be installed when bootstrapping an environment. .. function:: bootstrap(root=None, upgrade=False, user=False, \ @@ -100,7 +100,7 @@ Module API for the current environment. *upgrade* indicates whether or not to upgrade an existing installation - of an earlier version of ``pip`` to the bundled version. + of an earlier version of ``pip`` to the available version. *user* indicates whether to use the user scheme rather than installing globally. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 7edc552..7fe2b96 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -621,6 +621,18 @@ Build Changes don't build nor install test modules. (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) +* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If + specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` + wheel packages in this directory: if both are present, these wheel packages + are used instead of ensurepip bundled wheel packages. + + Some Linux distribution packaging policies recommend against bundling + dependencies. For example, Fedora installs wheel packages in the + ``/usr/share/python-wheels/`` directory and don't install the + ``ensurepip._bundled`` package. + + (Contributed by Victor Stinner in :issue:`42856`.) + C API Changes ============= diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index cb2882e..2276fd7 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,27 +1,82 @@ +import collections import os import os.path +import subprocess import sys -import runpy +import sysconfig import tempfile -import subprocess from importlib import resources -from . import _bundled - __all__ = ["version", "bootstrap"] - - +_PACKAGE_NAMES = ('setuptools', 'pip') _SETUPTOOLS_VERSION = "47.1.0" - _PIP_VERSION = "20.2.3" - _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION, "py3"), ("pip", _PIP_VERSION, "py2.py3"), ] +# Packages bundled in ensurepip._bundled have wheel_name set. +# Packages from WHEEL_PKG_DIR have wheel_path set. +_Package = collections.namedtuple('Package', + ('version', 'wheel_name', 'wheel_path')) + +# Directory of system wheel packages. Some Linux distribution packaging +# policies recommend against bundling dependencies. For example, Fedora +# installs wheel packages in the /usr/share/python-wheels/ directory and don't +# install the ensurepip._bundled package. +_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') + + +def _find_packages(path): + packages = {} + try: + filenames = os.listdir(path) + except OSError: + # Ignore: path doesn't exist or permission error + filenames = () + # Make the code deterministic if a directory contains multiple wheel files + # of the same package, but don't attempt to implement correct version + # comparison since this case should not happen. + filenames = sorted(filenames) + for filename in filenames: + # filename is like 'pip-20.2.3-py2.py3-none-any.whl' + if not filename.endswith(".whl"): + continue + for name in _PACKAGE_NAMES: + prefix = name + '-' + if filename.startswith(prefix): + break + else: + continue + + # Extract '20.2.2' from 'pip-20.2.2-py2.py3-none-any.whl' + version = filename.removeprefix(prefix).partition('-')[0] + wheel_path = os.path.join(path, filename) + packages[name] = _Package(version, None, wheel_path) + return packages + + +def _get_packages(): + global _PACKAGES, _WHEEL_PKG_DIR + if _PACKAGES is not None: + return _PACKAGES + + packages = {} + for name, version, py_tag in _PROJECTS: + wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" + packages[name] = _Package(version, wheel_name, None) + if _WHEEL_PKG_DIR: + dir_packages = _find_packages(_WHEEL_PKG_DIR) + # only used the wheel package directory if all packages are found there + if all(name in dir_packages for name in _PACKAGE_NAMES): + packages = dir_packages + _PACKAGES = packages + return packages +_PACKAGES = None + def _run_pip(args, additional_paths=None): # Run the bootstraping in a subprocess to avoid leaking any state that happens @@ -42,7 +97,8 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _PIP_VERSION + return _get_packages()['pip'].version + def _disable_pip_configuration_settings(): # We deliberately ignore all pip environment variables @@ -104,16 +160,23 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path additional_paths = [] - for project, version, py_tag in _PROJECTS: - wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag) - whl = resources.read_binary( - _bundled, - wheel_name, - ) - with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + for name, package in _get_packages().items(): + if package.wheel_name: + # Use bundled wheel package + from ensurepip import _bundled + wheel_name = package.wheel_name + whl = resources.read_binary(_bundled, wheel_name) + else: + # Use the wheel package directory + with open(package.wheel_path, "rb") as fp: + whl = fp.read() + wheel_name = os.path.basename(package.wheel_path) + + filename = os.path.join(tmpdir, wheel_name) + with open(filename, "wb") as fp: fp.write(whl) - additional_paths.append(os.path.join(tmpdir, wheel_name)) + additional_paths.append(filename) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -126,7 +189,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -139,11 +202,14 @@ def _uninstall_helper(*, verbosity=0): except ImportError: return - # If the pip version doesn't match the bundled one, leave it alone - if pip.__version__ != _PIP_VERSION: - msg = ("ensurepip will only uninstall a matching version " - "({!r} installed, {!r} bundled)") - print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + # If the installed pip version doesn't match the available one, + # leave it alone + available_version = version() + if pip.__version__ != available_version: + print(f"ensurepip will only uninstall a matching version " + f"({pip.__version__!r} installed, " + f"{available_version!r} available)", + file=sys.stderr) return _disable_pip_configuration_settings() @@ -153,7 +219,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) def _main(argv=None): diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 4786d28..bfca0cd 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -1,19 +1,68 @@ -import unittest -import unittest.mock -import test.support +import contextlib import os import os.path -import contextlib import sys +import tempfile +import test.support +import unittest +import unittest.mock import ensurepip import ensurepip._uninstall -class TestEnsurePipVersion(unittest.TestCase): +class TestPackages(unittest.TestCase): + def touch(self, directory, filename): + fullname = os.path.join(directory, filename) + open(fullname, "wb").close() + + def test_version(self): + # Test version() + with tempfile.TemporaryDirectory() as tmpdir: + self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl") + self.touch(tmpdir, "setuptools-49.1.3-py3-none-any.whl") + with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), + unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): + self.assertEqual(ensurepip.version(), '1.2.3b1') + + def test_get_packages_no_dir(self): + # Test _get_packages() without a wheel package directory + with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), + unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)): + packages = ensurepip._get_packages() + + # when bundled wheel packages are used, we get _PIP_VERSION + self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) + + # use bundled wheel packages + self.assertIsNotNone(packages['pip'].wheel_name) + self.assertIsNotNone(packages['setuptools'].wheel_name) + + def test_get_packages_with_dir(self): + # Test _get_packages() with a wheel package directory + setuptools_filename = "setuptools-49.1.3-py3-none-any.whl" + pip_filename = "pip-20.2.2-py2.py3-none-any.whl" + + with tempfile.TemporaryDirectory() as tmpdir: + self.touch(tmpdir, setuptools_filename) + self.touch(tmpdir, pip_filename) + # not used, make sure that it's ignored + self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl") + + with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), + unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): + packages = ensurepip._get_packages() + + self.assertEqual(packages['setuptools'].version, '49.1.3') + self.assertEqual(packages['setuptools'].wheel_path, + os.path.join(tmpdir, setuptools_filename)) + self.assertEqual(packages['pip'].version, '20.2.2') + self.assertEqual(packages['pip'].wheel_path, + os.path.join(tmpdir, pip_filename)) + + # wheel package is ignored + self.assertEqual(sorted(packages), ['pip', 'setuptools']) - def test_returns_version(self): - self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) class EnsurepipMixin: @@ -27,6 +76,8 @@ class EnsurepipMixin: real_devnull = os.devnull os_patch = unittest.mock.patch("ensurepip.os") patched_os = os_patch.start() + # But expose os.listdir() used by _find_packages() + patched_os.listdir = os.listdir self.addCleanup(os_patch.stop) patched_os.devnull = real_devnull patched_os.path = os.path @@ -147,7 +198,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull) @contextlib.contextmanager -def fake_pip(version=ensurepip._PIP_VERSION): +def fake_pip(version=ensurepip.version()): if version is None: pip = None else: @@ -243,7 +294,7 @@ class TestUninstall(EnsurepipMixin, unittest.TestCase): # Basic testing of the main functions and their argument parsing -EXPECTED_VERSION_OUTPUT = "pip " + ensurepip._PIP_VERSION +EXPECTED_VERSION_OUTPUT = "pip " + ensurepip.version() class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): diff --git a/Makefile.pre.in b/Makefile.pre.in index fa0b9d8..ca6b518 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -146,6 +146,8 @@ CONFINCLUDEDIR= $(exec_prefix)/include PLATLIBDIR= @PLATLIBDIR@ SCRIPTDIR= $(prefix)/$(PLATLIBDIR) ABIFLAGS= @ABIFLAGS@ +# Variable used by ensurepip +WHEEL_PKG_DIR= @WHEEL_PKG_DIR@ # Detailed destination directories BINLIBDEST= @BINLIBDEST@ diff --git a/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst b/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst new file mode 100644 index 0000000..6aab7a6 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst @@ -0,0 +1,9 @@ +Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If +specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` +wheel packages in this directory: if both are present, these wheel packages are +used instead of ensurepip bundled wheel packages. + +Some Linux distribution packaging policies recommend against bundling +dependencies. For example, Fedora installs wheel packages in the +``/usr/share/python-wheels/`` directory and don't install the +``ensurepip._bundled`` package. @@ -630,6 +630,7 @@ OPENSSL_INCLUDES ENSUREPIP SRCDIRS THREADHEADERS +WHEEL_PKG_DIR LIBPL PY_ENABLE_SHARED PLATLIBDIR @@ -847,6 +848,7 @@ with_libm with_libc enable_big_digits with_platlibdir +with_wheel_pkg_dir with_computed_gotos with_ensurepip with_openssl @@ -1576,6 +1578,9 @@ Optional Packages: system-dependent) --with-platlibdir=DIRNAME Python library directory name (default is "lib") + --with-wheel-pkg-dir=PATH + Directory of wheel packages used by ensurepip + (default: none) --with-computed-gotos enable computed gotos in evaluation loop (enabled by default on supported compilers) --with-ensurepip[=install|upgrade|no] @@ -15493,6 +15498,29 @@ else fi +# Check for --with-wheel-pkg-dir=PATH + +WHEEL_PKG_DIR="" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-wheel-pkg-dir" >&5 +$as_echo_n "checking for --with-wheel-pkg-dir... " >&6; } + +# Check whether --with-wheel-pkg-dir was given. +if test "${with_wheel_pkg_dir+set}" = set; then : + withval=$with_wheel_pkg_dir; +if test -n "$withval"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + WHEEL_PKG_DIR="$withval" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + # Check whether right shifting a negative integer extends the sign bit # or fills with zeros (like the Cray J90, according to Tim Peters). { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether right shift extends the sign bit" >&5 diff --git a/configure.ac b/configure.ac index 08c462a..99077e9 100644 --- a/configure.ac +++ b/configure.ac @@ -4838,6 +4838,22 @@ else fi AC_SUBST(LIBPL) +# Check for --with-wheel-pkg-dir=PATH +AC_SUBST(WHEEL_PKG_DIR) +WHEEL_PKG_DIR="" +AC_MSG_CHECKING(for --with-wheel-pkg-dir) +AC_ARG_WITH(wheel-pkg-dir, + AS_HELP_STRING([--with-wheel-pkg-dir=PATH], + [Directory of wheel packages used by ensurepip (default: none)]), +[ +if test -n "$withval"; then + AC_MSG_RESULT(yes) + WHEEL_PKG_DIR="$withval" +else + AC_MSG_RESULT(no) +fi], +[AC_MSG_RESULT(no)]) + # Check whether right shifting a negative integer extends the sign bit # or fills with zeros (like the Cray J90, according to Tim Peters). AC_MSG_CHECKING(whether right shift extends the sign bit) |