Close #19406: Initial implementation of ensurepip

Patch by Donald Stufft and Nick Coghlan

16 files changed, 488 insertions, 2 deletions

diff --git a/.hgeol b/.hgeol
index 4e03690..aad79c2 100644
--- a/.hgeol
+++ b/.hgeol
@@ -26,6 +26,7 @@
 **.psd  = BIN
 **.tar  = BIN
 **.wav  = BIN
+**.whl  = BIN
 **.xar  = BIN
 **.zip  = BIN
 
diff --git a/Doc/library/development.rst b/Doc/library/development.rst
index 2368769..06e7048 100644
--- a/Doc/library/development.rst
+++ b/Doc/library/development.rst
@@ -23,4 +23,3 @@ The list of modules described in this chapter is:
    unittest.mock-examples.rst
    2to3.rst
    test.rst
-   venv.rst
diff --git a/Doc/library/distribution.rst b/Doc/library/distribution.rst
new file mode 100644
index 0000000..fb3f5df
--- /dev/null
+++ b/Doc/library/distribution.rst
@@ -0,0 +1,14 @@
+***********************************
+Software Packaging and Distribution
+***********************************
+
+These libraries help you with publishing and installing Python software.
+While these modules are designed to work in conjunction with the
+`Python Package Index <https://pypi.python.org>`__, they can also be used
+with a local index server, or without any index server at all.
+
+.. toctree::
+
+   distutils.rst
+   ensurepip.rst
+   venv.rst
diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst
new file mode 100644
index 0000000..3756d4f
--- /dev/null
+++ b/Doc/library/ensurepip.rst
@@ -0,0 +1,125 @@
+:mod:`ensurepip` --- Bootstrapping the ``pip`` installer
+========================================================
+
+.. module:: ensurepip
+   :synopsis: Bootstrapping the ``pip`` installer into an existing Python
+              installation or virtual environment.
+
+The :mod:`ensurepip` package provides support for bootstrapping the ``pip``
+installer into an existing Python installation or virtual environment. This
+bootstrapping approach reflects the fact that ``pip`` is an independent
+project with its own release cycle, and the latest available stable version
+is bundled with maintenance and feature releases of the CPython reference
+interpreter.
+
+In most cases, end users of Python shouldn't need to invoke this module
+directly (as ``pip`` should be bootstrapped by default), but it may be
+needed if installing ``pip`` was skipped when installing Python (or
+when creating a virtual environment) or after explicitly uninstalling
+``pip``.
+
+.. versionadded:: 3.4
+
+.. note::
+
+   This module *does not* access the internet. All of the components
+   needed to bootstrap ``pip`` are included as internal parts of the
+   package.
+
+.. seealso::
+
+   :ref:`install-index`
+      The end user guide for installing Python packages
+
+   :pep:`453`: Explicit bootstrapping of pip in Python installations
+      The original rationale and specification for this module.
+
+
+Command line interface
+----------------------
+
+The command line interface is invoked using the interpreter's ``-m`` switch.
+
+The simplest possible invocation is::
+
+    python -m ensurepip
+
+This invocation will install ``pip`` if it is not already installed,
+but otherwise does nothing. To ensure the installed version of ``pip``
+is at least as recent as the one bundled with ``ensurepip``, pass the
+``--upgrade`` option::
+
+    python -m ensurepip --upgrade
+
+By default, ``pip`` is installed into the current virtual environment
+(if one is active) or into the system site packages (if there is no
+active virtual environment). The installation location can be controlled
+through two additional command line options:
+
+* ``--root <dir>``: Installs ``pip`` relative to the given root directory
+  rather than the root of the currently active virtual environment (if any)
+  or the default root for the current Python installation.
+* ``--user``: Installs ``pip`` into the user site packages directory rather
+  than globally for the current Python installation (this option is not
+  permitted inside an active virtual environment).
+
+By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where
+X.Y stands for the version of Python used to invoke ``ensurepip``). The
+scripts installed can be controlled through two additional command line
+options:
+
+* ``--altinstall``: if an alternate installation is requested, the ``pipX``
+  script will *not* be installed.
+
+* ``--default-pip``: if a "default pip" installation is requested, the
+   ``pip`` script will be installed in addition to the two regular scripts.
+
+Providing both of the script selection options will trigger an exception.
+
+
+Module API
+----------
+
+:mod:`ensurepip` exposes two functions for programmatic use:
+
+.. function:: version()
+
+   Returns a string specifying the bundled version of pip that will be
+   installed when bootstrapping an environment.
+
+.. function:: bootstrap(root=None, upgrade=False, user=False, \
+                        altinstall=False, default_pip=False, \
+                        verbosity=0)
+
+   Bootstraps ``pip`` into the current or designated environment.
+
+   *root* specifies an alternative root directory to install relative to.
+   If *root* is None, then installation uses the default install location
+   for the current environment.
+
+   *upgrade* indicates whether or not to upgrade an existing installation
+   of an earlier version of ``pip`` to the bundled version.
+
+   *user* indicates whether to use the user scheme rather than installing
+   globally.
+
+   By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where
+   X.Y stands for the current version of Python).
+
+   If *altinstall* is set, then ``pipX`` will *not* be installed.
+
+   If *default_pip* is set, then ``pip`` will be installed in addition to
+   the two regular scripts.
+
+   Setting both *altinstall* and *default_pip* will trigger
+   :exc:`ValueError`.
+
+   *verbosity* controls the level of output to :data:`sys.stdout` from the
+   bootstrapping operation.
+
+   .. note::
+
+      The bootstrapping process may install additional modules required by
+      ``pip``, but other software should not assume those dependencies will
+      always be present by default (as the dependencies may be removed in a
+      future version of ``pip``).
diff --git a/Doc/library/index.rst b/Doc/library/index.rst
index 1b25c6e..81289a5 100644
--- a/Doc/library/index.rst
+++ b/Doc/library/index.rst
@@ -65,6 +65,7 @@ the `Python Package Index <http://pypi.python.org/pypi>`_.
    tk.rst
    development.rst
    debug.rst
+   distribution.rst
    python.rst
    custominterp.rst
    modules.rst
diff --git a/Doc/library/python.rst b/Doc/library/python.rst
index b67fbfc..f307d7d 100644
--- a/Doc/library/python.rst
+++ b/Doc/library/python.rst
@@ -25,4 +25,3 @@ overview:
    inspect.rst
    site.rst
    fpectl.rst
-   distutils.rst
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
index ed6daee..dd992ed 100644
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -92,6 +92,7 @@ New library modules:
 
 * :mod:`asyncio`: New provisonal API for asynchronous IO (:pep:`3156`).
 * :mod:`enum`: Support for enumeration types (:pep:`435`).
+* :mod:`ensurepip`: Bootstrapping the pip installer (:pep:`453`).
 * :mod:`selectors`: High-level and efficient I/O multiplexing, built upon the
   :mod:`select` module primitives.
 * :mod:`statistics`: A basic numerically stable statistics library (:pep:`450`).
@@ -123,6 +124,34 @@ CPython implementation improvements:
 
 Please read on for a comprehensive list of user-facing changes.
 
+
+PEP 453: Explicit bootstrapping of pip in Python installations
+==============================================================
+
+The new :mod:`ensurepip` module (defined in :pep:`453`) provides a standard
+cross-platform mechanism to boostrap the pip installer into Python
+installations and virtual environments.
+
+.. note::
+
+   Only the first phase of PEP 453 has been implemented at this point.
+   This section will be fleshed out with additional details once those
+   other changes are implemented.
+
+   Refer to :issue:`19347` for the progress on additional steps:
+
+   * ``make install`` and ``make altinstall`` integration
+   * Windows installer integration
+   * Mac OS X installer integration
+   * :mod:`venv` module and :command:`pyvenv` integration
+
+.. seealso::
+
+   :pep:`453` - Explicit bootstrapping of pip in Python installations
+      PEP written by Donald Stufft and Nick Coghlan, implemented by
+      Donald Stufft, Nick Coghlan (and ...).
+
+
 .. _pep-446:
 
 PEP 446: Make newly created file descriptors non-inheritable
diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
new file mode 100644
index 0000000..bfc5bee
--- /dev/null
+++ b/Lib/ensurepip/__init__.py
@@ -0,0 +1,92 @@
+import os
+import os.path
+import pkgutil
+import sys
+import tempfile
+
+# TODO: Remove the --pre flag when a pip 1.5 final copy is available
+
+
+__all__ = ["version", "bootstrap"]
+
+
+_SETUPTOOLS_VERSION = "1.3.2"
+
+_PIP_VERSION = "1.5.dev1"
+
+_PROJECTS = [
+    ("setuptools", _SETUPTOOLS_VERSION),
+    ("pip", _PIP_VERSION),
+]
+
+
+def _run_pip(args, additional_paths):
+    # Add our bundled software to the sys.path so we can import it
+    sys.path = additional_paths + sys.path
+
+    # Install the bundled software
+    import pip
+    pip.main(args)
+
+
+def version():
+    """
+    Returns a string specifying the bundled version of pip.
+    """
+    return _PIP_VERSION
+
+
+def bootstrap(*, root=None, upgrade=False, user=False,
+              altinstall=False, default_pip=False,
+              verbosity=0):
+    """
+    Bootstrap pip into the current Python installation (or the given root
+    directory).
+    """
+    if altinstall and default_pip:
+        raise ValueError("Cannot use altinstall and default_pip together")
+
+    # By default, installing pip and setuptools installs all of the
+    # following scripts (X.Y == running Python version):
+    #
+    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
+    #
+    # pip 1.5+ allows ensurepip to request that some of those be left out
+    if altinstall:
+        # omit pip, pipX and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
+    elif not default_pip:
+        # omit pip and easy_install
+        os.environ["ENSUREPIP_OPTIONS"] = "install"
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # Put our bundled wheels into a temporary directory and construct the
+        # additional paths that need added to sys.path
+        additional_paths = []
+        for project, version in _PROJECTS:
+            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+            whl = pkgutil.get_data(
+                "ensurepip",
+                "_bundled/{}".format(wheel_name),
+            )
+            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+                fp.write(whl)
+
+            additional_paths.append(os.path.join(tmpdir, wheel_name))
+
+        # Construct the arguments to be passed to the pip command
+        args = [
+            "install", "--no-index", "--find-links", tmpdir,
+            # Temporary until pip 1.5 is final
+            "--pre",
+        ]
+        if root:
+            args += ["--root", root]
+        if upgrade:
+            args += ["--upgrade"]
+        if user:
+            args += ["--user"]
+        if verbosity:
+            args += ["-" + "v" * verbosity]
+
+        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
diff --git a/Lib/ensurepip/__main__.py b/Lib/ensurepip/__main__.py
new file mode 100644
index 0000000..53e8459
--- /dev/null
+++ b/Lib/ensurepip/__main__.py
@@ -0,0 +1,66 @@
+import argparse
+import ensurepip
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="python -m ensurepip")
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="pip {}".format(ensurepip.version()),
+        help="Show the version of pip that is bundled with this Python.",
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=0,
+        dest="verbosity",
+        help=("Give more output. Option is additive, and can be used up to 3 "
+              "times."),
+    )
+    parser.add_argument(
+        "-U", "--upgrade",
+        action="store_true",
+        default=False,
+        help="Upgrade pip and dependencies, even if already installed.",
+    )
+    parser.add_argument(
+        "--user",
+        action="store_true",
+        default=False,
+        help="Install using the user scheme.",
+    )
+    parser.add_argument(
+        "--root",
+        default=None,
+        help="Install everything relative to this alternate root directory.",
+    )
+    parser.add_argument(
+        "--altinstall",
+        action="store_true",
+        default=False,
+        help=("Make an alternate install, installing only the X.Y versioned"
+              "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
+    )
+    parser.add_argument(
+        "--default-pip",
+        action="store_true",
+        default=False,
+        help=("Make a default pip install, installing the unqualified pip "
+              "and easy_install in addition to the versioned scripts"),
+    )
+
+    args = parser.parse_args()
+
+    ensurepip.bootstrap(
+        root=args.root,
+        upgrade=args.upgrade,
+        user=args.user,
+        verbosity=args.verbosity,
+        altinstall=args.altinstall,
+        default_pip=args.default_pip,
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/Lib/ensurepip/_bundled/pip-1.5.dev1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-1.5.dev1-py2.py3-none-any.whl
new file mode 100644
index 0000000..65e3548
--- /dev/null
+++ b/Lib/ensurepip/_bundled/pip-1.5.dev1-py2.py3-none-any.whl
diff --git a/Lib/ensurepip/_bundled/setuptools-1.3.2-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-1.3.2-py2.py3-none-any.whl
new file mode 100644
index 0000000..81962b1
--- /dev/null
+++ b/Lib/ensurepip/_bundled/setuptools-1.3.2-py2.py3-none-any.whl
diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py
new file mode 100644
index 0000000..e87f476
--- /dev/null
+++ b/Lib/test/test_ensurepip.py
@@ -0,0 +1,123 @@
+import unittest
+import unittest.mock
+import ensurepip
+import test.support
+
+
+class TestEnsurePipVersion(unittest.TestCase):
+
+    def test_returns_version(self):
+        self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version())
+
+
+class TestBootstrap(unittest.TestCase):
+
+    def setUp(self):
+        run_pip_patch = unittest.mock.patch("ensurepip._run_pip")
+        self.run_pip = run_pip_patch.start()
+        self.addCleanup(run_pip_patch.stop)
+
+        os_environ_patch = unittest.mock.patch("ensurepip.os.environ", {})
+        self.os_environ = os_environ_patch.start()
+        self.addCleanup(os_environ_patch.stop)
+
+    def test_basic_bootstrapping(self):
+        ensurepip.bootstrap()
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--pre", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+        additional_paths = self.run_pip.call_args[0][1]
+        self.assertEqual(len(additional_paths), 2)
+
+    def test_bootstrapping_with_root(self):
+        ensurepip.bootstrap(root="/foo/bar/")
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--pre", "--root", "/foo/bar/",
+                "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    def test_bootstrapping_with_user(self):
+        ensurepip.bootstrap(user=True)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--pre", "--user", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    def test_bootstrapping_with_upgrade(self):
+        ensurepip.bootstrap(upgrade=True)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--pre", "--upgrade", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    def test_bootstrapping_with_verbosity_1(self):
+        ensurepip.bootstrap(verbosity=1)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--pre", "-v", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    def test_bootstrapping_with_verbosity_2(self):
+        ensurepip.bootstrap(verbosity=2)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--pre", "-vv", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    def test_bootstrapping_with_verbosity_3(self):
+        ensurepip.bootstrap(verbosity=3)
+
+        self.run_pip.assert_called_once_with(
+            [
+                "install", "--no-index", "--find-links",
+                unittest.mock.ANY, "--pre", "-vvv", "setuptools", "pip",
+            ],
+            unittest.mock.ANY,
+        )
+
+    def test_bootstrapping_with_regular_install(self):
+        ensurepip.bootstrap()
+        self.assertEqual(self.os_environ["ENSUREPIP_OPTIONS"], "install")
+
+    def test_bootstrapping_with_alt_install(self):
+        ensurepip.bootstrap(altinstall=True)
+        self.assertEqual(self.os_environ["ENSUREPIP_OPTIONS"], "altinstall")
+
+    def test_bootstrapping_with_default_pip(self):
+        ensurepip.bootstrap(default_pip=True)
+        self.assertNotIn("ENSUREPIP_OPTIONS", self.os_environ)
+
+    def test_altinstall_default_pip_conflict(self):
+        with self.assertRaises(ValueError):
+            ensurepip.bootstrap(altinstall=True, default_pip=True)
+
+
+if __name__ == "__main__":
+    test.support.run_unittest(__name__)
diff --git a/Makefile.pre.in b/Makefile.pre.in
index f88f66b..eae8d75 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1091,6 +1091,7 @@ LIBSUBDIRS=	tkinter tkinter/test tkinter/test/test_tkinter \
 		test/test_asyncio \
 		collections concurrent concurrent/futures encodings \
 		email email/mime test/test_email test/test_email/data \
+		ensurepip ensurepip/_bundled \
 		html json test/test_json http dbm xmlrpc \
 		sqlite3 sqlite3/test \
 		logging csv wsgiref urllib \
diff --git a/Misc/ACKS b/Misc/ACKS
index 5038eeb..34fac02 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1244,6 +1244,7 @@ Michael Stone
 Serhiy Storchaka
 Ken Stox
 Dan Stromberg
+Donald Stufft
 Daniel Stutzbach
 Andreas Stührk
 Colin Su
diff --git a/Misc/NEWS b/Misc/NEWS
index 563c7fe..0b97d23 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -34,6 +34,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #19406: implementation of the ensurepip module (part of PEP 453).
+  Patch by Donald Stufft and Nick Coghlan.
+
 - Issue #19544 and Issue #6286: Restore use of urllib over http allowing use
   of http_proxy for Distutils upload command, a feature accidentally lost
   in the rollback of distutils2.
diff --git a/Tools/scripts/checkpip.py b/Tools/scripts/checkpip.py
new file mode 100644
index 0000000..835101e
--- /dev/null
+++ b/Tools/scripts/checkpip.py
@@ -0,0 +1,32 @@
+#/usr/bin/env python3
+"""
+Checks that the version of the projects bundled in ensurepip are the latest
+versions available.
+"""
+import ensurepip
+import json
+import urllib.request
+import sys
+
+
+def main():
+    outofdate = False
+
+    for project, version in ensurepip._PROJECTS:
+        data = json.loads(urllib.request.urlopen(
+            "https://pypi.python.org/pypi/{}/json".format(project),
+            cadefault=True,
+        ).read().decode("utf8"))
+        upstream_version = data["info"]["version"]
+
+        if version != upstream_version:
+            outofdate = True
+            print("The latest version of {} on PyPI is {}, but ensurepip "
+                  "has {}".format(project, upstream_version, version))
+
+    if outofdate:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()