Make a new urllib package .

It consists of code from urllib, urllib2, urlparse, and robotparser.
The old modules have all been removed.  The new package has five
submodules: urllib.parse, urllib.request, urllib.response,
urllib.error, and urllib.robotparser.  The urllib.request.urlopen()
function uses the url opener from urllib2.

Note that the unittests have not been renamed for the
beta, but they will be renamed in the future.

Joint work with Senthil Kumaran.

1 files changed, 0 insertions, 1351 deletions

diff --git a/Lib/urllib2.py b/Lib/urllib2.py
deleted file mode 100644
index 4bcd397..0000000
--- a/Lib/urllib2.py
+++ /dev/null
@@ -1,1351 +0,0 @@
-"""An extensible library for opening URLs using a variety of protocols
-
-The simplest way to use this module is to call the urlopen function,
-which accepts a string containing a URL or a Request object (described
-below).  It opens the URL and returns the results as file-like
-object; the returned object has some extra methods described below.
-
-The OpenerDirector manages a collection of Handler objects that do
-all the actual work.  Each Handler implements a particular protocol or
-option.  The OpenerDirector is a composite object that invokes the
-Handlers needed to open the requested URL.  For example, the
-HTTPHandler performs HTTP GET and POST requests and deals with
-non-error returns.  The HTTPRedirectHandler automatically deals with
-HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
-deals with digest authentication.
-
-urlopen(url, data=None) -- Basic usage is the same as original
-urllib.  pass the url and optionally data to post to an HTTP URL, and
-get a file-like object back.  One difference is that you can also pass
-a Request instance instead of URL.  Raises a URLError (subclass of
-IOError); for HTTP errors, raises an HTTPError, which can also be
-treated as a valid response.
-
-build_opener -- Function that creates a new OpenerDirector instance.
-Will install the default handlers.  Accepts one or more Handlers as
-arguments, either instances or Handler classes that it will
-instantiate.  If one of the argument is a subclass of the default
-handler, the argument will be installed instead of the default.
-
-install_opener -- Installs a new opener as the default opener.
-
-objects of interest:
-OpenerDirector --
-
-Request -- An object that encapsulates the state of a request.  The
-state can be as simple as the URL.  It can also include extra HTTP
-headers, e.g. a User-Agent.
-
-BaseHandler --
-
-exceptions:
-URLError -- A subclass of IOError, individual protocols have their own
-specific subclass.
-
-HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
-as an exceptional event or valid response.
-
-internals:
-BaseHandler and parent
-_call_chain conventions
-
-Example usage:
-
-import urllib2
-
-# set up authentication info
-authinfo = urllib2.HTTPBasicAuthHandler()
-authinfo.add_password(realm='PDQ Application',
-                      uri='https://mahler:8092/site-updates.py',
-                      user='klem',
-                      passwd='geheim$parole')
-
-proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
-
-# build a new opener that adds authentication and caching FTP handlers
-opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
-
-# install it
-urllib2.install_opener(opener)
-
-f = urllib2.urlopen('http://www.python.org/')
-
-
-"""
-
-# XXX issues:
-# If an authentication error handler that tries to perform
-# authentication for some reason but fails, how should the error be
-# signalled?  The client needs to know the HTTP error code.  But if
-# the handler knows that the problem was, e.g., that it didn't know
-# that hash algo that requested in the challenge, it would be good to
-# pass that information along to the client, too.
-# ftp errors aren't handled cleanly
-# check digest against correct (i.e. non-apache) implementation
-
-# Possible extensions:
-# complex proxies  XXX not sure what exactly was meant by this
-# abstract factory for opener
-
-import base64
-import hashlib
-import http.client
-import io
-import email
-import os
-import posixpath
-import random
-import re
-import socket
-import sys
-import time
-import urlparse
-import bisect
-
-from io import StringIO
-
-from urllib import (unwrap, unquote, splittype, splithost, quote,
-     addinfourl, splitport, splitquery,
-     splitattr, ftpwrapper, noheaders, splituser, splitpasswd, splitvalue)
-
-# support for FileHandler, proxies via environment variables
-from urllib import localhost, url2pathname, getproxies
-
-# used in User-Agent header sent
-__version__ = sys.version[:3]
-
-_opener = None
-def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
-    global _opener
-    if _opener is None:
-        _opener = build_opener()
-    return _opener.open(url, data, timeout)
-
-def install_opener(opener):
-    global _opener
-    _opener = opener
-
-# do these error classes make sense?
-# make sure all of the IOError stuff is overridden.  we just want to be
-# subtypes.
-
-class URLError(IOError):
-    # URLError is a sub-type of IOError, but it doesn't share any of
-    # the implementation.  need to override __init__ and __str__.
-    # It sets self.args for compatibility with other EnvironmentError
-    # subclasses, but args doesn't have the typical format with errno in
-    # slot 0 and strerror in slot 1.  This may be better than nothing.
-    def __init__(self, reason):
-        self.args = reason,
-        self.reason = reason
-
-    def __str__(self):
-        return '<urlopen error %s>' % self.reason
-
-class HTTPError(URLError, addinfourl):
-    """Raised when HTTP error occurs, but also acts like non-error return"""
-    __super_init = addinfourl.__init__
-
-    def __init__(self, url, code, msg, hdrs, fp):
-        self.code = code
-        self.msg = msg
-        self.hdrs = hdrs
-        self.fp = fp
-        self.filename = url
-        # The addinfourl classes depend on fp being a valid file
-        # object.  In some cases, the HTTPError may not have a valid
-        # file object.  If this happens, the simplest workaround is to
-        # not initialize the base classes.
-        if fp is not None:
-            self.__super_init(fp, hdrs, url, code)
-
-    def __str__(self):
-        return 'HTTP Error %s: %s' % (self.code, self.msg)
-
-# copied from cookielib.py
-_cut_port_re = re.compile(r":\d+$")
-def request_host(request):
-    """Return request-host, as defined by RFC 2965.
-
-    Variation from RFC: returned value is lowercased, for convenient
-    comparison.
-
-    """
-    url = request.get_full_url()
-    host = urlparse.urlparse(url)[1]
-    if host == "":
-        host = request.get_header("Host", "")
-
-    # remove port, if present
-    host = _cut_port_re.sub("", host, 1)
-    return host.lower()
-
-class Request:
-
-    def __init__(self, url, data=None, headers={},
-                 origin_req_host=None, unverifiable=False):
-        # unwrap('<URL:type://host/path>') --> 'type://host/path'
-        self.__original = unwrap(url)
-        self.type = None
-        # self.__r_type is what's left after doing the splittype
-        self.host = None
-        self.port = None
-        self.data = data
-        self.headers = {}
-        for key, value in headers.items():
-            self.add_header(key, value)
-        self.unredirected_hdrs = {}
-        if origin_req_host is None:
-            origin_req_host = request_host(self)
-        self.origin_req_host = origin_req_host
-        self.unverifiable = unverifiable
-
-    def __getattr__(self, attr):
-        # XXX this is a fallback mechanism to guard against these
-        # methods getting called in a non-standard order.  this may be
-        # too complicated and/or unnecessary.
-        # XXX should the __r_XXX attributes be public?
-        if attr[:12] == '_Request__r_':
-            name = attr[12:]
-            if hasattr(Request, 'get_' + name):
-                getattr(self, 'get_' + name)()
-                return getattr(self, attr)
-        raise AttributeError(attr)
-
-    def get_method(self):
-        if self.has_data():
-            return "POST"
-        else:
-            return "GET"
-
-    # XXX these helper methods are lame
-
-    def add_data(self, data):
-        self.data = data
-
-    def has_data(self):
-        return self.data is not None
-
-    def get_data(self):
-        return self.data
-
-    def get_full_url(self):
-        return self.__original
-
-    def get_type(self):
-        if self.type is None:
-            self.type, self.__r_type = splittype(self.__original)
-            if self.type is None:
-                raise ValueError("unknown url type: %s" % self.__original)
-        return self.type
-
-    def get_host(self):
-        if self.host is None:
-            self.host, self.__r_host = splithost(self.__r_type)
-            if self.host:
-                self.host = unquote(self.host)
-        return self.host
-
-    def get_selector(self):
-        return self.__r_host
-
-    def set_proxy(self, host, type):
-        self.host, self.type = host, type
-        self.__r_host = self.__original
-
-    def get_origin_req_host(self):
-        return self.origin_req_host
-
-    def is_unverifiable(self):
-        return self.unverifiable
-
-    def add_header(self, key, val):
-        # useful for something like authentication
-        self.headers[key.capitalize()] = val
-
-    def add_unredirected_header(self, key, val):
-        # will not be added to a redirected request
-        self.unredirected_hdrs[key.capitalize()] = val
-
-    def has_header(self, header_name):
-        return (header_name in self.headers or
-                header_name in self.unredirected_hdrs)
-
-    def get_header(self, header_name, default=None):
-        return self.headers.get(
-            header_name,
-            self.unredirected_hdrs.get(header_name, default))
-
-    def header_items(self):
-        hdrs = self.unredirected_hdrs.copy()
-        hdrs.update(self.headers)
-        return list(hdrs.items())
-
-class OpenerDirector:
-    def __init__(self):
-        client_version = "Python-urllib/%s" % __version__
-        self.addheaders = [('User-agent', client_version)]
-        # manage the individual handlers
-        self.handlers = []
-        self.handle_open = {}
-        self.handle_error = {}
-        self.process_response = {}
-        self.process_request = {}
-
-    def add_handler(self, handler):
-        if not hasattr(handler, "add_parent"):
-            raise TypeError("expected BaseHandler instance, got %r" %
-                            type(handler))
-
-        added = False
-        for meth in dir(handler):
-            if meth in ["redirect_request", "do_open", "proxy_open"]:
-                # oops, coincidental match
-                continue
-
-            i = meth.find("_")
-            protocol = meth[:i]
-            condition = meth[i+1:]
-
-            if condition.startswith("error"):
-                j = condition.find("_") + i + 1
-                kind = meth[j+1:]
-                try:
-                    kind = int(kind)
-                except ValueError:
-                    pass
-                lookup = self.handle_error.get(protocol, {})
-                self.handle_error[protocol] = lookup
-            elif condition == "open":
-                kind = protocol
-                lookup = self.handle_open
-            elif condition == "response":
-                kind = protocol
-                lookup = self.process_response
-            elif condition == "request":
-                kind = protocol
-                lookup = self.process_request
-            else:
-                continue
-
-            handlers = lookup.setdefault(kind, [])
-            if handlers:
-                bisect.insort(handlers, handler)
-            else:
-                handlers.append(handler)
-            added = True
-
-        if added:
-            # the handlers must work in an specific order, the order
-            # is specified in a Handler attribute
-            bisect.insort(self.handlers, handler)
-            handler.add_parent(self)
-
-    def close(self):
-        # Only exists for backwards compatibility.
-        pass
-
-    def _call_chain(self, chain, kind, meth_name, *args):
-        # Handlers raise an exception if no one else should try to handle
-        # the request, or return None if they can't but another handler
-        # could.  Otherwise, they return the response.
-        handlers = chain.get(kind, ())
-        for handler in handlers:
-            func = getattr(handler, meth_name)
-
-            result = func(*args)
-            if result is not None:
-                return result
-
-    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
-        # accept a URL or a Request object
-        if isinstance(fullurl, str):
-            req = Request(fullurl, data)
-        else:
-            req = fullurl
-            if data is not None:
-                req.add_data(data)
-
-        req.timeout = timeout
-        protocol = req.get_type()
-
-        # pre-process request
-        meth_name = protocol+"_request"
-        for processor in self.process_request.get(protocol, []):
-            meth = getattr(processor, meth_name)
-            req = meth(req)
-
-        response = self._open(req, data)
-
-        # post-process response
-        meth_name = protocol+"_response"
-        for processor in self.process_response.get(protocol, []):
-            meth = getattr(processor, meth_name)
-            response = meth(req, response)
-
-        return response
-
-    def _open(self, req, data=None):
-        result = self._call_chain(self.handle_open, 'default',
-                                  'default_open', req)
-        if result:
-            return result
-
-        protocol = req.get_type()
-        result = self._call_chain(self.handle_open, protocol, protocol +
-                                  '_open', req)
-        if result:
-            return result
-
-        return self._call_chain(self.handle_open, 'unknown',
-                                'unknown_open', req)
-
-    def error(self, proto, *args):
-        if proto in ('http', 'https'):
-            # XXX http[s] protocols are special-cased
-            dict = self.handle_error['http'] # https is not different than http
-            proto = args[2]  # YUCK!
-            meth_name = 'http_error_%s' % proto
-            http_err = 1
-            orig_args = args
-        else:
-            dict = self.handle_error
-            meth_name = proto + '_error'
-            http_err = 0
-        args = (dict, proto, meth_name) + args
-        result = self._call_chain(*args)
-        if result:
-            return result
-
-        if http_err:
-            args = (dict, 'default', 'http_error_default') + orig_args
-            return self._call_chain(*args)
-
-# XXX probably also want an abstract factory that knows when it makes
-# sense to skip a superclass in favor of a subclass and when it might
-# make sense to include both
-
-def build_opener(*handlers):
-    """Create an opener object from a list of handlers.
-
-    The opener will use several default handlers, including support
-    for HTTP and FTP.
-
-    If any of the handlers passed as arguments are subclasses of the
-    default handlers, the default handlers will not be used.
-    """
-    def isclass(obj):
-        return isinstance(obj, type) or hasattr(obj, "__bases__")
-
-    opener = OpenerDirector()
-    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
-                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
-                       FTPHandler, FileHandler, HTTPErrorProcessor]
-    if hasattr(http.client, 'HTTPS'):
-        default_classes.append(HTTPSHandler)
-    skip = set()
-    for klass in default_classes:
-        for check in handlers:
-            if isclass(check):
-                if issubclass(check, klass):
-                    skip.add(klass)
-            elif isinstance(check, klass):
-                skip.add(klass)
-    for klass in skip:
-        default_classes.remove(klass)
-
-    for klass in default_classes:
-        opener.add_handler(klass())
-
-    for h in handlers:
-        if isclass(h):
-            h = h()
-        opener.add_handler(h)
-    return opener
-
-class BaseHandler:
-    handler_order = 500
-
-    def add_parent(self, parent):
-        self.parent = parent
-
-    def close(self):
-        # Only exists for backwards compatibility
-        pass
-
-    def __lt__(self, other):
-        if not hasattr(other, "handler_order"):
-            # Try to preserve the old behavior of having custom classes
-            # inserted after default ones (works only for custom user
-            # classes which are not aware of handler_order).
-            return True
-        return self.handler_order < other.handler_order
-
-
-class HTTPErrorProcessor(BaseHandler):
-    """Process HTTP error responses."""
-    handler_order = 1000  # after all other processing
-
-    def http_response(self, request, response):
-        code, msg, hdrs = response.code, response.msg, response.info()
-
-        # According to RFC 2616, "2xx" code indicates that the client's
-        # request was successfully received, understood, and accepted.
-        if not (200 <= code < 300):
-            response = self.parent.error(
-                'http', request, response, code, msg, hdrs)
-
-        return response
-
-    https_response = http_response
-
-class HTTPDefaultErrorHandler(BaseHandler):
-    def http_error_default(self, req, fp, code, msg, hdrs):
-        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
-
-class HTTPRedirectHandler(BaseHandler):
-    # maximum number of redirections to any single URL
-    # this is needed because of the state that cookies introduce
-    max_repeats = 4
-    # maximum total number of redirections (regardless of URL) before
-    # assuming we're in a loop
-    max_redirections = 10
-
-    def redirect_request(self, req, fp, code, msg, headers, newurl):
-        """Return a Request or None in response to a redirect.
-
-        This is called by the http_error_30x methods when a
-        redirection response is received.  If a redirection should
-        take place, return a new Request to allow http_error_30x to
-        perform the redirect.  Otherwise, raise HTTPError if no-one
-        else should try to handle this url.  Return None if you can't
-        but another Handler might.
-        """
-        m = req.get_method()
-        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
-            or code in (301, 302, 303) and m == "POST"):
-            # Strictly (according to RFC 2616), 301 or 302 in response
-            # to a POST MUST NOT cause a redirection without confirmation
-            # from the user (of urllib2, in this case).  In practice,
-            # essentially all clients do redirect in this case, so we
-            # do the same.
-            # be conciliant with URIs containing a space
-            newurl = newurl.replace(' ', '%20')
-            newheaders = dict((k,v) for k,v in req.headers.items()
-                              if k.lower() not in ("content-length", "content-type")
-                             )
-            return Request(newurl,
-                           headers=newheaders,
-                           origin_req_host=req.get_origin_req_host(),
-                           unverifiable=True)
-        else:
-            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
-
-    # Implementation note: To avoid the server sending us into an
-    # infinite loop, the request object needs to track what URLs we
-    # have already seen.  Do this by adding a handler-specific
-    # attribute to the Request object.
-    def http_error_302(self, req, fp, code, msg, headers):
-        # Some servers (incorrectly) return multiple Location headers
-        # (so probably same goes for URI).  Use first header.
-        if 'location' in headers:
-            newurl = headers['location']
-        elif 'uri' in headers:
-            newurl = headers['uri']
-        else:
-            return
-        newurl = urlparse.urljoin(req.get_full_url(), newurl)
-
-        # XXX Probably want to forget about the state of the current
-        # request, although that might interact poorly with other
-        # handlers that also use handler-specific request attributes
-        new = self.redirect_request(req, fp, code, msg, headers, newurl)
-        if new is None:
-            return
-
-        # loop detection
-        # .redirect_dict has a key url if url was previously visited.
-        if hasattr(req, 'redirect_dict'):
-            visited = new.redirect_dict = req.redirect_dict
-            if (visited.get(newurl, 0) >= self.max_repeats or
-                len(visited) >= self.max_redirections):
-                raise HTTPError(req.get_full_url(), code,
-                                self.inf_msg + msg, headers, fp)
-        else:
-            visited = new.redirect_dict = req.redirect_dict = {}
-        visited[newurl] = visited.get(newurl, 0) + 1
-
-        # Don't close the fp until we are sure that we won't use it
-        # with HTTPError.
-        fp.read()
-        fp.close()
-
-        return self.parent.open(new)
-
-    http_error_301 = http_error_303 = http_error_307 = http_error_302
-
-    inf_msg = "The HTTP server returned a redirect error that would " \
-              "lead to an infinite loop.\n" \
-              "The last 30x error message was:\n"
-
-
-def _parse_proxy(proxy):
-    """Return (scheme, user, password, host/port) given a URL or an authority.
-
-    If a URL is supplied, it must have an authority (host:port) component.
-    According to RFC 3986, having an authority component means the URL must
-    have two slashes after the scheme:
-
-    >>> _parse_proxy('file:/ftp.example.com/')
-    Traceback (most recent call last):
-    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
-
-    The first three items of the returned tuple may be None.
-
-    Examples of authority parsing:
-
-    >>> _parse_proxy('proxy.example.com')
-    (None, None, None, 'proxy.example.com')
-    >>> _parse_proxy('proxy.example.com:3128')
-    (None, None, None, 'proxy.example.com:3128')
-
-    The authority component may optionally include userinfo (assumed to be
-    username:password):
-
-    >>> _parse_proxy('joe:password@proxy.example.com')
-    (None, 'joe', 'password', 'proxy.example.com')
-    >>> _parse_proxy('joe:password@proxy.example.com:3128')
-    (None, 'joe', 'password', 'proxy.example.com:3128')
-
-    Same examples, but with URLs instead:
-
-    >>> _parse_proxy('http://proxy.example.com/')
-    ('http', None, None, 'proxy.example.com')
-    >>> _parse_proxy('http://proxy.example.com:3128/')
-    ('http', None, None, 'proxy.example.com:3128')
-    >>> _parse_proxy('http://joe:password@proxy.example.com/')
-    ('http', 'joe', 'password', 'proxy.example.com')
-    >>> _parse_proxy('http://joe:password@proxy.example.com:3128')
-    ('http', 'joe', 'password', 'proxy.example.com:3128')
-
-    Everything after the authority is ignored:
-
-    >>> _parse_proxy('ftp://joe:password@proxy.example.com/rubbish:3128')
-    ('ftp', 'joe', 'password', 'proxy.example.com')
-
-    Test for no trailing '/' case:
-
-    >>> _parse_proxy('http://joe:password@proxy.example.com')
-    ('http', 'joe', 'password', 'proxy.example.com')
-
-    """
-    scheme, r_scheme = splittype(proxy)
-    if not r_scheme.startswith("/"):
-        # authority
-        scheme = None
-        authority = proxy
-    else:
-        # URL
-        if not r_scheme.startswith("//"):
-            raise ValueError("proxy URL with no authority: %r" % proxy)
-        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
-        # and 3.3.), path is empty or starts with '/'
-        end = r_scheme.find("/", 2)
-        if end == -1:
-            end = None
-        authority = r_scheme[2:end]
-    userinfo, hostport = splituser(authority)
-    if userinfo is not None:
-        user, password = splitpasswd(userinfo)
-    else:
-        user = password = None
-    return scheme, user, password, hostport
-
-class ProxyHandler(BaseHandler):
-    # Proxies must be in front
-    handler_order = 100
-
-    def __init__(self, proxies=None):
-        if proxies is None:
-            proxies = getproxies()
-        assert hasattr(proxies, 'keys'), "proxies must be a mapping"
-        self.proxies = proxies
-        for type, url in proxies.items():
-            setattr(self, '%s_open' % type,
-                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
-                    meth(r, proxy, type))
-
-    def proxy_open(self, req, proxy, type):
-        orig_type = req.get_type()
-        proxy_type, user, password, hostport = _parse_proxy(proxy)
-        if proxy_type is None:
-            proxy_type = orig_type
-        if user and password:
-            user_pass = '%s:%s' % (unquote(user), unquote(password))
-            creds = base64.b64encode(user_pass.encode()).decode("ascii")
-            req.add_header('Proxy-authorization', 'Basic ' + creds)
-        hostport = unquote(hostport)
-        req.set_proxy(hostport, proxy_type)
-        if orig_type == proxy_type:
-            # let other handlers take care of it
-            return None
-        else:
-            # need to start over, because the other handlers don't
-            # grok the proxy's URL type
-            # e.g. if we have a constructor arg proxies like so:
-            # {'http': 'ftp://proxy.example.com'}, we may end up turning
-            # a request for http://acme.example.com/a into one for
-            # ftp://proxy.example.com/a
-            return self.parent.open(req)
-
-class HTTPPasswordMgr:
-
-    def __init__(self):
-        self.passwd = {}
-
-    def add_password(self, realm, uri, user, passwd):
-        # uri could be a single URI or a sequence
-        if isinstance(uri, str):
-            uri = [uri]
-        if not realm in self.passwd:
-            self.passwd[realm] = {}
-        for default_port in True, False:
-            reduced_uri = tuple(
-                [self.reduce_uri(u, default_port) for u in uri])
-            self.passwd[realm][reduced_uri] = (user, passwd)
-
-    def find_user_password(self, realm, authuri):
-        domains = self.passwd.get(realm, {})
-        for default_port in True, False:
-            reduced_authuri = self.reduce_uri(authuri, default_port)
-            for uris, authinfo in domains.items():
-                for uri in uris:
-                    if self.is_suburi(uri, reduced_authuri):
-                        return authinfo
-        return None, None
-
-    def reduce_uri(self, uri, default_port=True):
-        """Accept authority or URI and extract only the authority and path."""
-        # note HTTP URLs do not have a userinfo component
-        parts = urlparse.urlsplit(uri)
-        if parts[1]:
-            # URI
-            scheme = parts[0]
-            authority = parts[1]
-            path = parts[2] or '/'
-        else:
-            # host or host:port
-            scheme = None
-            authority = uri
-            path = '/'
-        host, port = splitport(authority)
-        if default_port and port is None and scheme is not None:
-            dport = {"http": 80,
-                     "https": 443,
-                     }.get(scheme)
-            if dport is not None:
-                authority = "%s:%d" % (host, dport)
-        return authority, path
-
-    def is_suburi(self, base, test):
-        """Check if test is below base in a URI tree
-
-        Both args must be URIs in reduced form.
-        """
-        if base == test:
-            return True
-        if base[0] != test[0]:
-            return False
-        common = posixpath.commonprefix((base[1], test[1]))
-        if len(common) == len(base[1]):
-            return True
-        return False
-
-
-class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
-
-    def find_user_password(self, realm, authuri):
-        user, password = HTTPPasswordMgr.find_user_password(self, realm,
-                                                            authuri)
-        if user is not None:
-            return user, password
-        return HTTPPasswordMgr.find_user_password(self, None, authuri)
-
-
-class AbstractBasicAuthHandler:
-
-    # XXX this allows for multiple auth-schemes, but will stupidly pick
-    # the last one with a realm specified.
-
-    # allow for double- and single-quoted realm values
-    # (single quotes are a violation of the RFC, but appear in the wild)
-    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
-                    'realm=(["\'])(.*?)\\2', re.I)
-
-    # XXX could pre-emptively send auth info already accepted (RFC 2617,
-    # end of section 2, and section 1.2 immediately after "credentials"
-    # production).
-
-    def __init__(self, password_mgr=None):
-        if password_mgr is None:
-            password_mgr = HTTPPasswordMgr()
-        self.passwd = password_mgr
-        self.add_password = self.passwd.add_password
-
-    def http_error_auth_reqed(self, authreq, host, req, headers):
-        # host may be an authority (without userinfo) or a URL with an
-        # authority
-        # XXX could be multiple headers
-        authreq = headers.get(authreq, None)
-        if authreq:
-            mo = AbstractBasicAuthHandler.rx.search(authreq)
-            if mo:
-                scheme, quote, realm = mo.groups()
-                if scheme.lower() == 'basic':
-                    return self.retry_http_basic_auth(host, req, realm)
-
-    def retry_http_basic_auth(self, host, req, realm):
-        user, pw = self.passwd.find_user_password(realm, host)
-        if pw is not None:
-            raw = "%s:%s" % (user, pw)
-            auth = "Basic " + base64.b64encode(raw.encode()).decode("ascii")
-            if req.headers.get(self.auth_header, None) == auth:
-                return None
-            req.add_header(self.auth_header, auth)
-            return self.parent.open(req)
-        else:
-            return None
-
-
-class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
-
-    auth_header = 'Authorization'
-
-    def http_error_401(self, req, fp, code, msg, headers):
-        url = req.get_full_url()
-        return self.http_error_auth_reqed('www-authenticate',
-                                          url, req, headers)
-
-
-class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
-
-    auth_header = 'Proxy-authorization'
-
-    def http_error_407(self, req, fp, code, msg, headers):
-        # http_error_auth_reqed requires that there is no userinfo component in
-        # authority.  Assume there isn't one, since urllib2 does not (and
-        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
-        # userinfo.
-        authority = req.get_host()
-        return self.http_error_auth_reqed('proxy-authenticate',
-                                          authority, req, headers)
-
-
-def randombytes(n):
-    """Return n random bytes."""
-    return os.urandom(n)
-
-class AbstractDigestAuthHandler:
-    # Digest authentication is specified in RFC 2617.
-
-    # XXX The client does not inspect the Authentication-Info header
-    # in a successful response.
-
-    # XXX It should be possible to test this implementation against
-    # a mock server that just generates a static set of challenges.
-
-    # XXX qop="auth-int" supports is shaky
-
-    def __init__(self, passwd=None):
-        if passwd is None:
-            passwd = HTTPPasswordMgr()
-        self.passwd = passwd
-        self.add_password = self.passwd.add_password
-        self.retried = 0
-        self.nonce_count = 0
-
-    def reset_retry_count(self):
-        self.retried = 0
-
-    def http_error_auth_reqed(self, auth_header, host, req, headers):
-        authreq = headers.get(auth_header, None)
-        if self.retried > 5:
-            # Don't fail endlessly - if we failed once, we'll probably
-            # fail a second time. Hm. Unless the Password Manager is
-            # prompting for the information. Crap. This isn't great
-            # but it's better than the current 'repeat until recursion
-            # depth exceeded' approach <wink>
-            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
-                            headers, None)
-        else:
-            self.retried += 1
-        if authreq:
-            scheme = authreq.split()[0]
-            if scheme.lower() == 'digest':
-                return self.retry_http_digest_auth(req, authreq)
-
-    def retry_http_digest_auth(self, req, auth):
-        token, challenge = auth.split(' ', 1)
-        chal = parse_keqv_list(parse_http_list(challenge))
-        auth = self.get_authorization(req, chal)
-        if auth:
-            auth_val = 'Digest %s' % auth
-            if req.headers.get(self.auth_header, None) == auth_val:
-                return None
-            req.add_unredirected_header(self.auth_header, auth_val)
-            resp = self.parent.open(req)
-            return resp
-
-    def get_cnonce(self, nonce):
-        # The cnonce-value is an opaque
-        # quoted string value provided by the client and used by both client
-        # and server to avoid chosen plaintext attacks, to provide mutual
-        # authentication, and to provide some message integrity protection.
-        # This isn't a fabulous effort, but it's probably Good Enough.
-        s = "%s:%s:%s:" % (self.nonce_count, nonce, time.ctime())
-        b = s.encode("ascii") + randombytes(8)
-        dig = hashlib.sha1(b).hexdigest()
-        return dig[:16]
-
-    def get_authorization(self, req, chal):
-        try:
-            realm = chal['realm']
-            nonce = chal['nonce']
-            qop = chal.get('qop')
-            algorithm = chal.get('algorithm', 'MD5')
-            # mod_digest doesn't send an opaque, even though it isn't
-            # supposed to be optional
-            opaque = chal.get('opaque', None)
-        except KeyError:
-            return None
-
-        H, KD = self.get_algorithm_impls(algorithm)
-        if H is None:
-            return None
-
-        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
-        if user is None:
-            return None
-
-        # XXX not implemented yet
-        if req.has_data():
-            entdig = self.get_entity_digest(req.get_data(), chal)
-        else:
-            entdig = None
-
-        A1 = "%s:%s:%s" % (user, realm, pw)
-        A2 = "%s:%s" % (req.get_method(),
-                        # XXX selector: what about proxies and full urls
-                        req.get_selector())
-        if qop == 'auth':
-            self.nonce_count += 1
-            ncvalue = '%08x' % self.nonce_count
-            cnonce = self.get_cnonce(nonce)
-            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
-            respdig = KD(H(A1), noncebit)
-        elif qop is None:
-            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
-        else:
-            # XXX handle auth-int.
-            raise URLError("qop '%s' is not supported." % qop)
-
-        # XXX should the partial digests be encoded too?
-
-        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
-               'response="%s"' % (user, realm, nonce, req.get_selector(),
-                                  respdig)
-        if opaque:
-            base += ', opaque="%s"' % opaque
-        if entdig:
-            base += ', digest="%s"' % entdig
-        base += ', algorithm="%s"' % algorithm
-        if qop:
-            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
-        return base
-
-    def get_algorithm_impls(self, algorithm):
-        # algorithm should be case-insensitive according to RFC2617
-        algorithm = algorithm.upper()
-        # lambdas assume digest modules are imported at the top level
-        if algorithm == 'MD5':
-            H = lambda x: hashlib.md5(x.encode("ascii")).hexdigest()
-        elif algorithm == 'SHA':
-            H = lambda x: hashlib.sha1(x.encode("ascii")).hexdigest()
-        # XXX MD5-sess
-        KD = lambda s, d: H("%s:%s" % (s, d))
-        return H, KD
-
-    def get_entity_digest(self, data, chal):
-        # XXX not implemented yet
-        return None
-
-
-class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
-    """An authentication protocol defined by RFC 2069
-
-    Digest authentication improves on basic authentication because it
-    does not transmit passwords in the clear.
-    """
-
-    auth_header = 'Authorization'
-    handler_order = 490  # before Basic auth
-
-    def http_error_401(self, req, fp, code, msg, headers):
-        host = urlparse.urlparse(req.get_full_url())[1]
-        retry = self.http_error_auth_reqed('www-authenticate',
-                                           host, req, headers)
-        self.reset_retry_count()
-        return retry
-
-
-class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
-
-    auth_header = 'Proxy-Authorization'
-    handler_order = 490  # before Basic auth
-
-    def http_error_407(self, req, fp, code, msg, headers):
-        host = req.get_host()
-        retry = self.http_error_auth_reqed('proxy-authenticate',
-                                           host, req, headers)
-        self.reset_retry_count()
-        return retry
-
-class AbstractHTTPHandler(BaseHandler):
-
-    def __init__(self, debuglevel=0):
-        self._debuglevel = debuglevel
-
-    def set_http_debuglevel(self, level):
-        self._debuglevel = level
-
-    def do_request_(self, request):
-        host = request.get_host()
-        if not host:
-            raise URLError('no host given')
-
-        if request.has_data():  # POST
-            data = request.get_data()
-            if not request.has_header('Content-type'):
-                request.add_unredirected_header(
-                    'Content-type',
-                    'application/x-www-form-urlencoded')
-            if not request.has_header('Content-length'):
-                request.add_unredirected_header(
-                    'Content-length', '%d' % len(data))
-
-        scheme, sel = splittype(request.get_selector())
-        sel_host, sel_path = splithost(sel)
-        if not request.has_header('Host'):
-            request.add_unredirected_header('Host', sel_host or host)
-        for name, value in self.parent.addheaders:
-            name = name.capitalize()
-            if not request.has_header(name):
-                request.add_unredirected_header(name, value)
-
-        return request
-
-    def do_open(self, http_class, req):
-        """Return an addinfourl object for the request, using http_class.
-
-        http_class must implement the HTTPConnection API from http.client.
-        The addinfourl return value is a file-like object.  It also
-        has methods and attributes including:
-            - info(): return a email.message.Message object for the headers
-            - geturl(): return the original request URL
-            - code: HTTP status code
-        """
-        host = req.get_host()
-        if not host:
-            raise URLError('no host given')
-
-        h = http_class(host, timeout=req.timeout) # will parse host:port
-        h.set_debuglevel(self._debuglevel)
-
-        headers = dict(req.headers)
-        headers.update(req.unredirected_hdrs)
-        # We want to make an HTTP/1.1 request, but the addinfourl
-        # class isn't prepared to deal with a persistent connection.
-        # It will try to read all remaining data from the socket,
-        # which will block while the server waits for the next request.
-        # So make sure the connection gets closed after the (only)
-        # request.
-        headers["Connection"] = "close"
-        headers = dict(
-            (name.title(), val) for name, val in headers.items())
-        try:
-            h.request(req.get_method(), req.get_selector(), req.data, headers)
-            r = h.getresponse()
-        except socket.error as err: # XXX what error?
-            raise URLError(err)
-
-        # Pick apart the HTTPResponse object to get the addinfourl
-        # object initialized properly.
-
-        # XXX Should an HTTPResponse object really be passed to
-        # BufferedReader?  If so, we should change http.client to support
-        # this use directly.
-
-        # Add some fake methods to the reader to satisfy BufferedReader.
-        r.readable = lambda: True
-        r.writable = r.seekable = lambda: False
-        r._checkReadable = lambda: True
-        r._checkWritable = lambda: False
-        fp = io.BufferedReader(r)
-
-        resp = addinfourl(fp, r.msg, req.get_full_url())
-        resp.code = r.status
-        resp.msg = r.reason
-        return resp
-
-
-class HTTPHandler(AbstractHTTPHandler):
-
-    def http_open(self, req):
-        return self.do_open(http.client.HTTPConnection, req)
-
-    http_request = AbstractHTTPHandler.do_request_
-
-if hasattr(http.client, 'HTTPS'):
-    class HTTPSHandler(AbstractHTTPHandler):
-
-        def https_open(self, req):
-            return self.do_open(http.client.HTTPSConnection, req)
-
-        https_request = AbstractHTTPHandler.do_request_
-
-class HTTPCookieProcessor(BaseHandler):
-    def __init__(self, cookiejar=None):
-        import http.cookiejar
-        if cookiejar is None:
-            cookiejar = http.cookiejar.CookieJar()
-        self.cookiejar = cookiejar
-
-    def http_request(self, request):
-        self.cookiejar.add_cookie_header(request)
-        return request
-
-    def http_response(self, request, response):
-        self.cookiejar.extract_cookies(response, request)
-        return response
-
-    https_request = http_request
-    https_response = http_response
-
-class UnknownHandler(BaseHandler):
-    def unknown_open(self, req):
-        type = req.get_type()
-        raise URLError('unknown url type: %s' % type)
-
-def parse_keqv_list(l):
-    """Parse list of key=value strings where keys are not duplicated."""
-    parsed = {}
-    for elt in l:
-        # Because of a trailing comma in the auth string, elt could be the
-        # empty string.
-        if not elt:
-            continue
-        k, v = elt.split('=', 1)
-        if v[0] == '"' and v[-1] == '"':
-            v = v[1:-1]
-        parsed[k] = v
-    return parsed
-
-def parse_http_list(s):
-    """Parse lists as described by RFC 2068 Section 2.
-
-    In particular, parse comma-separated lists where the elements of
-    the list may include quoted-strings.  A quoted-string could
-    contain a comma.  A non-quoted string could have quotes in the
-    middle.  Neither commas nor quotes count if they are escaped.
-    Only double-quotes count, not single-quotes.
-    """
-    res = []
-    part = ''
-
-    escape = quote = False
-    for cur in s:
-        if escape:
-            part += cur
-            escape = False
-            continue
-        if quote:
-            if cur == '\\':
-                escape = True
-                continue
-            elif cur == '"':
-                quote = False
-            part += cur
-            continue
-
-        if cur == ',':
-            res.append(part)
-            part = ''
-            continue
-
-        if cur == '"':
-            quote = True
-
-        part += cur
-
-    # append last part
-    if part:
-        res.append(part)
-
-    return [part.strip() for part in res]
-
-class FileHandler(BaseHandler):
-    # Use local file or FTP depending on form of URL
-    def file_open(self, req):
-        url = req.get_selector()
-        if url[:2] == '//' and url[2:3] != '/':
-            req.type = 'ftp'
-            return self.parent.open(req)
-        else:
-            return self.open_local_file(req)
-
-    # names for the localhost
-    names = None
-    def get_names(self):
-        if FileHandler.names is None:
-            try:
-                FileHandler.names = (socket.gethostbyname('localhost'),
-                                    socket.gethostbyname(socket.gethostname()))
-            except socket.gaierror:
-                FileHandler.names = (socket.gethostbyname('localhost'),)
-        return FileHandler.names
-
-    # not entirely sure what the rules are here
-    def open_local_file(self, req):
-        import email.utils
-        import mimetypes
-        host = req.get_host()
-        file = req.get_selector()
-        localfile = url2pathname(file)
-        try:
-            stats = os.stat(localfile)
-            size = stats.st_size
-            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
-            mtype = mimetypes.guess_type(file)[0]
-            headers = email.message_from_string(
-                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
-                (mtype or 'text/plain', size, modified))
-            if host:
-                host, port = splitport(host)
-            if not host or \
-                (not port and _safe_gethostbyname(host) in self.get_names()):
-                return addinfourl(open(localfile, 'rb'),
-                                  headers, 'file:'+file)
-        except OSError as msg:
-            # urllib2 users shouldn't expect OSErrors coming from urlopen()
-            raise URLError(msg)
-        raise URLError('file not on local host')
-
-def _safe_gethostbyname(host):
-    try:
-        return socket.gethostbyname(host)
-    except socket.gaierror:
-        return None
-
-class FTPHandler(BaseHandler):
-    def ftp_open(self, req):
-        import ftplib
-        import mimetypes
-        host = req.get_host()
-        if not host:
-            raise URLError('ftp error: no host given')
-        host, port = splitport(host)
-        if port is None:
-            port = ftplib.FTP_PORT
-        else:
-            port = int(port)
-
-        # username/password handling
-        user, host = splituser(host)
-        if user:
-            user, passwd = splitpasswd(user)
-        else:
-            passwd = None
-        host = unquote(host)
-        user = unquote(user or '')
-        passwd = unquote(passwd or '')
-
-        try:
-            host = socket.gethostbyname(host)
-        except socket.error as msg:
-            raise URLError(msg)
-        path, attrs = splitattr(req.get_selector())
-        dirs = path.split('/')
-        dirs = list(map(unquote, dirs))
-        dirs, file = dirs[:-1], dirs[-1]
-        if dirs and not dirs[0]:
-            dirs = dirs[1:]
-        try:
-            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
-            type = file and 'I' or 'D'
-            for attr in attrs:
-                attr, value = splitvalue(attr)
-                if attr.lower() == 'type' and \
-                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
-                    type = value.upper()
-            fp, retrlen = fw.retrfile(file, type)
-            headers = ""
-            mtype = mimetypes.guess_type(req.get_full_url())[0]
-            if mtype:
-                headers += "Content-type: %s\n" % mtype
-            if retrlen is not None and retrlen >= 0:
-                headers += "Content-length: %d\n" % retrlen
-            headers = email.message_from_string(headers)
-            sf = StringIO(str(headers))
-            return addinfourl(fp, headers, req.get_full_url())
-        except ftplib.all_errors as msg:
-            raise URLError('ftp error: %s' % msg).with_traceback(sys.exc_info()[2])
-
-    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
-        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
-        return fw
-
-class CacheFTPHandler(FTPHandler):
-    # XXX would be nice to have pluggable cache strategies
-    # XXX this stuff is definitely not thread safe
-    def __init__(self):
-        self.cache = {}
-        self.timeout = {}
-        self.soonest = 0
-        self.delay = 60
-        self.max_conns = 16
-
-    def setTimeout(self, t):
-        self.delay = t
-
-    def setMaxConns(self, m):
-        self.max_conns = m
-
-    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
-        key = user, host, port, '/'.join(dirs), timeout
-        if key in self.cache:
-            self.timeout[key] = time.time() + self.delay
-        else:
-            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
-            self.timeout[key] = time.time() + self.delay
-        self.check_cache()
-        return self.cache[key]
-
-    def check_cache(self):
-        # first check for old ones
-        t = time.time()
-        if self.soonest <= t:
-            for k, v in list(self.timeout.items()):
-                if v < t:
-                    self.cache[k].close()
-                    del self.cache[k]
-                    del self.timeout[k]
-        self.soonest = min(list(self.timeout.values()))
-
-        # then check the size
-        if len(self.cache) == self.max_conns:
-            for k, v in list(self.timeout.items()):
-                if v == self.soonest:
-                    del self.cache[k]
-                    del self.timeout[k]
-                    break
-            self.soonest = min(list(self.timeout.values()))