bpo-27485: Rename and deprecate undocumented functions in urllib.parse (GH-2205)

2 files changed, 152 insertions, 57 deletions

diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py
index 3f8cfe5..f21b8eb 100644
--- a/Lib/urllib/parse.py
+++ b/Lib/urllib/parse.py
@@ -30,6 +30,7 @@ test_urlparse.py provides a good indicator of parsing behavior.
 import re
 import sys
 import collections
+import warnings
 
 __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag",
            "urlsplit", "urlunsplit", "urlencode", "parse_qs",
@@ -288,7 +289,7 @@ by reference to a primary resource and additional identifying information.
 """
 
 _ParseResultBase.__doc__ = """
-ParseResult(scheme, netloc, path, params,  query, fragment)
+ParseResult(scheme, netloc, path, params, query, fragment)
 
 A 6-tuple that contains components of a parsed URL.
 """
@@ -913,7 +914,14 @@ def urlencode(query, doseq=False, safe='', encoding=None, errors=None,
                         l.append(k + '=' + elt)
     return '&'.join(l)
 
+
 def to_bytes(url):
+    warnings.warn("urllib.parse.to_bytes() is deprecated as of 3.8",
+                  DeprecationWarning, stacklevel=2)
+    return _to_bytes(url)
+
+
+def _to_bytes(url):
     """to_bytes(u"URL") --> 'URL'."""
     # Most URL schemes require ASCII. If that changes, the conversion
     # can be relaxed.
@@ -926,7 +934,14 @@ def to_bytes(url):
                                " contains non-ASCII characters")
     return url
 
+
 def unwrap(url):
+    warnings.warn("urllib.parse.unwrap() is deprecated as of 3.8",
+                  DeprecationWarning, stacklevel=2)
+    return _unwrap(url)
+
+
+def _unwrap(url):
     """unwrap('<URL:type://host/path>') --> 'type://host/path'."""
     url = str(url).strip()
     if url[:1] == '<' and url[-1:] == '>':
@@ -934,8 +949,16 @@ def unwrap(url):
     if url[:4] == 'URL:': url = url[4:].strip()
     return url
 
-_typeprog = None
+
 def splittype(url):
+    warnings.warn("urllib.parse.splittype() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splittype(url)
+
+
+_typeprog = None
+def _splittype(url):
     """splittype('type:opaquestring') --> 'type', 'opaquestring'."""
     global _typeprog
     if _typeprog is None:
@@ -947,8 +970,16 @@ def splittype(url):
         return scheme.lower(), data
     return None, url
 
-_hostprog = None
+
 def splithost(url):
+    warnings.warn("urllib.parse.splithost() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splithost(url)
+
+
+_hostprog = None
+def _splithost(url):
     """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
     global _hostprog
     if _hostprog is None:
@@ -962,19 +993,43 @@ def splithost(url):
         return host_port, path
     return None, url
 
+
 def splituser(host):
+    warnings.warn("urllib.parse.splituser() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splituser(host)
+
+
+def _splituser(host):
     """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
     user, delim, host = host.rpartition('@')
     return (user if delim else None), host
 
+
 def splitpasswd(user):
+    warnings.warn("urllib.parse.splitpasswd() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitpasswd(user)
+
+
+def _splitpasswd(user):
     """splitpasswd('user:passwd') -> 'user', 'passwd'."""
     user, delim, passwd = user.partition(':')
     return user, (passwd if delim else None)
 
+
+def splitport(host):
+    warnings.warn("urllib.parse.splitport() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitport(host)
+
+
 # splittag('/path#tag') --> '/path', 'tag'
 _portprog = None
-def splitport(host):
+def _splitport(host):
     """splitport('host:port') --> 'host', 'port'."""
     global _portprog
     if _portprog is None:
@@ -987,7 +1042,15 @@ def splitport(host):
             return host, port
     return host, None
 
+
 def splitnport(host, defport=-1):
+    warnings.warn("urllib.parse.splitnport() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitnport(host, defport)
+
+
+def _splitnport(host, defport=-1):
     """Split host and port, returning numeric port.
     Return given default port if no ':' found; defaults to -1.
     Return numerical port if a valid number are found after ':'.
@@ -1003,27 +1066,59 @@ def splitnport(host, defport=-1):
         return host, nport
     return host, defport
 
+
 def splitquery(url):
+    warnings.warn("urllib.parse.splitquery() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitquery(url)
+
+
+def _splitquery(url):
     """splitquery('/path?query') --> '/path', 'query'."""
     path, delim, query = url.rpartition('?')
     if delim:
         return path, query
     return url, None
 
+
 def splittag(url):
+    warnings.warn("urllib.parse.splittag() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splittag(url)
+
+
+def _splittag(url):
     """splittag('/path#tag') --> '/path', 'tag'."""
     path, delim, tag = url.rpartition('#')
     if delim:
         return path, tag
     return url, None
 
+
 def splitattr(url):
+    warnings.warn("urllib.parse.splitattr() is deprecated as of 3.8, "
+                  "use urllib.parse.urlparse() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitattr(url)
+
+
+def _splitattr(url):
     """splitattr('/path;attr1=value1;attr2=value2;...') ->
         '/path', ['attr1=value1', 'attr2=value2', ...]."""
     words = url.split(';')
     return words[0], words[1:]
 
+
 def splitvalue(attr):
+    warnings.warn("urllib.parse.splitvalue() is deprecated as of 3.8, "
+                  "use urllib.parse.parse_qsl() instead",
+                  DeprecationWarning, stacklevel=2)
+    return _splitvalue(attr)
+
+
+def _splitvalue(attr):
     """splitvalue('attr=value') --> 'attr', 'value'."""
     attr, delim, value = attr.partition('=')
     return attr, (value if delim else None)
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index 5b962f7..9d50b68 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -101,9 +101,9 @@ import warnings
 
 from urllib.error import URLError, HTTPError, ContentTooShortError
 from urllib.parse import (
-    urlparse, urlsplit, urljoin, unwrap, quote, unquote,
-    splittype, splithost, splitport, splituser, splitpasswd,
-    splitattr, splitquery, splitvalue, splittag, to_bytes,
+    urlparse, urlsplit, urljoin, _unwrap, quote, unquote,
+    _splittype, _splithost, _splitport, _splituser, _splitpasswd,
+    _splitattr, _splitquery, _splitvalue, _splittag, _to_bytes,
     unquote_to_bytes, urlunparse)
 from urllib.response import addinfourl, addclosehook
 
@@ -242,7 +242,7 @@ def urlretrieve(url, filename=None, reporthook=None, data=None):
     Returns a tuple containing the path to the newly created
     data file as well as the resulting HTTPMessage object.
     """
-    url_type, path = splittype(url)
+    url_type, path = _splittype(url)
 
     with contextlib.closing(urlopen(url, data)) as fp:
         headers = fp.info()
@@ -349,8 +349,8 @@ class Request:
     @full_url.setter
     def full_url(self, url):
         # unwrap('<URL:type://host/path>') --> 'type://host/path'
-        self._full_url = unwrap(url)
-        self._full_url, self.fragment = splittag(self._full_url)
+        self._full_url = _unwrap(url)
+        self._full_url, self.fragment = _splittag(self._full_url)
         self._parse()
 
     @full_url.deleter
@@ -378,10 +378,10 @@ class Request:
         self.data = None
 
     def _parse(self):
-        self.type, rest = splittype(self._full_url)
+        self.type, rest = _splittype(self._full_url)
         if self.type is None:
             raise ValueError("unknown url type: %r" % self.full_url)
-        self.host, self.selector = splithost(rest)
+        self.host, self.selector = _splithost(rest)
         if self.host:
             self.host = unquote(self.host)
 
@@ -768,7 +768,7 @@ def _parse_proxy(proxy):
     According to RFC 3986, having an authority component means the URL must
     have two slashes after the scheme.
     """
-    scheme, r_scheme = splittype(proxy)
+    scheme, r_scheme = _splittype(proxy)
     if not r_scheme.startswith("/"):
         # authority
         scheme = None
@@ -783,9 +783,9 @@ def _parse_proxy(proxy):
         if end == -1:
             end = None
         authority = r_scheme[2:end]
-    userinfo, hostport = splituser(authority)
+    userinfo, hostport = _splituser(authority)
     if userinfo is not None:
-        user, password = splitpasswd(userinfo)
+        user, password = _splitpasswd(userinfo)
     else:
         user = password = None
     return scheme, user, password, hostport
@@ -872,7 +872,7 @@ class HTTPPasswordMgr:
             scheme = None
             authority = uri
             path = '/'
-        host, port = splitport(authority)
+        host, port = _splitport(authority)
         if default_port and port is None and scheme is not None:
             dport = {"http": 80,
                      "https": 443,
@@ -1261,8 +1261,8 @@ class AbstractHTTPHandler(BaseHandler):
 
         sel_host = host
         if request.has_proxy():
-            scheme, sel = splittype(request.selector)
-            sel_host, sel_path = splithost(sel)
+            scheme, sel = _splittype(request.selector)
+            sel_host, sel_path = _splithost(sel)
         if not request.has_header('Host'):
             request.add_unredirected_header('Host', sel_host)
         for name, value in self.parent.addheaders:
@@ -1478,7 +1478,7 @@ class FileHandler(BaseHandler):
                 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
                 (mtype or 'text/plain', size, modified))
             if host:
-                host, port = splitport(host)
+                host, port = _splitport(host)
             if not host or \
                 (not port and _safe_gethostbyname(host) in self.get_names()):
                 if host:
@@ -1503,16 +1503,16 @@ class FTPHandler(BaseHandler):
         host = req.host
         if not host:
             raise URLError('ftp error: no host given')
-        host, port = splitport(host)
+        host, port = _splitport(host)
         if port is None:
             port = ftplib.FTP_PORT
         else:
             port = int(port)
 
         # username/password handling
-        user, host = splituser(host)
+        user, host = _splituser(host)
         if user:
-            user, passwd = splitpasswd(user)
+            user, passwd = _splitpasswd(user)
         else:
             passwd = None
         host = unquote(host)
@@ -1523,7 +1523,7 @@ class FTPHandler(BaseHandler):
             host = socket.gethostbyname(host)
         except OSError as msg:
             raise URLError(msg)
-        path, attrs = splitattr(req.selector)
+        path, attrs = _splitattr(req.selector)
         dirs = path.split('/')
         dirs = list(map(unquote, dirs))
         dirs, file = dirs[:-1], dirs[-1]
@@ -1533,7 +1533,7 @@ class FTPHandler(BaseHandler):
             fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
             type = file and 'I' or 'D'
             for attr in attrs:
-                attr, value = splitvalue(attr)
+                attr, value = _splitvalue(attr)
                 if attr.lower() == 'type' and \
                    value in ('a', 'A', 'i', 'I', 'd', 'D'):
                     type = value.upper()
@@ -1727,19 +1727,19 @@ class URLopener:
     # External interface
     def open(self, fullurl, data=None):
         """Use URLopener().open(file) instead of open(file, 'r')."""
-        fullurl = unwrap(to_bytes(fullurl))
+        fullurl = _unwrap(_to_bytes(fullurl))
         fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|")
         if self.tempcache and fullurl in self.tempcache:
             filename, headers = self.tempcache[fullurl]
             fp = open(filename, 'rb')
             return addinfourl(fp, headers, fullurl)
-        urltype, url = splittype(fullurl)
+        urltype, url = _splittype(fullurl)
         if not urltype:
             urltype = 'file'
         if urltype in self.proxies:
             proxy = self.proxies[urltype]
-            urltype, proxyhost = splittype(proxy)
-            host, selector = splithost(proxyhost)
+            urltype, proxyhost = _splittype(proxy)
+            host, selector = _splithost(proxyhost)
             url = (host, fullurl) # Signal special case to open_*()
         else:
             proxy = None
@@ -1763,22 +1763,22 @@ class URLopener:
 
     def open_unknown(self, fullurl, data=None):
         """Overridable interface to open unknown URL type."""
-        type, url = splittype(fullurl)
+        type, url = _splittype(fullurl)
         raise OSError('url error', 'unknown url type', type)
 
     def open_unknown_proxy(self, proxy, fullurl, data=None):
         """Overridable interface to open unknown URL type."""
-        type, url = splittype(fullurl)
+        type, url = _splittype(fullurl)
         raise OSError('url error', 'invalid proxy for %s' % type, proxy)
 
     # External interface
     def retrieve(self, url, filename=None, reporthook=None, data=None):
         """retrieve(url) returns (filename, headers) for a local object
         or (tempfilename, headers) for a remote object."""
-        url = unwrap(to_bytes(url))
+        url = _unwrap(_to_bytes(url))
         if self.tempcache and url in self.tempcache:
             return self.tempcache[url]
-        type, url1 = splittype(url)
+        type, url1 = _splittype(url)
         if filename is None and (not type or type == 'file'):
             try:
                 fp = self.open_local_file(url1)
@@ -1853,25 +1853,25 @@ class URLopener:
         user_passwd = None
         proxy_passwd= None
         if isinstance(url, str):
-            host, selector = splithost(url)
+            host, selector = _splithost(url)
             if host:
-                user_passwd, host = splituser(host)
+                user_passwd, host = _splituser(host)
                 host = unquote(host)
             realhost = host
         else:
             host, selector = url
             # check whether the proxy contains authorization information
-            proxy_passwd, host = splituser(host)
+            proxy_passwd, host = _splituser(host)
             # now we proceed with the url we want to obtain
-            urltype, rest = splittype(selector)
+            urltype, rest = _splittype(selector)
             url = rest
             user_passwd = None
             if urltype.lower() != 'http':
                 realhost = None
             else:
-                realhost, rest = splithost(rest)
+                realhost, rest = _splithost(rest)
                 if realhost:
-                    user_passwd, realhost = splituser(realhost)
+                    user_passwd, realhost = _splituser(realhost)
                 if user_passwd:
                     selector = "%s://%s%s" % (urltype, realhost, rest)
                 if proxy_bypass(realhost):
@@ -1977,7 +1977,7 @@ class URLopener:
         """Use local file."""
         import email.utils
         import mimetypes
-        host, file = splithost(url)
+        host, file = _splithost(url)
         localname = url2pathname(file)
         try:
             stats = os.stat(localname)
@@ -1994,7 +1994,7 @@ class URLopener:
             if file[:1] == '/':
                 urlfile = 'file://' + file
             return addinfourl(open(localname, 'rb'), headers, urlfile)
-        host, port = splitport(host)
+        host, port = _splitport(host)
         if (not port
            and socket.gethostbyname(host) in ((localhost(),) + thishost())):
             urlfile = file
@@ -2010,11 +2010,11 @@ class URLopener:
         if not isinstance(url, str):
             raise URLError('ftp error: proxy support for ftp protocol currently not implemented')
         import mimetypes
-        host, path = splithost(url)
+        host, path = _splithost(url)
         if not host: raise URLError('ftp error: no host given')
-        host, port = splitport(host)
-        user, host = splituser(host)
-        if user: user, passwd = splitpasswd(user)
+        host, port = _splitport(host)
+        user, host = _splituser(host)
+        if user: user, passwd = _splitpasswd(user)
         else: passwd = None
         host = unquote(host)
         user = unquote(user or '')
@@ -2025,7 +2025,7 @@ class URLopener:
             port = ftplib.FTP_PORT
         else:
             port = int(port)
-        path, attrs = splitattr(path)
+        path, attrs = _splitattr(path)
         path = unquote(path)
         dirs = path.split('/')
         dirs, file = dirs[:-1], dirs[-1]
@@ -2047,7 +2047,7 @@ class URLopener:
             if not file: type = 'D'
             else: type = 'I'
             for attr in attrs:
-                attr, value = splitvalue(attr)
+                attr, value = _splitvalue(attr)
                 if attr.lower() == 'type' and \
                    value in ('a', 'A', 'i', 'I', 'd', 'D'):
                     type = value.upper()
@@ -2230,11 +2230,11 @@ class FancyURLopener(URLopener):
             return getattr(self,name)(url, realm, data)
 
     def retry_proxy_http_basic_auth(self, url, realm, data=None):
-        host, selector = splithost(url)
+        host, selector = _splithost(url)
         newurl = 'http://' + host + selector
         proxy = self.proxies['http']
-        urltype, proxyhost = splittype(proxy)
-        proxyhost, proxyselector = splithost(proxyhost)
+        urltype, proxyhost = _splittype(proxy)
+        proxyhost, proxyselector = _splithost(proxyhost)
         i = proxyhost.find('@') + 1
         proxyhost = proxyhost[i:]
         user, passwd = self.get_user_passwd(proxyhost, realm, i)
@@ -2248,11 +2248,11 @@ class FancyURLopener(URLopener):
             return self.open(newurl, data)
 
     def retry_proxy_https_basic_auth(self, url, realm, data=None):
-        host, selector = splithost(url)
+        host, selector = _splithost(url)
         newurl = 'https://' + host + selector
         proxy = self.proxies['https']
-        urltype, proxyhost = splittype(proxy)
-        proxyhost, proxyselector = splithost(proxyhost)
+        urltype, proxyhost = _splittype(proxy)
+        proxyhost, proxyselector = _splithost(proxyhost)
         i = proxyhost.find('@') + 1
         proxyhost = proxyhost[i:]
         user, passwd = self.get_user_passwd(proxyhost, realm, i)
@@ -2266,7 +2266,7 @@ class FancyURLopener(URLopener):
             return self.open(newurl, data)
 
     def retry_http_basic_auth(self, url, realm, data=None):
-        host, selector = splithost(url)
+        host, selector = _splithost(url)
         i = host.find('@') + 1
         host = host[i:]
         user, passwd = self.get_user_passwd(host, realm, i)
@@ -2280,7 +2280,7 @@ class FancyURLopener(URLopener):
             return self.open(newurl, data)
 
     def retry_https_basic_auth(self, url, realm, data=None):
-        host, selector = splithost(url)
+        host, selector = _splithost(url)
         i = host.find('@') + 1
         host = host[i:]
         user, passwd = self.get_user_passwd(host, realm, i)
@@ -2502,7 +2502,7 @@ def proxy_bypass_environment(host, proxies=None):
     if no_proxy == '*':
         return 1
     # strip port off host
-    hostonly, port = splitport(host)
+    hostonly, port = _splitport(host)
     # check if the host ends with any of the DNS suffixes
     no_proxy_list = [proxy.strip() for proxy in no_proxy.split(',')]
     for name in no_proxy_list:
@@ -2533,7 +2533,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings):
     """
     from fnmatch import fnmatch
 
-    hostonly, port = splitport(host)
+    hostonly, port = _splitport(host)
 
     def ip2num(ipAddr):
         parts = ipAddr.split('.')
@@ -2687,7 +2687,7 @@ elif os.name == 'nt':
         if not proxyEnable or not proxyOverride:
             return 0
         # try to make a host list from name and IP address.
-        rawHost, port = splitport(host)
+        rawHost, port = _splitport(host)
         host = [rawHost]
         try:
             addr = socket.gethostbyname(rawHost)