Make a new urllib package .

It consists of code from urllib, urllib2, urlparse, and robotparser.
The old modules have all been removed.  The new package has five
submodules: urllib.parse, urllib.request, urllib.response,
urllib.error, and urllib.robotparser.  The urllib.request.urlopen()
function uses the url opener from urllib2.

Note that the unittests have not been renamed for the
beta, but they will be renamed in the future.

Joint work with Senthil Kumaran.

1 files changed, 47 insertions, 45 deletions

diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py
index c130190..1627923 100644
--- a/Lib/test/test_http_cookiejar.py
+++ b/Lib/test/test_http_cookiejar.py
@@ -1,6 +1,6 @@
 """Tests for http/cookiejar.py."""
 
-import re, os, time, urllib2
+import re, os, time, urllib.request
 from unittest import TestCase
 
 from test import support
@@ -206,7 +206,7 @@ def interact_netscape(cookiejar, url, *set_cookie_hdrs):
 
 def _interact(cookiejar, url, set_cookie_hdrs, hdr_name):
     """Perform a single request / response cycle, returning Cookie: header."""
-    req = urllib2.Request(url)
+    req = urllib.request.Request(url)
     cookiejar.add_cookie_header(req)
     cookie_hdr = req.get_header("Cookie", "")
     headers = []
@@ -330,7 +330,7 @@ class CookieTests(TestCase):
             ("http://foo/", "foo.local", True),
             ("http://foo/", ".local", True),
             ]:
-            request = urllib2.Request(url)
+            request = urllib.request.Request(url)
             r = pol.domain_return_ok(domain, request)
             if ok: self.assert_(r)
             else: self.assert_(not r)
@@ -547,46 +547,48 @@ class CookieTests(TestCase):
 
     def test_request_path(self):
         # with parameters
-        req = urllib2.Request("http://www.example.com/rheum/rhaponicum;"
-                              "foo=bar;sing=song?apples=pears&spam=eggs#ni")
+        req = urllib.request.Request(
+            "http://www.example.com/rheum/rhaponicum;"
+            "foo=bar;sing=song?apples=pears&spam=eggs#ni")
         self.assertEquals(request_path(req), "/rheum/rhaponicum;"
                      "foo=bar;sing=song?apples=pears&spam=eggs#ni")
         # without parameters
-        req = urllib2.Request("http://www.example.com/rheum/rhaponicum?"
-                              "apples=pears&spam=eggs#ni")
+        req = urllib.request.Request(
+            "http://www.example.com/rheum/rhaponicum?"
+            "apples=pears&spam=eggs#ni")
         self.assertEquals(request_path(req), "/rheum/rhaponicum?"
                      "apples=pears&spam=eggs#ni")
         # missing final slash
-        req = urllib2.Request("http://www.example.com")
+        req = urllib.request.Request("http://www.example.com")
         self.assertEquals(request_path(req), "/")
 
     def test_request_port(self):
-        req = urllib2.Request("http://www.acme.com:1234/",
-                              headers={"Host": "www.acme.com:4321"})
+        req = urllib.request.Request("http://www.acme.com:1234/",
+                                     headers={"Host": "www.acme.com:4321"})
         self.assertEquals(request_port(req), "1234")
-        req = urllib2.Request("http://www.acme.com/",
-                              headers={"Host": "www.acme.com:4321"})
+        req = urllib.request.Request("http://www.acme.com/",
+                                     headers={"Host": "www.acme.com:4321"})
         self.assertEquals(request_port(req), DEFAULT_HTTP_PORT)
 
     def test_request_host(self):
         # this request is illegal (RFC2616, 14.2.3)
-        req = urllib2.Request("http://1.1.1.1/",
-                              headers={"Host": "www.acme.com:80"})
+        req = urllib.request.Request("http://1.1.1.1/",
+                                     headers={"Host": "www.acme.com:80"})
         # libwww-perl wants this response, but that seems wrong (RFC 2616,
         # section 5.2, point 1., and RFC 2965 section 1, paragraph 3)
         #self.assertEquals(request_host(req), "www.acme.com")
         self.assertEquals(request_host(req), "1.1.1.1")
-        req = urllib2.Request("http://www.acme.com/",
-                              headers={"Host": "irrelevant.com"})
+        req = urllib.request.Request("http://www.acme.com/",
+                                     headers={"Host": "irrelevant.com"})
         self.assertEquals(request_host(req), "www.acme.com")
         # not actually sure this one is valid Request object, so maybe should
         # remove test for no host in url in request_host function?
-        req = urllib2.Request("/resource.html",
-                              headers={"Host": "www.acme.com"})
+        req = urllib.request.Request("/resource.html",
+                                     headers={"Host": "www.acme.com"})
         self.assertEquals(request_host(req), "www.acme.com")
         # port shouldn't be in request-host
-        req = urllib2.Request("http://www.acme.com:2345/resource.html",
-                              headers={"Host": "www.acme.com:5432"})
+        req = urllib.request.Request("http://www.acme.com:2345/resource.html",
+                                     headers={"Host": "www.acme.com:5432"})
         self.assertEquals(request_host(req), "www.acme.com")
 
     def test_is_HDN(self):
@@ -766,24 +768,24 @@ class CookieTests(TestCase):
             blocked_domains=["acme.com"],
             allowed_domains=["www.acme.com"]))
 
-        req = urllib2.Request("http://acme.com/")
+        req = urllib.request.Request("http://acme.com/")
         headers = ["Set-Cookie: CUSTOMER=WILE_E_COYOTE; path=/"]
         res = FakeResponse(headers, "http://acme.com/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 0)
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 1)
 
-        req = urllib2.Request("http://www.coyote.com/")
+        req = urllib.request.Request("http://www.coyote.com/")
         res = FakeResponse(headers, "http://www.coyote.com/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 1)
 
         # set a cookie with non-allowed domain...
-        req = urllib2.Request("http://www.coyote.com/")
+        req = urllib.request.Request("http://www.coyote.com/")
         res = FakeResponse(headers, "http://www.coyote.com/")
         cookies = c.make_cookies(res, req)
         c.set_cookie(cookies[0])
@@ -798,7 +800,7 @@ class CookieTests(TestCase):
         c = CookieJar(policy=pol)
         headers = ["Set-Cookie: CUSTOMER=WILE_E_COYOTE; path=/"]
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 0)
@@ -808,11 +810,11 @@ class CookieTests(TestCase):
         self.assertEquals(len(c), 1)
 
         c.clear()
-        req = urllib2.Request("http://www.roadrunner.net/")
+        req = urllib.request.Request("http://www.roadrunner.net/")
         res = FakeResponse(headers, "http://www.roadrunner.net/")
         c.extract_cookies(res, req)
         self.assertEquals(len(c), 1)
-        req = urllib2.Request("http://www.roadrunner.net/")
+        req = urllib.request.Request("http://www.roadrunner.net/")
         c.add_cookie_header(req)
         self.assert_((req.has_header("Cookie") and
                       req.has_header("Cookie2")))
@@ -823,7 +825,7 @@ class CookieTests(TestCase):
         self.assertEquals(len(c), 1)
 
         # set a cookie with blocked domain...
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         res = FakeResponse(headers, "http://www.acme.com/")
         cookies = c.make_cookies(res, req)
         c.set_cookie(cookies[0])
@@ -866,7 +868,7 @@ class CookieTests(TestCase):
         url = "http://www.acme.com"
         c = CookieJar(DefaultCookiePolicy(rfc2965=True))
         interact_2965(c, url, "foo=bar; Version=1")
-        req = urllib2.Request(url)
+        req = urllib.request.Request(url)
         self.assertEquals(len(c), 1)
         c.add_cookie_header(req)
         self.assert_(req.has_header("Cookie"))
@@ -1009,7 +1011,7 @@ class CookieTests(TestCase):
 
         def cookiejar_from_cookie_headers(headers):
             c = CookieJar()
-            req = urllib2.Request("http://www.example.com/")
+            req = urllib.request.Request("http://www.example.com/")
             r = FakeResponse(headers, "http://www.example.com/")
             c.extract_cookies(r, req)
             return c
@@ -1080,9 +1082,9 @@ class LWPCookieTests(TestCase):
 
         c = CookieJar(DefaultCookiePolicy(rfc2965 = True))
 
-        #req = urllib2.Request("http://1.1.1.1/",
+        #req = urllib.request.Request("http://1.1.1.1/",
         #              headers={"Host": "www.acme.com:80"})
-        req = urllib2.Request("http://www.acme.com:80/",
+        req = urllib.request.Request("http://www.acme.com:80/",
                       headers={"Host": "www.acme.com:80"})
 
         headers.append(
@@ -1091,7 +1093,7 @@ class LWPCookieTests(TestCase):
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         c.add_cookie_header(req)
 
         self.assertEqual(req.get_header("Cookie"), "CUSTOMER=WILE_E_COYOTE")
@@ -1101,7 +1103,7 @@ class LWPCookieTests(TestCase):
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/foo/bar")
+        req = urllib.request.Request("http://www.acme.com/foo/bar")
         c.add_cookie_header(req)
 
         h = req.get_header("Cookie")
@@ -1112,7 +1114,7 @@ class LWPCookieTests(TestCase):
         res = FakeResponse(headers, "http://www.acme.com")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         c.add_cookie_header(req)
 
         h = req.get_header("Cookie")
@@ -1120,7 +1122,7 @@ class LWPCookieTests(TestCase):
                      "CUSTOMER=WILE_E_COYOTE" in h and
                      "SHIPPING=FEDEX" not in h)
 
-        req = urllib2.Request("http://www.acme.com/foo/")
+        req = urllib.request.Request("http://www.acme.com/foo/")
         c.add_cookie_header(req)
 
         h = req.get_header("Cookie")
@@ -1155,13 +1157,13 @@ class LWPCookieTests(TestCase):
         c = CookieJar()
         headers = []
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         headers.append("Set-Cookie: PART_NUMBER=ROCKET_LAUNCHER_0001; path=/")
         res = FakeResponse(headers, "http://www.acme.com/")
 
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/")
+        req = urllib.request.Request("http://www.acme.com/")
         c.add_cookie_header(req)
 
         self.assertEquals(req.get_header("Cookie"),
@@ -1172,7 +1174,7 @@ class LWPCookieTests(TestCase):
         res = FakeResponse(headers, "http://www.acme.com/")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.acme.com/ammo")
+        req = urllib.request.Request("http://www.acme.com/ammo")
         c.add_cookie_header(req)
 
         self.assert_(re.search(r"PART_NUMBER=RIDING_ROCKET_0023;\s*"
@@ -1503,7 +1505,7 @@ class LWPCookieTests(TestCase):
         # Some additional Netscape cookies tests.
         c = CookieJar()
         headers = []
-        req = urllib2.Request("http://foo.bar.acme.com/foo")
+        req = urllib.request.Request("http://foo.bar.acme.com/foo")
 
         # Netscape allows a host part that contains dots
         headers.append("Set-Cookie: Customer=WILE_E_COYOTE; domain=.acme.com")
@@ -1517,7 +1519,7 @@ class LWPCookieTests(TestCase):
         res = FakeResponse(headers, "http://www.acme.com/foo")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://foo.bar.acme.com/foo")
+        req = urllib.request.Request("http://foo.bar.acme.com/foo")
         c.add_cookie_header(req)
         self.assert_(
             "PART_NUMBER=3,4" in req.get_header("Cookie") and
@@ -1559,12 +1561,12 @@ class LWPCookieTests(TestCase):
         c = CookieJar(DefaultCookiePolicy(rfc2965 = True))
         headers = []
 
-        req = urllib2.Request("http://www.ants.com/")
+        req = urllib.request.Request("http://www.ants.com/")
         headers.append("Set-Cookie: JSESSIONID=ABCDERANDOM123; Path=")
         res = FakeResponse(headers, "http://www.ants.com/")
         c.extract_cookies(res, req)
 
-        req = urllib2.Request("http://www.ants.com/")
+        req = urllib.request.Request("http://www.ants.com/")
         c.add_cookie_header(req)
 
         self.assertEquals(req.get_header("Cookie"),
@@ -1572,7 +1574,7 @@ class LWPCookieTests(TestCase):
         self.assertEquals(req.get_header("Cookie2"), '$Version="1"')
 
         # missing path in the request URI
-        req = urllib2.Request("http://www.ants.com:8080")
+        req = urllib.request.Request("http://www.ants.com:8080")
         c.add_cookie_header(req)
 
         self.assertEquals(req.get_header("Cookie"),
@@ -1585,7 +1587,7 @@ class LWPCookieTests(TestCase):
         # Check session cookies are deleted properly by
         # CookieJar.clear_session_cookies method
 
-        req = urllib2.Request('http://www.perlmeister.com/scripts')
+        req = urllib.request.Request('http://www.perlmeister.com/scripts')
         headers = []
         headers.append("Set-Cookie: s1=session;Path=/scripts")
         headers.append("Set-Cookie: p1=perm; Domain=.perlmeister.com;"