diff options
Diffstat (limited to 'Lib/test/test_urllib2.py')
| -rw-r--r-- | Lib/test/test_urllib2.py | 223 |
1 files changed, 166 insertions, 57 deletions
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index d069139..7d41ea1 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1,5 +1,6 @@ import unittest from test import support +from test import test_urllib import os import io @@ -10,8 +11,10 @@ import sys import urllib.request # The proxy bypass method imported below has logic specific to the OSX # proxy config data structure but is testable on all platforms. -from urllib.request import Request, OpenerDirector, _proxy_bypass_macosx_sysconf +from urllib.request import Request, OpenerDirector, _parse_proxy, _proxy_bypass_macosx_sysconf +from urllib.parse import urlparse import urllib.error +import http.client # XXX # Request @@ -41,7 +44,7 @@ class TrivialTests(unittest.TestCase): self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url') # XXX Name hacking to get this to work on Windows. - fname = os.path.abspath(urllib.request.__file__).replace('\\', '/') + fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/') if os.name == 'nt': file_url = "file:///%s" % fname @@ -118,6 +121,15 @@ class RequestHdrsTests(unittest.TestCase): self.assertIsNone(req.get_header("Not-there")) self.assertEqual(req.get_header("Not-there", "default"), "default") + req.remove_header("Spam-eggs") + self.assertFalse(req.has_header("Spam-eggs")) + + req.add_unredirected_header("Unredirected-spam", "Eggs") + self.assertTrue(req.has_header("Unredirected-spam")) + + req.remove_header("Unredirected-spam") + self.assertFalse(req.has_header("Unredirected-spam")) + def test_password_manager(self): mgr = urllib.request.HTTPPasswordMgr() @@ -311,8 +323,7 @@ class MockHTTPClass: if body: self.data = body if self.raise_on_endheaders: - import socket - raise socket.error() + raise OSError() def getresponse(self): return MockHTTPResponse(MockFile(), {}, 200, "OK") @@ -597,27 +608,6 @@ class OpenerDirectorTests(unittest.TestCase): if args[1] is not None: self.assertIsInstance(args[1], MockResponse) - def test_method_deprecations(self): - req = Request("http://www.example.com") - - with self.assertWarns(DeprecationWarning): - req.add_data("data") - with self.assertWarns(DeprecationWarning): - req.get_data() - with self.assertWarns(DeprecationWarning): - req.has_data() - with self.assertWarns(DeprecationWarning): - req.get_host() - with self.assertWarns(DeprecationWarning): - req.get_selector() - with self.assertWarns(DeprecationWarning): - req.is_unverifiable() - with self.assertWarns(DeprecationWarning): - req.get_origin_req_host() - with self.assertWarns(DeprecationWarning): - req.get_type() - - def sanepathname2url(path): try: path.encode("utf-8") @@ -690,7 +680,7 @@ class HandlerTests(unittest.TestCase): self.assertEqual(int(headers["Content-length"]), len(data)) def test_file(self): - import email.utils, socket + import email.utils h = urllib.request.FileHandler() o = h.parent = MockOpener() @@ -737,6 +727,7 @@ class HandlerTests(unittest.TestCase): for url in [ "file://localhost:80%s" % urlpath, "file:///file_does_not_exist.txt", + "file://not-a-local-host.com//dir/file.txt", "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), os.getcwd(), TESTFN), "file://somerandomhost.ontheinternet.com%s/%s" % @@ -812,7 +803,7 @@ class HandlerTests(unittest.TestCase): ("Foo", "bar"), ("Spam", "eggs")]) self.assertEqual(http.data, data) - # check socket.error converted to URLError + # check OSError converted to URLError http.raise_on_endheaders = True self.assertRaises(urllib.error.URLError, h.do_open, http, req) @@ -917,6 +908,36 @@ class HandlerTests(unittest.TestCase): p_ds_req = h.do_request_(ds_req) self.assertEqual(p_ds_req.unredirected_hdrs["Host"],"example.com") + def test_full_url_setter(self): + # Checks to ensure that components are set correctly after setting the + # full_url of a Request object + + urls = [ + 'http://example.com?foo=bar#baz', + 'http://example.com?foo=bar&spam=eggs#bash', + 'http://example.com', + ] + + # testing a reusable request instance, but the url parameter is + # required, so just use a dummy one to instantiate + r = Request('http://example.com') + for url in urls: + r.full_url = url + parsed = urlparse(url) + + self.assertEqual(r.get_full_url(), url) + # full_url setter uses splittag to split into components. + # splittag sets the fragment as None while urlparse sets it to '' + self.assertEqual(r.fragment or '', parsed.fragment) + self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) + + def test_full_url_deleter(self): + r = Request('http://www.example.com') + del r.full_url + self.assertIsNone(r.full_url) + self.assertIsNone(r.fragment) + self.assertEqual(r.selector, '') + def test_fixpath_in_weirdurls(self): # Issue4493: urllib2 to supply '/' when to urls where path does not # start with'/' @@ -1374,6 +1395,33 @@ class HandlerTests(unittest.TestCase): self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) + def test_http_closed(self): + """Test the connection is cleaned up when the response is closed""" + for (transfer, data) in ( + ("Connection: close", b"data"), + ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"), + ("Content-Length: 4", b"data"), + ): + header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer) + conn = test_urllib.fakehttp(header.encode() + data) + handler = urllib.request.AbstractHTTPHandler() + req = Request("http://dummy/") + req.timeout = None + with handler.do_open(conn, req) as resp: + resp.read() + self.assertTrue(conn.fakesock.closed, + "Connection not closed with {!r}".format(transfer)) + + def test_invalid_closed(self): + """Test the connection is cleaned up after an invalid response""" + conn = test_urllib.fakehttp(b"") + handler = urllib.request.AbstractHTTPHandler() + req = Request("http://dummy/") + req.timeout = None + with self.assertRaises(http.client.BadStatusLine): + handler.do_open(conn, req) + self.assertTrue(conn.fakesock.closed, "Connection not closed") + class MiscTests(unittest.TestCase): @@ -1418,6 +1466,22 @@ class MiscTests(unittest.TestCase): self.opener_has_handler(o, MyHTTPHandler) self.opener_has_handler(o, MyOtherHTTPHandler) + @unittest.skipUnless(support.is_resource_enabled('network'), + 'test requires network access') + def test_issue16464(self): + with support.transient_internet("http://www.example.com/"): + opener = urllib.request.build_opener() + request = urllib.request.Request("http://www.example.com/") + self.assertEqual(None, request.data) + + opener.open(request, "1".encode("us-ascii")) + self.assertEqual(b"1", request.data) + self.assertEqual("1", request.get_header("Content-length")) + + opener.open(request, "1234567890".encode("us-ascii")) + self.assertEqual(b"1234567890", request.data) + self.assertEqual("10", request.get_header("Content-length")) + def test_HTTPError_interface(self): """ Issue 13211 reveals that HTTPError didn't implement the URLError @@ -1429,23 +1493,68 @@ class MiscTests(unittest.TestCase): err = urllib.error.HTTPError(url, code, msg, hdrs, fp) self.assertTrue(hasattr(err, 'reason')) self.assertEqual(err.reason, 'something bad happened') - self.assertTrue(hasattr(err, 'hdrs')) - self.assertEqual(err.hdrs, 'Content-Length: 42') + self.assertTrue(hasattr(err, 'headers')) + self.assertEqual(err.headers, 'Content-Length: 42') expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) self.assertEqual(str(err), expected_errmsg) + def test_parse_proxy(self): + parse_proxy_test_cases = [ + ('proxy.example.com', + (None, None, None, 'proxy.example.com')), + ('proxy.example.com:3128', + (None, None, None, 'proxy.example.com:3128')), + ('proxy.example.com', (None, None, None, 'proxy.example.com')), + ('proxy.example.com:3128', + (None, None, None, 'proxy.example.com:3128')), + # The authority component may optionally include userinfo + # (assumed to be # username:password): + ('joe:password@proxy.example.com', + (None, 'joe', 'password', 'proxy.example.com')), + ('joe:password@proxy.example.com:3128', + (None, 'joe', 'password', 'proxy.example.com:3128')), + #Examples with URLS + ('http://proxy.example.com/', + ('http', None, None, 'proxy.example.com')), + ('http://proxy.example.com:3128/', + ('http', None, None, 'proxy.example.com:3128')), + ('http://joe:password@proxy.example.com/', + ('http', 'joe', 'password', 'proxy.example.com')), + ('http://joe:password@proxy.example.com:3128', + ('http', 'joe', 'password', 'proxy.example.com:3128')), + # Everything after the authority is ignored + ('ftp://joe:password@proxy.example.com/rubbish:3128', + ('ftp', 'joe', 'password', 'proxy.example.com')), + # Test for no trailing '/' case + ('http://joe:password@proxy.example.com', + ('http', 'joe', 'password', 'proxy.example.com')) + ] + + for tc, expected in parse_proxy_test_cases: + self.assertEqual(_parse_proxy(tc), expected) + + self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), class RequestTests(unittest.TestCase): + class PutRequest(Request): + method='PUT' def setUp(self): self.get = Request("http://www.python.org/~jeremy/") self.post = Request("http://www.python.org/~jeremy/", "data", headers={"X-Test": "test"}) + self.head = Request("http://www.python.org/~jeremy/", method='HEAD') + self.put = self.PutRequest("http://www.python.org/~jeremy/") + self.force_post = self.PutRequest("http://www.python.org/~jeremy/", + method="POST") def test_method(self): self.assertEqual("POST", self.post.get_method()) self.assertEqual("GET", self.get.get_method()) + self.assertEqual("HEAD", self.head.get_method()) + self.assertEqual("PUT", self.put.get_method()) + self.assertEqual("POST", self.force_post.get_method()) def test_data(self): self.assertFalse(self.get.data) @@ -1454,6 +1563,25 @@ class RequestTests(unittest.TestCase): self.assertTrue(self.get.data) self.assertEqual("POST", self.get.get_method()) + # issue 16464 + # if we change data we need to remove content-length header + # (cause it's most probably calculated for previous value) + def test_setting_data_should_remove_content_length(self): + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + self.get.add_unredirected_header("Content-length", 42) + self.assertEqual(42, self.get.unredirected_hdrs["Content-length"]) + self.get.data = "spam" + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + + # issue 17485 same for deleting data. + def test_deleting_data_should_remove_content_length(self): + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + self.get.data = 'foo' + self.get.add_unredirected_header("Content-length", 3) + self.assertEqual(3, self.get.unredirected_hdrs["Content-length"]) + del self.get.data + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + def test_get_full_url(self): self.assertEqual("http://www.python.org/~jeremy/", self.get.get_full_url()) @@ -1495,33 +1623,14 @@ class RequestTests(unittest.TestCase): req = Request(url) self.assertEqual(req.get_full_url(), url) - def test_HTTPError_interface_call(self): - """ - Issue 15701 - HTTPError interface has info method available from URLError - """ - err = urllib.request.HTTPError(msg="something bad happened", url=None, - code=None, hdrs='Content-Length:42', fp=None) - self.assertTrue(hasattr(err, 'reason')) - assert hasattr(err, 'reason') - assert hasattr(err, 'info') - assert callable(err.info) - try: - err.info() - except AttributeError: - self.fail('err.info call failed.') - self.assertEqual(err.info(), "Content-Length:42") - -def test_main(verbose=None): - from test import test_urllib2 - support.run_doctest(test_urllib2, verbose) - support.run_doctest(urllib.request, verbose) - tests = (TrivialTests, - OpenerDirectorTests, - HandlerTests, - MiscTests, - RequestTests, - RequestHdrsTests) - support.run_unittest(*tests) + def test_url_fullurl_get_full_url(self): + urls = ['http://docs.python.org', + 'http://docs.python.org/library/urllib2.html#OK', + 'http://www.python.org/?qs=query#fragment=true' ] + for url in urls: + req = Request(url) + self.assertEqual(req.get_full_url(), req.full_url) + if __name__ == "__main__": - test_main(verbose=True) + unittest.main() |