diff options
| -rw-r--r-- | Doc/library/http.client.rst | 12 | ||||
| -rw-r--r-- | Doc/library/urllib.request.rst | 16 | ||||
| -rw-r--r-- | Lib/http/client.py | 17 | ||||
| -rw-r--r-- | Lib/test/test_httplib.py | 16 | ||||
| -rw-r--r-- | Lib/test/test_urllib2.py | 51 | ||||
| -rw-r--r-- | Lib/urllib/request.py | 14 | ||||
| -rw-r--r-- | Misc/NEWS | 3 |
7 files changed, 112 insertions, 17 deletions
diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 67d7271..cba5907 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -393,14 +393,18 @@ HTTPConnection Objects string. The *body* may also be an open :term:`file object`, in which case the - contents of the file is sent; this file object should support - ``fileno()`` and ``read()`` methods. The header Content-Length is - automatically set to the length of the file as reported by - stat. + contents of the file is sent; this file object should support ``fileno()`` + and ``read()`` methods. The header Content-Length is automatically set to + the length of the file as reported by stat. The *body* argument may also be + an iterable and Contet-Length header should be explicitly provided when the + body is an iterable. The *headers* argument should be a mapping of extra HTTP headers to send with the request. + .. versionadded:: 3.2 + *body* can be an iterable + .. method:: HTTPConnection.getresponse() Should be called after a request is sent to get the response from the server. diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 730d258..6aa9e15 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -21,13 +21,14 @@ The :mod:`urllib.request` module defines the following functions: :class:`Request` object. *data* may be a string specifying additional data to send to the - server, or ``None`` if no such data is needed. Currently HTTP - requests are the only ones that use *data*; the HTTP request will - be a POST instead of a GET when the *data* parameter is provided. - *data* should be a buffer in the standard + server, or ``None`` if no such data is needed. *data* may also be an + iterable object and in that case Content-Length value must be specified in + the headers. Currently HTTP requests are the only ones that use *data*; the + HTTP request will be a POST instead of a GET when the *data* parameter is + provided. *data* should be a buffer in the standard :mimetype:`application/x-www-form-urlencoded` format. The - :func:`urllib.parse.urlencode` function takes a mapping or sequence - of 2-tuples and returns a string in this format. urllib.request module uses + :func:`urllib.parse.urlencode` function takes a mapping or sequence of + 2-tuples and returns a string in this format. urllib.request module uses HTTP/1.1 and includes ``Connection:close`` header in its HTTP requests. The optional *timeout* parameter specifies a timeout in seconds for @@ -76,6 +77,9 @@ The :mod:`urllib.request` module defines the following functions: HTTPS virtual hosts are now supported if possible (that is, if :data:`ssl.HAS_SNI` is true). + .. versionadded:: 3.2 + *data* can be an iterable object. + .. function:: install_opener(opener) Install an :class:`OpenerDirector` instance as the default global opener. diff --git a/Lib/http/client.py b/Lib/http/client.py index 8ea75ce..8d62aa5 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -71,6 +71,7 @@ import email.message import io import os import socket +import collections from urllib.parse import urlsplit import warnings @@ -730,7 +731,11 @@ class HTTPConnection: self.__state = _CS_IDLE def send(self, data): - """Send `data' to the server.""" + """Send `data' to the server. + ``data`` can be a string object, a bytes object, an array object, a + file-like object that supports a .read() method, or an iterable object. + """ + if self.sock is None: if self.auto_open: self.connect() @@ -762,8 +767,16 @@ class HTTPConnection: if encode: datablock = datablock.encode("iso-8859-1") self.sock.sendall(datablock) - else: + + try: self.sock.sendall(data) + except TypeError: + if isinstance(data, collections.Iterable): + for d in data: + self.sock.sendall(d) + else: + raise TypeError("data should be byte-like object\ + or an iterable, got %r " % type(it)) def _output(self, s): """Add a line of output to the current request buffer. diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 7dae65d..79fc6cc 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -230,6 +230,22 @@ class BasicTest(TestCase): conn.send(io.BytesIO(expected)) self.assertEqual(expected, sock.data) + def test_send_iter(self): + expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ + b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \ + b'\r\nonetwothree' + + def body(): + yield b"one" + yield b"two" + yield b"three" + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.request('GET', '/foo', body(), {'Content-Length': '11'}) + self.assertEquals(sock.data, expected) + def test_chunked(self): chunked_start = ( 'HTTP/1.1 200 OK\r\n' diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 9cc9697..1ce88af 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -4,6 +4,7 @@ from test import support import os import io import socket +import array import urllib.request from urllib.request import Request, OpenerDirector @@ -765,7 +766,7 @@ class HandlerTests(unittest.TestCase): o = h.parent = MockOpener() url = "http://example.com/" - for method, data in [("GET", None), ("POST", "blah")]: + for method, data in [("GET", None), ("POST", b"blah")]: req = Request(url, data, {"Foo": "bar"}) req.timeout = None req.add_unredirected_header("Spam", "eggs") @@ -795,7 +796,7 @@ class HandlerTests(unittest.TestCase): # check adding of standard headers o.addheaders = [("Spam", "eggs")] - for data in "", None: # POST, GET + for data in b"", None: # POST, GET req = Request("http://example.com/", data) r = MockResponse(200, "OK", {}, "") newreq = h.do_request_(req) @@ -821,6 +822,50 @@ class HandlerTests(unittest.TestCase): self.assertEqual(req.unredirected_hdrs["Host"], "baz") self.assertEqual(req.unredirected_hdrs["Spam"], "foo") + # Check iterable body support + def iterable_body(): + yield b"one" + yield b"two" + yield b"three" + + for headers in {}, {"Content-Length": 11}: + req = Request("http://example.com/", iterable_body(), headers) + if not headers: + # Having an iterable body without a Content-Length should + # raise an exception + self.assertRaises(ValueError, h.do_request_, req) + else: + newreq = h.do_request_(req) + + # A file object + + """ + file_obj = io.StringIO() + file_obj.write("Something\nSomething\nSomething\n") + + for headers in {}, {"Content-Length": 30}: + req = Request("http://example.com/", file_obj, headers) + if not headers: + # Having an iterable body without a Content-Length should + # raise an exception + self.assertRaises(ValueError, h.do_request_, req) + else: + newreq = h.do_request_(req) + self.assertEqual(int(newreq.get_header('Content-length')),30) + + file_obj.close() + + # array.array Iterable - Content Length is calculated + + iterable_array = array.array("I",[1,2,3,4]) + + for headers in {}, {"Content-Length": 16}: + req = Request("http://example.com/", iterable_array, headers) + newreq = h.do_request_(req) + self.assertEqual(int(newreq.get_header('Content-length')),16) + """ + + def test_http_doubleslash(self): # Checks the presence of any unnecessary double slash in url does not # break anything. Previously, a double slash directly after the host @@ -828,7 +873,7 @@ class HandlerTests(unittest.TestCase): h = urllib.request.AbstractHTTPHandler() o = h.parent = MockOpener() - data = "" + data = b"" ds_urls = [ "http://example.com/foo/bar/baz.html", "http://example.com//foo/bar/baz.html", diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index fe66a67..732c112 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -94,6 +94,7 @@ import re import socket import sys import time +import collections from urllib.error import URLError, HTTPError, ContentTooShortError from urllib.parse import ( @@ -1053,8 +1054,17 @@ class AbstractHTTPHandler(BaseHandler): 'Content-type', 'application/x-www-form-urlencoded') if not request.has_header('Content-length'): - request.add_unredirected_header( - 'Content-length', '%d' % len(data)) + try: + mv = memoryview(data) + except TypeError: + print(data) + if isinstance(data, collections.Iterable): + raise ValueError("Content-Length should be specified \ + for iterable data of type %r %r" % (type(data), + data)) + else: + request.add_unredirected_header( + 'Content-length', '%d' % len(mv) * mv.itemsize) sel_host = host if request.has_proxy(): @@ -23,6 +23,9 @@ Core and Builtins Library ------- +- Issue #3243: Support iterable bodies in httplib. Patch Contributions by + Xuanji Li and Chris AtLee. + - Issue #10611: SystemExit exception will no longer kill a unittest run. - Issue #9857: It is now possible to skip a test in a setUp, tearDown or clean |