Issue3243 - Support iterable bodies in httplib. Patch contributions by Xuanji Li and Chris AtLee.

4 files changed, 91 insertions, 7 deletions

diff --git a/Lib/http/client.py b/Lib/http/client.py
index 8ea75ce..8d62aa5 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -71,6 +71,7 @@ import email.message
 import io
 import os
 import socket
+import collections
 from urllib.parse import urlsplit
 import warnings
 
@@ -730,7 +731,11 @@ class HTTPConnection:
         self.__state = _CS_IDLE
 
     def send(self, data):
-        """Send `data' to the server."""
+        """Send `data' to the server.
+        ``data`` can be a string object, a bytes object, an array object, a
+        file-like object that supports a .read() method, or an iterable object.
+        """
+
         if self.sock is None:
             if self.auto_open:
                 self.connect()
@@ -762,8 +767,16 @@ class HTTPConnection:
                 if encode:
                     datablock = datablock.encode("iso-8859-1")
                 self.sock.sendall(datablock)
-        else:
+
+        try:
             self.sock.sendall(data)
+        except TypeError:
+            if isinstance(data, collections.Iterable):
+                for d in data:
+                    self.sock.sendall(d)
+            else:
+                raise TypeError("data should be byte-like object\
+                        or an iterable, got %r " % type(it))
 
     def _output(self, s):
         """Add a line of output to the current request buffer.
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
index 7dae65d..79fc6cc 100644
--- a/Lib/test/test_httplib.py
+++ b/Lib/test/test_httplib.py
@@ -230,6 +230,22 @@ class BasicTest(TestCase):
         conn.send(io.BytesIO(expected))
         self.assertEqual(expected, sock.data)
 
+    def test_send_iter(self):
+        expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \
+                   b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \
+                   b'\r\nonetwothree'
+
+        def body():
+            yield b"one"
+            yield b"two"
+            yield b"three"
+
+        conn = client.HTTPConnection('example.com')
+        sock = FakeSocket("")
+        conn.sock = sock
+        conn.request('GET', '/foo', body(), {'Content-Length': '11'})
+        self.assertEquals(sock.data, expected)
+
     def test_chunked(self):
         chunked_start = (
             'HTTP/1.1 200 OK\r\n'
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index 9cc9697..1ce88af 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -4,6 +4,7 @@ from test import support
 import os
 import io
 import socket
+import array
 
 import urllib.request
 from urllib.request import Request, OpenerDirector
@@ -765,7 +766,7 @@ class HandlerTests(unittest.TestCase):
         o = h.parent = MockOpener()
 
         url = "http://example.com/"
-        for method, data in [("GET", None), ("POST", "blah")]:
+        for method, data in [("GET", None), ("POST", b"blah")]:
             req = Request(url, data, {"Foo": "bar"})
             req.timeout = None
             req.add_unredirected_header("Spam", "eggs")
@@ -795,7 +796,7 @@ class HandlerTests(unittest.TestCase):
 
         # check adding of standard headers
         o.addheaders = [("Spam", "eggs")]
-        for data in "", None:  # POST, GET
+        for data in b"", None:  # POST, GET
             req = Request("http://example.com/", data)
             r = MockResponse(200, "OK", {}, "")
             newreq = h.do_request_(req)
@@ -821,6 +822,50 @@ class HandlerTests(unittest.TestCase):
             self.assertEqual(req.unredirected_hdrs["Host"], "baz")
             self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
 
+        # Check iterable body support
+        def iterable_body():
+            yield b"one"
+            yield b"two"
+            yield b"three"
+
+        for headers in {}, {"Content-Length": 11}:
+            req = Request("http://example.com/", iterable_body(), headers)
+            if not headers:
+                # Having an iterable body without a Content-Length should
+                # raise an exception
+                self.assertRaises(ValueError, h.do_request_, req)
+            else:
+                newreq = h.do_request_(req)
+
+        # A file object
+
+        """
+        file_obj = io.StringIO()
+        file_obj.write("Something\nSomething\nSomething\n")
+
+        for headers in {}, {"Content-Length": 30}:
+            req = Request("http://example.com/", file_obj, headers)
+            if not headers:
+                # Having an iterable body without a Content-Length should
+                # raise an exception
+                self.assertRaises(ValueError, h.do_request_, req)
+            else:
+                newreq = h.do_request_(req)
+                self.assertEqual(int(newreq.get_header('Content-length')),30)
+
+        file_obj.close()
+
+        # array.array Iterable - Content Length is calculated
+
+        iterable_array = array.array("I",[1,2,3,4])
+
+        for headers in {}, {"Content-Length": 16}:
+            req = Request("http://example.com/", iterable_array, headers)
+            newreq = h.do_request_(req)
+            self.assertEqual(int(newreq.get_header('Content-length')),16)
+        """
+
+
     def test_http_doubleslash(self):
         # Checks the presence of any unnecessary double slash in url does not
         # break anything. Previously, a double slash directly after the host
@@ -828,7 +873,7 @@ class HandlerTests(unittest.TestCase):
         h = urllib.request.AbstractHTTPHandler()
         o = h.parent = MockOpener()
 
-        data = ""
+        data = b""
         ds_urls = [
             "http://example.com/foo/bar/baz.html",
             "http://example.com//foo/bar/baz.html",
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index fe66a67..732c112 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -94,6 +94,7 @@ import re
 import socket
 import sys
 import time
+import collections
 
 from urllib.error import URLError, HTTPError, ContentTooShortError
 from urllib.parse import (
@@ -1053,8 +1054,17 @@ class AbstractHTTPHandler(BaseHandler):
                     'Content-type',
                     'application/x-www-form-urlencoded')
             if not request.has_header('Content-length'):
-                request.add_unredirected_header(
-                    'Content-length', '%d' % len(data))
+                try:
+                    mv = memoryview(data)
+                except TypeError:
+                    print(data)
+                    if isinstance(data, collections.Iterable):
+                        raise ValueError("Content-Length should be specified \
+                                for iterable data of type %r %r" % (type(data),
+                                data))
+                else:
+                    request.add_unredirected_header(
+                            'Content-length', '%d' % len(mv) * mv.itemsize)
 
         sel_host = host
         if request.has_proxy():