bpo-31945: Configurable blocksize in HTTP(S)Connection (#4279)

blocksize was hardcoded to 8192, preventing efficient upload when using
file-like body. Add blocksize argument to __init__, so users can
configure the blocksize to fit their needs.

I tested this uploading data from /dev/zero to a web server dropping the
received data, to test the overhead of the HTTPConnection.send() with a
file-like object.

Here is an example 10g upload with the default buffer size (8192):

$ time ~/src/cpython/release/python upload-httplib.py 10 https://localhost:8000/
Uploaded 10.00g in 17.53 seconds (584.00m/s)

real	0m17.574s
user	0m8.887s
sys	0m5.971s

Same with 512k blocksize:

$ time ~/src/cpython/release/python upload-httplib.py 10 https://localhost:8000/
Uploaded 10.00g in 6.60 seconds (1551.15m/s)

real	0m6.641s
user	0m3.426s
sys	0m2.162s

In real world usage the difference will be smaller, depending on the
local and remote storage and the network.

See https://github.com/nirs/http-bench for more info.

1 files changed, 7 insertions, 7 deletions

diff --git a/Lib/http/client.py b/Lib/http/client.py
index bbb3152..70eadae 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -825,9 +825,10 @@ class HTTPConnection:
         return None
 
     def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
-                 source_address=None):
+                 source_address=None, blocksize=8192):
         self.timeout = timeout
         self.source_address = source_address
+        self.blocksize = blocksize
         self.sock = None
         self._buffer = []
         self.__response = None
@@ -958,7 +959,6 @@ class HTTPConnection:
 
         if self.debuglevel > 0:
             print("send:", repr(data))
-        blocksize = 8192
         if hasattr(data, "read") :
             if self.debuglevel > 0:
                 print("sendIng a read()able")
@@ -966,7 +966,7 @@ class HTTPConnection:
             if encode and self.debuglevel > 0:
                 print("encoding file using iso-8859-1")
             while 1:
-                datablock = data.read(blocksize)
+                datablock = data.read(self.blocksize)
                 if not datablock:
                     break
                 if encode:
@@ -991,14 +991,13 @@ class HTTPConnection:
         self._buffer.append(s)
 
     def _read_readable(self, readable):
-        blocksize = 8192
         if self.debuglevel > 0:
             print("sendIng a read()able")
         encode = self._is_textIO(readable)
         if encode and self.debuglevel > 0:
             print("encoding file using iso-8859-1")
         while True:
-            datablock = readable.read(blocksize)
+            datablock = readable.read(self.blocksize)
             if not datablock:
                 break
             if encode:
@@ -1353,9 +1352,10 @@ else:
         def __init__(self, host, port=None, key_file=None, cert_file=None,
                      timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
                      source_address=None, *, context=None,
-                     check_hostname=None):
+                     check_hostname=None, blocksize=8192):
             super(HTTPSConnection, self).__init__(host, port, timeout,
-                                                  source_address)
+                                                  source_address,
+                                                  blocksize=blocksize)
             if (key_file is not None or cert_file is not None or
                         check_hostname is not None):
                 import warnings