From bccacd19fa7b56dcf2fbfab15992b6b94ab6666b Mon Sep 17 00:00:00 2001 From: Alexander Buchkovsky Date: Tue, 6 Nov 2018 21:38:34 +0200 Subject: bpo-17560: Too small type for struct.pack/unpack in mutliprocessing.Connection (GH-10305) Allow sending more than 2 GB at once on a multiprocessing connection on non-Windows systems. --- Lib/multiprocessing/connection.py | 30 ++++++++++++++-------- .../2018-11-03-10-12-04.bpo-35152.xpqskp.rst | 1 + 2 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 1f3ea50..c9f995e 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -389,23 +389,33 @@ class Connection(_ConnectionBase): def _send_bytes(self, buf): n = len(buf) - # For wire compatibility with 3.2 and lower - header = struct.pack("!i", n) - if n > 16384: - # The payload is large so Nagle's algorithm won't be triggered - # and we'd better avoid the cost of concatenation. + if n > 0x7fffffff: + pre_header = struct.pack("!i", -1) + header = struct.pack("!Q", n) + self._send(pre_header) self._send(header) self._send(buf) else: - # Issue #20540: concatenate before sending, to avoid delays due - # to Nagle's algorithm on a TCP socket. - # Also note we want to avoid sending a 0-length buffer separately, - # to avoid "broken pipe" errors if the other end closed the pipe. - self._send(header + buf) + # For wire compatibility with 3.7 and lower + header = struct.pack("!i", n) + if n > 16384: + # The payload is large so Nagle's algorithm won't be triggered + # and we'd better avoid the cost of concatenation. + self._send(header) + self._send(buf) + else: + # Issue #20540: concatenate before sending, to avoid delays due + # to Nagle's algorithm on a TCP socket. + # Also note we want to avoid sending a 0-length buffer separately, + # to avoid "broken pipe" errors if the other end closed the pipe. + self._send(header + buf) def _recv_bytes(self, maxsize=None): buf = self._recv(4) size, = struct.unpack("!i", buf.getvalue()) + if size == -1: + buf = self._recv(8) + size, = struct.unpack("!Q", buf.getvalue()) if maxsize is not None and size > maxsize: return None return self._recv(size) diff --git a/Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst b/Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst new file mode 100644 index 0000000..7cc9ed3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-11-03-10-12-04.bpo-35152.xpqskp.rst @@ -0,0 +1 @@ +Allow sending more than 2 GB at once on a multiprocessing connection on non-Windows systems. \ No newline at end of file -- cgit v0.12