From 13f61bf7f1fb9b9b109b089610e845d98e6dc937 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 3 Sep 2024 12:32:38 +0900 Subject: gh-121313: multiprocessing: simplify by increasing the connection buffer size to 64KiB (GH-123559) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Increases the multiprocessing connection buffer size from 8k to 64k for efficiency, without overallocating. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Victor Stinner --- Lib/multiprocessing/connection.py | 20 ++++---------------- .../2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst | 1 - .../2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst | 1 + 3 files changed, 5 insertions(+), 17 deletions(-) delete mode 100644 Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst create mode 100644 Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index d84b52f..7e901cf 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -13,7 +13,6 @@ import errno import io import itertools import os -import stat import sys import socket import struct @@ -40,7 +39,9 @@ except ImportError: # # -BUFSIZE = 8192 +# 64 KiB is the default PIPE buffer size of most POSIX platforms. +BUFSIZE = 64 * 1024 + # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. @@ -361,11 +362,6 @@ if _winapi: f.write(ov.getbuffer()) return f -""" -The default size of a pipe on Linux systems is 16 times the base page size: -https://man7.org/linux/man-pages/man7/pipe.7.html -""" -PAGES_PER_PIPE = 16 class Connection(_ConnectionBase): """ @@ -378,14 +374,11 @@ class Connection(_ConnectionBase): _close(self._handle) _write = _multiprocessing.send _read = _multiprocessing.recv - _default_pipe_size = 0 else: def _close(self, _close=os.close): _close(self._handle) _write = os.write _read = os.read - _base_page_size = os.sysconf(os.sysconf_names['SC_PAGESIZE']) - _default_pipe_size = _base_page_size * PAGES_PER_PIPE def _send(self, buf, write=_write): remaining = len(buf) @@ -400,13 +393,8 @@ class Connection(_ConnectionBase): buf = io.BytesIO() handle = self._handle remaining = size - is_pipe = False - if size > self._default_pipe_size > 0: - mode = os.fstat(handle).st_mode - is_pipe = stat.S_ISFIFO(mode) - limit = self._default_pipe_size if is_pipe else remaining while remaining > 0: - to_read = min(limit, remaining) + to_read = min(BUFSIZE, remaining) chunk = read(handle, to_read) n = len(chunk) if n == 0: diff --git a/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst deleted file mode 100644 index 06abce9..0000000 --- a/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst +++ /dev/null @@ -1 +0,0 @@ -Limit reading size in multiprocessing connection._recv for pipes to default pipe size of 16 times base page size, in order to avoid memory overallocation and unnecessary memory management system calls. diff --git a/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst new file mode 100644 index 0000000..bb41063 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst @@ -0,0 +1 @@ +Limit the reading size in the :class:`multiprocessing.connection.Connection` class to 64 KiB to prevent memory overallocation and unnecessary memory management system calls. -- cgit v0.12