From 7165754b6b5f3b7c07050d921fa1c58bba5f0ff1 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Mon, 28 May 2018 18:31:55 -0400 Subject: bpo-32410: Avoid blocking on file IO in sendfile fallback code (GH-7172) --- Lib/asyncio/base_events.py | 7 +++++-- Lib/asyncio/constants.py | 4 ++++ Lib/test/test_asyncio/test_base_events.py | 3 +++ Lib/test/test_asyncio/test_events.py | 11 +++++++++++ .../next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst | 1 + 5 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index a0243f5..ffd2513 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -800,7 +800,10 @@ class BaseEventLoop(events.AbstractEventLoop): async def _sock_sendfile_fallback(self, sock, file, offset, count): if offset: file.seek(offset) - blocksize = min(count, 16384) if count else 16384 + blocksize = ( + min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) + if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE + ) buf = bytearray(blocksize) total_sent = 0 try: @@ -810,7 +813,7 @@ class BaseEventLoop(events.AbstractEventLoop): if blocksize <= 0: break view = memoryview(buf)[:blocksize] - read = file.readinto(view) + read = await self.run_in_executor(None, file.readinto, view) if not read: break # EOF await self.sock_sendall(sock, view) diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py index 739b0a7..d7ba496 100644 --- a/Lib/asyncio/constants.py +++ b/Lib/asyncio/constants.py @@ -14,6 +14,10 @@ DEBUG_STACK_DEPTH = 10 # Number of seconds to wait for SSL handshake to complete SSL_HANDSHAKE_TIMEOUT = 10.0 +# Used in sendfile fallback code. We use fallback for platforms +# that don't support sendfile, or for TLS connections. +SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 256 + # The enum should be here to break circular dependencies between # base_events and sslproto class _SendfileMode(enum.Enum): diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 8566a9d..11e9465 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -1818,12 +1818,15 @@ class BaseLoopSockSendfileTests(test_utils.TestCase): @classmethod def setUpClass(cls): + cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16 with open(support.TESTFN, 'wb') as fp: fp.write(cls.DATA) super().setUpClass() @classmethod def tearDownClass(cls): + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize support.unlink(support.TESTFN) super().tearDownClass() diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 39d85e8..e7c4fa6 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -2160,6 +2160,17 @@ class SockSendfileMixin(SendfileBase): async def wait_closed(self): await self.fut + @classmethod + def setUpClass(cls): + cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16 + super().setUpClass() + + @classmethod + def tearDownClass(cls): + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize + super().tearDownClass() + def set_socket_opts(self, sock): # On macOS, SO_SNDBUF is reset by connect(). So this method # should be called after the socket is connected. diff --git a/Misc/NEWS.d/next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst b/Misc/NEWS.d/next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst new file mode 100644 index 0000000..2d7bb20 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst @@ -0,0 +1 @@ +Avoid blocking on file IO in sendfile fallback code -- cgit v0.12