From 0c98f95c45cda4b17f3f414dcf9ba854198ed40f Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 5 Dec 2008 08:32:07 +0000 Subject: Merged revisions 67559 via svnmerge from svn+ssh://pythondev@svn.python.org/python/branches/py3k ........ r67559 | christian.heimes | 2008-12-05 09:26:55 +0100 (Fri, 05 Dec 2008) | 1 line Fixed issue #4533: File read operation was dreadfully slow ........ --- Misc/NEWS | 5 +++++ Modules/_fileio.c | 31 +++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index be55e58..b4b5a13 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,11 @@ What's New in Python 3.0.1? Core and Builtins ----------------- +- Issue #4533: File read operation was dreadfully slow due to a slowly + growing read buffer. Fixed by using the same growth rate algorithm as + Python 2.x. + + Library ------- diff --git a/Modules/_fileio.c b/Modules/_fileio.c index 0a34eb3..c23d5a3 100644 --- a/Modules/_fileio.c +++ b/Modules/_fileio.c @@ -27,6 +27,20 @@ #include #endif +#if BUFSIZ < (8*1024) +#define SMALLCHUNK (8*1024) +#elif (BUFSIZ >= (2 << 25)) +#error "unreasonable BUFSIZ > 64MB defined" +#else +#define SMALLCHUNK BUFSIZ +#endif + +#if SIZEOF_INT < 4 +#define BIGCHUNK (512 * 32) +#else +#define BIGCHUNK (512 * 1024) +#endif + typedef struct { PyObject_HEAD int fd; @@ -387,8 +401,6 @@ fileio_readinto(PyFileIOObject *self, PyObject *args) return PyLong_FromSsize_t(n); } -#define DEFAULT_BUFFER_SIZE (8*1024) - static PyObject * fileio_readall(PyFileIOObject *self) { @@ -396,12 +408,23 @@ fileio_readall(PyFileIOObject *self) Py_ssize_t total = 0; int n; - result = PyBytes_FromStringAndSize(NULL, DEFAULT_BUFFER_SIZE); + result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); if (result == NULL) return NULL; while (1) { - Py_ssize_t newsize = total + DEFAULT_BUFFER_SIZE; + Py_ssize_t newsize = (total < SMALLCHUNK) ? SMALLCHUNK : total; + + /* Keep doubling until we reach BIGCHUNK; + then keep adding BIGCHUNK. */ + if (newsize <= BIGCHUNK) { + newsize += newsize; + } + else { + /* NOTE: overflow impossible due to limits on BUFSIZ */ + newsize += BIGCHUNK; + } + if (PyBytes_GET_SIZE(result) < newsize) { if (_PyBytes_Resize(&result, newsize) < 0) { if (total == 0) { -- cgit v0.12