diff options
| author | Christian Heimes <christian@cheimes.de> | 2008-12-05 08:26:55 (GMT) |
|---|---|---|
| committer | Christian Heimes <christian@cheimes.de> | 2008-12-05 08:26:55 (GMT) |
| commit | a872de55dc9d4e292c4b8fe7e09741081890305e (patch) | |
| tree | ec5f79f9bd3037b42eca04cb246e831b092801c5 /Modules/_fileio.c | |
| parent | 7a259ca8e3fa23c4259f12ab65b53059a0c1df77 (diff) | |
| download | cpython-a872de55dc9d4e292c4b8fe7e09741081890305e.zip cpython-a872de55dc9d4e292c4b8fe7e09741081890305e.tar.gz cpython-a872de55dc9d4e292c4b8fe7e09741081890305e.tar.bz2 | |
Fixed issue #4533: File read operation was dreadfully slow
Diffstat (limited to 'Modules/_fileio.c')
| -rw-r--r-- | Modules/_fileio.c | 31 |
1 files changed, 27 insertions, 4 deletions
diff --git a/Modules/_fileio.c b/Modules/_fileio.c index 0a34eb3..c23d5a3 100644 --- a/Modules/_fileio.c +++ b/Modules/_fileio.c @@ -27,6 +27,20 @@ #include <windows.h> #endif +#if BUFSIZ < (8*1024) +#define SMALLCHUNK (8*1024) +#elif (BUFSIZ >= (2 << 25)) +#error "unreasonable BUFSIZ > 64MB defined" +#else +#define SMALLCHUNK BUFSIZ +#endif + +#if SIZEOF_INT < 4 +#define BIGCHUNK (512 * 32) +#else +#define BIGCHUNK (512 * 1024) +#endif + typedef struct { PyObject_HEAD int fd; @@ -387,8 +401,6 @@ fileio_readinto(PyFileIOObject *self, PyObject *args) return PyLong_FromSsize_t(n); } -#define DEFAULT_BUFFER_SIZE (8*1024) - static PyObject * fileio_readall(PyFileIOObject *self) { @@ -396,12 +408,23 @@ fileio_readall(PyFileIOObject *self) Py_ssize_t total = 0; int n; - result = PyBytes_FromStringAndSize(NULL, DEFAULT_BUFFER_SIZE); + result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); if (result == NULL) return NULL; while (1) { - Py_ssize_t newsize = total + DEFAULT_BUFFER_SIZE; + Py_ssize_t newsize = (total < SMALLCHUNK) ? SMALLCHUNK : total; + + /* Keep doubling until we reach BIGCHUNK; + then keep adding BIGCHUNK. */ + if (newsize <= BIGCHUNK) { + newsize += newsize; + } + else { + /* NOTE: overflow impossible due to limits on BUFSIZ */ + newsize += BIGCHUNK; + } + if (PyBytes_GET_SIZE(result) < newsize) { if (_PyBytes_Resize(&result, newsize) < 0) { if (total == 0) { |