diff options
| -rw-r--r-- | Include/fileutils.h | 13 | ||||
| -rw-r--r-- | Lib/test/test_largefile.py | 13 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst | 1 | ||||
| -rw-r--r-- | Modules/_io/fileio.c | 8 | ||||
| -rw-r--r-- | Python/fileutils.c | 24 |
5 files changed, 33 insertions, 26 deletions
diff --git a/Include/fileutils.h b/Include/fileutils.h index f0a8e2c..232d966 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -81,6 +81,19 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx( #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _Py_device_encoding(int); +#if defined(MS_WINDOWS) || defined(__APPLE__) + /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). + On macOS 10.13, read() and write() with more than INT_MAX bytes + fail with EINVAL (bpo-24658). */ +# define _PY_READ_MAX INT_MAX +# define _PY_WRITE_MAX INT_MAX +#else + /* write() should truncate the input to PY_SSIZE_T_MAX bytes, + but it's safer to do it ourself to have a portable behaviour */ +# define _PY_READ_MAX PY_SSIZE_T_MAX +# define _PY_WRITE_MAX PY_SSIZE_T_MAX +#endif + #ifdef MS_WINDOWS struct _Py_stat_struct { unsigned long st_dev; diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py index 21296cc..8870c72 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_largefile.py @@ -5,12 +5,12 @@ import os import stat import sys import unittest -from test.support import TESTFN, requires, unlink +from test.support import TESTFN, requires, unlink, bigmemtest import io # C implementation of io import _pyio as pyio # Python implementation of io # size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes) -size = 2500000000 +size = 2_500_000_000 class LargeFileTest: """Test that each file function works as expected for large @@ -45,6 +45,15 @@ class LargeFileTest: raise cls.failureException('File was not truncated by opening ' 'with mode "wb"') + # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, + # so memuse=2 is needed + @bigmemtest(size=size, memuse=2, dry_run=False) + def test_large_read(self, _size): + # bpo-24658: Test that a read greater than 2GB does not fail. + with self.open(TESTFN, "rb") as f: + self.assertEqual(len(f.read()), size + 1) + self.assertEqual(f.tell(), size + 1) + def test_osstat(self): self.assertEqual(os.stat(TESTFN)[stat.ST_SIZE], size+1) diff --git a/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst new file mode 100644 index 0000000..ff660a1 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2018-10-17-14-36-08.bpo-24658.Naddgx.rst @@ -0,0 +1 @@ +On macOS, fix reading from and writing into a file with a size larger than 2 GiB.
\ No newline at end of file diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index c0e43e0..44d51c9 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -791,11 +791,9 @@ _io_FileIO_read_impl(fileio *self, Py_ssize_t size) if (size < 0) return _io_FileIO_readall_impl(self); -#ifdef MS_WINDOWS - /* On Windows, the count parameter of read() is an int */ - if (size > INT_MAX) - size = INT_MAX; -#endif + if (size > _PY_READ_MAX) { + size = _PY_READ_MAX; + } bytes = PyBytes_FromStringAndSize(NULL, size); if (bytes == NULL) diff --git a/Python/fileutils.c b/Python/fileutils.c index 0486f86..2c4061e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1471,18 +1471,9 @@ _Py_read(int fd, void *buf, size_t count) * handler raised an exception. */ assert(!PyErr_Occurred()); -#ifdef MS_WINDOWS - if (count > INT_MAX) { - /* On Windows, the count parameter of read() is an int */ - count = INT_MAX; - } -#else - if (count > PY_SSIZE_T_MAX) { - /* if count is greater than PY_SSIZE_T_MAX, - * read() result is undefined */ - count = PY_SSIZE_T_MAX; + if (count > _PY_READ_MAX) { + count = _PY_READ_MAX; } -#endif _Py_BEGIN_SUPPRESS_IPH do { @@ -1533,15 +1524,10 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) depending on heap usage). */ count = 32767; } - else if (count > INT_MAX) - count = INT_MAX; -#else - if (count > PY_SSIZE_T_MAX) { - /* write() should truncate count to PY_SSIZE_T_MAX, but it's safer - * to do it ourself to have a portable behaviour. */ - count = PY_SSIZE_T_MAX; - } #endif + if (count > _PY_WRITE_MAX) { + count = _PY_WRITE_MAX; + } if (gil_held) { do { |