diff options
| -rw-r--r-- | Lib/test/test_io.py | 44 | ||||
| -rw-r--r-- | Misc/NEWS | 3 | ||||
| -rw-r--r-- | Modules/_io/bufferedio.c | 101 |
3 files changed, 102 insertions, 46 deletions
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index c11aafd..1129484 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -1401,15 +1401,18 @@ class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest): rw.seek(0, 0) self.assertEqual(b"asdf", rw.read(4)) - rw.write(b"asdf") + rw.write(b"123f") rw.seek(0, 0) - self.assertEqual(b"asdfasdfl", rw.read()) + self.assertEqual(b"asdf123fl", rw.read()) self.assertEqual(9, rw.tell()) rw.seek(-4, 2) self.assertEqual(5, rw.tell()) rw.seek(2, 1) self.assertEqual(7, rw.tell()) self.assertEqual(b"fl", rw.read(11)) + rw.flush() + self.assertEqual(b"asdf123fl", raw.getvalue()) + self.assertRaises(TypeError, rw.seek, 0.0) def check_flush_and_read(self, read_func): @@ -1554,6 +1557,43 @@ class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest): BufferedReaderTest.test_misbehaved_io(self) BufferedWriterTest.test_misbehaved_io(self) + def test_interleaved_read_write(self): + # Test for issue #12213 + with self.BytesIO(b'abcdefgh') as raw: + with self.tp(raw, 100) as f: + f.write(b"1") + self.assertEqual(f.read(1), b'b') + f.write(b'2') + self.assertEqual(f.read1(1), b'd') + f.write(b'3') + buf = bytearray(1) + f.readinto(buf) + self.assertEqual(buf, b'f') + f.write(b'4') + self.assertEqual(f.peek(1), b'h') + f.flush() + self.assertEqual(raw.getvalue(), b'1b2d3f4h') + + with self.BytesIO(b'abc') as raw: + with self.tp(raw, 100) as f: + self.assertEqual(f.read(1), b'a') + f.write(b"2") + self.assertEqual(f.read(1), b'c') + f.flush() + self.assertEqual(raw.getvalue(), b'a2c') + + def test_interleaved_readline_write(self): + with self.BytesIO(b'ab\ncdef\ng\n') as raw: + with self.tp(raw) as f: + f.write(b'1') + self.assertEqual(f.readline(), b'b\n') + f.write(b'2') + self.assertEqual(f.readline(), b'def\n') + f.write(b'3') + self.assertEqual(f.readline(), b'\n') + f.flush() + self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n') + # You can't construct a BufferedRandom over a non-seekable stream. test_unseekable = None @@ -265,6 +265,9 @@ Core and Builtins Library ------- +- Issue #12213: Fix a buffering bug with interleaved reads and writes that + could appear on BufferedRandom streams. + - Issue #12778: Reduce memory consumption when JSON-encoding a large container of many small objects. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index cdaa36e..a8631e0 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -753,25 +753,38 @@ _trap_eintr(void) */ static PyObject * -buffered_flush(buffered *self, PyObject *args) +buffered_flush_and_rewind_unlocked(buffered *self) { PyObject *res; - CHECK_INITIALIZED(self) - CHECK_CLOSED(self, "flush of closed file") - - if (!ENTER_BUFFERED(self)) - return NULL; res = _bufferedwriter_flush_unlocked(self, 0); - if (res != NULL && self->readable) { + if (res == NULL) + return NULL; + Py_DECREF(res); + + if (self->readable) { /* Rewind the raw stream so that its position corresponds to the current logical position. */ Py_off_t n; n = _buffered_raw_seek(self, -RAW_OFFSET(self), 1); - if (n == -1) - Py_CLEAR(res); _bufferedreader_reset_buf(self); + if (n == -1) + return NULL; } + Py_RETURN_NONE; +} + +static PyObject * +buffered_flush(buffered *self, PyObject *args) +{ + PyObject *res; + + CHECK_INITIALIZED(self) + CHECK_CLOSED(self, "flush of closed file") + + if (!ENTER_BUFFERED(self)) + return NULL; + res = buffered_flush_and_rewind_unlocked(self); LEAVE_BUFFERED(self) return res; @@ -792,7 +805,7 @@ buffered_peek(buffered *self, PyObject *args) return NULL; if (self->writable) { - res = _bufferedwriter_flush_unlocked(self, 1); + res = buffered_flush_and_rewind_unlocked(self); if (res == NULL) goto end; Py_CLEAR(res); @@ -827,19 +840,18 @@ buffered_read(buffered *self, PyObject *args) if (!ENTER_BUFFERED(self)) return NULL; res = _bufferedreader_read_all(self); - LEAVE_BUFFERED(self) } else { res = _bufferedreader_read_fast(self, n); - if (res == Py_None) { - Py_DECREF(res); - if (!ENTER_BUFFERED(self)) - return NULL; - res = _bufferedreader_read_generic(self, n); - LEAVE_BUFFERED(self) - } + if (res != Py_None) + return res; + Py_DECREF(res); + if (!ENTER_BUFFERED(self)) + return NULL; + res = _bufferedreader_read_generic(self, n); } + LEAVE_BUFFERED(self) return res; } @@ -865,13 +877,6 @@ buffered_read1(buffered *self, PyObject *args) if (!ENTER_BUFFERED(self)) return NULL; - if (self->writable) { - res = _bufferedwriter_flush_unlocked(self, 1); - if (res == NULL) - goto end; - Py_CLEAR(res); - } - /* Return up to n bytes. If at least one byte is buffered, we only return buffered bytes. Otherwise, we do one raw read. */ @@ -891,6 +896,13 @@ buffered_read1(buffered *self, PyObject *args) goto end; } + if (self->writable) { + res = buffered_flush_and_rewind_unlocked(self); + if (res == NULL) + goto end; + Py_DECREF(res); + } + /* Fill the buffer from the raw stream, and copy it to the result. */ _bufferedreader_reset_buf(self); r = _bufferedreader_fill_buffer(self); @@ -939,7 +951,7 @@ buffered_readinto(buffered *self, PyObject *args) goto end_unlocked; if (self->writable) { - res = _bufferedwriter_flush_unlocked(self, 0); + res = buffered_flush_and_rewind_unlocked(self); if (res == NULL) goto end; Py_CLEAR(res); @@ -1022,12 +1034,6 @@ _buffered_readline(buffered *self, Py_ssize_t limit) goto end_unlocked; /* Now we try to get some more from the raw stream */ - if (self->writable) { - res = _bufferedwriter_flush_unlocked(self, 1); - if (res == NULL) - goto end; - Py_CLEAR(res); - } chunks = PyList_New(0); if (chunks == NULL) goto end; @@ -1041,9 +1047,16 @@ _buffered_readline(buffered *self, Py_ssize_t limit) } Py_CLEAR(res); written += n; + self->pos += n; if (limit >= 0) limit -= n; } + if (self->writable) { + PyObject *r = buffered_flush_and_rewind_unlocked(self); + if (r == NULL) + goto end; + Py_DECREF(r); + } for (;;) { _bufferedreader_reset_buf(self); @@ -1212,20 +1225,11 @@ buffered_truncate(buffered *self, PyObject *args) return NULL; if (self->writable) { - res = _bufferedwriter_flush_unlocked(self, 0); + res = buffered_flush_and_rewind_unlocked(self); if (res == NULL) goto end; Py_CLEAR(res); } - if (self->readable) { - if (pos == Py_None) { - /* Rewind the raw stream so that its position corresponds to - the current logical position. */ - if (_buffered_raw_seek(self, -RAW_OFFSET(self), 1) == -1) - goto end; - } - _bufferedreader_reset_buf(self); - } res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_truncate, pos, NULL); if (res == NULL) goto end; @@ -1416,15 +1420,16 @@ _bufferedreader_read_all(buffered *self) self->buffer + self->pos, current_size); if (data == NULL) return NULL; + self->pos += current_size; } - _bufferedreader_reset_buf(self); /* We're going past the buffer's bounds, flush it */ if (self->writable) { - res = _bufferedwriter_flush_unlocked(self, 1); + res = buffered_flush_and_rewind_unlocked(self); if (res == NULL) return NULL; Py_CLEAR(res); } + _bufferedreader_reset_buf(self); if (PyObject_HasAttr(self->raw, _PyIO_str_readall)) { chunk = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readall, NULL); @@ -1540,6 +1545,14 @@ _bufferedreader_read_generic(buffered *self, Py_ssize_t n) memcpy(out, self->buffer + self->pos, current_size); remaining -= current_size; written += current_size; + self->pos += current_size; + } + /* Flush the write buffer if necessary */ + if (self->writable) { + PyObject *r = buffered_flush_and_rewind_unlocked(self); + if (r == NULL) + goto error; + Py_DECREF(r); } _bufferedreader_reset_buf(self); while (remaining > 0) { |