diff options
| -rw-r--r-- | Lib/io.py | 36 | ||||
| -rw-r--r-- | Lib/test/test_io.py | 67 | ||||
| -rw-r--r-- | Modules/_fileio.c | 276 |
3 files changed, 205 insertions, 174 deletions
@@ -1,9 +1,15 @@ -"""New I/O library. +"""New I/O library conforming to PEP 3116. This is an early prototype; eventually some of this will be reimplemented in C and the rest may be turned into a package. -See PEP 3116. +Conformance of alternative implementations: all arguments are intended +to be positional-only except the arguments of the open() function. +Argument names except those of the open() function are not part of the +specification. Instance variables and methods whose name starts with +a leading underscore are not part of the specification (except "magic" +names like __iter__). Only the top-level names listed in the __all__ +variable are part of the specification. XXX need to default buffer size to 1 if isatty() XXX need to support 1 meaning line-buffered @@ -142,6 +148,9 @@ class IOBase: This does not define read(), readinto() and write(), nor readline() and friends, since their signatures vary per layer. + + Not that calling any method (even inquiries) on a closed file is + undefined. Implementations may raise IOError in this case. """ ### Internal ### @@ -153,19 +162,20 @@ class IOBase: ### Positioning ### - def seek(self, pos: int, whence: int = 0) -> None: - """seek(pos: int, whence: int = 0) -> None. Change stream position. + def seek(self, pos: int, whence: int = 0) -> int: + """seek(pos: int, whence: int = 0) -> int. Change stream position. Seek to byte offset pos relative to position indicated by whence: 0 Start of stream (the default). pos should be >= 0; 1 Current position - whence may be negative; 2 End of stream - whence usually negative. + Returns the new absolute position. """ self._unsupported("seek") def tell(self) -> int: """tell() -> int. Return current stream position.""" - self._unsupported("tell") + return self.seek(0, 1) def truncate(self, pos: int = None) -> None: """truncate(size: int = None) -> None. Truncate file to size bytes. @@ -432,7 +442,7 @@ class _BufferedIOMixin(BufferedIOBase): ### Positioning ### def seek(self, pos, whence=0): - self.raw.seek(pos, whence) + return self.raw.seek(pos, whence) def tell(self): return self.raw.tell() @@ -515,6 +525,7 @@ class _MemoryIOMixin(BufferedIOBase): self._pos = max(0, len(self._buffer) + pos) else: raise IOError("invalid whence value") + return self._pos def tell(self): return self._pos @@ -620,8 +631,9 @@ class BufferedReader(_BufferedIOMixin): def seek(self, pos, whence=0): if whence == 1: pos -= len(self._read_buf) - self.raw.seek(pos, whence) + pos = self.raw.seek(pos, whence) self._read_buf = b"" + return pos class BufferedWriter(_BufferedIOMixin): @@ -679,7 +691,7 @@ class BufferedWriter(_BufferedIOMixin): def seek(self, pos, whence=0): self.flush() - self.raw.seek(pos, whence) + return self.raw.seek(pos, whence) class BufferedRWPair(BufferedIOBase): @@ -750,13 +762,9 @@ class BufferedRandom(BufferedWriter, BufferedReader): self.flush() # First do the raw seek, then empty the read buffer, so that # if the raw seek fails, we don't lose buffered data forever. - self.raw.seek(pos, whence) + pos = self.raw.seek(pos, whence) self._read_buf = b"" - # XXX I suppose we could implement some magic here to move through the - # existing read buffer in the case of seek(<some small +ve number>, 1) - # XXX OTOH it might be good to *guarantee* that the buffer is - # empty after a seek or flush; for small relative forward - # seeks one might as well use small reads instead. + return pos def tell(self): if (self._write_buf): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 2ca1e70..87d6df6 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4,10 +4,10 @@ import unittest from itertools import chain from test import test_support -import io +import io # The module under test -class MockIO(io.RawIOBase): +class MockRawIO(io.RawIOBase): def __init__(self, read_stack=()): self._read_stack = list(read_stack) @@ -56,13 +56,13 @@ class MockFileIO(io.BytesIO): class MockNonBlockWriterIO(io.RawIOBase): - def __init__(self, blockingScript): - self.bs = list(blockingScript) + def __init__(self, blocking_script): + self._blocking_script = list(blocking_script) self._write_stack = [] def write(self, b): self._write_stack.append(b[:]) - n = self.bs.pop(0) + n = self._blocking_script.pop(0) if (n < 0): raise io.BlockingIOError(0, "test blocking", -n) else: @@ -90,6 +90,23 @@ class IOTest(unittest.TestCase): f.seek(-2, 2) f.truncate() + def large_file_ops(self, f): + assert f.readable() + assert f.writable() + self.assertEqual(f.seek(2**32), 2**32) + self.assertEqual(f.tell(), 2**32) + self.assertEqual(f.write(b"xxx"), 3) + self.assertEqual(f.tell(), 2**32 + 3) + self.assertEqual(f.seek(-1, 1), 2**32 + 2) + f.truncate() + self.assertEqual(f.tell(), 2**32 + 2) + self.assertEqual(f.seek(0, 2), 2**32 + 2) + f.truncate(2**32 + 1) + self.assertEqual(f.tell(), 2**32 + 1) + self.assertEqual(f.seek(0, 2), 2**32 + 1) + self.assertEqual(f.seek(-1, 2), 2**32) + self.assertEqual(f.read(2), b"x") + def read_ops(self, f): data = f.read(5) self.assertEqual(data, b"hello") @@ -130,19 +147,9 @@ class IOTest(unittest.TestCase): f = io.BytesIO(data) self.read_ops(f) - def test_fileio_FileIO(self): - import _fileio - f = _fileio._FileIO(test_support.TESTFN, "w") - self.assertEqual(f.readable(), False) - self.assertEqual(f.writable(), True) - self.assertEqual(f.seekable(), True) - self.write_ops(f) - f.close() - f = _fileio._FileIO(test_support.TESTFN, "r") - self.assertEqual(f.readable(), True) - self.assertEqual(f.writable(), False) - self.assertEqual(f.seekable(), True) - self.read_ops(f) + def test_large_file_ops(self): + f = io.open(test_support.TESTFN, "w+b", buffering=0) + self.large_file_ops(f) f.close() @@ -205,7 +212,7 @@ class StringIOTest(MemorySeekTestMixin, unittest.TestCase): class BufferedReaderTest(unittest.TestCase): def testRead(self): - rawio = MockIO((b"abc", b"d", b"efg")) + rawio = MockRawIO((b"abc", b"d", b"efg")) bufio = io.BufferedReader(rawio) self.assertEquals(b"abcdef", bufio.read(6)) @@ -231,7 +238,7 @@ class BufferedReaderTest(unittest.TestCase): def testReadNonBlocking(self): # Inject some None's in there to simulate EWOULDBLOCK - rawio = MockIO((b"abc", b"d", None, b"efg", None, None)) + rawio = MockRawIO((b"abc", b"d", None, b"efg", None, None)) bufio = io.BufferedReader(rawio) self.assertEquals(b"abcd", bufio.read(6)) @@ -241,19 +248,19 @@ class BufferedReaderTest(unittest.TestCase): self.assertEquals(b"", bufio.read()) def testReadToEof(self): - rawio = MockIO((b"abc", b"d", b"efg")) + rawio = MockRawIO((b"abc", b"d", b"efg")) bufio = io.BufferedReader(rawio) self.assertEquals(b"abcdefg", bufio.read(9000)) def testReadNoArgs(self): - rawio = MockIO((b"abc", b"d", b"efg")) + rawio = MockRawIO((b"abc", b"d", b"efg")) bufio = io.BufferedReader(rawio) self.assertEquals(b"abcdefg", bufio.read()) def testFileno(self): - rawio = MockIO((b"abc", b"d", b"efg")) + rawio = MockRawIO((b"abc", b"d", b"efg")) bufio = io.BufferedReader(rawio) self.assertEquals(42, bufio.fileno()) @@ -268,7 +275,7 @@ class BufferedWriterTest(unittest.TestCase): def testWrite(self): # Write to the buffered IO but don't overflow the buffer. - writer = MockIO() + writer = MockRawIO() bufio = io.BufferedWriter(writer, 8) bufio.write(b"abc") @@ -276,7 +283,7 @@ class BufferedWriterTest(unittest.TestCase): self.assertFalse(writer._write_stack) def testWriteOverflow(self): - writer = MockIO() + writer = MockRawIO() bufio = io.BufferedWriter(writer, 8) bufio.write(b"abc") @@ -305,13 +312,13 @@ class BufferedWriterTest(unittest.TestCase): # later. def testFileno(self): - rawio = MockIO((b"abc", b"d", b"efg")) + rawio = MockRawIO((b"abc", b"d", b"efg")) bufio = io.BufferedWriter(rawio) self.assertEquals(42, bufio.fileno()) def testFlush(self): - writer = MockIO() + writer = MockRawIO() bufio = io.BufferedWriter(writer, 8) bufio.write(b"abc") @@ -323,8 +330,8 @@ class BufferedWriterTest(unittest.TestCase): class BufferedRWPairTest(unittest.TestCase): def testRWPair(self): - r = MockIO(()) - w = MockIO() + r = MockRawIO(()) + w = MockRawIO() pair = io.BufferedRWPair(r, w) # XXX need implementation @@ -333,7 +340,7 @@ class BufferedRWPairTest(unittest.TestCase): class BufferedRandomTest(unittest.TestCase): def testReadAndWrite(self): - raw = MockIO((b"asdf", b"ghjk")) + raw = MockRawIO((b"asdf", b"ghjk")) rw = io.BufferedRandom(raw, 8, 12) self.assertEqual(b"as", rw.read(2)) diff --git a/Modules/_fileio.c b/Modules/_fileio.c index 88ce2f1..985fa81 100644 --- a/Modules/_fileio.c +++ b/Modules/_fileio.c @@ -18,12 +18,6 @@ * To Do: * * - autoconfify header file inclusion - * - Make the ABC RawIO type and inherit from it. - * - * Unanswered questions: - * - * - Add mode and name properties a la Python 2 file objects? - * - Check for readable/writable before attempting to read/write? */ #ifdef MS_WINDOWS @@ -36,7 +30,6 @@ typedef struct { PyObject_HEAD int fd; - int own_fd; /* 1 means we should close fd */ int readable; int writable; int seekable; /* -1 means unknown */ @@ -80,8 +73,6 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kews) if (self != NULL) { self->fd = -1; self->weakreflist = NULL; - self->own_fd = 1; - self->seekable = -1; } return (PyObject *) self; @@ -107,7 +98,7 @@ dircheck(PyFileIOObject* self) PyObject *exc; PyObject *closeresult = fileio_close(self); Py_DECREF(closeresult); - + exc = PyObject_CallFunction(PyExc_IOError, "(is)", EISDIR, msg); PyErr_SetObject(PyExc_IOError, exc); @@ -126,7 +117,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) static char *kwlist[] = {"file", "mode", NULL}; char *name = NULL; char *mode = "r"; - char *s; + char *s; int wideargument = 0; int ret = 0; int rwa = 0, plus = 0, append = 0; @@ -183,7 +174,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) } self->readable = self->writable = 0; - s = mode; + self->seekable = -1; + s = mode; while (*s) { switch (*s++) { case 'r': @@ -240,7 +232,6 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) if (fd >= 0) { self->fd = fd; - /* XXX Should we set self->own_fd = 0 ??? */ } else { Py_BEGIN_ALLOW_THREADS @@ -257,7 +248,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) error: ret = -1; - + done: PyMem_Free(name); return ret; @@ -269,15 +260,16 @@ fileio_dealloc(PyFileIOObject *self) if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); - if (self->fd >= 0 && self->own_fd) { + if (self->fd >= 0) { PyObject *closeresult = fileio_close(self); if (closeresult == NULL) { #ifdef HAVE_STRERROR - PySys_WriteStderr("close failed: [Errno %d] %s\n", errno, strerror(errno)); + PySys_WriteStderr("close failed: [Errno %d] %s\n", + errno, strerror(errno)); #else PySys_WriteStderr("close failed: [Errno %d]\n", errno); #endif - } else + } else Py_DECREF(closeresult); } @@ -292,6 +284,13 @@ err_closed(void) } static PyObject * +err_mode(char *action) +{ + PyErr_Format(PyExc_ValueError, "File not open for %s", action); + return NULL; +} + +static PyObject * fileio_fileno(PyFileIOObject *self) { if (self->fd < 0) @@ -338,9 +337,12 @@ fileio_readinto(PyFileIOObject *self, PyObject *args) { char *ptr; Py_ssize_t n; - + if (self->fd < 0) return err_closed(); + if (!self->readable) + return err_mode("reading"); + if (!PyArg_ParseTuple(args, "w#", &ptr, &n)) return NULL; @@ -367,6 +369,8 @@ fileio_read(PyFileIOObject *self, PyObject *args) if (self->fd < 0) return err_closed(); + if (!self->readable) + return err_mode("reading"); if (!PyArg_ParseTuple(args, "i", &size)) return NULL; @@ -391,9 +395,9 @@ fileio_read(PyFileIOObject *self, PyObject *args) if (n != size) { if (PyBytes_Resize(bytes, n) < 0) { Py_DECREF(bytes); - return NULL; + return NULL; } - } + } return (PyObject *) bytes; } @@ -406,6 +410,9 @@ fileio_write(PyFileIOObject *self, PyObject *args) if (self->fd < 0) return err_closed(); + if (!self->writable) + return err_mode("writing"); + if (!PyArg_ParseTuple(args, "s#", &ptr, &n)) return NULL; @@ -424,146 +431,163 @@ fileio_write(PyFileIOObject *self, PyObject *args) return PyInt_FromLong(n); } +/* XXX Windows support below is likely incomplete */ + +#if defined(MS_WIN64) || defined(MS_WINDOWS) +typedef PY_LONG_LONG Py_off_t; +#else +typedef off_t Py_off_t; +#endif + +/* Cribbed from posix_lseek() */ static PyObject * -fileio_seek(PyFileIOObject *self, PyObject *args) +portable_lseek(int fd, PyObject *posobj, int whence) { - Py_ssize_t offset; - Py_ssize_t whence = 0; + Py_off_t pos, res; - if (self->fd < 0) - return err_closed(); +#ifdef SEEK_SET + /* Turn 0, 1, 2 into SEEK_{SET,CUR,END} */ + switch (whence) { +#if SEEK_SET != 0 + case 0: whence = SEEK_SET; break; +#endif +#if SEEK_CUR != 1 + case 1: whence = SEEK_CUR; break; +#endif +#if SEEL_END != 2 + case 2: whence = SEEK_END; break; +#endif + } +#endif /* SEEK_SET */ - if (!PyArg_ParseTuple(args, "i|i", &offset, &whence)) - return NULL; + if (posobj == NULL) + pos = 0; + else { +#if !defined(HAVE_LARGEFILE_SUPPORT) + pos = PyInt_AsLong(posobj); +#else + pos = PyLong_Check(posobj) ? + PyLong_AsLongLong(posobj) : PyInt_AsLong(posobj); +#endif + if (PyErr_Occurred()) + return NULL; + } Py_BEGIN_ALLOW_THREADS - errno = 0; - offset = lseek(self->fd, offset, whence); +#if defined(MS_WIN64) || defined(MS_WINDOWS) + res = _lseeki64(fd, pos, whence); +#else + res = lseek(fd, pos, whence); +#endif Py_END_ALLOW_THREADS + if (res < 0) + return PyErr_SetFromErrno(PyExc_IOError); - if (offset < 0) { - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - - Py_RETURN_NONE; +#if !defined(HAVE_LARGEFILE_SUPPORT) + return PyInt_FromLong(res); +#else + return PyLong_FromLongLong(res); +#endif } static PyObject * -fileio_tell(PyFileIOObject *self, PyObject *args) +fileio_seek(PyFileIOObject *self, PyObject *args) { - Py_ssize_t offset; + PyObject *posobj; + int whence = 0; if (self->fd < 0) return err_closed(); - Py_BEGIN_ALLOW_THREADS - errno = 0; - offset = lseek(self->fd, 0, SEEK_CUR); - Py_END_ALLOW_THREADS - - if (offset < 0) { - PyErr_SetFromErrno(PyExc_IOError); + if (!PyArg_ParseTuple(args, "O|i", &posobj, &whence)) return NULL; - } - return PyInt_FromLong(offset); + return portable_lseek(self->fd, posobj, whence); } -#ifdef HAVE_FTRUNCATE static PyObject * -fileio_truncate(PyFileIOObject *self, PyObject *args) +fileio_tell(PyFileIOObject *self, PyObject *args) { - Py_ssize_t length; - int ret; - if (self->fd < 0) return err_closed(); - /* Setup default value */ - Py_BEGIN_ALLOW_THREADS - errno = 0; - length = lseek(self->fd, 0, SEEK_CUR); - Py_END_ALLOW_THREADS + return portable_lseek(self->fd, NULL, 1); +} - if (length < 0) { - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - - if (!PyArg_ParseTuple(args, "|i", &length)) +static PyObject * +fileio_truncate(PyFileIOObject *self, PyObject *args) +{ + PyObject *posobj = NULL; + Py_off_t pos; + int fd, whence; + + fd = self->fd; + if (fd < 0) + return err_closed(); + if (!self->writable) + return err_mode("writing"); + + if (!PyArg_ParseTuple(args, "|O", &posobj)) return NULL; -#ifdef MS_WINDOWS - /* MS _chsize doesn't work if newsize doesn't fit in 32 bits, - so don't even try using it. */ - { - HANDLE hFile; - Py_ssize_t initialpos; + if (posobj == NULL) + whence = 1; + else + whence = 0; - /* Have to move current pos to desired endpoint on Windows. */ - Py_BEGIN_ALLOW_THREADS - errno = 0; - ret = _portable_fseek(f->f_fp, newsize, SEEK_SET) != 0; - Py_END_ALLOW_THREADS - if (ret) - goto onioerror; + posobj = portable_lseek(fd, posobj, whence); + if (posobj == NULL) + return NULL; + +#if !defined(HAVE_LARGEFILE_SUPPORT) + pos = PyInt_AsLong(posobj); +#else + pos = PyLong_Check(posobj) ? + PyLong_AsLongLong(posobj) : PyInt_AsLong(posobj); +#endif + Py_DECREF(posobj); + if (PyErr_Occurred()) + return NULL; - /* Truncate. Note that this may grow the file! */ - Py_BEGIN_ALLOW_THREADS - errno = 0; - hFile = (HANDLE)_get_osfhandle(fileno(f->f_fp)); - ret = hFile == (HANDLE)-1; - if (ret == 0) { - ret = SetEndOfFile(hFile) == 0; - if (ret) - errno = EACCES; - } - Py_END_ALLOW_THREADS - if (ret) - goto onioerror; - } -#else Py_BEGIN_ALLOW_THREADS errno = 0; - ret = ftruncate(self->fd, length); + pos = ftruncate(fd, pos); Py_END_ALLOW_THREADS -#endif /* !MS_WINDOWS */ - if (ret < 0) { - onioerror: + if (errno < 0) PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - /* Return to initial position */ - Py_BEGIN_ALLOW_THREADS - errno = 0; - ret = lseek(self->fd, length, SEEK_SET); - Py_END_ALLOW_THREADS - if (ret < 0) - goto onioerror; - Py_RETURN_NONE; } -#endif + +static char * +mode_string(PyFileIOObject *self) +{ + if (self->readable) { + if (self->writable) + return "r+"; + else + return "r"; + } + else + return "w"; +} static PyObject * fileio_repr(PyFileIOObject *self) { - PyObject *ret = NULL; + if (self->fd < 0) + return PyString_FromFormat("_fileio._FileIO(-1)"); - ret = PyString_FromFormat("<%s file at %p>", - self->fd < 0 ? "closed" : "open", - self); - return ret; + return PyString_FromFormat("_fileio._FileIO(%d, '%s')", + self->fd, mode_string(self)); } static PyObject * fileio_isatty(PyFileIOObject *self) { long res; - + if (self->fd < 0) return err_closed(); Py_BEGIN_ALLOW_THREADS @@ -572,14 +596,6 @@ fileio_isatty(PyFileIOObject *self) return PyBool_FromLong(res); } -static PyObject * -fileio_self(PyFileIOObject *self) -{ - if (self->fd < 0) - return err_closed(); - Py_INCREF(self); - return (PyObject *)self; -} PyDoc_STRVAR(fileio_doc, "file(name: str[, mode: str]) -> file IO object\n" @@ -639,12 +655,6 @@ PyDoc_STRVAR(close_doc, PyDoc_STRVAR(isatty_doc, "isatty() -> bool. True if the file is connected to a tty device."); -PyDoc_STRVAR(enter_doc, -"__enter__() -> self."); - -PyDoc_STRVAR(exit_doc, -"__exit__(*excinfo) -> None. Closes the file."); - PyDoc_STRVAR(seekable_doc, "seekable() -> bool. True if file supports random-access."); @@ -667,20 +677,26 @@ static PyMethodDef fileio_methods[] = { {"writable", (PyCFunction)fileio_writable, METH_NOARGS, writable_doc}, {"fileno", (PyCFunction)fileio_fileno, METH_NOARGS, fileno_doc}, {"isatty", (PyCFunction)fileio_isatty, METH_NOARGS, isatty_doc}, - {"__enter__",(PyCFunction)fileio_self, METH_NOARGS, enter_doc}, - {"__exit__", (PyCFunction)fileio_close, METH_VARARGS, exit_doc}, {NULL, NULL} /* sentinel */ }; -/* 'closed' is an attribute for backwards compatibility reasons. */ +/* 'closed' and 'mode' are attributes for backwards compatibility reasons. */ + +static PyObject * +get_closed(PyFileIOObject *self, void *closure) +{ + return PyBool_FromLong((long)(self->fd < 0)); +} + static PyObject * -get_closed(PyFileIOObject *f, void *closure) +get_mode(PyFileIOObject *self, void *closure) { - return PyBool_FromLong((long)(f->fd < 0)); + return PyString_FromString(mode_string(self)); } static PyGetSetDef fileio_getsetlist[] = { {"closed", (getter)get_closed, NULL, "True if the file is closed"}, + {"mode", (getter)get_mode, NULL, "String giving the file mode"}, {0}, }; |