From 8d8d6300034864d9975aedfd0560db8b7bace00f Mon Sep 17 00:00:00 2001 From: Alexandre Vassalotti Date: Sat, 4 Apr 2009 19:58:40 +0000 Subject: Issue 5682: Move _io module into its own subdirectory. Reviewed by: Antoine Pitrou --- Makefile.pre.in | 15 +- Modules/Setup.dist | 2 +- Modules/_bufferedio.c | 2139 --------------------------------------- Modules/_bytesio.c | 757 -------------- Modules/_fileio.c | 1034 ------------------- Modules/_io/_iomodule.c | 760 ++++++++++++++ Modules/_io/_iomodule.h | 150 +++ Modules/_io/bufferedio.c | 2139 +++++++++++++++++++++++++++++++++++++++ Modules/_io/bytesio.c | 757 ++++++++++++++ Modules/_io/fileio.c | 1034 +++++++++++++++++++ Modules/_io/iobase.c | 894 ++++++++++++++++ Modules/_io/stringio.c | 769 ++++++++++++++ Modules/_io/textio.c | 2420 ++++++++++++++++++++++++++++++++++++++++++++ Modules/_iobase.c | 894 ---------------- Modules/_iomodule.h | 150 --- Modules/_stringio.c | 769 -------------- Modules/_textio.c | 2420 -------------------------------------------- Modules/io.c | 760 -------------- PC/VC6/pythoncore.dsp | 48 +- PC/VS7.1/pythoncore.vcproj | 35 +- PC/VS8.0/pythoncore.vcproj | 60 +- PCbuild/pythoncore.vcproj | 68 +- 22 files changed, 9050 insertions(+), 9024 deletions(-) delete mode 100644 Modules/_bufferedio.c delete mode 100644 Modules/_bytesio.c delete mode 100644 Modules/_fileio.c create mode 100644 Modules/_io/_iomodule.c create mode 100644 Modules/_io/_iomodule.h create mode 100644 Modules/_io/bufferedio.c create mode 100644 Modules/_io/bytesio.c create mode 100644 Modules/_io/fileio.c create mode 100644 Modules/_io/iobase.c create mode 100644 Modules/_io/stringio.c create mode 100644 Modules/_io/textio.c delete mode 100644 Modules/_iobase.c delete mode 100644 Modules/_iomodule.h delete mode 100644 Modules/_stringio.c delete mode 100644 Modules/_textio.c delete mode 100644 Modules/io.c diff --git a/Makefile.pre.in b/Makefile.pre.in index dc1d197..0025307 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -193,15 +193,16 @@ MODULE_OBJS= \ # Used of signalmodule.o is not available SIGNAL_OBJS= @SIGNAL_OBJS@ -IO_H= Modules/_iomodule.h +IO_H= Modules/_io/_iomodule.h IO_OBJS= \ - Modules/io.o \ - Modules/_iobase.o \ - Modules/_fileio.o \ - Modules/_bufferedio.o \ - Modules/_textio.o \ - Modules/_bytesio.o + Modules/_io/_iomodule.o \ + Modules/_io/iobase.o \ + Modules/_io/fileio.o \ + Modules/_io/bufferedio.o \ + Modules/_io/textio.o \ + Modules/_io/bytesio.o \ + Modules/_io/stringio.o ########################################################################## # Grammar diff --git a/Modules/Setup.dist b/Modules/Setup.dist index f8d3a83..824b614 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -114,7 +114,7 @@ _codecs _codecsmodule.c # access to the builtin codecs and codec registry _weakref _weakref.c # weak references # Standard I/O baseline -_io io.c _iobase.c _fileio.c _bytesio.c _bufferedio.c _textio.c _stringio.c +_io -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c # The zipimport module is always imported at startup. Having it as a # builtin module avoids some bootstrapping problems and reduces overhead. diff --git a/Modules/_bufferedio.c b/Modules/_bufferedio.c deleted file mode 100644 index 9960dba..0000000 --- a/Modules/_bufferedio.c +++ /dev/null @@ -1,2139 +0,0 @@ -/* - An implementation of Buffered I/O as defined by PEP 3116 - "New I/O" - - Classes defined here: BufferedIOBase, BufferedReader, BufferedWriter, - BufferedRandom. - - Written by Amaury Forgeot d'Arc and Antoine Pitrou -*/ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#include "pythread.h" -#include "_iomodule.h" - -/* - * BufferedIOBase class, inherits from IOBase. - */ -PyDoc_STRVAR(BufferedIOBase_doc, - "Base class for buffered IO objects.\n" - "\n" - "The main difference with RawIOBase is that the read() method\n" - "supports omitting the size argument, and does not have a default\n" - "implementation that defers to readinto().\n" - "\n" - "In addition, read(), readinto() and write() may raise\n" - "BlockingIOError if the underlying raw stream is in non-blocking\n" - "mode and not ready; unlike their raw counterparts, they will never\n" - "return None.\n" - "\n" - "A typical implementation should not inherit from a RawIOBase\n" - "implementation, but wrap one.\n" - ); - -static PyObject * -BufferedIOBase_readinto(PyObject *self, PyObject *args) -{ - Py_buffer buf; - Py_ssize_t len; - PyObject *data; - - if (!PyArg_ParseTuple(args, "w*:readinto", &buf)) { - return NULL; - } - - data = PyObject_CallMethod(self, "read", "n", buf.len); - if (data == NULL) - goto error; - - if (!PyBytes_Check(data)) { - Py_DECREF(data); - PyErr_SetString(PyExc_TypeError, "read() should return bytes"); - goto error; - } - - len = Py_SIZE(data); - memcpy(buf.buf, PyBytes_AS_STRING(data), len); - - PyBuffer_Release(&buf); - Py_DECREF(data); - - return PyLong_FromSsize_t(len); - - error: - PyBuffer_Release(&buf); - return NULL; -} - -static PyObject * -BufferedIOBase_unsupported(const char *message) -{ - PyErr_SetString(IO_STATE->unsupported_operation, message); - return NULL; -} - -PyDoc_STRVAR(BufferedIOBase_read_doc, - "Read and return up to n bytes.\n" - "\n" - "If the argument is omitted, None, or negative, reads and\n" - "returns all data until EOF.\n" - "\n" - "If the argument is positive, and the underlying raw stream is\n" - "not 'interactive', multiple raw reads may be issued to satisfy\n" - "the byte count (unless EOF is reached first). But for\n" - "interactive raw streams (as well as sockets and pipes), at most\n" - "one raw read will be issued, and a short result does not imply\n" - "that EOF is imminent.\n" - "\n" - "Returns an empty bytes object on EOF.\n" - "\n" - "Returns None if the underlying raw stream was open in non-blocking\n" - "mode and no data is available at the moment.\n"); - -static PyObject * -BufferedIOBase_read(PyObject *self, PyObject *args) -{ - return BufferedIOBase_unsupported("read"); -} - -PyDoc_STRVAR(BufferedIOBase_read1_doc, - "Read and return up to n bytes, with at most one read() call\n" - "to the underlying raw stream. A short result does not imply\n" - "that EOF is imminent.\n" - "\n" - "Returns an empty bytes object on EOF.\n"); - -static PyObject * -BufferedIOBase_read1(PyObject *self, PyObject *args) -{ - return BufferedIOBase_unsupported("read1"); -} - -PyDoc_STRVAR(BufferedIOBase_write_doc, - "Write the given buffer to the IO stream.\n" - "\n" - "Returns the number of bytes written, which is never less than\n" - "len(b).\n" - "\n" - "Raises BlockingIOError if the buffer is full and the\n" - "underlying raw stream cannot accept more data at the moment.\n"); - -static PyObject * -BufferedIOBase_write(PyObject *self, PyObject *args) -{ - return BufferedIOBase_unsupported("write"); -} - - -static PyMethodDef BufferedIOBase_methods[] = { - {"read", BufferedIOBase_read, METH_VARARGS, BufferedIOBase_read_doc}, - {"read1", BufferedIOBase_read1, METH_VARARGS, BufferedIOBase_read1_doc}, - {"readinto", BufferedIOBase_readinto, METH_VARARGS, NULL}, - {"write", BufferedIOBase_write, METH_VARARGS, BufferedIOBase_write_doc}, - {NULL, NULL} -}; - -PyTypeObject PyBufferedIOBase_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._BufferedIOBase", /*tp_name*/ - 0, /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - BufferedIOBase_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - BufferedIOBase_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyIOBase_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - - -typedef struct { - PyObject_HEAD - - PyObject *raw; - int ok; /* Initialized? */ - int readable; - int writable; - - /* Absolute position inside the raw stream (-1 if unknown). */ - Py_off_t abs_pos; - - /* A static buffer of size `buffer_size` */ - char *buffer; - /* Current logical position in the buffer. */ - Py_off_t pos; - /* Position of the raw stream in the buffer. */ - Py_off_t raw_pos; - - /* Just after the last buffered byte in the buffer, or -1 if the buffer - isn't ready for reading. */ - Py_off_t read_end; - - /* Just after the last byte actually written */ - Py_off_t write_pos; - /* Just after the last byte waiting to be written, or -1 if the buffer - isn't ready for writing. */ - Py_off_t write_end; - - PyThread_type_lock lock; - - Py_ssize_t buffer_size; - Py_ssize_t buffer_mask; - - PyObject *dict; - PyObject *weakreflist; -} BufferedObject; - -/* - Implementation notes: - - * BufferedReader, BufferedWriter and BufferedRandom try to share most - methods (this is helped by the members `readable` and `writable`, which - are initialized in the respective constructors) - * They also share a single buffer for reading and writing. This enables - interleaved reads and writes without flushing. It also makes the logic - a bit trickier to get right. - * The absolute position of the raw stream is cached, if possible, in the - `abs_pos` member. It must be updated every time an operation is done - on the raw stream. If not sure, it can be reinitialized by calling - _Buffered_raw_tell(), which queries the raw stream (_Buffered_raw_seek() - also does it). To read it, use RAW_TELL(). - * Three helpers, _BufferedReader_raw_read, _BufferedWriter_raw_write and - _BufferedWriter_flush_unlocked do a lot of useful housekeeping. - - NOTE: we should try to maintain block alignment of reads and writes to the - raw stream (according to the buffer size), but for now it is only done - in read() and friends. - - XXX: method naming is a bit messy. -*/ - -/* These macros protect the BufferedObject against concurrent operations. */ - -#define ENTER_BUFFERED(self) \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock(self->lock, 1); \ - Py_END_ALLOW_THREADS - -#define LEAVE_BUFFERED(self) \ - PyThread_release_lock(self->lock); - -#define CHECK_INITIALIZED(self) \ - if (self->ok <= 0) { \ - PyErr_SetString(PyExc_ValueError, \ - "I/O operation on uninitialized object"); \ - return NULL; \ - } - -#define CHECK_INITIALIZED_INT(self) \ - if (self->ok <= 0) { \ - PyErr_SetString(PyExc_ValueError, \ - "I/O operation on uninitialized object"); \ - return -1; \ - } - -#define VALID_READ_BUFFER(self) \ - (self->readable && self->read_end != -1) - -#define VALID_WRITE_BUFFER(self) \ - (self->writable && self->write_end != -1) - -#define ADJUST_POSITION(self, _new_pos) \ - do { \ - self->pos = _new_pos; \ - if (VALID_READ_BUFFER(self) && self->read_end < self->pos) \ - self->read_end = self->pos; \ - } while(0) - -#define READAHEAD(self) \ - ((self->readable && VALID_READ_BUFFER(self)) \ - ? (self->read_end - self->pos) : 0) - -#define RAW_OFFSET(self) \ - (((VALID_READ_BUFFER(self) || VALID_WRITE_BUFFER(self)) \ - && self->raw_pos >= 0) ? self->raw_pos - self->pos : 0) - -#define RAW_TELL(self) \ - (self->abs_pos != -1 ? self->abs_pos : _Buffered_raw_tell(self)) - -#define MINUS_LAST_BLOCK(self, size) \ - (self->buffer_mask ? \ - (size & ~self->buffer_mask) : \ - (self->buffer_size * (size / self->buffer_size))) - - -static void -BufferedObject_dealloc(BufferedObject *self) -{ - if (self->ok && _PyIOBase_finalize((PyObject *) self) < 0) - return; - _PyObject_GC_UNTRACK(self); - self->ok = 0; - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)self); - Py_CLEAR(self->raw); - if (self->buffer) { - PyMem_Free(self->buffer); - self->buffer = NULL; - } - if (self->lock) { - PyThread_free_lock(self->lock); - self->lock = NULL; - } - Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static int -Buffered_traverse(BufferedObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->raw); - Py_VISIT(self->dict); - return 0; -} - -static int -Buffered_clear(BufferedObject *self) -{ - if (self->ok && _PyIOBase_finalize((PyObject *) self) < 0) - return -1; - self->ok = 0; - Py_CLEAR(self->raw); - Py_CLEAR(self->dict); - return 0; -} - -/* - * _BufferedIOMixin methods - * This is not a class, just a collection of methods that will be reused - * by BufferedReader and BufferedWriter - */ - -/* Flush and close */ - -static PyObject * -BufferedIOMixin_flush(BufferedObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self) - return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_flush, NULL); -} - -static int -BufferedIOMixin_closed(BufferedObject *self) -{ - int closed; - PyObject *res; - CHECK_INITIALIZED_INT(self) - res = PyObject_GetAttr(self->raw, _PyIO_str_closed); - if (res == NULL) - return -1; - closed = PyObject_IsTrue(res); - Py_DECREF(res); - return closed; -} - -static PyObject * -BufferedIOMixin_closed_get(BufferedObject *self, void *context) -{ - CHECK_INITIALIZED(self) - return PyObject_GetAttr(self->raw, _PyIO_str_closed); -} - -static PyObject * -BufferedIOMixin_close(BufferedObject *self, PyObject *args) -{ - PyObject *res = NULL; - int r; - - CHECK_INITIALIZED(self) - ENTER_BUFFERED(self) - - r = BufferedIOMixin_closed(self); - if (r < 0) - goto end; - if (r > 0) { - res = Py_None; - Py_INCREF(res); - goto end; - } - /* flush() will most probably re-take the lock, so drop it first */ - LEAVE_BUFFERED(self) - res = PyObject_CallMethodObjArgs((PyObject *)self, _PyIO_str_flush, NULL); - ENTER_BUFFERED(self) - if (res == NULL) { - /* If flush() fails, just give up */ - if (PyErr_ExceptionMatches(PyExc_IOError)) - PyErr_Clear(); - else - goto end; - } - Py_XDECREF(res); - - res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_close, NULL); - -end: - LEAVE_BUFFERED(self) - return res; -} - -/* Inquiries */ - -static PyObject * -BufferedIOMixin_seekable(BufferedObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self) - return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_seekable, NULL); -} - -static PyObject * -BufferedIOMixin_readable(BufferedObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self) - return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readable, NULL); -} - -static PyObject * -BufferedIOMixin_writable(BufferedObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self) - return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_writable, NULL); -} - -static PyObject * -BufferedIOMixin_name_get(BufferedObject *self, void *context) -{ - CHECK_INITIALIZED(self) - return PyObject_GetAttrString(self->raw, "name"); -} - -static PyObject * -BufferedIOMixin_mode_get(BufferedObject *self, void *context) -{ - CHECK_INITIALIZED(self) - return PyObject_GetAttrString(self->raw, "mode"); -} - -/* Lower-level APIs */ - -static PyObject * -BufferedIOMixin_fileno(BufferedObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self) - return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_fileno, NULL); -} - -static PyObject * -BufferedIOMixin_isatty(BufferedObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self) - return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_isatty, NULL); -} - - -/* Forward decls */ -static PyObject * -_BufferedWriter_flush_unlocked(BufferedObject *, int); -static Py_ssize_t -_BufferedReader_fill_buffer(BufferedObject *self); -static void -_BufferedReader_reset_buf(BufferedObject *self); -static void -_BufferedWriter_reset_buf(BufferedObject *self); -static PyObject * -_BufferedReader_peek_unlocked(BufferedObject *self, Py_ssize_t); -static PyObject * -_BufferedReader_read_unlocked(BufferedObject *self, Py_ssize_t); - - -/* - * Helpers - */ - -/* Returns the address of the `written` member if a BlockingIOError was - raised, NULL otherwise. The error is always re-raised. */ -static Py_ssize_t * -_Buffered_check_blocking_error(void) -{ - PyObject *t, *v, *tb; - PyBlockingIOErrorObject *err; - - PyErr_Fetch(&t, &v, &tb); - if (v == NULL || !PyErr_GivenExceptionMatches(v, PyExc_BlockingIOError)) { - PyErr_Restore(t, v, tb); - return NULL; - } - err = (PyBlockingIOErrorObject *) v; - /* TODO: sanity check (err->written >= 0) */ - PyErr_Restore(t, v, tb); - return &err->written; -} - -static Py_off_t -_Buffered_raw_tell(BufferedObject *self) -{ - PyObject *res; - Py_off_t n; - res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_tell, NULL); - if (res == NULL) - return -1; - n = PyNumber_AsOff_t(res, PyExc_ValueError); - Py_DECREF(res); - if (n < 0) { - if (!PyErr_Occurred()) - PyErr_Format(PyExc_IOError, - "Raw stream returned invalid position %zd", n); - return -1; - } - self->abs_pos = n; - return n; -} - -static Py_off_t -_Buffered_raw_seek(BufferedObject *self, Py_off_t target, int whence) -{ - PyObject *res, *posobj, *whenceobj; - Py_off_t n; - - posobj = PyLong_FromOff_t(target); - if (posobj == NULL) - return -1; - whenceobj = PyLong_FromLong(whence); - if (whenceobj == NULL) { - Py_DECREF(posobj); - return -1; - } - res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_seek, - posobj, whenceobj, NULL); - Py_DECREF(posobj); - Py_DECREF(whenceobj); - if (res == NULL) - return -1; - n = PyNumber_AsOff_t(res, PyExc_ValueError); - Py_DECREF(res); - if (n < 0) { - if (!PyErr_Occurred()) - PyErr_Format(PyExc_IOError, - "Raw stream returned invalid position %zd", n); - return -1; - } - self->abs_pos = n; - return n; -} - -static int -_Buffered_init(BufferedObject *self) -{ - Py_ssize_t n; - if (self->buffer_size <= 0) { - PyErr_SetString(PyExc_ValueError, - "buffer size must be strictly positive"); - return -1; - } - if (self->buffer) - PyMem_Free(self->buffer); - self->buffer = PyMem_Malloc(self->buffer_size); - if (self->buffer == NULL) { - PyErr_NoMemory(); - return -1; - } - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - PyErr_SetString(PyExc_RuntimeError, "can't allocate read lock"); - return -1; - } - /* Find out whether buffer_size is a power of 2 */ - /* XXX is this optimization useful? */ - for (n = self->buffer_size - 1; n & 1; n >>= 1) - ; - if (n == 0) - self->buffer_mask = self->buffer_size - 1; - else - self->buffer_mask = 0; - if (_Buffered_raw_tell(self) == -1) - PyErr_Clear(); - return 0; -} - -/* - * Shared methods and wrappers - */ - -static PyObject * -Buffered_flush(BufferedObject *self, PyObject *args) -{ - PyObject *res; - - CHECK_INITIALIZED(self) - if (BufferedIOMixin_closed(self)) { - PyErr_SetString(PyExc_ValueError, "flush of closed file"); - return NULL; - } - - ENTER_BUFFERED(self) - res = _BufferedWriter_flush_unlocked(self, 0); - if (res != NULL && self->readable) { - /* Rewind the raw stream so that its position corresponds to - the current logical position. */ - Py_off_t n; - n = _Buffered_raw_seek(self, -RAW_OFFSET(self), 1); - if (n == -1) - Py_CLEAR(res); - _BufferedReader_reset_buf(self); - } - LEAVE_BUFFERED(self) - - return res; -} - -static PyObject * -Buffered_peek(BufferedObject *self, PyObject *args) -{ - Py_ssize_t n = 0; - PyObject *res = NULL; - - CHECK_INITIALIZED(self) - if (!PyArg_ParseTuple(args, "|n:peek", &n)) { - return NULL; - } - - ENTER_BUFFERED(self) - - if (self->writable) { - res = _BufferedWriter_flush_unlocked(self, 1); - if (res == NULL) - goto end; - Py_CLEAR(res); - } - res = _BufferedReader_peek_unlocked(self, n); - -end: - LEAVE_BUFFERED(self) - return res; -} - -static PyObject * -Buffered_read(BufferedObject *self, PyObject *args) -{ - Py_ssize_t n = -1; - PyObject *res; - - CHECK_INITIALIZED(self) - if (!PyArg_ParseTuple(args, "|n:read", &n)) { - return NULL; - } - if (n < -1) { - PyErr_SetString(PyExc_ValueError, - "read length must be positive or -1"); - return NULL; - } - - if (BufferedIOMixin_closed(self)) { - PyErr_SetString(PyExc_ValueError, "read of closed file"); - return NULL; - } - - ENTER_BUFFERED(self) - res = _BufferedReader_read_unlocked(self, n); - LEAVE_BUFFERED(self) - - return res; -} - -static PyObject * -Buffered_read1(BufferedObject *self, PyObject *args) -{ - Py_ssize_t n, have, r; - PyObject *res = NULL; - - CHECK_INITIALIZED(self) - if (!PyArg_ParseTuple(args, "n:read1", &n)) { - return NULL; - } - - if (n < 0) { - PyErr_SetString(PyExc_ValueError, - "read length must be positive"); - return NULL; - } - if (n == 0) - return PyBytes_FromStringAndSize(NULL, 0); - - ENTER_BUFFERED(self) - - if (self->writable) { - res = _BufferedWriter_flush_unlocked(self, 1); - if (res == NULL) - goto end; - Py_CLEAR(res); - } - - /* Return up to n bytes. If at least one byte is buffered, we - only return buffered bytes. Otherwise, we do one raw read. */ - - /* XXX: this mimicks the io.py implementation but is probably wrong. - If we need to read from the raw stream, then we could actually read - all `n` bytes asked by the caller (and possibly more, so as to fill - our buffer for the next reads). */ - - have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); - if (have > 0) { - if (n > have) - n = have; - res = PyBytes_FromStringAndSize(self->buffer + self->pos, n); - if (res == NULL) - goto end; - self->pos += n; - goto end; - } - - /* Fill the buffer from the raw stream, and copy it to the result. */ - _BufferedReader_reset_buf(self); - r = _BufferedReader_fill_buffer(self); - if (r == -1) - goto end; - if (r == -2) - r = 0; - if (n > r) - n = r; - res = PyBytes_FromStringAndSize(self->buffer, n); - if (res == NULL) - goto end; - self->pos = n; - -end: - LEAVE_BUFFERED(self) - return res; -} - -static PyObject * -Buffered_readinto(BufferedObject *self, PyObject *args) -{ - PyObject *res = NULL; - - CHECK_INITIALIZED(self) - - /* TODO: use raw.readinto() instead! */ - if (self->writable) { - ENTER_BUFFERED(self) - res = _BufferedWriter_flush_unlocked(self, 0); - LEAVE_BUFFERED(self) - if (res == NULL) - goto end; - Py_DECREF(res); - } - res = BufferedIOBase_readinto((PyObject *)self, args); - -end: - return res; -} - -static PyObject * -_Buffered_readline(BufferedObject *self, Py_ssize_t limit) -{ - PyObject *res = NULL; - PyObject *chunks = NULL; - Py_ssize_t n, written = 0; - const char *start, *s, *end; - - if (BufferedIOMixin_closed(self)) { - PyErr_SetString(PyExc_ValueError, "readline of closed file"); - return NULL; - } - - ENTER_BUFFERED(self) - - /* First, try to find a line in the buffer */ - n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); - if (limit >= 0 && n > limit) - n = limit; - start = self->buffer + self->pos; - end = start + n; - s = start; - while (s < end) { - if (*s++ == '\n') { - res = PyBytes_FromStringAndSize(start, s - start); - if (res != NULL) - self->pos += s - start; - goto end; - } - } - if (n == limit) { - res = PyBytes_FromStringAndSize(start, n); - if (res != NULL) - self->pos += n; - goto end; - } - - /* Now we try to get some more from the raw stream */ - if (self->writable) { - res = _BufferedWriter_flush_unlocked(self, 1); - if (res == NULL) - goto end; - Py_CLEAR(res); - } - chunks = PyList_New(0); - if (chunks == NULL) - goto end; - if (n > 0) { - res = PyBytes_FromStringAndSize(start, n); - if (res == NULL) - goto end; - if (PyList_Append(chunks, res) < 0) { - Py_CLEAR(res); - goto end; - } - Py_CLEAR(res); - written += n; - if (limit >= 0) - limit -= n; - } - - for (;;) { - _BufferedReader_reset_buf(self); - n = _BufferedReader_fill_buffer(self); - if (n == -1) - goto end; - if (n <= 0) - break; - if (limit >= 0 && n > limit) - n = limit; - start = self->buffer; - end = start + n; - s = start; - while (s < end) { - if (*s++ == '\n') { - res = PyBytes_FromStringAndSize(start, s - start); - if (res == NULL) - goto end; - self->pos = s - start; - goto found; - } - } - res = PyBytes_FromStringAndSize(start, n); - if (res == NULL) - goto end; - if (n == limit) { - self->pos = n; - break; - } - if (PyList_Append(chunks, res) < 0) { - Py_CLEAR(res); - goto end; - } - Py_CLEAR(res); - written += n; - if (limit >= 0) - limit -= n; - } -found: - if (res != NULL && PyList_Append(chunks, res) < 0) { - Py_CLEAR(res); - goto end; - } - Py_CLEAR(res); - res = _PyBytes_Join(_PyIO_empty_bytes, chunks); - -end: - LEAVE_BUFFERED(self) - Py_XDECREF(chunks); - return res; -} - -static PyObject * -Buffered_readline(BufferedObject *self, PyObject *args) -{ - Py_ssize_t limit = -1; - - CHECK_INITIALIZED(self) - - if (!PyArg_ParseTuple(args, "|n:readline", &limit)) { - return NULL; - } - return _Buffered_readline(self, limit); -} - - -static PyObject * -Buffered_tell(BufferedObject *self, PyObject *args) -{ - Py_off_t pos; - - CHECK_INITIALIZED(self) - pos = _Buffered_raw_tell(self); - if (pos == -1) - return NULL; - pos -= RAW_OFFSET(self); - /* TODO: sanity check (pos >= 0) */ - return PyLong_FromOff_t(pos); -} - -static PyObject * -Buffered_seek(BufferedObject *self, PyObject *args) -{ - Py_off_t target, n; - int whence = 0; - PyObject *targetobj, *res = NULL; - - CHECK_INITIALIZED(self) - if (!PyArg_ParseTuple(args, "O|i:seek", &targetobj, &whence)) { - return NULL; - } - - if (whence < 0 || whence > 2) { - PyErr_Format(PyExc_ValueError, - "whence must be between 0 and 2, not %d", whence); - return NULL; - } - target = PyNumber_AsOff_t(targetobj, PyExc_ValueError); - if (target == -1 && PyErr_Occurred()) - return NULL; - - ENTER_BUFFERED(self) - - if (whence != 2 && self->readable) { - Py_off_t current, avail; - /* Check if seeking leaves us inside the current buffer, - so as to return quickly if possible. - Don't know how to do that when whence == 2, though. */ - current = RAW_TELL(self); - avail = READAHEAD(self); - if (avail > 0) { - Py_off_t offset; - if (whence == 0) - offset = target - (current - RAW_OFFSET(self)); - else - offset = target; - if (offset >= -self->pos && offset <= avail) { - self->pos += offset; - res = PyLong_FromOff_t(current - avail + offset); - goto end; - } - } - } - - /* Fallback: invoke raw seek() method and clear buffer */ - if (self->writable) { - res = _BufferedWriter_flush_unlocked(self, 0); - if (res == NULL) - goto end; - Py_CLEAR(res); - _BufferedWriter_reset_buf(self); - } - - /* TODO: align on block boundary and read buffer if needed? */ - if (whence == 1) - target -= RAW_OFFSET(self); - n = _Buffered_raw_seek(self, target, whence); - if (n == -1) - goto end; - self->raw_pos = -1; - res = PyLong_FromOff_t(n); - if (res != NULL && self->readable) - _BufferedReader_reset_buf(self); - -end: - LEAVE_BUFFERED(self) - return res; -} - -static PyObject * -Buffered_truncate(BufferedObject *self, PyObject *args) -{ - PyObject *pos = Py_None; - PyObject *res = NULL; - - CHECK_INITIALIZED(self) - if (!PyArg_ParseTuple(args, "|O:truncate", &pos)) { - return NULL; - } - - ENTER_BUFFERED(self) - - if (self->writable) { - res = _BufferedWriter_flush_unlocked(self, 0); - if (res == NULL) - goto end; - Py_CLEAR(res); - } - if (self->readable) { - if (pos == Py_None) { - /* Rewind the raw stream so that its position corresponds to - the current logical position. */ - if (_Buffered_raw_seek(self, -RAW_OFFSET(self), 1) == -1) - goto end; - } - _BufferedReader_reset_buf(self); - } - res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_truncate, pos, NULL); - if (res == NULL) - goto end; - /* Reset cached position */ - if (_Buffered_raw_tell(self) == -1) - PyErr_Clear(); - -end: - LEAVE_BUFFERED(self) - return res; -} - -static PyObject * -Buffered_iternext(BufferedObject *self) -{ - PyObject *line; - PyTypeObject *tp; - - CHECK_INITIALIZED(self); - - tp = Py_TYPE(self); - if (tp == &PyBufferedReader_Type || - tp == &PyBufferedRandom_Type) { - /* Skip method call overhead for speed */ - line = _Buffered_readline(self, -1); - } - else { - line = PyObject_CallMethodObjArgs((PyObject *)self, - _PyIO_str_readline, NULL); - if (line && !PyBytes_Check(line)) { - PyErr_Format(PyExc_IOError, - "readline() should have returned a bytes object, " - "not '%.200s'", Py_TYPE(line)->tp_name); - Py_DECREF(line); - return NULL; - } - } - - if (line == NULL) - return NULL; - - if (PyBytes_GET_SIZE(line) == 0) { - /* Reached EOF or would have blocked */ - Py_DECREF(line); - return NULL; - } - - return line; -} - -/* - * class BufferedReader - */ - -PyDoc_STRVAR(BufferedReader_doc, - "Create a new buffered reader using the given readable raw IO object."); - -static void _BufferedReader_reset_buf(BufferedObject *self) -{ - self->read_end = -1; -} - -static int -BufferedReader_init(BufferedObject *self, PyObject *args, PyObject *kwds) -{ - char *kwlist[] = {"raw", "buffer_size", NULL}; - Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - PyObject *raw; - - self->ok = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|n:BufferedReader", kwlist, - &raw, &buffer_size)) { - return -1; - } - - if (_PyIOBase_checkReadable(raw, Py_True) == NULL) - return -1; - - Py_CLEAR(self->raw); - Py_INCREF(raw); - self->raw = raw; - self->buffer_size = buffer_size; - self->readable = 1; - self->writable = 0; - - if (_Buffered_init(self) < 0) - return -1; - _BufferedReader_reset_buf(self); - - self->ok = 1; - return 0; -} - -static Py_ssize_t -_BufferedReader_raw_read(BufferedObject *self, char *start, Py_ssize_t len) -{ - Py_buffer buf; - PyObject *memobj, *res; - Py_ssize_t n; - /* NOTE: the buffer needn't be released as its object is NULL. */ - if (PyBuffer_FillInfo(&buf, NULL, start, len, 0, PyBUF_CONTIG) == -1) - return -1; - memobj = PyMemoryView_FromBuffer(&buf); - if (memobj == NULL) - return -1; - res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readinto, memobj, NULL); - Py_DECREF(memobj); - if (res == NULL) - return -1; - if (res == Py_None) { - /* Non-blocking stream would have blocked. Special return code! */ - Py_DECREF(res); - return -2; - } - n = PyNumber_AsSsize_t(res, PyExc_ValueError); - Py_DECREF(res); - if (n < 0 || n > len) { - PyErr_Format(PyExc_IOError, - "raw readinto() returned invalid length %zd " - "(should have been between 0 and %zd)", n, len); - return -1; - } - if (n > 0 && self->abs_pos != -1) - self->abs_pos += n; - return n; -} - -static Py_ssize_t -_BufferedReader_fill_buffer(BufferedObject *self) -{ - Py_ssize_t start, len, n; - if (VALID_READ_BUFFER(self)) - start = Py_SAFE_DOWNCAST(self->read_end, Py_off_t, Py_ssize_t); - else - start = 0; - len = self->buffer_size - start; - n = _BufferedReader_raw_read(self, self->buffer + start, len); - if (n <= 0) - return n; - self->read_end = start + n; - self->raw_pos = start + n; - return n; -} - -static PyObject * -_BufferedReader_read_unlocked(BufferedObject *self, Py_ssize_t n) -{ - PyObject *data, *res = NULL; - Py_ssize_t current_size, remaining, written; - char *out; - - /* Special case for when the number of bytes to read is unspecified. */ - if (n == -1) { - PyObject *chunks = PyList_New(0); - if (chunks == NULL) - return NULL; - - /* First copy what we have in the current buffer. */ - current_size = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); - data = NULL; - if (current_size) { - data = PyBytes_FromStringAndSize( - self->buffer + self->pos, current_size); - if (data == NULL) { - Py_DECREF(chunks); - return NULL; - } - } - _BufferedReader_reset_buf(self); - /* We're going past the buffer's bounds, flush it */ - if (self->writable) { - res = _BufferedWriter_flush_unlocked(self, 1); - if (res == NULL) { - Py_DECREF(chunks); - return NULL; - } - Py_CLEAR(res); - } - while (1) { - if (data) { - if (PyList_Append(chunks, data) < 0) { - Py_DECREF(data); - Py_DECREF(chunks); - return NULL; - } - Py_DECREF(data); - } - - /* Read until EOF or until read() would block. */ - data = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_read, NULL); - if (data == NULL) { - Py_DECREF(chunks); - return NULL; - } - if (data != Py_None && !PyBytes_Check(data)) { - Py_DECREF(data); - Py_DECREF(chunks); - PyErr_SetString(PyExc_TypeError, "read() should return bytes"); - return NULL; - } - if (data == Py_None || PyBytes_GET_SIZE(data) == 0) { - if (current_size == 0) { - Py_DECREF(chunks); - return data; - } - else { - res = _PyBytes_Join(_PyIO_empty_bytes, chunks); - Py_DECREF(data); - Py_DECREF(chunks); - return res; - } - } - current_size += PyBytes_GET_SIZE(data); - if (self->abs_pos != -1) - self->abs_pos += PyBytes_GET_SIZE(data); - } - } - - /* The number of bytes to read is specified, return at most n bytes. */ - current_size = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); - if (n <= current_size) { - /* Fast path: the data to read is fully buffered. */ - res = PyBytes_FromStringAndSize(self->buffer + self->pos, n); - if (res == NULL) - goto error; - self->pos += n; - return res; - } - - /* Slow path: read from the stream until enough bytes are read, - * or until an EOF occurs or until read() would block. - */ - res = PyBytes_FromStringAndSize(NULL, n); - if (res == NULL) - goto error; - out = PyBytes_AS_STRING(res); - remaining = n; - written = 0; - if (current_size > 0) { - memcpy(out, self->buffer + self->pos, current_size); - remaining -= current_size; - written += current_size; - } - _BufferedReader_reset_buf(self); - while (remaining > 0) { - /* We want to read a whole block at the end into buffer. - If we had readv() we could do this in one pass. */ - Py_ssize_t r = MINUS_LAST_BLOCK(self, remaining); - if (r == 0) - break; - r = _BufferedReader_raw_read(self, out + written, r); - if (r == -1) - goto error; - if (r == 0 || r == -2) { - /* EOF occurred or read() would block. */ - if (r == 0 || written > 0) { - if (_PyBytes_Resize(&res, written)) - goto error; - return res; - } - Py_DECREF(res); - Py_INCREF(Py_None); - return Py_None; - } - remaining -= r; - written += r; - } - assert(remaining <= self->buffer_size); - self->pos = 0; - self->raw_pos = 0; - self->read_end = 0; - while (self->read_end < self->buffer_size) { - Py_ssize_t r = _BufferedReader_fill_buffer(self); - if (r == -1) - goto error; - if (r == 0 || r == -2) { - /* EOF occurred or read() would block. */ - if (r == 0 || written > 0) { - if (_PyBytes_Resize(&res, written)) - goto error; - return res; - } - Py_DECREF(res); - Py_INCREF(Py_None); - return Py_None; - } - if (remaining > r) { - memcpy(out + written, self->buffer + self->pos, r); - written += r; - self->pos += r; - remaining -= r; - } - else if (remaining > 0) { - memcpy(out + written, self->buffer + self->pos, remaining); - written += remaining; - self->pos += remaining; - remaining = 0; - } - if (remaining == 0) - break; - } - - return res; - -error: - Py_XDECREF(res); - return NULL; -} - -static PyObject * -_BufferedReader_peek_unlocked(BufferedObject *self, Py_ssize_t n) -{ - Py_ssize_t have, r; - - have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); - /* Constraints: - 1. we don't want to advance the file position. - 2. we don't want to lose block alignment, so we can't shift the buffer - to make some place. - Therefore, we either return `have` bytes (if > 0), or a full buffer. - */ - if (have > 0) { - return PyBytes_FromStringAndSize(self->buffer + self->pos, have); - } - - /* Fill the buffer from the raw stream, and copy it to the result. */ - _BufferedReader_reset_buf(self); - r = _BufferedReader_fill_buffer(self); - if (r == -1) - return NULL; - if (r == -2) - r = 0; - self->pos = 0; - return PyBytes_FromStringAndSize(self->buffer, r); -} - -static PyMethodDef BufferedReader_methods[] = { - /* BufferedIOMixin methods */ - {"flush", (PyCFunction)BufferedIOMixin_flush, METH_NOARGS}, - {"close", (PyCFunction)BufferedIOMixin_close, METH_NOARGS}, - {"seekable", (PyCFunction)BufferedIOMixin_seekable, METH_NOARGS}, - {"readable", (PyCFunction)BufferedIOMixin_readable, METH_NOARGS}, - {"writable", (PyCFunction)BufferedIOMixin_writable, METH_NOARGS}, - {"fileno", (PyCFunction)BufferedIOMixin_fileno, METH_NOARGS}, - {"isatty", (PyCFunction)BufferedIOMixin_isatty, METH_NOARGS}, - - {"read", (PyCFunction)Buffered_read, METH_VARARGS}, - {"peek", (PyCFunction)Buffered_peek, METH_VARARGS}, - {"read1", (PyCFunction)Buffered_read1, METH_VARARGS}, - {"readline", (PyCFunction)Buffered_readline, METH_VARARGS}, - {"seek", (PyCFunction)Buffered_seek, METH_VARARGS}, - {"tell", (PyCFunction)Buffered_tell, METH_NOARGS}, - {"truncate", (PyCFunction)Buffered_truncate, METH_VARARGS}, - {NULL, NULL} -}; - -static PyMemberDef BufferedReader_members[] = { - {"raw", T_OBJECT, offsetof(BufferedObject, raw), 0}, - {NULL} -}; - -static PyGetSetDef BufferedReader_getset[] = { - {"closed", (getter)BufferedIOMixin_closed_get, NULL, NULL}, - {"name", (getter)BufferedIOMixin_name_get, NULL, NULL}, - {"mode", (getter)BufferedIOMixin_mode_get, NULL, NULL}, - {0} -}; - - -PyTypeObject PyBufferedReader_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BufferedReader", /*tp_name*/ - sizeof(BufferedObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BufferedObject_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - BufferedReader_doc, /* tp_doc */ - (traverseproc)Buffered_traverse, /* tp_traverse */ - (inquiry)Buffered_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(BufferedObject, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - (iternextfunc)Buffered_iternext, /* tp_iternext */ - BufferedReader_methods, /* tp_methods */ - BufferedReader_members, /* tp_members */ - BufferedReader_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(BufferedObject, dict), /* tp_dictoffset */ - (initproc)BufferedReader_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ -}; - - - -static int -complain_about_max_buffer_size(void) -{ - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "max_buffer_size is deprecated", 1) < 0) - return 0; - return 1; -} - -/* - * class BufferedWriter - */ -PyDoc_STRVAR(BufferedWriter_doc, - "A buffer for a writeable sequential RawIO object.\n" - "\n" - "The constructor creates a BufferedWriter for the given writeable raw\n" - "stream. If the buffer_size is not given, it defaults to\n" - "DEFAULT_BUFFER_SIZE. max_buffer_size isn't used anymore.\n" - ); - -static void -_BufferedWriter_reset_buf(BufferedObject *self) -{ - self->write_pos = 0; - self->write_end = -1; -} - -static int -BufferedWriter_init(BufferedObject *self, PyObject *args, PyObject *kwds) -{ - /* TODO: properly deprecate max_buffer_size */ - char *kwlist[] = {"raw", "buffer_size", "max_buffer_size", NULL}; - Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - Py_ssize_t max_buffer_size = -234; - PyObject *raw; - - self->ok = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|nn:BufferedReader", kwlist, - &raw, &buffer_size, &max_buffer_size)) { - return -1; - } - - if (max_buffer_size != -234 && !complain_about_max_buffer_size()) - return -1; - - if (_PyIOBase_checkWritable(raw, Py_True) == NULL) - return -1; - - Py_CLEAR(self->raw); - Py_INCREF(raw); - self->raw = raw; - self->readable = 0; - self->writable = 1; - - self->buffer_size = buffer_size; - if (_Buffered_init(self) < 0) - return -1; - _BufferedWriter_reset_buf(self); - self->pos = 0; - - self->ok = 1; - return 0; -} - -static Py_ssize_t -_BufferedWriter_raw_write(BufferedObject *self, char *start, Py_ssize_t len) -{ - Py_buffer buf; - PyObject *memobj, *res; - Py_ssize_t n; - /* NOTE: the buffer needn't be released as its object is NULL. */ - if (PyBuffer_FillInfo(&buf, NULL, start, len, 1, PyBUF_CONTIG_RO) == -1) - return -1; - memobj = PyMemoryView_FromBuffer(&buf); - if (memobj == NULL) - return -1; - res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_write, memobj, NULL); - Py_DECREF(memobj); - if (res == NULL) - return -1; - n = PyNumber_AsSsize_t(res, PyExc_ValueError); - Py_DECREF(res); - if (n < 0 || n > len) { - PyErr_Format(PyExc_IOError, - "raw write() returned invalid length %zd " - "(should have been between 0 and %zd)", n, len); - return -1; - } - if (n > 0 && self->abs_pos != -1) - self->abs_pos += n; - return n; -} - -/* `restore_pos` is 1 if we need to restore the raw stream position at - the end, 0 otherwise. */ -static PyObject * -_BufferedWriter_flush_unlocked(BufferedObject *self, int restore_pos) -{ - Py_ssize_t written = 0; - Py_off_t n, rewind; - - if (!VALID_WRITE_BUFFER(self) || self->write_pos == self->write_end) - goto end; - /* First, rewind */ - rewind = RAW_OFFSET(self) + (self->pos - self->write_pos); - if (rewind != 0) { - n = _Buffered_raw_seek(self, -rewind, 1); - if (n < 0) { - goto error; - } - self->raw_pos -= rewind; - } - while (self->write_pos < self->write_end) { - n = _BufferedWriter_raw_write(self, - self->buffer + self->write_pos, - Py_SAFE_DOWNCAST(self->write_end - self->write_pos, - Py_off_t, Py_ssize_t)); - if (n == -1) { - Py_ssize_t *w = _Buffered_check_blocking_error(); - if (w == NULL) - goto error; - self->write_pos += *w; - self->raw_pos = self->write_pos; - written += *w; - *w = written; - /* Already re-raised */ - goto error; - } - self->write_pos += n; - self->raw_pos = self->write_pos; - written += Py_SAFE_DOWNCAST(n, Py_off_t, Py_ssize_t); - } - - if (restore_pos) { - Py_off_t forward = rewind - written; - if (forward != 0) { - n = _Buffered_raw_seek(self, forward, 1); - if (n < 0) { - goto error; - } - self->raw_pos += forward; - } - } - _BufferedWriter_reset_buf(self); - -end: - Py_RETURN_NONE; - -error: - return NULL; -} - -static PyObject * -BufferedWriter_write(BufferedObject *self, PyObject *args) -{ - PyObject *res = NULL; - Py_buffer buf; - Py_ssize_t written, avail, remaining, n; - - CHECK_INITIALIZED(self) - if (!PyArg_ParseTuple(args, "y*:write", &buf)) { - return NULL; - } - - if (BufferedIOMixin_closed(self)) { - PyErr_SetString(PyExc_ValueError, "write to closed file"); - PyBuffer_Release(&buf); - return NULL; - } - - ENTER_BUFFERED(self) - - /* Fast path: the data to write can be fully buffered. */ - if (!VALID_READ_BUFFER(self) && !VALID_WRITE_BUFFER(self)) { - self->pos = 0; - self->raw_pos = 0; - } - avail = Py_SAFE_DOWNCAST(self->buffer_size - self->pos, Py_off_t, Py_ssize_t); - if (buf.len <= avail) { - memcpy(self->buffer + self->pos, buf.buf, buf.len); - if (!VALID_WRITE_BUFFER(self)) { - self->write_pos = self->pos; - } - ADJUST_POSITION(self, self->pos + buf.len); - if (self->pos > self->write_end) - self->write_end = self->pos; - written = buf.len; - goto end; - } - - /* First write the current buffer */ - res = _BufferedWriter_flush_unlocked(self, 0); - if (res == NULL) { - Py_ssize_t *w = _Buffered_check_blocking_error(); - if (w == NULL) - goto error; - if (self->readable) - _BufferedReader_reset_buf(self); - /* Make some place by shifting the buffer. */ - assert(VALID_WRITE_BUFFER(self)); - memmove(self->buffer, self->buffer + self->write_pos, - Py_SAFE_DOWNCAST(self->write_end - self->write_pos, - Py_off_t, Py_ssize_t)); - self->write_end -= self->write_pos; - self->raw_pos -= self->write_pos; - self->pos -= self->write_pos; - self->write_pos = 0; - avail = Py_SAFE_DOWNCAST(self->buffer_size - self->write_end, - Py_off_t, Py_ssize_t); - if (buf.len <= avail) { - /* Everything can be buffered */ - PyErr_Clear(); - memcpy(self->buffer + self->write_end, buf.buf, buf.len); - self->write_end += buf.len; - written = buf.len; - goto end; - } - /* Buffer as much as possible. */ - memcpy(self->buffer + self->write_end, buf.buf, avail); - self->write_end += avail; - /* Already re-raised */ - *w = avail; - goto error; - } - Py_CLEAR(res); - - /* Then write buf itself. At this point the buffer has been emptied. */ - remaining = buf.len; - written = 0; - while (remaining > self->buffer_size) { - n = _BufferedWriter_raw_write( - self, (char *) buf.buf + written, buf.len - written); - if (n == -1) { - Py_ssize_t *w = _Buffered_check_blocking_error(); - if (w == NULL) - goto error; - written += *w; - remaining -= *w; - if (remaining > self->buffer_size) { - /* Can't buffer everything, still buffer as much as possible */ - memcpy(self->buffer, - (char *) buf.buf + written, self->buffer_size); - self->raw_pos = 0; - ADJUST_POSITION(self, self->buffer_size); - self->write_end = self->buffer_size; - *w = written + self->buffer_size; - /* Already re-raised */ - goto error; - } - PyErr_Clear(); - break; - } - written += n; - remaining -= n; - } - if (self->readable) - _BufferedReader_reset_buf(self); - if (remaining > 0) { - memcpy(self->buffer, (char *) buf.buf + written, remaining); - written += remaining; - } - self->write_pos = 0; - /* TODO: sanity check (remaining >= 0) */ - self->write_end = remaining; - ADJUST_POSITION(self, remaining); - self->raw_pos = 0; - -end: - res = PyLong_FromSsize_t(written); - -error: - LEAVE_BUFFERED(self) - PyBuffer_Release(&buf); - return res; -} - -static PyMethodDef BufferedWriter_methods[] = { - /* BufferedIOMixin methods */ - {"close", (PyCFunction)BufferedIOMixin_close, METH_NOARGS}, - {"seekable", (PyCFunction)BufferedIOMixin_seekable, METH_NOARGS}, - {"readable", (PyCFunction)BufferedIOMixin_readable, METH_NOARGS}, - {"writable", (PyCFunction)BufferedIOMixin_writable, METH_NOARGS}, - {"fileno", (PyCFunction)BufferedIOMixin_fileno, METH_NOARGS}, - {"isatty", (PyCFunction)BufferedIOMixin_isatty, METH_NOARGS}, - - {"write", (PyCFunction)BufferedWriter_write, METH_VARARGS}, - {"truncate", (PyCFunction)Buffered_truncate, METH_VARARGS}, - {"flush", (PyCFunction)Buffered_flush, METH_NOARGS}, - {"seek", (PyCFunction)Buffered_seek, METH_VARARGS}, - {"tell", (PyCFunction)Buffered_tell, METH_NOARGS}, - {NULL, NULL} -}; - -static PyMemberDef BufferedWriter_members[] = { - {"raw", T_OBJECT, offsetof(BufferedObject, raw), 0}, - {NULL} -}; - -static PyGetSetDef BufferedWriter_getset[] = { - {"closed", (getter)BufferedIOMixin_closed_get, NULL, NULL}, - {"name", (getter)BufferedIOMixin_name_get, NULL, NULL}, - {"mode", (getter)BufferedIOMixin_mode_get, NULL, NULL}, - {0} -}; - - -PyTypeObject PyBufferedWriter_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BufferedWriter", /*tp_name*/ - sizeof(BufferedObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BufferedObject_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - BufferedWriter_doc, /* tp_doc */ - (traverseproc)Buffered_traverse, /* tp_traverse */ - (inquiry)Buffered_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(BufferedObject, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - 0, /* tp_iternext */ - BufferedWriter_methods, /* tp_methods */ - BufferedWriter_members, /* tp_members */ - BufferedWriter_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(BufferedObject, dict), /* tp_dictoffset */ - (initproc)BufferedWriter_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ -}; - - - -/* - * BufferedRWPair - */ - -PyDoc_STRVAR(BufferedRWPair_doc, - "A buffered reader and writer object together.\n" - "\n" - "A buffered reader object and buffered writer object put together to\n" - "form a sequential IO object that can read and write. This is typically\n" - "used with a socket or two-way pipe.\n" - "\n" - "reader and writer are RawIOBase objects that are readable and\n" - "writeable respectively. If the buffer_size is omitted it defaults to\n" - "DEFAULT_BUFFER_SIZE.\n" - ); - -/* XXX The usefulness of this (compared to having two separate IO objects) is - * questionable. - */ - -typedef struct { - PyObject_HEAD - BufferedObject *reader; - BufferedObject *writer; - PyObject *dict; - PyObject *weakreflist; -} BufferedRWPairObject; - -static int -BufferedRWPair_init(BufferedRWPairObject *self, PyObject *args, - PyObject *kwds) -{ - PyObject *reader, *writer; - Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - Py_ssize_t max_buffer_size = -234; - - if (!PyArg_ParseTuple(args, "OO|nn:BufferedRWPair", &reader, &writer, - &buffer_size, &max_buffer_size)) { - return -1; - } - - if (max_buffer_size != -234 && !complain_about_max_buffer_size()) - return -1; - - if (_PyIOBase_checkReadable(reader, Py_True) == NULL) - return -1; - if (_PyIOBase_checkWritable(writer, Py_True) == NULL) - return -1; - - args = Py_BuildValue("(n)", buffer_size); - if (args == NULL) { - Py_CLEAR(self->reader); - return -1; - } - self->reader = (BufferedObject *)PyType_GenericNew( - &PyBufferedReader_Type, args, NULL); - Py_DECREF(args); - if (self->reader == NULL) - return -1; - - args = Py_BuildValue("(n)", buffer_size); - if (args == NULL) { - Py_CLEAR(self->reader); - return -1; - } - self->writer = (BufferedObject *)PyType_GenericNew( - &PyBufferedWriter_Type, args, NULL); - Py_DECREF(args); - if (self->writer == NULL) { - Py_CLEAR(self->reader); - return -1; - } - return 0; -} - -static int -BufferedRWPair_traverse(BufferedRWPairObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->dict); - return 0; -} - -static int -BufferedRWPair_clear(BufferedRWPairObject *self) -{ - Py_CLEAR(self->reader); - Py_CLEAR(self->writer); - Py_CLEAR(self->dict); - return 0; -} - -static void -BufferedRWPair_dealloc(BufferedRWPairObject *self) -{ - _PyObject_GC_UNTRACK(self); - Py_CLEAR(self->reader); - Py_CLEAR(self->writer); - Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -static PyObject * -_forward_call(BufferedObject *self, const char *name, PyObject *args) -{ - PyObject *func = PyObject_GetAttrString((PyObject *)self, name); - PyObject *ret; - - if (func == NULL) { - PyErr_SetString(PyExc_AttributeError, name); - return NULL; - } - - ret = PyObject_CallObject(func, args); - Py_DECREF(func); - return ret; -} - -static PyObject * -BufferedRWPair_read(BufferedRWPairObject *self, PyObject *args) -{ - return _forward_call(self->reader, "read", args); -} - -static PyObject * -BufferedRWPair_peek(BufferedRWPairObject *self, PyObject *args) -{ - return _forward_call(self->reader, "peek", args); -} - -static PyObject * -BufferedRWPair_read1(BufferedRWPairObject *self, PyObject *args) -{ - return _forward_call(self->reader, "read1", args); -} - -static PyObject * -BufferedRWPair_write(BufferedRWPairObject *self, PyObject *args) -{ - return _forward_call(self->writer, "write", args); -} - -static PyObject * -BufferedRWPair_flush(BufferedRWPairObject *self, PyObject *args) -{ - return _forward_call(self->writer, "flush", args); -} - -static PyObject * -BufferedRWPair_readable(BufferedRWPairObject *self, PyObject *args) -{ - return _forward_call(self->reader, "readable", args); -} - -static PyObject * -BufferedRWPair_writable(BufferedRWPairObject *self, PyObject *args) -{ - return _forward_call(self->writer, "writable", args); -} - -static PyObject * -BufferedRWPair_close(BufferedRWPairObject *self, PyObject *args) -{ - PyObject *ret = _forward_call(self->writer, "close", args); - if (ret == NULL) - return NULL; - Py_DECREF(ret); - - return _forward_call(self->reader, "close", args); -} - -static PyObject * -BufferedRWPair_isatty(BufferedRWPairObject *self, PyObject *args) -{ - PyObject *ret = _forward_call(self->writer, "isatty", args); - - if (ret != Py_False) { - /* either True or exception */ - return ret; - } - Py_DECREF(ret); - - return _forward_call(self->reader, "isatty", args); -} - - -static PyMethodDef BufferedRWPair_methods[] = { - {"read", (PyCFunction)BufferedRWPair_read, METH_VARARGS}, - {"peek", (PyCFunction)BufferedRWPair_peek, METH_VARARGS}, - {"read1", (PyCFunction)BufferedRWPair_read1, METH_VARARGS}, - {"readinto", (PyCFunction)Buffered_readinto, METH_VARARGS}, - - {"write", (PyCFunction)BufferedRWPair_write, METH_VARARGS}, - {"flush", (PyCFunction)BufferedRWPair_flush, METH_NOARGS}, - - {"readable", (PyCFunction)BufferedRWPair_readable, METH_NOARGS}, - {"writable", (PyCFunction)BufferedRWPair_writable, METH_NOARGS}, - - {"close", (PyCFunction)BufferedRWPair_close, METH_NOARGS}, - {"isatty", (PyCFunction)BufferedRWPair_isatty, METH_NOARGS}, - - {NULL, NULL} -}; - -PyTypeObject PyBufferedRWPair_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BufferedRWPair", /*tp_name*/ - sizeof(BufferedRWPairObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BufferedRWPair_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - BufferedRWPair_doc, /* tp_doc */ - (traverseproc)BufferedRWPair_traverse, /* tp_traverse */ - (inquiry)BufferedRWPair_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(BufferedRWPairObject, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - 0, /* tp_iternext */ - BufferedRWPair_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(BufferedRWPairObject, dict), /* tp_dictoffset */ - (initproc)BufferedRWPair_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ -}; - - - -/* - * BufferedRandom - */ - -PyDoc_STRVAR(BufferedRandom_doc, - "A buffered interface to random access streams.\n" - "\n" - "The constructor creates a reader and writer for a seekable stream,\n" - "raw, given in the first argument. If the buffer_size is omitted it\n" - "defaults to DEFAULT_BUFFER_SIZE. max_buffer_size isn't used anymore.\n" - ); - -static int -BufferedRandom_init(BufferedObject *self, PyObject *args, PyObject *kwds) -{ - char *kwlist[] = {"raw", "buffer_size", "max_buffer_size", NULL}; - Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; - Py_ssize_t max_buffer_size = -234; - PyObject *raw; - - self->ok = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|nn:BufferedReader", kwlist, - &raw, &buffer_size, &max_buffer_size)) { - return -1; - } - - if (max_buffer_size != -234 && !complain_about_max_buffer_size()) - return -1; - - if (_PyIOBase_checkSeekable(raw, Py_True) == NULL) - return -1; - if (_PyIOBase_checkReadable(raw, Py_True) == NULL) - return -1; - if (_PyIOBase_checkWritable(raw, Py_True) == NULL) - return -1; - - Py_CLEAR(self->raw); - Py_INCREF(raw); - self->raw = raw; - self->buffer_size = buffer_size; - self->readable = 1; - self->writable = 1; - - if (_Buffered_init(self) < 0) - return -1; - _BufferedReader_reset_buf(self); - _BufferedWriter_reset_buf(self); - self->pos = 0; - - self->ok = 1; - return 0; -} - -static PyMethodDef BufferedRandom_methods[] = { - /* BufferedIOMixin methods */ - {"close", (PyCFunction)BufferedIOMixin_close, METH_NOARGS}, - {"seekable", (PyCFunction)BufferedIOMixin_seekable, METH_NOARGS}, - {"readable", (PyCFunction)BufferedIOMixin_readable, METH_NOARGS}, - {"writable", (PyCFunction)BufferedIOMixin_writable, METH_NOARGS}, - {"fileno", (PyCFunction)BufferedIOMixin_fileno, METH_NOARGS}, - {"isatty", (PyCFunction)BufferedIOMixin_isatty, METH_NOARGS}, - - {"flush", (PyCFunction)Buffered_flush, METH_NOARGS}, - - {"seek", (PyCFunction)Buffered_seek, METH_VARARGS}, - {"tell", (PyCFunction)Buffered_tell, METH_NOARGS}, - {"truncate", (PyCFunction)Buffered_truncate, METH_VARARGS}, - {"read", (PyCFunction)Buffered_read, METH_VARARGS}, - {"read1", (PyCFunction)Buffered_read1, METH_VARARGS}, - {"readinto", (PyCFunction)Buffered_readinto, METH_VARARGS}, - {"readline", (PyCFunction)Buffered_readline, METH_VARARGS}, - {"peek", (PyCFunction)Buffered_peek, METH_VARARGS}, - {"write", (PyCFunction)BufferedWriter_write, METH_VARARGS}, - {NULL, NULL} -}; - -static PyMemberDef BufferedRandom_members[] = { - {"raw", T_OBJECT, offsetof(BufferedObject, raw), 0}, - {NULL} -}; - -static PyGetSetDef BufferedRandom_getset[] = { - {"closed", (getter)BufferedIOMixin_closed_get, NULL, NULL}, - {"name", (getter)BufferedIOMixin_name_get, NULL, NULL}, - {"mode", (getter)BufferedIOMixin_mode_get, NULL, NULL}, - {0} -}; - - -PyTypeObject PyBufferedRandom_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BufferedRandom", /*tp_name*/ - sizeof(BufferedObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BufferedObject_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - BufferedRandom_doc, /* tp_doc */ - (traverseproc)Buffered_traverse, /* tp_traverse */ - (inquiry)Buffered_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(BufferedObject, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - (iternextfunc)Buffered_iternext, /* tp_iternext */ - BufferedRandom_methods, /* tp_methods */ - BufferedRandom_members, /* tp_members */ - BufferedRandom_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /*tp_dict*/ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(BufferedObject, dict), /*tp_dictoffset*/ - (initproc)BufferedRandom_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ -}; - diff --git a/Modules/_bytesio.c b/Modules/_bytesio.c deleted file mode 100644 index 0068cb7..0000000 --- a/Modules/_bytesio.c +++ /dev/null @@ -1,757 +0,0 @@ -#include "Python.h" -#include "structmember.h" /* for offsetof() */ -#include "_iomodule.h" - -typedef struct { - PyObject_HEAD - char *buf; - Py_ssize_t pos; - Py_ssize_t string_size; - size_t buf_size; - PyObject *dict; - PyObject *weakreflist; -} BytesIOObject; - -#define CHECK_CLOSED(self) \ - if ((self)->buf == NULL) { \ - PyErr_SetString(PyExc_ValueError, \ - "I/O operation on closed file."); \ - return NULL; \ - } - -/* Internal routine to get a line from the buffer of a BytesIO - object. Returns the length between the current position to the - next newline character. */ -static Py_ssize_t -get_line(BytesIOObject *self, char **output) -{ - char *n; - const char *str_end; - Py_ssize_t len; - - assert(self->buf != NULL); - - /* Move to the end of the line, up to the end of the string, s. */ - str_end = self->buf + self->string_size; - for (n = self->buf + self->pos; - n < str_end && *n != '\n'; - n++); - - /* Skip the newline character */ - if (n < str_end) - n++; - - /* Get the length from the current position to the end of the line. */ - len = n - (self->buf + self->pos); - *output = self->buf + self->pos; - - assert(len >= 0); - assert(self->pos < PY_SSIZE_T_MAX - len); - self->pos += len; - - return len; -} - -/* Internal routine for changing the size of the buffer of BytesIO objects. - The caller should ensure that the 'size' argument is non-negative. Returns - 0 on success, -1 otherwise. */ -static int -resize_buffer(BytesIOObject *self, size_t size) -{ - /* Here, unsigned types are used to avoid dealing with signed integer - overflow, which is undefined in C. */ - size_t alloc = self->buf_size; - char *new_buf = NULL; - - assert(self->buf != NULL); - - /* For simplicity, stay in the range of the signed type. Anyway, Python - doesn't allow strings to be longer than this. */ - if (size > PY_SSIZE_T_MAX) - goto overflow; - - if (size < alloc / 2) { - /* Major downsize; resize down to exact size. */ - alloc = size + 1; - } - else if (size < alloc) { - /* Within allocated size; quick exit */ - return 0; - } - else if (size <= alloc * 1.125) { - /* Moderate upsize; overallocate similar to list_resize() */ - alloc = size + (size >> 3) + (size < 9 ? 3 : 6); - } - else { - /* Major upsize; resize up to exact size */ - alloc = size + 1; - } - - if (alloc > ((size_t)-1) / sizeof(char)) - goto overflow; - new_buf = (char *)PyMem_Realloc(self->buf, alloc * sizeof(char)); - if (new_buf == NULL) { - PyErr_NoMemory(); - return -1; - } - self->buf_size = alloc; - self->buf = new_buf; - - return 0; - - overflow: - PyErr_SetString(PyExc_OverflowError, - "new buffer size too large"); - return -1; -} - -/* Internal routine for writing a string of bytes to the buffer of a BytesIO - object. Returns the number of bytes wrote, or -1 on error. */ -static Py_ssize_t -write_bytes(BytesIOObject *self, const char *bytes, Py_ssize_t len) -{ - assert(self->buf != NULL); - assert(self->pos >= 0); - assert(len >= 0); - - if ((size_t)self->pos + len > self->buf_size) { - if (resize_buffer(self, (size_t)self->pos + len) < 0) - return -1; - } - - if (self->pos > self->string_size) { - /* In case of overseek, pad with null bytes the buffer region between - the end of stream and the current position. - - 0 lo string_size hi - | |<---used--->|<----------available----------->| - | | <--to pad-->|<---to write---> | - 0 buf position - */ - memset(self->buf + self->string_size, '\0', - (self->pos - self->string_size) * sizeof(char)); - } - - /* Copy the data to the internal buffer, overwriting some of the existing - data if self->pos < self->string_size. */ - memcpy(self->buf + self->pos, bytes, len); - self->pos += len; - - /* Set the new length of the internal string if it has changed. */ - if (self->string_size < self->pos) { - self->string_size = self->pos; - } - - return len; -} - -static PyObject * -bytesio_get_closed(BytesIOObject *self) -{ - if (self->buf == NULL) { - Py_RETURN_TRUE; - } - else { - Py_RETURN_FALSE; - } -} - -/* Generic getter for the writable, readable and seekable properties */ -static PyObject * -return_true(BytesIOObject *self) -{ - Py_RETURN_TRUE; -} - -PyDoc_STRVAR(flush_doc, -"flush() -> None. Does nothing."); - -static PyObject * -bytesio_flush(BytesIOObject *self) -{ - Py_RETURN_NONE; -} - -PyDoc_STRVAR(getval_doc, -"getvalue() -> bytes.\n" -"\n" -"Retrieve the entire contents of the BytesIO object."); - -static PyObject * -bytesio_getvalue(BytesIOObject *self) -{ - CHECK_CLOSED(self); - return PyBytes_FromStringAndSize(self->buf, self->string_size); -} - -PyDoc_STRVAR(isatty_doc, -"isatty() -> False.\n" -"\n" -"Always returns False since BytesIO objects are not connected\n" -"to a tty-like device."); - -static PyObject * -bytesio_isatty(BytesIOObject *self) -{ - CHECK_CLOSED(self); - Py_RETURN_FALSE; -} - -PyDoc_STRVAR(tell_doc, -"tell() -> current file position, an integer\n"); - -static PyObject * -bytesio_tell(BytesIOObject *self) -{ - CHECK_CLOSED(self); - return PyLong_FromSsize_t(self->pos); -} - -PyDoc_STRVAR(read_doc, -"read([size]) -> read at most size bytes, returned as a string.\n" -"\n" -"If the size argument is negative, read until EOF is reached.\n" -"Return an empty string at EOF."); - -static PyObject * -bytesio_read(BytesIOObject *self, PyObject *args) -{ - Py_ssize_t size, n; - char *output; - PyObject *arg = Py_None; - - CHECK_CLOSED(self); - - if (!PyArg_ParseTuple(args, "|O:read", &arg)) - return NULL; - - if (PyLong_Check(arg)) { - size = PyLong_AsSsize_t(arg); - if (size == -1 && PyErr_Occurred()) - return NULL; - } - else if (arg == Py_None) { - /* Read until EOF is reached, by default. */ - size = -1; - } - else { - PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", - Py_TYPE(arg)->tp_name); - return NULL; - } - - /* adjust invalid sizes */ - n = self->string_size - self->pos; - if (size < 0 || size > n) { - size = n; - if (size < 0) - size = 0; - } - - assert(self->buf != NULL); - output = self->buf + self->pos; - self->pos += size; - - return PyBytes_FromStringAndSize(output, size); -} - - -PyDoc_STRVAR(read1_doc, -"read1(size) -> read at most size bytes, returned as a string.\n" -"\n" -"If the size argument is negative or omitted, read until EOF is reached.\n" -"Return an empty string at EOF."); - -static PyObject * -bytesio_read1(BytesIOObject *self, PyObject *n) -{ - PyObject *arg, *res; - - arg = PyTuple_Pack(1, n); - if (arg == NULL) - return NULL; - res = bytesio_read(self, arg); - Py_DECREF(arg); - return res; -} - -PyDoc_STRVAR(readline_doc, -"readline([size]) -> next line from the file, as a string.\n" -"\n" -"Retain newline. A non-negative size argument limits the maximum\n" -"number of bytes to return (an incomplete line may be returned then).\n" -"Return an empty string at EOF.\n"); - -static PyObject * -bytesio_readline(BytesIOObject *self, PyObject *args) -{ - Py_ssize_t size, n; - char *output; - PyObject *arg = Py_None; - - CHECK_CLOSED(self); - - if (!PyArg_ParseTuple(args, "|O:readline", &arg)) - return NULL; - - if (PyLong_Check(arg)) { - size = PyLong_AsSsize_t(arg); - if (size == -1 && PyErr_Occurred()) - return NULL; - } - else if (arg == Py_None) { - /* No size limit, by default. */ - size = -1; - } - else { - PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", - Py_TYPE(arg)->tp_name); - return NULL; - } - - n = get_line(self, &output); - - if (size >= 0 && size < n) { - size = n - size; - n -= size; - self->pos -= size; - } - - return PyBytes_FromStringAndSize(output, n); -} - -PyDoc_STRVAR(readlines_doc, -"readlines([size]) -> list of strings, each a line from the file.\n" -"\n" -"Call readline() repeatedly and return a list of the lines so read.\n" -"The optional size argument, if given, is an approximate bound on the\n" -"total number of bytes in the lines returned.\n"); - -static PyObject * -bytesio_readlines(BytesIOObject *self, PyObject *args) -{ - Py_ssize_t maxsize, size, n; - PyObject *result, *line; - char *output; - PyObject *arg = Py_None; - - CHECK_CLOSED(self); - - if (!PyArg_ParseTuple(args, "|O:readlines", &arg)) - return NULL; - - if (PyLong_Check(arg)) { - maxsize = PyLong_AsSsize_t(arg); - if (maxsize == -1 && PyErr_Occurred()) - return NULL; - } - else if (arg == Py_None) { - /* No size limit, by default. */ - maxsize = -1; - } - else { - PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", - Py_TYPE(arg)->tp_name); - return NULL; - } - - size = 0; - result = PyList_New(0); - if (!result) - return NULL; - - while ((n = get_line(self, &output)) != 0) { - line = PyBytes_FromStringAndSize(output, n); - if (!line) - goto on_error; - if (PyList_Append(result, line) == -1) { - Py_DECREF(line); - goto on_error; - } - Py_DECREF(line); - size += n; - if (maxsize > 0 && size >= maxsize) - break; - } - return result; - - on_error: - Py_DECREF(result); - return NULL; -} - -PyDoc_STRVAR(readinto_doc, -"readinto(bytearray) -> int. Read up to len(b) bytes into b.\n" -"\n" -"Returns number of bytes read (0 for EOF), or None if the object\n" -"is set not to block as has no data to read."); - -static PyObject * -bytesio_readinto(BytesIOObject *self, PyObject *buffer) -{ - void *raw_buffer; - Py_ssize_t len; - - CHECK_CLOSED(self); - - if (PyObject_AsWriteBuffer(buffer, &raw_buffer, &len) == -1) - return NULL; - - if (self->pos + len > self->string_size) - len = self->string_size - self->pos; - - memcpy(raw_buffer, self->buf + self->pos, len); - assert(self->pos + len < PY_SSIZE_T_MAX); - assert(len >= 0); - self->pos += len; - - return PyLong_FromSsize_t(len); -} - -PyDoc_STRVAR(truncate_doc, -"truncate([size]) -> int. Truncate the file to at most size bytes.\n" -"\n" -"Size defaults to the current file position, as returned by tell().\n" -"Returns the new size. Imply an absolute seek to the position size."); - -static PyObject * -bytesio_truncate(BytesIOObject *self, PyObject *args) -{ - Py_ssize_t size; - PyObject *arg = Py_None; - - CHECK_CLOSED(self); - - if (!PyArg_ParseTuple(args, "|O:truncate", &arg)) - return NULL; - - if (PyLong_Check(arg)) { - size = PyLong_AsSsize_t(arg); - if (size == -1 && PyErr_Occurred()) - return NULL; - } - else if (arg == Py_None) { - /* Truncate to current position if no argument is passed. */ - size = self->pos; - } - else { - PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", - Py_TYPE(arg)->tp_name); - return NULL; - } - - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "negative size value %zd", size); - return NULL; - } - - if (size < self->string_size) { - self->string_size = size; - if (resize_buffer(self, size) < 0) - return NULL; - } - self->pos = size; - - return PyLong_FromSsize_t(size); -} - -static PyObject * -bytesio_iternext(BytesIOObject *self) -{ - char *next; - Py_ssize_t n; - - CHECK_CLOSED(self); - - n = get_line(self, &next); - - if (!next || n == 0) - return NULL; - - return PyBytes_FromStringAndSize(next, n); -} - -PyDoc_STRVAR(seek_doc, -"seek(pos, whence=0) -> int. Change stream position.\n" -"\n" -"Seek to byte offset pos relative to position indicated by whence:\n" -" 0 Start of stream (the default). pos should be >= 0;\n" -" 1 Current position - pos may be negative;\n" -" 2 End of stream - pos usually negative.\n" -"Returns the new absolute position."); - -static PyObject * -bytesio_seek(BytesIOObject *self, PyObject *args) -{ - Py_ssize_t pos; - int mode = 0; - - CHECK_CLOSED(self); - - if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &mode)) - return NULL; - - if (pos < 0 && mode == 0) { - PyErr_Format(PyExc_ValueError, - "negative seek value %zd", pos); - return NULL; - } - - /* mode 0: offset relative to beginning of the string. - mode 1: offset relative to current position. - mode 2: offset relative the end of the string. */ - if (mode == 1) { - if (pos > PY_SSIZE_T_MAX - self->pos) { - PyErr_SetString(PyExc_OverflowError, - "new position too large"); - return NULL; - } - pos += self->pos; - } - else if (mode == 2) { - if (pos > PY_SSIZE_T_MAX - self->string_size) { - PyErr_SetString(PyExc_OverflowError, - "new position too large"); - return NULL; - } - pos += self->string_size; - } - else if (mode != 0) { - PyErr_Format(PyExc_ValueError, - "invalid whence (%i, should be 0, 1 or 2)", mode); - return NULL; - } - - if (pos < 0) - pos = 0; - self->pos = pos; - - return PyLong_FromSsize_t(self->pos); -} - -PyDoc_STRVAR(write_doc, -"write(bytes) -> int. Write bytes to file.\n" -"\n" -"Return the number of bytes written."); - -static PyObject * -bytesio_write(BytesIOObject *self, PyObject *obj) -{ - Py_ssize_t n = 0; - Py_buffer buf; - PyObject *result = NULL; - - CHECK_CLOSED(self); - - if (PyObject_GetBuffer(obj, &buf, PyBUF_CONTIG_RO) < 0) - return NULL; - - if (buf.len != 0) - n = write_bytes(self, buf.buf, buf.len); - if (n >= 0) - result = PyLong_FromSsize_t(n); - - PyBuffer_Release(&buf); - return result; -} - -PyDoc_STRVAR(writelines_doc, -"writelines(sequence_of_strings) -> None. Write strings to the file.\n" -"\n" -"Note that newlines are not added. The sequence can be any iterable\n" -"object producing strings. This is equivalent to calling write() for\n" -"each string."); - -static PyObject * -bytesio_writelines(BytesIOObject *self, PyObject *v) -{ - PyObject *it, *item; - PyObject *ret; - - CHECK_CLOSED(self); - - it = PyObject_GetIter(v); - if (it == NULL) - return NULL; - - while ((item = PyIter_Next(it)) != NULL) { - ret = bytesio_write(self, item); - Py_DECREF(item); - if (ret == NULL) { - Py_DECREF(it); - return NULL; - } - Py_DECREF(ret); - } - Py_DECREF(it); - - /* See if PyIter_Next failed */ - if (PyErr_Occurred()) - return NULL; - - Py_RETURN_NONE; -} - -PyDoc_STRVAR(close_doc, -"close() -> None. Disable all I/O operations."); - -static PyObject * -bytesio_close(BytesIOObject *self) -{ - if (self->buf != NULL) { - PyMem_Free(self->buf); - self->buf = NULL; - } - Py_RETURN_NONE; -} - -static void -bytesio_dealloc(BytesIOObject *self) -{ - if (self->buf != NULL) { - PyMem_Free(self->buf); - self->buf = NULL; - } - Py_TYPE(self)->tp_clear((PyObject *)self); - Py_TYPE(self)->tp_free(self); -} - -static PyObject * -bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - BytesIOObject *self; - - assert(type != NULL && type->tp_alloc != NULL); - self = (BytesIOObject *)type->tp_alloc(type, 0); - if (self == NULL) - return NULL; - - self->string_size = 0; - self->pos = 0; - self->buf_size = 0; - self->buf = (char *)PyMem_Malloc(0); - if (self->buf == NULL) { - Py_DECREF(self); - return PyErr_NoMemory(); - } - - return (PyObject *)self; -} - -static int -bytesio_init(BytesIOObject *self, PyObject *args, PyObject *kwds) -{ - PyObject *initvalue = NULL; - - if (!PyArg_ParseTuple(args, "|O:BytesIO", &initvalue)) - return -1; - - /* In case, __init__ is called multiple times. */ - self->string_size = 0; - self->pos = 0; - - if (initvalue && initvalue != Py_None) { - PyObject *res; - res = bytesio_write(self, initvalue); - if (res == NULL) - return -1; - Py_DECREF(res); - self->pos = 0; - } - - return 0; -} - -static int -bytesio_traverse(BytesIOObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->dict); - Py_VISIT(self->weakreflist); - return 0; -} - -static int -bytesio_clear(BytesIOObject *self) -{ - Py_CLEAR(self->dict); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)self); - return 0; -} - - -static PyGetSetDef bytesio_getsetlist[] = { - {"closed", (getter)bytesio_get_closed, NULL, - "True if the file is closed."}, - {0}, /* sentinel */ -}; - -static struct PyMethodDef bytesio_methods[] = { - {"readable", (PyCFunction)return_true, METH_NOARGS, NULL}, - {"seekable", (PyCFunction)return_true, METH_NOARGS, NULL}, - {"writable", (PyCFunction)return_true, METH_NOARGS, NULL}, - {"close", (PyCFunction)bytesio_close, METH_NOARGS, close_doc}, - {"flush", (PyCFunction)bytesio_flush, METH_NOARGS, flush_doc}, - {"isatty", (PyCFunction)bytesio_isatty, METH_NOARGS, isatty_doc}, - {"tell", (PyCFunction)bytesio_tell, METH_NOARGS, tell_doc}, - {"write", (PyCFunction)bytesio_write, METH_O, write_doc}, - {"writelines", (PyCFunction)bytesio_writelines, METH_O, writelines_doc}, - {"read1", (PyCFunction)bytesio_read1, METH_O, read1_doc}, - {"readinto", (PyCFunction)bytesio_readinto, METH_O, readinto_doc}, - {"readline", (PyCFunction)bytesio_readline, METH_VARARGS, readline_doc}, - {"readlines", (PyCFunction)bytesio_readlines, METH_VARARGS, readlines_doc}, - {"read", (PyCFunction)bytesio_read, METH_VARARGS, read_doc}, - {"getvalue", (PyCFunction)bytesio_getvalue, METH_VARARGS, getval_doc}, - {"seek", (PyCFunction)bytesio_seek, METH_VARARGS, seek_doc}, - {"truncate", (PyCFunction)bytesio_truncate, METH_VARARGS, truncate_doc}, - {NULL, NULL} /* sentinel */ -}; - -PyDoc_STRVAR(bytesio_doc, -"BytesIO([buffer]) -> object\n" -"\n" -"Create a buffered I/O implementation using an in-memory bytes\n" -"buffer, ready for reading and writing."); - -PyTypeObject PyBytesIO_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.BytesIO", /*tp_name*/ - sizeof(BytesIOObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)bytesio_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - bytesio_doc, /*tp_doc*/ - (traverseproc)bytesio_traverse, /*tp_traverse*/ - (inquiry)bytesio_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - offsetof(BytesIOObject, weakreflist), /*tp_weaklistoffset*/ - PyObject_SelfIter, /*tp_iter*/ - (iternextfunc)bytesio_iternext, /*tp_iternext*/ - bytesio_methods, /*tp_methods*/ - 0, /*tp_members*/ - bytesio_getsetlist, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - offsetof(BytesIOObject, dict), /*tp_dictoffset*/ - (initproc)bytesio_init, /*tp_init*/ - 0, /*tp_alloc*/ - bytesio_new, /*tp_new*/ -}; diff --git a/Modules/_fileio.c b/Modules/_fileio.c deleted file mode 100644 index c0f5c90..0000000 --- a/Modules/_fileio.c +++ /dev/null @@ -1,1034 +0,0 @@ -/* Author: Daniel Stutzbach */ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include -#include -#include -#include /* For offsetof */ -#include "_iomodule.h" - -/* - * Known likely problems: - * - * - Files larger then 2**32-1 - * - Files with unicode filenames - * - Passing numbers greater than 2**32-1 when an integer is expected - * - Making it work on Windows and other oddball platforms - * - * To Do: - * - * - autoconfify header file inclusion - */ - -#ifdef MS_WINDOWS -/* can simulate truncate with Win32 API functions; see file_truncate */ -#define HAVE_FTRUNCATE -#define WIN32_LEAN_AND_MEAN -#include -#endif - -#if BUFSIZ < (8*1024) -#define SMALLCHUNK (8*1024) -#elif (BUFSIZ >= (2 << 25)) -#error "unreasonable BUFSIZ > 64MB defined" -#else -#define SMALLCHUNK BUFSIZ -#endif - -#if SIZEOF_INT < 4 -#define BIGCHUNK (512 * 32) -#else -#define BIGCHUNK (512 * 1024) -#endif - -typedef struct { - PyObject_HEAD - int fd; - unsigned readable : 1; - unsigned writable : 1; - int seekable : 2; /* -1 means unknown */ - int closefd : 1; - PyObject *weakreflist; - PyObject *dict; -} PyFileIOObject; - -PyTypeObject PyFileIO_Type; - -#define PyFileIO_Check(op) (PyObject_TypeCheck((op), &PyFileIO_Type)) - -int -_PyFileIO_closed(PyObject *self) -{ - return ((PyFileIOObject *)self)->fd < 0; -} - -static PyObject * -portable_lseek(int fd, PyObject *posobj, int whence); - -static PyObject *portable_lseek(int fd, PyObject *posobj, int whence); - -/* Returns 0 on success, -1 with exception set on failure. */ -static int -internal_close(PyFileIOObject *self) -{ - int err = 0; - int save_errno = 0; - if (self->fd >= 0) { - int fd = self->fd; - self->fd = -1; - /* fd is accessible and someone else may have closed it */ - if (_PyVerify_fd(fd)) { - Py_BEGIN_ALLOW_THREADS - err = close(fd); - if (err < 0) - save_errno = errno; - Py_END_ALLOW_THREADS - } else { - save_errno = errno; - err = -1; - } - } - if (err < 0) { - errno = save_errno; - PyErr_SetFromErrno(PyExc_IOError); - return -1; - } - return 0; -} - -static PyObject * -fileio_close(PyFileIOObject *self) -{ - if (!self->closefd) { - self->fd = -1; - Py_RETURN_NONE; - } - errno = internal_close(self); - if (errno < 0) - return NULL; - - return PyObject_CallMethod((PyObject*)&PyRawIOBase_Type, - "close", "O", self); -} - -static PyObject * -fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyFileIOObject *self; - - assert(type != NULL && type->tp_alloc != NULL); - - self = (PyFileIOObject *) type->tp_alloc(type, 0); - if (self != NULL) { - self->fd = -1; - self->readable = 0; - self->writable = 0; - self->seekable = -1; - self->closefd = 1; - self->weakreflist = NULL; - } - - return (PyObject *) self; -} - -/* On Unix, open will succeed for directories. - In Python, there should be no file objects referring to - directories, so we need a check. */ - -static int -dircheck(PyFileIOObject* self, const char *name) -{ -#if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR) - struct stat buf; - if (self->fd < 0) - return 0; - if (fstat(self->fd, &buf) == 0 && S_ISDIR(buf.st_mode)) { - char *msg = strerror(EISDIR); - PyObject *exc; - if (internal_close(self)) - return -1; - - exc = PyObject_CallFunction(PyExc_IOError, "(iss)", - EISDIR, msg, name); - PyErr_SetObject(PyExc_IOError, exc); - Py_XDECREF(exc); - return -1; - } -#endif - return 0; -} - -static int -check_fd(int fd) -{ -#if defined(HAVE_FSTAT) - struct stat buf; - if (!_PyVerify_fd(fd) || (fstat(fd, &buf) < 0 && errno == EBADF)) { - PyObject *exc; - char *msg = strerror(EBADF); - exc = PyObject_CallFunction(PyExc_OSError, "(is)", - EBADF, msg); - PyErr_SetObject(PyExc_OSError, exc); - Py_XDECREF(exc); - return -1; - } -#endif - return 0; -} - - -static int -fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) -{ - PyFileIOObject *self = (PyFileIOObject *) oself; - static char *kwlist[] = {"file", "mode", "closefd", NULL}; - const char *name = NULL; - PyObject *nameobj, *stringobj = NULL; - char *mode = "r"; - char *s; -#ifdef MS_WINDOWS - Py_UNICODE *widename = NULL; -#endif - int ret = 0; - int rwa = 0, plus = 0, append = 0; - int flags = 0; - int fd = -1; - int closefd = 1; - - assert(PyFileIO_Check(oself)); - if (self->fd >= 0) { - /* Have to close the existing file first. */ - if (internal_close(self) < 0) - return -1; - } - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|si:fileio", - kwlist, &nameobj, &mode, &closefd)) - return -1; - - if (PyFloat_Check(nameobj)) { - PyErr_SetString(PyExc_TypeError, - "integer argument expected, got float"); - return -1; - } - - fd = PyLong_AsLong(nameobj); - if (fd < 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Negative filedescriptor"); - return -1; - } - PyErr_Clear(); - } - -#ifdef Py_WIN_WIDE_FILENAMES - if (GetVersion() < 0x80000000) { - /* On NT, so wide API available */ - if (PyUnicode_Check(nameobj)) - widename = PyUnicode_AS_UNICODE(nameobj); - } - if (widename == NULL) -#endif - if (fd < 0) - { - if (PyBytes_Check(nameobj) || PyByteArray_Check(nameobj)) { - Py_ssize_t namelen; - if (PyObject_AsCharBuffer(nameobj, &name, &namelen) < 0) - return -1; - } - else { - PyObject *u = PyUnicode_FromObject(nameobj); - - if (u == NULL) - return -1; - - stringobj = PyUnicode_AsEncodedString( - u, Py_FileSystemDefaultEncoding, NULL); - Py_DECREF(u); - if (stringobj == NULL) - return -1; - if (!PyBytes_Check(stringobj)) { - PyErr_SetString(PyExc_TypeError, - "encoder failed to return bytes"); - goto error; - } - name = PyBytes_AS_STRING(stringobj); - } - } - - s = mode; - while (*s) { - switch (*s++) { - case 'r': - if (rwa) { - bad_mode: - PyErr_SetString(PyExc_ValueError, - "Must have exactly one of read/write/append mode"); - goto error; - } - rwa = 1; - self->readable = 1; - break; - case 'w': - if (rwa) - goto bad_mode; - rwa = 1; - self->writable = 1; - flags |= O_CREAT | O_TRUNC; - break; - case 'a': - if (rwa) - goto bad_mode; - rwa = 1; - self->writable = 1; - flags |= O_CREAT; - append = 1; - break; - case 'b': - break; - case '+': - if (plus) - goto bad_mode; - self->readable = self->writable = 1; - plus = 1; - break; - default: - PyErr_Format(PyExc_ValueError, - "invalid mode: %.200s", mode); - goto error; - } - } - - if (!rwa) - goto bad_mode; - - if (self->readable && self->writable) - flags |= O_RDWR; - else if (self->readable) - flags |= O_RDONLY; - else - flags |= O_WRONLY; - -#ifdef O_BINARY - flags |= O_BINARY; -#endif - -#ifdef O_APPEND - if (append) - flags |= O_APPEND; -#endif - - if (fd >= 0) { - if (check_fd(fd)) - goto error; - self->fd = fd; - self->closefd = closefd; - } - else { - self->closefd = 1; - if (!closefd) { - PyErr_SetString(PyExc_ValueError, - "Cannot use closefd=False with file name"); - goto error; - } - - Py_BEGIN_ALLOW_THREADS - errno = 0; -#ifdef MS_WINDOWS - if (widename != NULL) - self->fd = _wopen(widename, flags, 0666); - else -#endif - self->fd = open(name, flags, 0666); - Py_END_ALLOW_THREADS - if (self->fd < 0) { -#ifdef MS_WINDOWS - if (widename != NULL) - PyErr_SetFromErrnoWithUnicodeFilename(PyExc_IOError, widename); - else -#endif - PyErr_SetFromErrnoWithFilename(PyExc_IOError, name); - goto error; - } - if(dircheck(self, name) < 0) - goto error; - } - - if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0) - goto error; - - if (append) { - /* For consistent behaviour, we explicitly seek to the - end of file (otherwise, it might be done only on the - first write()). */ - PyObject *pos = portable_lseek(self->fd, NULL, 2); - if (pos == NULL) - goto error; - Py_DECREF(pos); - } - - goto done; - - error: - ret = -1; - - done: - Py_CLEAR(stringobj); - return ret; -} - -static int -fileio_traverse(PyFileIOObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->dict); - return 0; -} - -static int -fileio_clear(PyFileIOObject *self) -{ - Py_CLEAR(self->dict); - return 0; -} - -static void -fileio_dealloc(PyFileIOObject *self) -{ - if (_PyIOBase_finalize((PyObject *) self) < 0) - return; - _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static PyObject * -err_closed(void) -{ - PyErr_SetString(PyExc_ValueError, "I/O operation on closed file"); - return NULL; -} - -static PyObject * -err_mode(char *action) -{ - PyErr_Format(PyExc_ValueError, "File not open for %s", action); - return NULL; -} - -static PyObject * -fileio_fileno(PyFileIOObject *self) -{ - if (self->fd < 0) - return err_closed(); - return PyLong_FromLong((long) self->fd); -} - -static PyObject * -fileio_readable(PyFileIOObject *self) -{ - if (self->fd < 0) - return err_closed(); - return PyBool_FromLong((long) self->readable); -} - -static PyObject * -fileio_writable(PyFileIOObject *self) -{ - if (self->fd < 0) - return err_closed(); - return PyBool_FromLong((long) self->writable); -} - -static PyObject * -fileio_seekable(PyFileIOObject *self) -{ - if (self->fd < 0) - return err_closed(); - if (self->seekable < 0) { - PyObject *pos = portable_lseek(self->fd, NULL, SEEK_CUR); - if (pos == NULL) { - PyErr_Clear(); - self->seekable = 0; - } else { - Py_DECREF(pos); - self->seekable = 1; - } - } - return PyBool_FromLong((long) self->seekable); -} - -static PyObject * -fileio_readinto(PyFileIOObject *self, PyObject *args) -{ - Py_buffer pbuf; - Py_ssize_t n; - - if (self->fd < 0) - return err_closed(); - if (!self->readable) - return err_mode("reading"); - - if (!PyArg_ParseTuple(args, "w*", &pbuf)) - return NULL; - - if (_PyVerify_fd(self->fd)) { - Py_BEGIN_ALLOW_THREADS - errno = 0; - n = read(self->fd, pbuf.buf, pbuf.len); - Py_END_ALLOW_THREADS - } else - n = -1; - PyBuffer_Release(&pbuf); - if (n < 0) { - if (errno == EAGAIN) - Py_RETURN_NONE; - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - - return PyLong_FromSsize_t(n); -} - -static size_t -new_buffersize(PyFileIOObject *self, size_t currentsize) -{ -#ifdef HAVE_FSTAT - off_t pos, end; - struct stat st; - if (fstat(self->fd, &st) == 0) { - end = st.st_size; - pos = lseek(self->fd, 0L, SEEK_CUR); - /* Files claiming a size smaller than SMALLCHUNK may - actually be streaming pseudo-files. In this case, we - apply the more aggressive algorithm below. - */ - if (end >= SMALLCHUNK && end >= pos && pos >= 0) { - /* Add 1 so if the file were to grow we'd notice. */ - return currentsize + end - pos + 1; - } - } -#endif - if (currentsize > SMALLCHUNK) { - /* Keep doubling until we reach BIGCHUNK; - then keep adding BIGCHUNK. */ - if (currentsize <= BIGCHUNK) - return currentsize + currentsize; - else - return currentsize + BIGCHUNK; - } - return currentsize + SMALLCHUNK; -} - -static PyObject * -fileio_readall(PyFileIOObject *self) -{ - PyObject *result; - Py_ssize_t total = 0; - int n; - - if (!_PyVerify_fd(self->fd)) - return PyErr_SetFromErrno(PyExc_IOError); - - result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); - if (result == NULL) - return NULL; - - while (1) { - size_t newsize = new_buffersize(self, total); - if (newsize > PY_SSIZE_T_MAX || newsize <= 0) { - PyErr_SetString(PyExc_OverflowError, - "unbounded read returned more bytes " - "than a Python string can hold "); - Py_DECREF(result); - return NULL; - } - - if (PyBytes_GET_SIZE(result) < newsize) { - if (_PyBytes_Resize(&result, newsize) < 0) { - if (total == 0) { - Py_DECREF(result); - return NULL; - } - PyErr_Clear(); - break; - } - } - Py_BEGIN_ALLOW_THREADS - errno = 0; - n = read(self->fd, - PyBytes_AS_STRING(result) + total, - newsize - total); - Py_END_ALLOW_THREADS - if (n == 0) - break; - if (n < 0) { - if (total > 0) - break; - if (errno == EAGAIN) { - Py_DECREF(result); - Py_RETURN_NONE; - } - Py_DECREF(result); - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - total += n; - } - - if (PyBytes_GET_SIZE(result) > total) { - if (_PyBytes_Resize(&result, total) < 0) { - /* This should never happen, but just in case */ - Py_DECREF(result); - return NULL; - } - } - return result; -} - -static PyObject * -fileio_read(PyFileIOObject *self, PyObject *args) -{ - char *ptr; - Py_ssize_t n; - Py_ssize_t size = -1; - PyObject *bytes; - - if (self->fd < 0) - return err_closed(); - if (!self->readable) - return err_mode("reading"); - - if (!PyArg_ParseTuple(args, "|n", &size)) - return NULL; - - if (size < 0) { - return fileio_readall(self); - } - - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) - return NULL; - ptr = PyBytes_AS_STRING(bytes); - - if (_PyVerify_fd(self->fd)) { - Py_BEGIN_ALLOW_THREADS - errno = 0; - n = read(self->fd, ptr, size); - Py_END_ALLOW_THREADS - } else - n = -1; - - if (n < 0) { - Py_DECREF(bytes); - if (errno == EAGAIN) - Py_RETURN_NONE; - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - - if (n != size) { - if (_PyBytes_Resize(&bytes, n) < 0) { - Py_DECREF(bytes); - return NULL; - } - } - - return (PyObject *) bytes; -} - -static PyObject * -fileio_write(PyFileIOObject *self, PyObject *args) -{ - Py_buffer pbuf; - Py_ssize_t n; - - if (self->fd < 0) - return err_closed(); - if (!self->writable) - return err_mode("writing"); - - if (!PyArg_ParseTuple(args, "s*", &pbuf)) - return NULL; - - if (_PyVerify_fd(self->fd)) { - Py_BEGIN_ALLOW_THREADS - errno = 0; - n = write(self->fd, pbuf.buf, pbuf.len); - Py_END_ALLOW_THREADS - } else - n = -1; - - PyBuffer_Release(&pbuf); - - if (n < 0) { - if (errno == EAGAIN) - Py_RETURN_NONE; - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - - return PyLong_FromSsize_t(n); -} - -/* XXX Windows support below is likely incomplete */ - -/* Cribbed from posix_lseek() */ -static PyObject * -portable_lseek(int fd, PyObject *posobj, int whence) -{ - Py_off_t pos, res; - -#ifdef SEEK_SET - /* Turn 0, 1, 2 into SEEK_{SET,CUR,END} */ - switch (whence) { -#if SEEK_SET != 0 - case 0: whence = SEEK_SET; break; -#endif -#if SEEK_CUR != 1 - case 1: whence = SEEK_CUR; break; -#endif -#if SEEK_END != 2 - case 2: whence = SEEK_END; break; -#endif - } -#endif /* SEEK_SET */ - - if (posobj == NULL) - pos = 0; - else { - if(PyFloat_Check(posobj)) { - PyErr_SetString(PyExc_TypeError, "an integer is required"); - return NULL; - } -#if defined(HAVE_LARGEFILE_SUPPORT) - pos = PyLong_AsLongLong(posobj); -#else - pos = PyLong_AsLong(posobj); -#endif - if (PyErr_Occurred()) - return NULL; - } - - if (_PyVerify_fd(fd)) { - Py_BEGIN_ALLOW_THREADS -#if defined(MS_WIN64) || defined(MS_WINDOWS) - res = _lseeki64(fd, pos, whence); -#else - res = lseek(fd, pos, whence); -#endif - Py_END_ALLOW_THREADS - } else - res = -1; - if (res < 0) - return PyErr_SetFromErrno(PyExc_IOError); - -#if defined(HAVE_LARGEFILE_SUPPORT) - return PyLong_FromLongLong(res); -#else - return PyLong_FromLong(res); -#endif -} - -static PyObject * -fileio_seek(PyFileIOObject *self, PyObject *args) -{ - PyObject *posobj; - int whence = 0; - - if (self->fd < 0) - return err_closed(); - - if (!PyArg_ParseTuple(args, "O|i", &posobj, &whence)) - return NULL; - - return portable_lseek(self->fd, posobj, whence); -} - -static PyObject * -fileio_tell(PyFileIOObject *self, PyObject *args) -{ - if (self->fd < 0) - return err_closed(); - - return portable_lseek(self->fd, NULL, 1); -} - -#ifdef HAVE_FTRUNCATE -static PyObject * -fileio_truncate(PyFileIOObject *self, PyObject *args) -{ - PyObject *posobj = NULL; - Py_off_t pos; - int ret; - int fd; - - fd = self->fd; - if (fd < 0) - return err_closed(); - if (!self->writable) - return err_mode("writing"); - - if (!PyArg_ParseTuple(args, "|O", &posobj)) - return NULL; - - if (posobj == Py_None || posobj == NULL) { - /* Get the current position. */ - posobj = portable_lseek(fd, NULL, 1); - if (posobj == NULL) - return NULL; - } - else { - /* Move to the position to be truncated. */ - posobj = portable_lseek(fd, posobj, 0); - } - if (posobj == NULL) - return NULL; - -#if defined(HAVE_LARGEFILE_SUPPORT) - pos = PyLong_AsLongLong(posobj); -#else - pos = PyLong_AsLong(posobj); -#endif - if (pos == -1 && PyErr_Occurred()) - return NULL; - -#ifdef MS_WINDOWS - /* MS _chsize doesn't work if newsize doesn't fit in 32 bits, - so don't even try using it. */ - { - HANDLE hFile; - - /* Truncate. Note that this may grow the file! */ - Py_BEGIN_ALLOW_THREADS - errno = 0; - hFile = (HANDLE)_get_osfhandle(fd); - ret = hFile == (HANDLE)-1; - if (ret == 0) { - ret = SetEndOfFile(hFile) == 0; - if (ret) - errno = EACCES; - } - Py_END_ALLOW_THREADS - } -#else - Py_BEGIN_ALLOW_THREADS - errno = 0; - ret = ftruncate(fd, pos); - Py_END_ALLOW_THREADS -#endif /* !MS_WINDOWS */ - - if (ret != 0) { - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - - return posobj; -} -#endif - -static char * -mode_string(PyFileIOObject *self) -{ - if (self->readable) { - if (self->writable) - return "rb+"; - else - return "rb"; - } - else - return "wb"; -} - -static PyObject * -fileio_repr(PyFileIOObject *self) -{ - if (self->fd < 0) - return PyUnicode_FromFormat("io.FileIO(-1)"); - - return PyUnicode_FromFormat("io.FileIO(%d, '%s')", - self->fd, mode_string(self)); -} - -static PyObject * -fileio_isatty(PyFileIOObject *self) -{ - long res; - - if (self->fd < 0) - return err_closed(); - Py_BEGIN_ALLOW_THREADS - res = isatty(self->fd); - Py_END_ALLOW_THREADS - return PyBool_FromLong(res); -} - - -PyDoc_STRVAR(fileio_doc, -"file(name: str[, mode: str]) -> file IO object\n" -"\n" -"Open a file. The mode can be 'r', 'w' or 'a' for reading (default),\n" -"writing or appending. The file will be created if it doesn't exist\n" -"when opened for writing or appending; it will be truncated when\n" -"opened for writing. Add a '+' to the mode to allow simultaneous\n" -"reading and writing."); - -PyDoc_STRVAR(read_doc, -"read(size: int) -> bytes. read at most size bytes, returned as bytes.\n" -"\n" -"Only makes one system call, so less data may be returned than requested\n" -"In non-blocking mode, returns None if no data is available.\n" -"On end-of-file, returns ''."); - -PyDoc_STRVAR(readall_doc, -"readall() -> bytes. read all data from the file, returned as bytes.\n" -"\n" -"In non-blocking mode, returns as much as is immediately available,\n" -"or None if no data is available. On end-of-file, returns ''."); - -PyDoc_STRVAR(write_doc, -"write(b: bytes) -> int. Write bytes b to file, return number written.\n" -"\n" -"Only makes one system call, so not all of the data may be written.\n" -"The number of bytes actually written is returned."); - -PyDoc_STRVAR(fileno_doc, -"fileno() -> int. \"file descriptor\".\n" -"\n" -"This is needed for lower-level file interfaces, such the fcntl module."); - -PyDoc_STRVAR(seek_doc, -"seek(offset: int[, whence: int]) -> None. Move to new file position.\n" -"\n" -"Argument offset is a byte count. Optional argument whence defaults to\n" -"0 (offset from start of file, offset should be >= 0); other values are 1\n" -"(move relative to current position, positive or negative), and 2 (move\n" -"relative to end of file, usually negative, although many platforms allow\n" -"seeking beyond the end of a file)." -"\n" -"Note that not all file objects are seekable."); - -#ifdef HAVE_FTRUNCATE -PyDoc_STRVAR(truncate_doc, -"truncate([size: int]) -> None. Truncate the file to at most size bytes.\n" -"\n" -"Size defaults to the current file position, as returned by tell()." -"The current file position is changed to the value of size."); -#endif - -PyDoc_STRVAR(tell_doc, -"tell() -> int. Current file position"); - -PyDoc_STRVAR(readinto_doc, -"readinto() -> Same as RawIOBase.readinto()."); - -PyDoc_STRVAR(close_doc, -"close() -> None. Close the file.\n" -"\n" -"A closed file cannot be used for further I/O operations. close() may be\n" -"called more than once without error. Changes the fileno to -1."); - -PyDoc_STRVAR(isatty_doc, -"isatty() -> bool. True if the file is connected to a tty device."); - -PyDoc_STRVAR(seekable_doc, -"seekable() -> bool. True if file supports random-access."); - -PyDoc_STRVAR(readable_doc, -"readable() -> bool. True if file was opened in a read mode."); - -PyDoc_STRVAR(writable_doc, -"writable() -> bool. True if file was opened in a write mode."); - -static PyMethodDef fileio_methods[] = { - {"read", (PyCFunction)fileio_read, METH_VARARGS, read_doc}, - {"readall", (PyCFunction)fileio_readall, METH_NOARGS, readall_doc}, - {"readinto", (PyCFunction)fileio_readinto, METH_VARARGS, readinto_doc}, - {"write", (PyCFunction)fileio_write, METH_VARARGS, write_doc}, - {"seek", (PyCFunction)fileio_seek, METH_VARARGS, seek_doc}, - {"tell", (PyCFunction)fileio_tell, METH_VARARGS, tell_doc}, -#ifdef HAVE_FTRUNCATE - {"truncate", (PyCFunction)fileio_truncate, METH_VARARGS, truncate_doc}, -#endif - {"close", (PyCFunction)fileio_close, METH_NOARGS, close_doc}, - {"seekable", (PyCFunction)fileio_seekable, METH_NOARGS, seekable_doc}, - {"readable", (PyCFunction)fileio_readable, METH_NOARGS, readable_doc}, - {"writable", (PyCFunction)fileio_writable, METH_NOARGS, writable_doc}, - {"fileno", (PyCFunction)fileio_fileno, METH_NOARGS, fileno_doc}, - {"isatty", (PyCFunction)fileio_isatty, METH_NOARGS, isatty_doc}, - {NULL, NULL} /* sentinel */ -}; - -/* 'closed' and 'mode' are attributes for backwards compatibility reasons. */ - -static PyObject * -get_closed(PyFileIOObject *self, void *closure) -{ - return PyBool_FromLong((long)(self->fd < 0)); -} - -static PyObject * -get_closefd(PyFileIOObject *self, void *closure) -{ - return PyBool_FromLong((long)(self->closefd)); -} - -static PyObject * -get_mode(PyFileIOObject *self, void *closure) -{ - return PyUnicode_FromString(mode_string(self)); -} - -static PyGetSetDef fileio_getsetlist[] = { - {"closed", (getter)get_closed, NULL, "True if the file is closed"}, - {"closefd", (getter)get_closefd, NULL, - "True if the file descriptor will be closed"}, - {"mode", (getter)get_mode, NULL, "String giving the file mode"}, - {0}, -}; - -PyTypeObject PyFileIO_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.FileIO", - sizeof(PyFileIOObject), - 0, - (destructor)fileio_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - (reprfunc)fileio_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - fileio_doc, /* tp_doc */ - (traverseproc)fileio_traverse, /* tp_traverse */ - (inquiry)fileio_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(PyFileIOObject, weakreflist), /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - fileio_methods, /* tp_methods */ - 0, /* tp_members */ - fileio_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(PyFileIOObject, dict), /* tp_dictoffset */ - fileio_init, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - fileio_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ -}; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c new file mode 100644 index 0000000..58b5798 --- /dev/null +++ b/Modules/_io/_iomodule.c @@ -0,0 +1,760 @@ +/* + An implementation of the new I/O lib as defined by PEP 3116 - "New I/O" + + Classes defined here: UnsupportedOperation, BlockingIOError. + Functions defined here: open(). + + Mostly written by Amaury Forgeot d'Arc +*/ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#include "_iomodule.h" + +#ifdef HAVE_SYS_TYPES_H +#include +#endif /* HAVE_SYS_TYPES_H */ + +#ifdef HAVE_SYS_STAT_H +#include +#endif /* HAVE_SYS_STAT_H */ + + +/* Various interned strings */ + +PyObject *_PyIO_str_close; +PyObject *_PyIO_str_closed; +PyObject *_PyIO_str_decode; +PyObject *_PyIO_str_encode; +PyObject *_PyIO_str_fileno; +PyObject *_PyIO_str_flush; +PyObject *_PyIO_str_getstate; +PyObject *_PyIO_str_isatty; +PyObject *_PyIO_str_newlines; +PyObject *_PyIO_str_nl; +PyObject *_PyIO_str_read; +PyObject *_PyIO_str_read1; +PyObject *_PyIO_str_readable; +PyObject *_PyIO_str_readinto; +PyObject *_PyIO_str_readline; +PyObject *_PyIO_str_reset; +PyObject *_PyIO_str_seek; +PyObject *_PyIO_str_seekable; +PyObject *_PyIO_str_tell; +PyObject *_PyIO_str_truncate; +PyObject *_PyIO_str_writable; +PyObject *_PyIO_str_write; + +PyObject *_PyIO_empty_str; +PyObject *_PyIO_empty_bytes; + + +PyDoc_STRVAR(module_doc, +"The io module provides the Python interfaces to stream handling. The\n" +"builtin open function is defined in this module.\n" +"\n" +"At the top of the I/O hierarchy is the abstract base class IOBase. It\n" +"defines the basic interface to a stream. Note, however, that there is no\n" +"seperation between reading and writing to streams; implementations are\n" +"allowed to throw an IOError if they do not support a given operation.\n" +"\n" +"Extending IOBase is RawIOBase which deals simply with the reading and\n" +"writing of raw bytes to a stream. FileIO subc lasses RawIOBase to provide\n" +"an interface to OS files.\n" +"\n" +"BufferedIOBase deals with buffering on a raw byte stream (RawIOBase). Its\n" +"subclasses, BufferedWriter, BufferedReader, and BufferedRWPair buffer\n" +"streams that are readable, writable, and both respectively.\n" +"BufferedRandom provides a buffered interface to random access\n" +"streams. BytesIO is a simple stream of in-memory bytes.\n" +"\n" +"Another IOBase subclass, TextIOBase, deals with the encoding and decoding\n" +"of streams into text. TextIOWrapper, which extends it, is a buffered text\n" +"interface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO\n" +"is a in-memory stream for text.\n" +"\n" +"Argument names are not part of the specification, and only the arguments\n" +"of open() are intended to be used as keyword arguments.\n" +"\n" +"data:\n" +"\n" +"DEFAULT_BUFFER_SIZE\n" +"\n" +" An int containing the default buffer size used by the module's buffered\n" +" I/O classes. open() uses the file's blksize (as obtained by os.stat) if\n" +" possible.\n" + ); + + +/* + * BlockingIOError extends IOError + */ + +static int +BlockingIOError_init(PyBlockingIOErrorObject *self, PyObject *args, + PyObject *kwds) +{ + PyObject *myerrno = NULL, *strerror = NULL; + PyObject *baseargs = NULL; + Py_ssize_t written = 0; + + assert(PyTuple_Check(args)); + + self->written = 0; + if (!PyArg_ParseTuple(args, "OO|n:BlockingIOError", + &myerrno, &strerror, &written)) + return -1; + + baseargs = PyTuple_Pack(2, myerrno, strerror); + if (baseargs == NULL) + return -1; + /* This will take care of initializing of myerrno and strerror members */ + if (((PyTypeObject *)PyExc_IOError)->tp_init( + (PyObject *)self, baseargs, kwds) == -1) { + Py_DECREF(baseargs); + return -1; + } + Py_DECREF(baseargs); + + self->written = written; + return 0; +} + +static PyMemberDef BlockingIOError_members[] = { + {"characters_written", T_PYSSIZET, offsetof(PyBlockingIOErrorObject, written), 0}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject _PyExc_BlockingIOError = { + PyVarObject_HEAD_INIT(NULL, 0) + "BlockingIOError", /*tp_name*/ + sizeof(PyBlockingIOErrorObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + PyDoc_STR("Exception raised when I/O would block " + "on a non-blocking I/O stream"), /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + BlockingIOError_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)BlockingIOError_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; +PyObject *PyExc_BlockingIOError = (PyObject *)&_PyExc_BlockingIOError; + + +/* + * The main open() function + */ +PyDoc_STRVAR(open_doc, +"Open file and return a stream. Raise IOError upon failure.\n" +"\n" +"file is either a text or byte string giving the name (and the path\n" +"if the file isn't in the current working directory) of the file to\n" +"be opened or an integer file descriptor of the file to be\n" +"wrapped. (If a file descriptor is given, it is closed when the\n" +"returned I/O object is closed, unless closefd is set to False.)\n" +"\n" +"mode is an optional string that specifies the mode in which the file\n" +"is opened. It defaults to 'r' which means open for reading in text\n" +"mode. Other common values are 'w' for writing (truncating the file if\n" +"it already exists), and 'a' for appending (which on some Unix systems,\n" +"means that all writes append to the end of the file regardless of the\n" +"current seek position). In text mode, if encoding is not specified the\n" +"encoding used is platform dependent. (For reading and writing raw\n" +"bytes use binary mode and leave encoding unspecified.) The available\n" +"modes are:\n" +"\n" +"========= ===============================================================\n" +"Character Meaning\n" +"--------- ---------------------------------------------------------------\n" +"'r' open for reading (default)\n" +"'w' open for writing, truncating the file first\n" +"'a' open for writing, appending to the end of the file if it exists\n" +"'b' binary mode\n" +"'t' text mode (default)\n" +"'+' open a disk file for updating (reading and writing)\n" +"'U' universal newline mode (for backwards compatibility; unneeded\n" +" for new code)\n" +"========= ===============================================================\n" +"\n" +"The default mode is 'rt' (open for reading text). For binary random\n" +"access, the mode 'w+b' opens and truncates the file to 0 bytes, while\n" +"'r+b' opens the file without truncation.\n" +"\n" +"Python distinguishes between files opened in binary and text modes,\n" +"even when the underlying operating system doesn't. Files opened in\n" +"binary mode (appending 'b' to the mode argument) return contents as\n" +"bytes objects without any decoding. In text mode (the default, or when\n" +"'t' is appended to the mode argument), the contents of the file are\n" +"returned as strings, the bytes having been first decoded using a\n" +"platform-dependent encoding or using the specified encoding if given.\n" +"\n" +"buffering is an optional integer used to set the buffering policy. By\n" +"default full buffering is on. Pass 0 to switch buffering off (only\n" +"allowed in binary mode), 1 to set line buffering, and an integer > 1\n" +"for full buffering.\n" +"\n" +"encoding is the name of the encoding used to decode or encode the\n" +"file. This should only be used in text mode. The default encoding is\n" +"platform dependent, but any encoding supported by Python can be\n" +"passed. See the codecs module for the list of supported encodings.\n" +"\n" +"errors is an optional string that specifies how encoding errors are to\n" +"be handled---this argument should not be used in binary mode. Pass\n" +"'strict' to raise a ValueError exception if there is an encoding error\n" +"(the default of None has the same effect), or pass 'ignore' to ignore\n" +"errors. (Note that ignoring encoding errors can lead to data loss.)\n" +"See the documentation for codecs.register for a list of the permitted\n" +"encoding error strings.\n" +"\n" +"newline controls how universal newlines works (it only applies to text\n" +"mode). It can be None, '', '\\n', '\\r', and '\\r\\n'. It works as\n" +"follows:\n" +"\n" +"* On input, if newline is None, universal newlines mode is\n" +" enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n" +" these are translated into '\\n' before being returned to the\n" +" caller. If it is '', universal newline mode is enabled, but line\n" +" endings are returned to the caller untranslated. If it has any of\n" +" the other legal values, input lines are only terminated by the given\n" +" string, and the line ending is returned to the caller untranslated.\n" +"\n" +"* On output, if newline is None, any '\\n' characters written are\n" +" translated to the system default line separator, os.linesep. If\n" +" newline is '', no translation takes place. If newline is any of the\n" +" other legal values, any '\\n' characters written are translated to\n" +" the given string.\n" +"\n" +"If closefd is False, the underlying file descriptor will be kept open\n" +"when the file is closed. This does not work when a file name is given\n" +"and must be True in that case.\n" +"\n" +"open() returns a file object whose type depends on the mode, and\n" +"through which the standard file operations such as reading and writing\n" +"are performed. When open() is used to open a file in a text mode ('w',\n" +"'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open\n" +"a file in a binary mode, the returned class varies: in read binary\n" +"mode, it returns a BufferedReader; in write binary and append binary\n" +"modes, it returns a BufferedWriter, and in read/write mode, it returns\n" +"a BufferedRandom.\n" +"\n" +"It is also possible to use a string or bytearray as a file for both\n" +"reading and writing. For strings StringIO can be used like a file\n" +"opened in a text mode, and for bytes a BytesIO can be used like a file\n" +"opened in a binary mode.\n" + ); + +static PyObject * +io_open(PyObject *self, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"file", "mode", "buffering", + "encoding", "errors", "newline", + "closefd", NULL}; + PyObject *file; + char *mode = "r"; + int buffering = -1, closefd = 1; + char *encoding = NULL, *errors = NULL, *newline = NULL; + unsigned i; + + int reading = 0, writing = 0, appending = 0, updating = 0; + int text = 0, binary = 0, universal = 0; + + char rawmode[5], *m; + int line_buffering, isatty; + + PyObject *raw, *modeobj = NULL, *buffer = NULL, *wrapper = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|sizzzi:open", kwlist, + &file, &mode, &buffering, + &encoding, &errors, &newline, + &closefd)) { + return NULL; + } + + if (!PyUnicode_Check(file) && + !PyBytes_Check(file) && + !PyNumber_Check(file)) { + PyErr_Format(PyExc_TypeError, "invalid file: %R", file); + return NULL; + } + + /* Decode mode */ + for (i = 0; i < strlen(mode); i++) { + char c = mode[i]; + + switch (c) { + case 'r': + reading = 1; + break; + case 'w': + writing = 1; + break; + case 'a': + appending = 1; + break; + case '+': + updating = 1; + break; + case 't': + text = 1; + break; + case 'b': + binary = 1; + break; + case 'U': + universal = 1; + reading = 1; + break; + default: + goto invalid_mode; + } + + /* c must not be duplicated */ + if (strchr(mode+i+1, c)) { + invalid_mode: + PyErr_Format(PyExc_ValueError, "invalid mode: '%s'", mode); + return NULL; + } + + } + + m = rawmode; + if (reading) *(m++) = 'r'; + if (writing) *(m++) = 'w'; + if (appending) *(m++) = 'a'; + if (updating) *(m++) = '+'; + *m = '\0'; + + /* Parameters validation */ + if (universal) { + if (writing || appending) { + PyErr_SetString(PyExc_ValueError, + "can't use U and writing mode at once"); + return NULL; + } + reading = 1; + } + + if (text && binary) { + PyErr_SetString(PyExc_ValueError, + "can't have text and binary mode at once"); + return NULL; + } + + if (reading + writing + appending > 1) { + PyErr_SetString(PyExc_ValueError, + "must have exactly one of read/write/append mode"); + return NULL; + } + + if (binary && encoding != NULL) { + PyErr_SetString(PyExc_ValueError, + "binary mode doesn't take an encoding argument"); + return NULL; + } + + if (binary && errors != NULL) { + PyErr_SetString(PyExc_ValueError, + "binary mode doesn't take an errors argument"); + return NULL; + } + + if (binary && newline != NULL) { + PyErr_SetString(PyExc_ValueError, + "binary mode doesn't take a newline argument"); + return NULL; + } + + /* Create the Raw file stream */ + raw = PyObject_CallFunction((PyObject *)&PyFileIO_Type, + "Osi", file, rawmode, closefd); + if (raw == NULL) + return NULL; + + modeobj = PyUnicode_FromString(mode); + if (modeobj == NULL) + goto error; + + /* buffering */ + { + PyObject *res = PyObject_CallMethod(raw, "isatty", NULL); + if (res == NULL) + goto error; + isatty = PyLong_AsLong(res); + Py_DECREF(res); + if (isatty == -1 && PyErr_Occurred()) + goto error; + } + + if (buffering == 1 || (buffering < 0 && isatty)) { + buffering = -1; + line_buffering = 1; + } + else + line_buffering = 0; + + if (buffering < 0) { + buffering = DEFAULT_BUFFER_SIZE; +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE + { + struct stat st; + long fileno; + PyObject *res = PyObject_CallMethod(raw, "fileno", NULL); + if (res == NULL) + goto error; + + fileno = PyLong_AsLong(res); + Py_DECREF(res); + if (fileno == -1 && PyErr_Occurred()) + goto error; + + if (fstat(fileno, &st) >= 0) + buffering = st.st_blksize; + } +#endif + } + if (buffering < 0) { + PyErr_SetString(PyExc_ValueError, + "invalid buffering size"); + goto error; + } + + /* if not buffering, returns the raw file object */ + if (buffering == 0) { + if (!binary) { + PyErr_SetString(PyExc_ValueError, + "can't have unbuffered text I/O"); + goto error; + } + + Py_DECREF(modeobj); + return raw; + } + + /* wraps into a buffered file */ + { + PyObject *Buffered_class; + + if (updating) + Buffered_class = (PyObject *)&PyBufferedRandom_Type; + else if (writing || appending) + Buffered_class = (PyObject *)&PyBufferedWriter_Type; + else if (reading) + Buffered_class = (PyObject *)&PyBufferedReader_Type; + else { + PyErr_Format(PyExc_ValueError, + "unknown mode: '%s'", mode); + goto error; + } + + buffer = PyObject_CallFunction(Buffered_class, "Oi", raw, buffering); + } + Py_CLEAR(raw); + if (buffer == NULL) + goto error; + + + /* if binary, returns the buffered file */ + if (binary) { + Py_DECREF(modeobj); + return buffer; + } + + /* wraps into a TextIOWrapper */ + wrapper = PyObject_CallFunction((PyObject *)&PyTextIOWrapper_Type, + "Osssi", + buffer, + encoding, errors, newline, + line_buffering); + Py_CLEAR(buffer); + if (wrapper == NULL) + goto error; + + if (PyObject_SetAttrString(wrapper, "mode", modeobj) < 0) + goto error; + Py_DECREF(modeobj); + return wrapper; + + error: + Py_XDECREF(raw); + Py_XDECREF(modeobj); + Py_XDECREF(buffer); + Py_XDECREF(wrapper); + return NULL; +} + +/* + * Private helpers for the io module. + */ + +Py_off_t +PyNumber_AsOff_t(PyObject *item, PyObject *err) +{ + Py_off_t result; + PyObject *runerr; + PyObject *value = PyNumber_Index(item); + if (value == NULL) + return -1; + + /* We're done if PyLong_AsSsize_t() returns without error. */ + result = PyLong_AsOff_t(value); + if (result != -1 || !(runerr = PyErr_Occurred())) + goto finish; + + /* Error handling code -- only manage OverflowError differently */ + if (!PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) + goto finish; + + PyErr_Clear(); + /* If no error-handling desired then the default clipping + is sufficient. + */ + if (!err) { + assert(PyLong_Check(value)); + /* Whether or not it is less than or equal to + zero is determined by the sign of ob_size + */ + if (_PyLong_Sign(value) < 0) + result = PY_OFF_T_MIN; + else + result = PY_OFF_T_MAX; + } + else { + /* Otherwise replace the error with caller's error object. */ + PyErr_Format(err, + "cannot fit '%.200s' into an offset-sized integer", + item->ob_type->tp_name); + } + + finish: + Py_DECREF(value); + return result; +} + +static int +iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { + _PyIO_State *state = IO_MOD_STATE(mod); + if (!state->initialized) + return 0; + Py_VISIT(state->os_module); + if (state->locale_module != NULL) { + Py_VISIT(state->locale_module); + } + Py_VISIT(state->unsupported_operation); + return 0; +} + +static int +iomodule_clear(PyObject *mod) { + _PyIO_State *state = IO_MOD_STATE(mod); + if (!state->initialized) + return 0; + Py_CLEAR(state->os_module); + if (state->locale_module != NULL) + Py_CLEAR(state->locale_module); + Py_CLEAR(state->unsupported_operation); + return 0; +} + +static void +iomodule_free(PyObject *mod) { + iomodule_clear(mod); +} + +/* + * Module definition + */ + +static PyMethodDef module_methods[] = { + {"open", (PyCFunction)io_open, METH_VARARGS|METH_KEYWORDS, open_doc}, + {NULL, NULL} +}; + +struct PyModuleDef _PyIO_Module = { + PyModuleDef_HEAD_INIT, + "io", + module_doc, + sizeof(_PyIO_State), + module_methods, + NULL, + iomodule_traverse, + iomodule_clear, + (freefunc)iomodule_free, +}; + +PyMODINIT_FUNC +PyInit__io(void) +{ + PyObject *m = PyModule_Create(&_PyIO_Module); + _PyIO_State *state = NULL; + if (m == NULL) + return NULL; + state = IO_MOD_STATE(m); + state->initialized = 0; + + /* put os in the module state */ + state->os_module = PyImport_ImportModule("os"); + if (state->os_module == NULL) + goto fail; + +#define ADD_TYPE(type, name) \ + if (PyType_Ready(type) < 0) \ + goto fail; \ + Py_INCREF(type); \ + if (PyModule_AddObject(m, name, (PyObject *)type) < 0) { \ + Py_DECREF(type); \ + goto fail; \ + } + + /* DEFAULT_BUFFER_SIZE */ + if (PyModule_AddIntMacro(m, DEFAULT_BUFFER_SIZE) < 0) + goto fail; + + /* UnsupportedOperation inherits from ValueError and IOError */ + state->unsupported_operation = PyObject_CallFunction( + (PyObject *)&PyType_Type, "s(OO){}", + "UnsupportedOperation", PyExc_ValueError, PyExc_IOError); + if (state->unsupported_operation == NULL) + goto fail; + Py_INCREF(state->unsupported_operation); + if (PyModule_AddObject(m, "UnsupportedOperation", + state->unsupported_operation) < 0) + goto fail; + + /* BlockingIOError */ + _PyExc_BlockingIOError.tp_base = (PyTypeObject *) PyExc_IOError; + ADD_TYPE(&_PyExc_BlockingIOError, "BlockingIOError"); + + /* Concrete base types of the IO ABCs. + (the ABCs themselves are declared through inheritance in io.py) + */ + ADD_TYPE(&PyIOBase_Type, "_IOBase"); + ADD_TYPE(&PyRawIOBase_Type, "_RawIOBase"); + ADD_TYPE(&PyBufferedIOBase_Type, "_BufferedIOBase"); + ADD_TYPE(&PyTextIOBase_Type, "_TextIOBase"); + + /* Implementation of concrete IO objects. */ + /* FileIO */ + PyFileIO_Type.tp_base = &PyRawIOBase_Type; + ADD_TYPE(&PyFileIO_Type, "FileIO"); + + /* BytesIO */ + PyBytesIO_Type.tp_base = &PyBufferedIOBase_Type; + ADD_TYPE(&PyBytesIO_Type, "BytesIO"); + + /* StringIO */ + PyStringIO_Type.tp_base = &PyTextIOBase_Type; + ADD_TYPE(&PyStringIO_Type, "StringIO"); + + /* BufferedReader */ + PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type; + ADD_TYPE(&PyBufferedReader_Type, "BufferedReader"); + + /* BufferedWriter */ + PyBufferedWriter_Type.tp_base = &PyBufferedIOBase_Type; + ADD_TYPE(&PyBufferedWriter_Type, "BufferedWriter"); + + /* BufferedRWPair */ + PyBufferedRWPair_Type.tp_base = &PyBufferedIOBase_Type; + ADD_TYPE(&PyBufferedRWPair_Type, "BufferedRWPair"); + + /* BufferedRandom */ + PyBufferedRandom_Type.tp_base = &PyBufferedIOBase_Type; + ADD_TYPE(&PyBufferedRandom_Type, "BufferedRandom"); + + /* TextIOWrapper */ + PyTextIOWrapper_Type.tp_base = &PyTextIOBase_Type; + ADD_TYPE(&PyTextIOWrapper_Type, "TextIOWrapper"); + + /* IncrementalNewlineDecoder */ + ADD_TYPE(&PyIncrementalNewlineDecoder_Type, "IncrementalNewlineDecoder"); + + /* Interned strings */ + if (!(_PyIO_str_close = PyUnicode_InternFromString("close"))) + goto fail; + if (!(_PyIO_str_closed = PyUnicode_InternFromString("closed"))) + goto fail; + if (!(_PyIO_str_decode = PyUnicode_InternFromString("decode"))) + goto fail; + if (!(_PyIO_str_encode = PyUnicode_InternFromString("encode"))) + goto fail; + if (!(_PyIO_str_fileno = PyUnicode_InternFromString("fileno"))) + goto fail; + if (!(_PyIO_str_flush = PyUnicode_InternFromString("flush"))) + goto fail; + if (!(_PyIO_str_getstate = PyUnicode_InternFromString("getstate"))) + goto fail; + if (!(_PyIO_str_isatty = PyUnicode_InternFromString("isatty"))) + goto fail; + if (!(_PyIO_str_newlines = PyUnicode_InternFromString("newlines"))) + goto fail; + if (!(_PyIO_str_nl = PyUnicode_InternFromString("\n"))) + goto fail; + if (!(_PyIO_str_read = PyUnicode_InternFromString("read"))) + goto fail; + if (!(_PyIO_str_read1 = PyUnicode_InternFromString("read1"))) + goto fail; + if (!(_PyIO_str_readable = PyUnicode_InternFromString("readable"))) + goto fail; + if (!(_PyIO_str_readinto = PyUnicode_InternFromString("readinto"))) + goto fail; + if (!(_PyIO_str_readline = PyUnicode_InternFromString("readline"))) + goto fail; + if (!(_PyIO_str_reset = PyUnicode_InternFromString("reset"))) + goto fail; + if (!(_PyIO_str_seek = PyUnicode_InternFromString("seek"))) + goto fail; + if (!(_PyIO_str_seekable = PyUnicode_InternFromString("seekable"))) + goto fail; + if (!(_PyIO_str_tell = PyUnicode_InternFromString("tell"))) + goto fail; + if (!(_PyIO_str_truncate = PyUnicode_InternFromString("truncate"))) + goto fail; + if (!(_PyIO_str_write = PyUnicode_InternFromString("write"))) + goto fail; + if (!(_PyIO_str_writable = PyUnicode_InternFromString("writable"))) + goto fail; + + if (!(_PyIO_empty_str = PyUnicode_FromStringAndSize(NULL, 0))) + goto fail; + if (!(_PyIO_empty_bytes = PyBytes_FromStringAndSize(NULL, 0))) + goto fail; + + state->initialized = 1; + + return m; + + fail: + Py_XDECREF(state->os_module); + Py_XDECREF(state->unsupported_operation); + Py_DECREF(m); + return NULL; +} diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h new file mode 100644 index 0000000..a44f127 --- /dev/null +++ b/Modules/_io/_iomodule.h @@ -0,0 +1,150 @@ +/* + * Declarations shared between the different parts of the io module + */ + +/* ABCs */ +extern PyTypeObject PyIOBase_Type; +extern PyTypeObject PyRawIOBase_Type; +extern PyTypeObject PyBufferedIOBase_Type; +extern PyTypeObject PyTextIOBase_Type; + +/* Concrete classes */ +extern PyTypeObject PyFileIO_Type; +extern PyTypeObject PyBytesIO_Type; +extern PyTypeObject PyStringIO_Type; +extern PyTypeObject PyBufferedReader_Type; +extern PyTypeObject PyBufferedWriter_Type; +extern PyTypeObject PyBufferedRWPair_Type; +extern PyTypeObject PyBufferedRandom_Type; +extern PyTypeObject PyTextIOWrapper_Type; +extern PyTypeObject PyIncrementalNewlineDecoder_Type; + +/* These functions are used as METH_NOARGS methods, are normally called + * with args=NULL, and return a new reference. + * BUT when args=Py_True is passed, they return a borrowed reference. + */ +extern PyObject* _PyIOBase_checkReadable(PyObject *self, PyObject *args); +extern PyObject* _PyIOBase_checkWritable(PyObject *self, PyObject *args); +extern PyObject* _PyIOBase_checkSeekable(PyObject *self, PyObject *args); +extern PyObject* _PyIOBase_checkClosed(PyObject *self, PyObject *args); + +/* Helper for finalization. + This function will revive an object ready to be deallocated and try to + close() it. It returns 0 if the object can be destroyed, or -1 if it + is alive again. */ +extern int _PyIOBase_finalize(PyObject *self); + +/* Returns true if the given FileIO object is closed. + Doesn't check the argument type, so be careful! */ +extern int _PyFileIO_closed(PyObject *self); + +/* Shortcut to the core of the IncrementalNewlineDecoder.decode method */ +extern PyObject *_PyIncrementalNewlineDecoder_decode( + PyObject *self, PyObject *input, int final); + +/* Finds the first line ending between `start` and `end`. + If found, returns the index after the line ending and doesn't touch + `*consumed`. + If not found, returns -1 and sets `*consumed` to the number of characters + which can be safely put aside until another search. + + NOTE: for performance reasons, `end` must point to a NUL character ('\0'). + Otherwise, the function will scan further and return garbage. */ +extern Py_ssize_t _PyIO_find_line_ending( + int translated, int universal, PyObject *readnl, + Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed); + + +#define DEFAULT_BUFFER_SIZE (8 * 1024) /* bytes */ + +typedef struct { + PyException_HEAD + PyObject *myerrno; + PyObject *strerror; + PyObject *filename; /* Not used, but part of the IOError object */ + Py_ssize_t written; +} PyBlockingIOErrorObject; +PyAPI_DATA(PyObject *) PyExc_BlockingIOError; + +/* + * Offset type for positioning. + */ + +#if defined(MS_WIN64) || defined(MS_WINDOWS) + +/* Windows uses long long for offsets */ +typedef PY_LONG_LONG Py_off_t; +# define PyLong_AsOff_t PyLong_AsLongLong +# define PyLong_FromOff_t PyLong_FromLongLong +# define PY_OFF_T_MAX PY_LLONG_MAX +# define PY_OFF_T_MIN PY_LLONG_MIN + +#else + +/* Other platforms use off_t */ +typedef off_t Py_off_t; +#if (SIZEOF_OFF_T == SIZEOF_SIZE_T) +# define PyLong_AsOff_t PyLong_AsSsize_t +# define PyLong_FromOff_t PyLong_FromSsize_t +# define PY_OFF_T_MAX PY_SSIZE_T_MAX +# define PY_OFF_T_MIN PY_SSIZE_T_MIN +#elif (SIZEOF_OFF_T == SIZEOF_LONG_LONG) +# define PyLong_AsOff_t PyLong_AsLongLong +# define PyLong_FromOff_t PyLong_FromLongLong +# define PY_OFF_T_MAX PY_LLONG_MAX +# define PY_OFF_T_MIN PY_LLONG_MIN +#elif (SIZEOF_OFF_T == SIZEOF_LONG) +# define PyLong_AsOff_t PyLong_AsLong +# define PyLong_FromOff_t PyLong_FromLong +# define PY_OFF_T_MAX LONG_MAX +# define PY_OFF_T_MIN LONG_MIN +#else +# error off_t does not match either size_t, long, or long long! +#endif + +#endif + +extern Py_off_t PyNumber_AsOff_t(PyObject *item, PyObject *err); + +/* Implementation details */ + +/* IO module structure */ + +extern PyModuleDef _PyIO_Module; + +typedef struct { + int initialized; + PyObject *os_module; + PyObject *locale_module; + + PyObject *unsupported_operation; +} _PyIO_State; + +#define IO_MOD_STATE(mod) ((_PyIO_State *)PyModule_GetState(mod)) +#define IO_STATE IO_MOD_STATE(PyState_FindModule(&_PyIO_Module)) + +extern PyObject *_PyIO_str_close; +extern PyObject *_PyIO_str_closed; +extern PyObject *_PyIO_str_decode; +extern PyObject *_PyIO_str_encode; +extern PyObject *_PyIO_str_fileno; +extern PyObject *_PyIO_str_flush; +extern PyObject *_PyIO_str_getstate; +extern PyObject *_PyIO_str_isatty; +extern PyObject *_PyIO_str_newlines; +extern PyObject *_PyIO_str_nl; +extern PyObject *_PyIO_str_read; +extern PyObject *_PyIO_str_read1; +extern PyObject *_PyIO_str_readable; +extern PyObject *_PyIO_str_readinto; +extern PyObject *_PyIO_str_readline; +extern PyObject *_PyIO_str_reset; +extern PyObject *_PyIO_str_seek; +extern PyObject *_PyIO_str_seekable; +extern PyObject *_PyIO_str_tell; +extern PyObject *_PyIO_str_truncate; +extern PyObject *_PyIO_str_writable; +extern PyObject *_PyIO_str_write; + +extern PyObject *_PyIO_empty_str; +extern PyObject *_PyIO_empty_bytes; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c new file mode 100644 index 0000000..9960dba --- /dev/null +++ b/Modules/_io/bufferedio.c @@ -0,0 +1,2139 @@ +/* + An implementation of Buffered I/O as defined by PEP 3116 - "New I/O" + + Classes defined here: BufferedIOBase, BufferedReader, BufferedWriter, + BufferedRandom. + + Written by Amaury Forgeot d'Arc and Antoine Pitrou +*/ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#include "pythread.h" +#include "_iomodule.h" + +/* + * BufferedIOBase class, inherits from IOBase. + */ +PyDoc_STRVAR(BufferedIOBase_doc, + "Base class for buffered IO objects.\n" + "\n" + "The main difference with RawIOBase is that the read() method\n" + "supports omitting the size argument, and does not have a default\n" + "implementation that defers to readinto().\n" + "\n" + "In addition, read(), readinto() and write() may raise\n" + "BlockingIOError if the underlying raw stream is in non-blocking\n" + "mode and not ready; unlike their raw counterparts, they will never\n" + "return None.\n" + "\n" + "A typical implementation should not inherit from a RawIOBase\n" + "implementation, but wrap one.\n" + ); + +static PyObject * +BufferedIOBase_readinto(PyObject *self, PyObject *args) +{ + Py_buffer buf; + Py_ssize_t len; + PyObject *data; + + if (!PyArg_ParseTuple(args, "w*:readinto", &buf)) { + return NULL; + } + + data = PyObject_CallMethod(self, "read", "n", buf.len); + if (data == NULL) + goto error; + + if (!PyBytes_Check(data)) { + Py_DECREF(data); + PyErr_SetString(PyExc_TypeError, "read() should return bytes"); + goto error; + } + + len = Py_SIZE(data); + memcpy(buf.buf, PyBytes_AS_STRING(data), len); + + PyBuffer_Release(&buf); + Py_DECREF(data); + + return PyLong_FromSsize_t(len); + + error: + PyBuffer_Release(&buf); + return NULL; +} + +static PyObject * +BufferedIOBase_unsupported(const char *message) +{ + PyErr_SetString(IO_STATE->unsupported_operation, message); + return NULL; +} + +PyDoc_STRVAR(BufferedIOBase_read_doc, + "Read and return up to n bytes.\n" + "\n" + "If the argument is omitted, None, or negative, reads and\n" + "returns all data until EOF.\n" + "\n" + "If the argument is positive, and the underlying raw stream is\n" + "not 'interactive', multiple raw reads may be issued to satisfy\n" + "the byte count (unless EOF is reached first). But for\n" + "interactive raw streams (as well as sockets and pipes), at most\n" + "one raw read will be issued, and a short result does not imply\n" + "that EOF is imminent.\n" + "\n" + "Returns an empty bytes object on EOF.\n" + "\n" + "Returns None if the underlying raw stream was open in non-blocking\n" + "mode and no data is available at the moment.\n"); + +static PyObject * +BufferedIOBase_read(PyObject *self, PyObject *args) +{ + return BufferedIOBase_unsupported("read"); +} + +PyDoc_STRVAR(BufferedIOBase_read1_doc, + "Read and return up to n bytes, with at most one read() call\n" + "to the underlying raw stream. A short result does not imply\n" + "that EOF is imminent.\n" + "\n" + "Returns an empty bytes object on EOF.\n"); + +static PyObject * +BufferedIOBase_read1(PyObject *self, PyObject *args) +{ + return BufferedIOBase_unsupported("read1"); +} + +PyDoc_STRVAR(BufferedIOBase_write_doc, + "Write the given buffer to the IO stream.\n" + "\n" + "Returns the number of bytes written, which is never less than\n" + "len(b).\n" + "\n" + "Raises BlockingIOError if the buffer is full and the\n" + "underlying raw stream cannot accept more data at the moment.\n"); + +static PyObject * +BufferedIOBase_write(PyObject *self, PyObject *args) +{ + return BufferedIOBase_unsupported("write"); +} + + +static PyMethodDef BufferedIOBase_methods[] = { + {"read", BufferedIOBase_read, METH_VARARGS, BufferedIOBase_read_doc}, + {"read1", BufferedIOBase_read1, METH_VARARGS, BufferedIOBase_read1_doc}, + {"readinto", BufferedIOBase_readinto, METH_VARARGS, NULL}, + {"write", BufferedIOBase_write, METH_VARARGS, BufferedIOBase_write_doc}, + {NULL, NULL} +}; + +PyTypeObject PyBufferedIOBase_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io._BufferedIOBase", /*tp_name*/ + 0, /*tp_basicsize*/ + 0, /*tp_itemsize*/ + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + BufferedIOBase_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BufferedIOBase_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyIOBase_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +typedef struct { + PyObject_HEAD + + PyObject *raw; + int ok; /* Initialized? */ + int readable; + int writable; + + /* Absolute position inside the raw stream (-1 if unknown). */ + Py_off_t abs_pos; + + /* A static buffer of size `buffer_size` */ + char *buffer; + /* Current logical position in the buffer. */ + Py_off_t pos; + /* Position of the raw stream in the buffer. */ + Py_off_t raw_pos; + + /* Just after the last buffered byte in the buffer, or -1 if the buffer + isn't ready for reading. */ + Py_off_t read_end; + + /* Just after the last byte actually written */ + Py_off_t write_pos; + /* Just after the last byte waiting to be written, or -1 if the buffer + isn't ready for writing. */ + Py_off_t write_end; + + PyThread_type_lock lock; + + Py_ssize_t buffer_size; + Py_ssize_t buffer_mask; + + PyObject *dict; + PyObject *weakreflist; +} BufferedObject; + +/* + Implementation notes: + + * BufferedReader, BufferedWriter and BufferedRandom try to share most + methods (this is helped by the members `readable` and `writable`, which + are initialized in the respective constructors) + * They also share a single buffer for reading and writing. This enables + interleaved reads and writes without flushing. It also makes the logic + a bit trickier to get right. + * The absolute position of the raw stream is cached, if possible, in the + `abs_pos` member. It must be updated every time an operation is done + on the raw stream. If not sure, it can be reinitialized by calling + _Buffered_raw_tell(), which queries the raw stream (_Buffered_raw_seek() + also does it). To read it, use RAW_TELL(). + * Three helpers, _BufferedReader_raw_read, _BufferedWriter_raw_write and + _BufferedWriter_flush_unlocked do a lot of useful housekeeping. + + NOTE: we should try to maintain block alignment of reads and writes to the + raw stream (according to the buffer size), but for now it is only done + in read() and friends. + + XXX: method naming is a bit messy. +*/ + +/* These macros protect the BufferedObject against concurrent operations. */ + +#define ENTER_BUFFERED(self) \ + Py_BEGIN_ALLOW_THREADS \ + PyThread_acquire_lock(self->lock, 1); \ + Py_END_ALLOW_THREADS + +#define LEAVE_BUFFERED(self) \ + PyThread_release_lock(self->lock); + +#define CHECK_INITIALIZED(self) \ + if (self->ok <= 0) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on uninitialized object"); \ + return NULL; \ + } + +#define CHECK_INITIALIZED_INT(self) \ + if (self->ok <= 0) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on uninitialized object"); \ + return -1; \ + } + +#define VALID_READ_BUFFER(self) \ + (self->readable && self->read_end != -1) + +#define VALID_WRITE_BUFFER(self) \ + (self->writable && self->write_end != -1) + +#define ADJUST_POSITION(self, _new_pos) \ + do { \ + self->pos = _new_pos; \ + if (VALID_READ_BUFFER(self) && self->read_end < self->pos) \ + self->read_end = self->pos; \ + } while(0) + +#define READAHEAD(self) \ + ((self->readable && VALID_READ_BUFFER(self)) \ + ? (self->read_end - self->pos) : 0) + +#define RAW_OFFSET(self) \ + (((VALID_READ_BUFFER(self) || VALID_WRITE_BUFFER(self)) \ + && self->raw_pos >= 0) ? self->raw_pos - self->pos : 0) + +#define RAW_TELL(self) \ + (self->abs_pos != -1 ? self->abs_pos : _Buffered_raw_tell(self)) + +#define MINUS_LAST_BLOCK(self, size) \ + (self->buffer_mask ? \ + (size & ~self->buffer_mask) : \ + (self->buffer_size * (size / self->buffer_size))) + + +static void +BufferedObject_dealloc(BufferedObject *self) +{ + if (self->ok && _PyIOBase_finalize((PyObject *) self) < 0) + return; + _PyObject_GC_UNTRACK(self); + self->ok = 0; + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *)self); + Py_CLEAR(self->raw); + if (self->buffer) { + PyMem_Free(self->buffer); + self->buffer = NULL; + } + if (self->lock) { + PyThread_free_lock(self->lock); + self->lock = NULL; + } + Py_CLEAR(self->dict); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int +Buffered_traverse(BufferedObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->raw); + Py_VISIT(self->dict); + return 0; +} + +static int +Buffered_clear(BufferedObject *self) +{ + if (self->ok && _PyIOBase_finalize((PyObject *) self) < 0) + return -1; + self->ok = 0; + Py_CLEAR(self->raw); + Py_CLEAR(self->dict); + return 0; +} + +/* + * _BufferedIOMixin methods + * This is not a class, just a collection of methods that will be reused + * by BufferedReader and BufferedWriter + */ + +/* Flush and close */ + +static PyObject * +BufferedIOMixin_flush(BufferedObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self) + return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_flush, NULL); +} + +static int +BufferedIOMixin_closed(BufferedObject *self) +{ + int closed; + PyObject *res; + CHECK_INITIALIZED_INT(self) + res = PyObject_GetAttr(self->raw, _PyIO_str_closed); + if (res == NULL) + return -1; + closed = PyObject_IsTrue(res); + Py_DECREF(res); + return closed; +} + +static PyObject * +BufferedIOMixin_closed_get(BufferedObject *self, void *context) +{ + CHECK_INITIALIZED(self) + return PyObject_GetAttr(self->raw, _PyIO_str_closed); +} + +static PyObject * +BufferedIOMixin_close(BufferedObject *self, PyObject *args) +{ + PyObject *res = NULL; + int r; + + CHECK_INITIALIZED(self) + ENTER_BUFFERED(self) + + r = BufferedIOMixin_closed(self); + if (r < 0) + goto end; + if (r > 0) { + res = Py_None; + Py_INCREF(res); + goto end; + } + /* flush() will most probably re-take the lock, so drop it first */ + LEAVE_BUFFERED(self) + res = PyObject_CallMethodObjArgs((PyObject *)self, _PyIO_str_flush, NULL); + ENTER_BUFFERED(self) + if (res == NULL) { + /* If flush() fails, just give up */ + if (PyErr_ExceptionMatches(PyExc_IOError)) + PyErr_Clear(); + else + goto end; + } + Py_XDECREF(res); + + res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_close, NULL); + +end: + LEAVE_BUFFERED(self) + return res; +} + +/* Inquiries */ + +static PyObject * +BufferedIOMixin_seekable(BufferedObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self) + return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_seekable, NULL); +} + +static PyObject * +BufferedIOMixin_readable(BufferedObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self) + return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readable, NULL); +} + +static PyObject * +BufferedIOMixin_writable(BufferedObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self) + return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_writable, NULL); +} + +static PyObject * +BufferedIOMixin_name_get(BufferedObject *self, void *context) +{ + CHECK_INITIALIZED(self) + return PyObject_GetAttrString(self->raw, "name"); +} + +static PyObject * +BufferedIOMixin_mode_get(BufferedObject *self, void *context) +{ + CHECK_INITIALIZED(self) + return PyObject_GetAttrString(self->raw, "mode"); +} + +/* Lower-level APIs */ + +static PyObject * +BufferedIOMixin_fileno(BufferedObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self) + return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_fileno, NULL); +} + +static PyObject * +BufferedIOMixin_isatty(BufferedObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self) + return PyObject_CallMethodObjArgs(self->raw, _PyIO_str_isatty, NULL); +} + + +/* Forward decls */ +static PyObject * +_BufferedWriter_flush_unlocked(BufferedObject *, int); +static Py_ssize_t +_BufferedReader_fill_buffer(BufferedObject *self); +static void +_BufferedReader_reset_buf(BufferedObject *self); +static void +_BufferedWriter_reset_buf(BufferedObject *self); +static PyObject * +_BufferedReader_peek_unlocked(BufferedObject *self, Py_ssize_t); +static PyObject * +_BufferedReader_read_unlocked(BufferedObject *self, Py_ssize_t); + + +/* + * Helpers + */ + +/* Returns the address of the `written` member if a BlockingIOError was + raised, NULL otherwise. The error is always re-raised. */ +static Py_ssize_t * +_Buffered_check_blocking_error(void) +{ + PyObject *t, *v, *tb; + PyBlockingIOErrorObject *err; + + PyErr_Fetch(&t, &v, &tb); + if (v == NULL || !PyErr_GivenExceptionMatches(v, PyExc_BlockingIOError)) { + PyErr_Restore(t, v, tb); + return NULL; + } + err = (PyBlockingIOErrorObject *) v; + /* TODO: sanity check (err->written >= 0) */ + PyErr_Restore(t, v, tb); + return &err->written; +} + +static Py_off_t +_Buffered_raw_tell(BufferedObject *self) +{ + PyObject *res; + Py_off_t n; + res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_tell, NULL); + if (res == NULL) + return -1; + n = PyNumber_AsOff_t(res, PyExc_ValueError); + Py_DECREF(res); + if (n < 0) { + if (!PyErr_Occurred()) + PyErr_Format(PyExc_IOError, + "Raw stream returned invalid position %zd", n); + return -1; + } + self->abs_pos = n; + return n; +} + +static Py_off_t +_Buffered_raw_seek(BufferedObject *self, Py_off_t target, int whence) +{ + PyObject *res, *posobj, *whenceobj; + Py_off_t n; + + posobj = PyLong_FromOff_t(target); + if (posobj == NULL) + return -1; + whenceobj = PyLong_FromLong(whence); + if (whenceobj == NULL) { + Py_DECREF(posobj); + return -1; + } + res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_seek, + posobj, whenceobj, NULL); + Py_DECREF(posobj); + Py_DECREF(whenceobj); + if (res == NULL) + return -1; + n = PyNumber_AsOff_t(res, PyExc_ValueError); + Py_DECREF(res); + if (n < 0) { + if (!PyErr_Occurred()) + PyErr_Format(PyExc_IOError, + "Raw stream returned invalid position %zd", n); + return -1; + } + self->abs_pos = n; + return n; +} + +static int +_Buffered_init(BufferedObject *self) +{ + Py_ssize_t n; + if (self->buffer_size <= 0) { + PyErr_SetString(PyExc_ValueError, + "buffer size must be strictly positive"); + return -1; + } + if (self->buffer) + PyMem_Free(self->buffer); + self->buffer = PyMem_Malloc(self->buffer_size); + if (self->buffer == NULL) { + PyErr_NoMemory(); + return -1; + } + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + PyErr_SetString(PyExc_RuntimeError, "can't allocate read lock"); + return -1; + } + /* Find out whether buffer_size is a power of 2 */ + /* XXX is this optimization useful? */ + for (n = self->buffer_size - 1; n & 1; n >>= 1) + ; + if (n == 0) + self->buffer_mask = self->buffer_size - 1; + else + self->buffer_mask = 0; + if (_Buffered_raw_tell(self) == -1) + PyErr_Clear(); + return 0; +} + +/* + * Shared methods and wrappers + */ + +static PyObject * +Buffered_flush(BufferedObject *self, PyObject *args) +{ + PyObject *res; + + CHECK_INITIALIZED(self) + if (BufferedIOMixin_closed(self)) { + PyErr_SetString(PyExc_ValueError, "flush of closed file"); + return NULL; + } + + ENTER_BUFFERED(self) + res = _BufferedWriter_flush_unlocked(self, 0); + if (res != NULL && self->readable) { + /* Rewind the raw stream so that its position corresponds to + the current logical position. */ + Py_off_t n; + n = _Buffered_raw_seek(self, -RAW_OFFSET(self), 1); + if (n == -1) + Py_CLEAR(res); + _BufferedReader_reset_buf(self); + } + LEAVE_BUFFERED(self) + + return res; +} + +static PyObject * +Buffered_peek(BufferedObject *self, PyObject *args) +{ + Py_ssize_t n = 0; + PyObject *res = NULL; + + CHECK_INITIALIZED(self) + if (!PyArg_ParseTuple(args, "|n:peek", &n)) { + return NULL; + } + + ENTER_BUFFERED(self) + + if (self->writable) { + res = _BufferedWriter_flush_unlocked(self, 1); + if (res == NULL) + goto end; + Py_CLEAR(res); + } + res = _BufferedReader_peek_unlocked(self, n); + +end: + LEAVE_BUFFERED(self) + return res; +} + +static PyObject * +Buffered_read(BufferedObject *self, PyObject *args) +{ + Py_ssize_t n = -1; + PyObject *res; + + CHECK_INITIALIZED(self) + if (!PyArg_ParseTuple(args, "|n:read", &n)) { + return NULL; + } + if (n < -1) { + PyErr_SetString(PyExc_ValueError, + "read length must be positive or -1"); + return NULL; + } + + if (BufferedIOMixin_closed(self)) { + PyErr_SetString(PyExc_ValueError, "read of closed file"); + return NULL; + } + + ENTER_BUFFERED(self) + res = _BufferedReader_read_unlocked(self, n); + LEAVE_BUFFERED(self) + + return res; +} + +static PyObject * +Buffered_read1(BufferedObject *self, PyObject *args) +{ + Py_ssize_t n, have, r; + PyObject *res = NULL; + + CHECK_INITIALIZED(self) + if (!PyArg_ParseTuple(args, "n:read1", &n)) { + return NULL; + } + + if (n < 0) { + PyErr_SetString(PyExc_ValueError, + "read length must be positive"); + return NULL; + } + if (n == 0) + return PyBytes_FromStringAndSize(NULL, 0); + + ENTER_BUFFERED(self) + + if (self->writable) { + res = _BufferedWriter_flush_unlocked(self, 1); + if (res == NULL) + goto end; + Py_CLEAR(res); + } + + /* Return up to n bytes. If at least one byte is buffered, we + only return buffered bytes. Otherwise, we do one raw read. */ + + /* XXX: this mimicks the io.py implementation but is probably wrong. + If we need to read from the raw stream, then we could actually read + all `n` bytes asked by the caller (and possibly more, so as to fill + our buffer for the next reads). */ + + have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + if (have > 0) { + if (n > have) + n = have; + res = PyBytes_FromStringAndSize(self->buffer + self->pos, n); + if (res == NULL) + goto end; + self->pos += n; + goto end; + } + + /* Fill the buffer from the raw stream, and copy it to the result. */ + _BufferedReader_reset_buf(self); + r = _BufferedReader_fill_buffer(self); + if (r == -1) + goto end; + if (r == -2) + r = 0; + if (n > r) + n = r; + res = PyBytes_FromStringAndSize(self->buffer, n); + if (res == NULL) + goto end; + self->pos = n; + +end: + LEAVE_BUFFERED(self) + return res; +} + +static PyObject * +Buffered_readinto(BufferedObject *self, PyObject *args) +{ + PyObject *res = NULL; + + CHECK_INITIALIZED(self) + + /* TODO: use raw.readinto() instead! */ + if (self->writable) { + ENTER_BUFFERED(self) + res = _BufferedWriter_flush_unlocked(self, 0); + LEAVE_BUFFERED(self) + if (res == NULL) + goto end; + Py_DECREF(res); + } + res = BufferedIOBase_readinto((PyObject *)self, args); + +end: + return res; +} + +static PyObject * +_Buffered_readline(BufferedObject *self, Py_ssize_t limit) +{ + PyObject *res = NULL; + PyObject *chunks = NULL; + Py_ssize_t n, written = 0; + const char *start, *s, *end; + + if (BufferedIOMixin_closed(self)) { + PyErr_SetString(PyExc_ValueError, "readline of closed file"); + return NULL; + } + + ENTER_BUFFERED(self) + + /* First, try to find a line in the buffer */ + n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + if (limit >= 0 && n > limit) + n = limit; + start = self->buffer + self->pos; + end = start + n; + s = start; + while (s < end) { + if (*s++ == '\n') { + res = PyBytes_FromStringAndSize(start, s - start); + if (res != NULL) + self->pos += s - start; + goto end; + } + } + if (n == limit) { + res = PyBytes_FromStringAndSize(start, n); + if (res != NULL) + self->pos += n; + goto end; + } + + /* Now we try to get some more from the raw stream */ + if (self->writable) { + res = _BufferedWriter_flush_unlocked(self, 1); + if (res == NULL) + goto end; + Py_CLEAR(res); + } + chunks = PyList_New(0); + if (chunks == NULL) + goto end; + if (n > 0) { + res = PyBytes_FromStringAndSize(start, n); + if (res == NULL) + goto end; + if (PyList_Append(chunks, res) < 0) { + Py_CLEAR(res); + goto end; + } + Py_CLEAR(res); + written += n; + if (limit >= 0) + limit -= n; + } + + for (;;) { + _BufferedReader_reset_buf(self); + n = _BufferedReader_fill_buffer(self); + if (n == -1) + goto end; + if (n <= 0) + break; + if (limit >= 0 && n > limit) + n = limit; + start = self->buffer; + end = start + n; + s = start; + while (s < end) { + if (*s++ == '\n') { + res = PyBytes_FromStringAndSize(start, s - start); + if (res == NULL) + goto end; + self->pos = s - start; + goto found; + } + } + res = PyBytes_FromStringAndSize(start, n); + if (res == NULL) + goto end; + if (n == limit) { + self->pos = n; + break; + } + if (PyList_Append(chunks, res) < 0) { + Py_CLEAR(res); + goto end; + } + Py_CLEAR(res); + written += n; + if (limit >= 0) + limit -= n; + } +found: + if (res != NULL && PyList_Append(chunks, res) < 0) { + Py_CLEAR(res); + goto end; + } + Py_CLEAR(res); + res = _PyBytes_Join(_PyIO_empty_bytes, chunks); + +end: + LEAVE_BUFFERED(self) + Py_XDECREF(chunks); + return res; +} + +static PyObject * +Buffered_readline(BufferedObject *self, PyObject *args) +{ + Py_ssize_t limit = -1; + + CHECK_INITIALIZED(self) + + if (!PyArg_ParseTuple(args, "|n:readline", &limit)) { + return NULL; + } + return _Buffered_readline(self, limit); +} + + +static PyObject * +Buffered_tell(BufferedObject *self, PyObject *args) +{ + Py_off_t pos; + + CHECK_INITIALIZED(self) + pos = _Buffered_raw_tell(self); + if (pos == -1) + return NULL; + pos -= RAW_OFFSET(self); + /* TODO: sanity check (pos >= 0) */ + return PyLong_FromOff_t(pos); +} + +static PyObject * +Buffered_seek(BufferedObject *self, PyObject *args) +{ + Py_off_t target, n; + int whence = 0; + PyObject *targetobj, *res = NULL; + + CHECK_INITIALIZED(self) + if (!PyArg_ParseTuple(args, "O|i:seek", &targetobj, &whence)) { + return NULL; + } + + if (whence < 0 || whence > 2) { + PyErr_Format(PyExc_ValueError, + "whence must be between 0 and 2, not %d", whence); + return NULL; + } + target = PyNumber_AsOff_t(targetobj, PyExc_ValueError); + if (target == -1 && PyErr_Occurred()) + return NULL; + + ENTER_BUFFERED(self) + + if (whence != 2 && self->readable) { + Py_off_t current, avail; + /* Check if seeking leaves us inside the current buffer, + so as to return quickly if possible. + Don't know how to do that when whence == 2, though. */ + current = RAW_TELL(self); + avail = READAHEAD(self); + if (avail > 0) { + Py_off_t offset; + if (whence == 0) + offset = target - (current - RAW_OFFSET(self)); + else + offset = target; + if (offset >= -self->pos && offset <= avail) { + self->pos += offset; + res = PyLong_FromOff_t(current - avail + offset); + goto end; + } + } + } + + /* Fallback: invoke raw seek() method and clear buffer */ + if (self->writable) { + res = _BufferedWriter_flush_unlocked(self, 0); + if (res == NULL) + goto end; + Py_CLEAR(res); + _BufferedWriter_reset_buf(self); + } + + /* TODO: align on block boundary and read buffer if needed? */ + if (whence == 1) + target -= RAW_OFFSET(self); + n = _Buffered_raw_seek(self, target, whence); + if (n == -1) + goto end; + self->raw_pos = -1; + res = PyLong_FromOff_t(n); + if (res != NULL && self->readable) + _BufferedReader_reset_buf(self); + +end: + LEAVE_BUFFERED(self) + return res; +} + +static PyObject * +Buffered_truncate(BufferedObject *self, PyObject *args) +{ + PyObject *pos = Py_None; + PyObject *res = NULL; + + CHECK_INITIALIZED(self) + if (!PyArg_ParseTuple(args, "|O:truncate", &pos)) { + return NULL; + } + + ENTER_BUFFERED(self) + + if (self->writable) { + res = _BufferedWriter_flush_unlocked(self, 0); + if (res == NULL) + goto end; + Py_CLEAR(res); + } + if (self->readable) { + if (pos == Py_None) { + /* Rewind the raw stream so that its position corresponds to + the current logical position. */ + if (_Buffered_raw_seek(self, -RAW_OFFSET(self), 1) == -1) + goto end; + } + _BufferedReader_reset_buf(self); + } + res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_truncate, pos, NULL); + if (res == NULL) + goto end; + /* Reset cached position */ + if (_Buffered_raw_tell(self) == -1) + PyErr_Clear(); + +end: + LEAVE_BUFFERED(self) + return res; +} + +static PyObject * +Buffered_iternext(BufferedObject *self) +{ + PyObject *line; + PyTypeObject *tp; + + CHECK_INITIALIZED(self); + + tp = Py_TYPE(self); + if (tp == &PyBufferedReader_Type || + tp == &PyBufferedRandom_Type) { + /* Skip method call overhead for speed */ + line = _Buffered_readline(self, -1); + } + else { + line = PyObject_CallMethodObjArgs((PyObject *)self, + _PyIO_str_readline, NULL); + if (line && !PyBytes_Check(line)) { + PyErr_Format(PyExc_IOError, + "readline() should have returned a bytes object, " + "not '%.200s'", Py_TYPE(line)->tp_name); + Py_DECREF(line); + return NULL; + } + } + + if (line == NULL) + return NULL; + + if (PyBytes_GET_SIZE(line) == 0) { + /* Reached EOF or would have blocked */ + Py_DECREF(line); + return NULL; + } + + return line; +} + +/* + * class BufferedReader + */ + +PyDoc_STRVAR(BufferedReader_doc, + "Create a new buffered reader using the given readable raw IO object."); + +static void _BufferedReader_reset_buf(BufferedObject *self) +{ + self->read_end = -1; +} + +static int +BufferedReader_init(BufferedObject *self, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"raw", "buffer_size", NULL}; + Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; + PyObject *raw; + + self->ok = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|n:BufferedReader", kwlist, + &raw, &buffer_size)) { + return -1; + } + + if (_PyIOBase_checkReadable(raw, Py_True) == NULL) + return -1; + + Py_CLEAR(self->raw); + Py_INCREF(raw); + self->raw = raw; + self->buffer_size = buffer_size; + self->readable = 1; + self->writable = 0; + + if (_Buffered_init(self) < 0) + return -1; + _BufferedReader_reset_buf(self); + + self->ok = 1; + return 0; +} + +static Py_ssize_t +_BufferedReader_raw_read(BufferedObject *self, char *start, Py_ssize_t len) +{ + Py_buffer buf; + PyObject *memobj, *res; + Py_ssize_t n; + /* NOTE: the buffer needn't be released as its object is NULL. */ + if (PyBuffer_FillInfo(&buf, NULL, start, len, 0, PyBUF_CONTIG) == -1) + return -1; + memobj = PyMemoryView_FromBuffer(&buf); + if (memobj == NULL) + return -1; + res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readinto, memobj, NULL); + Py_DECREF(memobj); + if (res == NULL) + return -1; + if (res == Py_None) { + /* Non-blocking stream would have blocked. Special return code! */ + Py_DECREF(res); + return -2; + } + n = PyNumber_AsSsize_t(res, PyExc_ValueError); + Py_DECREF(res); + if (n < 0 || n > len) { + PyErr_Format(PyExc_IOError, + "raw readinto() returned invalid length %zd " + "(should have been between 0 and %zd)", n, len); + return -1; + } + if (n > 0 && self->abs_pos != -1) + self->abs_pos += n; + return n; +} + +static Py_ssize_t +_BufferedReader_fill_buffer(BufferedObject *self) +{ + Py_ssize_t start, len, n; + if (VALID_READ_BUFFER(self)) + start = Py_SAFE_DOWNCAST(self->read_end, Py_off_t, Py_ssize_t); + else + start = 0; + len = self->buffer_size - start; + n = _BufferedReader_raw_read(self, self->buffer + start, len); + if (n <= 0) + return n; + self->read_end = start + n; + self->raw_pos = start + n; + return n; +} + +static PyObject * +_BufferedReader_read_unlocked(BufferedObject *self, Py_ssize_t n) +{ + PyObject *data, *res = NULL; + Py_ssize_t current_size, remaining, written; + char *out; + + /* Special case for when the number of bytes to read is unspecified. */ + if (n == -1) { + PyObject *chunks = PyList_New(0); + if (chunks == NULL) + return NULL; + + /* First copy what we have in the current buffer. */ + current_size = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + data = NULL; + if (current_size) { + data = PyBytes_FromStringAndSize( + self->buffer + self->pos, current_size); + if (data == NULL) { + Py_DECREF(chunks); + return NULL; + } + } + _BufferedReader_reset_buf(self); + /* We're going past the buffer's bounds, flush it */ + if (self->writable) { + res = _BufferedWriter_flush_unlocked(self, 1); + if (res == NULL) { + Py_DECREF(chunks); + return NULL; + } + Py_CLEAR(res); + } + while (1) { + if (data) { + if (PyList_Append(chunks, data) < 0) { + Py_DECREF(data); + Py_DECREF(chunks); + return NULL; + } + Py_DECREF(data); + } + + /* Read until EOF or until read() would block. */ + data = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_read, NULL); + if (data == NULL) { + Py_DECREF(chunks); + return NULL; + } + if (data != Py_None && !PyBytes_Check(data)) { + Py_DECREF(data); + Py_DECREF(chunks); + PyErr_SetString(PyExc_TypeError, "read() should return bytes"); + return NULL; + } + if (data == Py_None || PyBytes_GET_SIZE(data) == 0) { + if (current_size == 0) { + Py_DECREF(chunks); + return data; + } + else { + res = _PyBytes_Join(_PyIO_empty_bytes, chunks); + Py_DECREF(data); + Py_DECREF(chunks); + return res; + } + } + current_size += PyBytes_GET_SIZE(data); + if (self->abs_pos != -1) + self->abs_pos += PyBytes_GET_SIZE(data); + } + } + + /* The number of bytes to read is specified, return at most n bytes. */ + current_size = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + if (n <= current_size) { + /* Fast path: the data to read is fully buffered. */ + res = PyBytes_FromStringAndSize(self->buffer + self->pos, n); + if (res == NULL) + goto error; + self->pos += n; + return res; + } + + /* Slow path: read from the stream until enough bytes are read, + * or until an EOF occurs or until read() would block. + */ + res = PyBytes_FromStringAndSize(NULL, n); + if (res == NULL) + goto error; + out = PyBytes_AS_STRING(res); + remaining = n; + written = 0; + if (current_size > 0) { + memcpy(out, self->buffer + self->pos, current_size); + remaining -= current_size; + written += current_size; + } + _BufferedReader_reset_buf(self); + while (remaining > 0) { + /* We want to read a whole block at the end into buffer. + If we had readv() we could do this in one pass. */ + Py_ssize_t r = MINUS_LAST_BLOCK(self, remaining); + if (r == 0) + break; + r = _BufferedReader_raw_read(self, out + written, r); + if (r == -1) + goto error; + if (r == 0 || r == -2) { + /* EOF occurred or read() would block. */ + if (r == 0 || written > 0) { + if (_PyBytes_Resize(&res, written)) + goto error; + return res; + } + Py_DECREF(res); + Py_INCREF(Py_None); + return Py_None; + } + remaining -= r; + written += r; + } + assert(remaining <= self->buffer_size); + self->pos = 0; + self->raw_pos = 0; + self->read_end = 0; + while (self->read_end < self->buffer_size) { + Py_ssize_t r = _BufferedReader_fill_buffer(self); + if (r == -1) + goto error; + if (r == 0 || r == -2) { + /* EOF occurred or read() would block. */ + if (r == 0 || written > 0) { + if (_PyBytes_Resize(&res, written)) + goto error; + return res; + } + Py_DECREF(res); + Py_INCREF(Py_None); + return Py_None; + } + if (remaining > r) { + memcpy(out + written, self->buffer + self->pos, r); + written += r; + self->pos += r; + remaining -= r; + } + else if (remaining > 0) { + memcpy(out + written, self->buffer + self->pos, remaining); + written += remaining; + self->pos += remaining; + remaining = 0; + } + if (remaining == 0) + break; + } + + return res; + +error: + Py_XDECREF(res); + return NULL; +} + +static PyObject * +_BufferedReader_peek_unlocked(BufferedObject *self, Py_ssize_t n) +{ + Py_ssize_t have, r; + + have = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + /* Constraints: + 1. we don't want to advance the file position. + 2. we don't want to lose block alignment, so we can't shift the buffer + to make some place. + Therefore, we either return `have` bytes (if > 0), or a full buffer. + */ + if (have > 0) { + return PyBytes_FromStringAndSize(self->buffer + self->pos, have); + } + + /* Fill the buffer from the raw stream, and copy it to the result. */ + _BufferedReader_reset_buf(self); + r = _BufferedReader_fill_buffer(self); + if (r == -1) + return NULL; + if (r == -2) + r = 0; + self->pos = 0; + return PyBytes_FromStringAndSize(self->buffer, r); +} + +static PyMethodDef BufferedReader_methods[] = { + /* BufferedIOMixin methods */ + {"flush", (PyCFunction)BufferedIOMixin_flush, METH_NOARGS}, + {"close", (PyCFunction)BufferedIOMixin_close, METH_NOARGS}, + {"seekable", (PyCFunction)BufferedIOMixin_seekable, METH_NOARGS}, + {"readable", (PyCFunction)BufferedIOMixin_readable, METH_NOARGS}, + {"writable", (PyCFunction)BufferedIOMixin_writable, METH_NOARGS}, + {"fileno", (PyCFunction)BufferedIOMixin_fileno, METH_NOARGS}, + {"isatty", (PyCFunction)BufferedIOMixin_isatty, METH_NOARGS}, + + {"read", (PyCFunction)Buffered_read, METH_VARARGS}, + {"peek", (PyCFunction)Buffered_peek, METH_VARARGS}, + {"read1", (PyCFunction)Buffered_read1, METH_VARARGS}, + {"readline", (PyCFunction)Buffered_readline, METH_VARARGS}, + {"seek", (PyCFunction)Buffered_seek, METH_VARARGS}, + {"tell", (PyCFunction)Buffered_tell, METH_NOARGS}, + {"truncate", (PyCFunction)Buffered_truncate, METH_VARARGS}, + {NULL, NULL} +}; + +static PyMemberDef BufferedReader_members[] = { + {"raw", T_OBJECT, offsetof(BufferedObject, raw), 0}, + {NULL} +}; + +static PyGetSetDef BufferedReader_getset[] = { + {"closed", (getter)BufferedIOMixin_closed_get, NULL, NULL}, + {"name", (getter)BufferedIOMixin_name_get, NULL, NULL}, + {"mode", (getter)BufferedIOMixin_mode_get, NULL, NULL}, + {0} +}; + + +PyTypeObject PyBufferedReader_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.BufferedReader", /*tp_name*/ + sizeof(BufferedObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)BufferedObject_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + BufferedReader_doc, /* tp_doc */ + (traverseproc)Buffered_traverse, /* tp_traverse */ + (inquiry)Buffered_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(BufferedObject, weakreflist), /*tp_weaklistoffset*/ + 0, /* tp_iter */ + (iternextfunc)Buffered_iternext, /* tp_iternext */ + BufferedReader_methods, /* tp_methods */ + BufferedReader_members, /* tp_members */ + BufferedReader_getset, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(BufferedObject, dict), /* tp_dictoffset */ + (initproc)BufferedReader_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + + + +static int +complain_about_max_buffer_size(void) +{ + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "max_buffer_size is deprecated", 1) < 0) + return 0; + return 1; +} + +/* + * class BufferedWriter + */ +PyDoc_STRVAR(BufferedWriter_doc, + "A buffer for a writeable sequential RawIO object.\n" + "\n" + "The constructor creates a BufferedWriter for the given writeable raw\n" + "stream. If the buffer_size is not given, it defaults to\n" + "DEFAULT_BUFFER_SIZE. max_buffer_size isn't used anymore.\n" + ); + +static void +_BufferedWriter_reset_buf(BufferedObject *self) +{ + self->write_pos = 0; + self->write_end = -1; +} + +static int +BufferedWriter_init(BufferedObject *self, PyObject *args, PyObject *kwds) +{ + /* TODO: properly deprecate max_buffer_size */ + char *kwlist[] = {"raw", "buffer_size", "max_buffer_size", NULL}; + Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; + Py_ssize_t max_buffer_size = -234; + PyObject *raw; + + self->ok = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|nn:BufferedReader", kwlist, + &raw, &buffer_size, &max_buffer_size)) { + return -1; + } + + if (max_buffer_size != -234 && !complain_about_max_buffer_size()) + return -1; + + if (_PyIOBase_checkWritable(raw, Py_True) == NULL) + return -1; + + Py_CLEAR(self->raw); + Py_INCREF(raw); + self->raw = raw; + self->readable = 0; + self->writable = 1; + + self->buffer_size = buffer_size; + if (_Buffered_init(self) < 0) + return -1; + _BufferedWriter_reset_buf(self); + self->pos = 0; + + self->ok = 1; + return 0; +} + +static Py_ssize_t +_BufferedWriter_raw_write(BufferedObject *self, char *start, Py_ssize_t len) +{ + Py_buffer buf; + PyObject *memobj, *res; + Py_ssize_t n; + /* NOTE: the buffer needn't be released as its object is NULL. */ + if (PyBuffer_FillInfo(&buf, NULL, start, len, 1, PyBUF_CONTIG_RO) == -1) + return -1; + memobj = PyMemoryView_FromBuffer(&buf); + if (memobj == NULL) + return -1; + res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_write, memobj, NULL); + Py_DECREF(memobj); + if (res == NULL) + return -1; + n = PyNumber_AsSsize_t(res, PyExc_ValueError); + Py_DECREF(res); + if (n < 0 || n > len) { + PyErr_Format(PyExc_IOError, + "raw write() returned invalid length %zd " + "(should have been between 0 and %zd)", n, len); + return -1; + } + if (n > 0 && self->abs_pos != -1) + self->abs_pos += n; + return n; +} + +/* `restore_pos` is 1 if we need to restore the raw stream position at + the end, 0 otherwise. */ +static PyObject * +_BufferedWriter_flush_unlocked(BufferedObject *self, int restore_pos) +{ + Py_ssize_t written = 0; + Py_off_t n, rewind; + + if (!VALID_WRITE_BUFFER(self) || self->write_pos == self->write_end) + goto end; + /* First, rewind */ + rewind = RAW_OFFSET(self) + (self->pos - self->write_pos); + if (rewind != 0) { + n = _Buffered_raw_seek(self, -rewind, 1); + if (n < 0) { + goto error; + } + self->raw_pos -= rewind; + } + while (self->write_pos < self->write_end) { + n = _BufferedWriter_raw_write(self, + self->buffer + self->write_pos, + Py_SAFE_DOWNCAST(self->write_end - self->write_pos, + Py_off_t, Py_ssize_t)); + if (n == -1) { + Py_ssize_t *w = _Buffered_check_blocking_error(); + if (w == NULL) + goto error; + self->write_pos += *w; + self->raw_pos = self->write_pos; + written += *w; + *w = written; + /* Already re-raised */ + goto error; + } + self->write_pos += n; + self->raw_pos = self->write_pos; + written += Py_SAFE_DOWNCAST(n, Py_off_t, Py_ssize_t); + } + + if (restore_pos) { + Py_off_t forward = rewind - written; + if (forward != 0) { + n = _Buffered_raw_seek(self, forward, 1); + if (n < 0) { + goto error; + } + self->raw_pos += forward; + } + } + _BufferedWriter_reset_buf(self); + +end: + Py_RETURN_NONE; + +error: + return NULL; +} + +static PyObject * +BufferedWriter_write(BufferedObject *self, PyObject *args) +{ + PyObject *res = NULL; + Py_buffer buf; + Py_ssize_t written, avail, remaining, n; + + CHECK_INITIALIZED(self) + if (!PyArg_ParseTuple(args, "y*:write", &buf)) { + return NULL; + } + + if (BufferedIOMixin_closed(self)) { + PyErr_SetString(PyExc_ValueError, "write to closed file"); + PyBuffer_Release(&buf); + return NULL; + } + + ENTER_BUFFERED(self) + + /* Fast path: the data to write can be fully buffered. */ + if (!VALID_READ_BUFFER(self) && !VALID_WRITE_BUFFER(self)) { + self->pos = 0; + self->raw_pos = 0; + } + avail = Py_SAFE_DOWNCAST(self->buffer_size - self->pos, Py_off_t, Py_ssize_t); + if (buf.len <= avail) { + memcpy(self->buffer + self->pos, buf.buf, buf.len); + if (!VALID_WRITE_BUFFER(self)) { + self->write_pos = self->pos; + } + ADJUST_POSITION(self, self->pos + buf.len); + if (self->pos > self->write_end) + self->write_end = self->pos; + written = buf.len; + goto end; + } + + /* First write the current buffer */ + res = _BufferedWriter_flush_unlocked(self, 0); + if (res == NULL) { + Py_ssize_t *w = _Buffered_check_blocking_error(); + if (w == NULL) + goto error; + if (self->readable) + _BufferedReader_reset_buf(self); + /* Make some place by shifting the buffer. */ + assert(VALID_WRITE_BUFFER(self)); + memmove(self->buffer, self->buffer + self->write_pos, + Py_SAFE_DOWNCAST(self->write_end - self->write_pos, + Py_off_t, Py_ssize_t)); + self->write_end -= self->write_pos; + self->raw_pos -= self->write_pos; + self->pos -= self->write_pos; + self->write_pos = 0; + avail = Py_SAFE_DOWNCAST(self->buffer_size - self->write_end, + Py_off_t, Py_ssize_t); + if (buf.len <= avail) { + /* Everything can be buffered */ + PyErr_Clear(); + memcpy(self->buffer + self->write_end, buf.buf, buf.len); + self->write_end += buf.len; + written = buf.len; + goto end; + } + /* Buffer as much as possible. */ + memcpy(self->buffer + self->write_end, buf.buf, avail); + self->write_end += avail; + /* Already re-raised */ + *w = avail; + goto error; + } + Py_CLEAR(res); + + /* Then write buf itself. At this point the buffer has been emptied. */ + remaining = buf.len; + written = 0; + while (remaining > self->buffer_size) { + n = _BufferedWriter_raw_write( + self, (char *) buf.buf + written, buf.len - written); + if (n == -1) { + Py_ssize_t *w = _Buffered_check_blocking_error(); + if (w == NULL) + goto error; + written += *w; + remaining -= *w; + if (remaining > self->buffer_size) { + /* Can't buffer everything, still buffer as much as possible */ + memcpy(self->buffer, + (char *) buf.buf + written, self->buffer_size); + self->raw_pos = 0; + ADJUST_POSITION(self, self->buffer_size); + self->write_end = self->buffer_size; + *w = written + self->buffer_size; + /* Already re-raised */ + goto error; + } + PyErr_Clear(); + break; + } + written += n; + remaining -= n; + } + if (self->readable) + _BufferedReader_reset_buf(self); + if (remaining > 0) { + memcpy(self->buffer, (char *) buf.buf + written, remaining); + written += remaining; + } + self->write_pos = 0; + /* TODO: sanity check (remaining >= 0) */ + self->write_end = remaining; + ADJUST_POSITION(self, remaining); + self->raw_pos = 0; + +end: + res = PyLong_FromSsize_t(written); + +error: + LEAVE_BUFFERED(self) + PyBuffer_Release(&buf); + return res; +} + +static PyMethodDef BufferedWriter_methods[] = { + /* BufferedIOMixin methods */ + {"close", (PyCFunction)BufferedIOMixin_close, METH_NOARGS}, + {"seekable", (PyCFunction)BufferedIOMixin_seekable, METH_NOARGS}, + {"readable", (PyCFunction)BufferedIOMixin_readable, METH_NOARGS}, + {"writable", (PyCFunction)BufferedIOMixin_writable, METH_NOARGS}, + {"fileno", (PyCFunction)BufferedIOMixin_fileno, METH_NOARGS}, + {"isatty", (PyCFunction)BufferedIOMixin_isatty, METH_NOARGS}, + + {"write", (PyCFunction)BufferedWriter_write, METH_VARARGS}, + {"truncate", (PyCFunction)Buffered_truncate, METH_VARARGS}, + {"flush", (PyCFunction)Buffered_flush, METH_NOARGS}, + {"seek", (PyCFunction)Buffered_seek, METH_VARARGS}, + {"tell", (PyCFunction)Buffered_tell, METH_NOARGS}, + {NULL, NULL} +}; + +static PyMemberDef BufferedWriter_members[] = { + {"raw", T_OBJECT, offsetof(BufferedObject, raw), 0}, + {NULL} +}; + +static PyGetSetDef BufferedWriter_getset[] = { + {"closed", (getter)BufferedIOMixin_closed_get, NULL, NULL}, + {"name", (getter)BufferedIOMixin_name_get, NULL, NULL}, + {"mode", (getter)BufferedIOMixin_mode_get, NULL, NULL}, + {0} +}; + + +PyTypeObject PyBufferedWriter_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.BufferedWriter", /*tp_name*/ + sizeof(BufferedObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)BufferedObject_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + BufferedWriter_doc, /* tp_doc */ + (traverseproc)Buffered_traverse, /* tp_traverse */ + (inquiry)Buffered_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(BufferedObject, weakreflist), /*tp_weaklistoffset*/ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BufferedWriter_methods, /* tp_methods */ + BufferedWriter_members, /* tp_members */ + BufferedWriter_getset, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(BufferedObject, dict), /* tp_dictoffset */ + (initproc)BufferedWriter_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + + + +/* + * BufferedRWPair + */ + +PyDoc_STRVAR(BufferedRWPair_doc, + "A buffered reader and writer object together.\n" + "\n" + "A buffered reader object and buffered writer object put together to\n" + "form a sequential IO object that can read and write. This is typically\n" + "used with a socket or two-way pipe.\n" + "\n" + "reader and writer are RawIOBase objects that are readable and\n" + "writeable respectively. If the buffer_size is omitted it defaults to\n" + "DEFAULT_BUFFER_SIZE.\n" + ); + +/* XXX The usefulness of this (compared to having two separate IO objects) is + * questionable. + */ + +typedef struct { + PyObject_HEAD + BufferedObject *reader; + BufferedObject *writer; + PyObject *dict; + PyObject *weakreflist; +} BufferedRWPairObject; + +static int +BufferedRWPair_init(BufferedRWPairObject *self, PyObject *args, + PyObject *kwds) +{ + PyObject *reader, *writer; + Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; + Py_ssize_t max_buffer_size = -234; + + if (!PyArg_ParseTuple(args, "OO|nn:BufferedRWPair", &reader, &writer, + &buffer_size, &max_buffer_size)) { + return -1; + } + + if (max_buffer_size != -234 && !complain_about_max_buffer_size()) + return -1; + + if (_PyIOBase_checkReadable(reader, Py_True) == NULL) + return -1; + if (_PyIOBase_checkWritable(writer, Py_True) == NULL) + return -1; + + args = Py_BuildValue("(n)", buffer_size); + if (args == NULL) { + Py_CLEAR(self->reader); + return -1; + } + self->reader = (BufferedObject *)PyType_GenericNew( + &PyBufferedReader_Type, args, NULL); + Py_DECREF(args); + if (self->reader == NULL) + return -1; + + args = Py_BuildValue("(n)", buffer_size); + if (args == NULL) { + Py_CLEAR(self->reader); + return -1; + } + self->writer = (BufferedObject *)PyType_GenericNew( + &PyBufferedWriter_Type, args, NULL); + Py_DECREF(args); + if (self->writer == NULL) { + Py_CLEAR(self->reader); + return -1; + } + return 0; +} + +static int +BufferedRWPair_traverse(BufferedRWPairObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->dict); + return 0; +} + +static int +BufferedRWPair_clear(BufferedRWPairObject *self) +{ + Py_CLEAR(self->reader); + Py_CLEAR(self->writer); + Py_CLEAR(self->dict); + return 0; +} + +static void +BufferedRWPair_dealloc(BufferedRWPairObject *self) +{ + _PyObject_GC_UNTRACK(self); + Py_CLEAR(self->reader); + Py_CLEAR(self->writer); + Py_CLEAR(self->dict); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +_forward_call(BufferedObject *self, const char *name, PyObject *args) +{ + PyObject *func = PyObject_GetAttrString((PyObject *)self, name); + PyObject *ret; + + if (func == NULL) { + PyErr_SetString(PyExc_AttributeError, name); + return NULL; + } + + ret = PyObject_CallObject(func, args); + Py_DECREF(func); + return ret; +} + +static PyObject * +BufferedRWPair_read(BufferedRWPairObject *self, PyObject *args) +{ + return _forward_call(self->reader, "read", args); +} + +static PyObject * +BufferedRWPair_peek(BufferedRWPairObject *self, PyObject *args) +{ + return _forward_call(self->reader, "peek", args); +} + +static PyObject * +BufferedRWPair_read1(BufferedRWPairObject *self, PyObject *args) +{ + return _forward_call(self->reader, "read1", args); +} + +static PyObject * +BufferedRWPair_write(BufferedRWPairObject *self, PyObject *args) +{ + return _forward_call(self->writer, "write", args); +} + +static PyObject * +BufferedRWPair_flush(BufferedRWPairObject *self, PyObject *args) +{ + return _forward_call(self->writer, "flush", args); +} + +static PyObject * +BufferedRWPair_readable(BufferedRWPairObject *self, PyObject *args) +{ + return _forward_call(self->reader, "readable", args); +} + +static PyObject * +BufferedRWPair_writable(BufferedRWPairObject *self, PyObject *args) +{ + return _forward_call(self->writer, "writable", args); +} + +static PyObject * +BufferedRWPair_close(BufferedRWPairObject *self, PyObject *args) +{ + PyObject *ret = _forward_call(self->writer, "close", args); + if (ret == NULL) + return NULL; + Py_DECREF(ret); + + return _forward_call(self->reader, "close", args); +} + +static PyObject * +BufferedRWPair_isatty(BufferedRWPairObject *self, PyObject *args) +{ + PyObject *ret = _forward_call(self->writer, "isatty", args); + + if (ret != Py_False) { + /* either True or exception */ + return ret; + } + Py_DECREF(ret); + + return _forward_call(self->reader, "isatty", args); +} + + +static PyMethodDef BufferedRWPair_methods[] = { + {"read", (PyCFunction)BufferedRWPair_read, METH_VARARGS}, + {"peek", (PyCFunction)BufferedRWPair_peek, METH_VARARGS}, + {"read1", (PyCFunction)BufferedRWPair_read1, METH_VARARGS}, + {"readinto", (PyCFunction)Buffered_readinto, METH_VARARGS}, + + {"write", (PyCFunction)BufferedRWPair_write, METH_VARARGS}, + {"flush", (PyCFunction)BufferedRWPair_flush, METH_NOARGS}, + + {"readable", (PyCFunction)BufferedRWPair_readable, METH_NOARGS}, + {"writable", (PyCFunction)BufferedRWPair_writable, METH_NOARGS}, + + {"close", (PyCFunction)BufferedRWPair_close, METH_NOARGS}, + {"isatty", (PyCFunction)BufferedRWPair_isatty, METH_NOARGS}, + + {NULL, NULL} +}; + +PyTypeObject PyBufferedRWPair_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.BufferedRWPair", /*tp_name*/ + sizeof(BufferedRWPairObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)BufferedRWPair_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + BufferedRWPair_doc, /* tp_doc */ + (traverseproc)BufferedRWPair_traverse, /* tp_traverse */ + (inquiry)BufferedRWPair_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(BufferedRWPairObject, weakreflist), /*tp_weaklistoffset*/ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BufferedRWPair_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(BufferedRWPairObject, dict), /* tp_dictoffset */ + (initproc)BufferedRWPair_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + + + +/* + * BufferedRandom + */ + +PyDoc_STRVAR(BufferedRandom_doc, + "A buffered interface to random access streams.\n" + "\n" + "The constructor creates a reader and writer for a seekable stream,\n" + "raw, given in the first argument. If the buffer_size is omitted it\n" + "defaults to DEFAULT_BUFFER_SIZE. max_buffer_size isn't used anymore.\n" + ); + +static int +BufferedRandom_init(BufferedObject *self, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"raw", "buffer_size", "max_buffer_size", NULL}; + Py_ssize_t buffer_size = DEFAULT_BUFFER_SIZE; + Py_ssize_t max_buffer_size = -234; + PyObject *raw; + + self->ok = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|nn:BufferedReader", kwlist, + &raw, &buffer_size, &max_buffer_size)) { + return -1; + } + + if (max_buffer_size != -234 && !complain_about_max_buffer_size()) + return -1; + + if (_PyIOBase_checkSeekable(raw, Py_True) == NULL) + return -1; + if (_PyIOBase_checkReadable(raw, Py_True) == NULL) + return -1; + if (_PyIOBase_checkWritable(raw, Py_True) == NULL) + return -1; + + Py_CLEAR(self->raw); + Py_INCREF(raw); + self->raw = raw; + self->buffer_size = buffer_size; + self->readable = 1; + self->writable = 1; + + if (_Buffered_init(self) < 0) + return -1; + _BufferedReader_reset_buf(self); + _BufferedWriter_reset_buf(self); + self->pos = 0; + + self->ok = 1; + return 0; +} + +static PyMethodDef BufferedRandom_methods[] = { + /* BufferedIOMixin methods */ + {"close", (PyCFunction)BufferedIOMixin_close, METH_NOARGS}, + {"seekable", (PyCFunction)BufferedIOMixin_seekable, METH_NOARGS}, + {"readable", (PyCFunction)BufferedIOMixin_readable, METH_NOARGS}, + {"writable", (PyCFunction)BufferedIOMixin_writable, METH_NOARGS}, + {"fileno", (PyCFunction)BufferedIOMixin_fileno, METH_NOARGS}, + {"isatty", (PyCFunction)BufferedIOMixin_isatty, METH_NOARGS}, + + {"flush", (PyCFunction)Buffered_flush, METH_NOARGS}, + + {"seek", (PyCFunction)Buffered_seek, METH_VARARGS}, + {"tell", (PyCFunction)Buffered_tell, METH_NOARGS}, + {"truncate", (PyCFunction)Buffered_truncate, METH_VARARGS}, + {"read", (PyCFunction)Buffered_read, METH_VARARGS}, + {"read1", (PyCFunction)Buffered_read1, METH_VARARGS}, + {"readinto", (PyCFunction)Buffered_readinto, METH_VARARGS}, + {"readline", (PyCFunction)Buffered_readline, METH_VARARGS}, + {"peek", (PyCFunction)Buffered_peek, METH_VARARGS}, + {"write", (PyCFunction)BufferedWriter_write, METH_VARARGS}, + {NULL, NULL} +}; + +static PyMemberDef BufferedRandom_members[] = { + {"raw", T_OBJECT, offsetof(BufferedObject, raw), 0}, + {NULL} +}; + +static PyGetSetDef BufferedRandom_getset[] = { + {"closed", (getter)BufferedIOMixin_closed_get, NULL, NULL}, + {"name", (getter)BufferedIOMixin_name_get, NULL, NULL}, + {"mode", (getter)BufferedIOMixin_mode_get, NULL, NULL}, + {0} +}; + + +PyTypeObject PyBufferedRandom_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.BufferedRandom", /*tp_name*/ + sizeof(BufferedObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)BufferedObject_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + BufferedRandom_doc, /* tp_doc */ + (traverseproc)Buffered_traverse, /* tp_traverse */ + (inquiry)Buffered_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(BufferedObject, weakreflist), /*tp_weaklistoffset*/ + 0, /* tp_iter */ + (iternextfunc)Buffered_iternext, /* tp_iternext */ + BufferedRandom_methods, /* tp_methods */ + BufferedRandom_members, /* tp_members */ + BufferedRandom_getset, /* tp_getset */ + 0, /* tp_base */ + 0, /*tp_dict*/ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(BufferedObject, dict), /*tp_dictoffset*/ + (initproc)BufferedRandom_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c new file mode 100644 index 0000000..0068cb7 --- /dev/null +++ b/Modules/_io/bytesio.c @@ -0,0 +1,757 @@ +#include "Python.h" +#include "structmember.h" /* for offsetof() */ +#include "_iomodule.h" + +typedef struct { + PyObject_HEAD + char *buf; + Py_ssize_t pos; + Py_ssize_t string_size; + size_t buf_size; + PyObject *dict; + PyObject *weakreflist; +} BytesIOObject; + +#define CHECK_CLOSED(self) \ + if ((self)->buf == NULL) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on closed file."); \ + return NULL; \ + } + +/* Internal routine to get a line from the buffer of a BytesIO + object. Returns the length between the current position to the + next newline character. */ +static Py_ssize_t +get_line(BytesIOObject *self, char **output) +{ + char *n; + const char *str_end; + Py_ssize_t len; + + assert(self->buf != NULL); + + /* Move to the end of the line, up to the end of the string, s. */ + str_end = self->buf + self->string_size; + for (n = self->buf + self->pos; + n < str_end && *n != '\n'; + n++); + + /* Skip the newline character */ + if (n < str_end) + n++; + + /* Get the length from the current position to the end of the line. */ + len = n - (self->buf + self->pos); + *output = self->buf + self->pos; + + assert(len >= 0); + assert(self->pos < PY_SSIZE_T_MAX - len); + self->pos += len; + + return len; +} + +/* Internal routine for changing the size of the buffer of BytesIO objects. + The caller should ensure that the 'size' argument is non-negative. Returns + 0 on success, -1 otherwise. */ +static int +resize_buffer(BytesIOObject *self, size_t size) +{ + /* Here, unsigned types are used to avoid dealing with signed integer + overflow, which is undefined in C. */ + size_t alloc = self->buf_size; + char *new_buf = NULL; + + assert(self->buf != NULL); + + /* For simplicity, stay in the range of the signed type. Anyway, Python + doesn't allow strings to be longer than this. */ + if (size > PY_SSIZE_T_MAX) + goto overflow; + + if (size < alloc / 2) { + /* Major downsize; resize down to exact size. */ + alloc = size + 1; + } + else if (size < alloc) { + /* Within allocated size; quick exit */ + return 0; + } + else if (size <= alloc * 1.125) { + /* Moderate upsize; overallocate similar to list_resize() */ + alloc = size + (size >> 3) + (size < 9 ? 3 : 6); + } + else { + /* Major upsize; resize up to exact size */ + alloc = size + 1; + } + + if (alloc > ((size_t)-1) / sizeof(char)) + goto overflow; + new_buf = (char *)PyMem_Realloc(self->buf, alloc * sizeof(char)); + if (new_buf == NULL) { + PyErr_NoMemory(); + return -1; + } + self->buf_size = alloc; + self->buf = new_buf; + + return 0; + + overflow: + PyErr_SetString(PyExc_OverflowError, + "new buffer size too large"); + return -1; +} + +/* Internal routine for writing a string of bytes to the buffer of a BytesIO + object. Returns the number of bytes wrote, or -1 on error. */ +static Py_ssize_t +write_bytes(BytesIOObject *self, const char *bytes, Py_ssize_t len) +{ + assert(self->buf != NULL); + assert(self->pos >= 0); + assert(len >= 0); + + if ((size_t)self->pos + len > self->buf_size) { + if (resize_buffer(self, (size_t)self->pos + len) < 0) + return -1; + } + + if (self->pos > self->string_size) { + /* In case of overseek, pad with null bytes the buffer region between + the end of stream and the current position. + + 0 lo string_size hi + | |<---used--->|<----------available----------->| + | | <--to pad-->|<---to write---> | + 0 buf position + */ + memset(self->buf + self->string_size, '\0', + (self->pos - self->string_size) * sizeof(char)); + } + + /* Copy the data to the internal buffer, overwriting some of the existing + data if self->pos < self->string_size. */ + memcpy(self->buf + self->pos, bytes, len); + self->pos += len; + + /* Set the new length of the internal string if it has changed. */ + if (self->string_size < self->pos) { + self->string_size = self->pos; + } + + return len; +} + +static PyObject * +bytesio_get_closed(BytesIOObject *self) +{ + if (self->buf == NULL) { + Py_RETURN_TRUE; + } + else { + Py_RETURN_FALSE; + } +} + +/* Generic getter for the writable, readable and seekable properties */ +static PyObject * +return_true(BytesIOObject *self) +{ + Py_RETURN_TRUE; +} + +PyDoc_STRVAR(flush_doc, +"flush() -> None. Does nothing."); + +static PyObject * +bytesio_flush(BytesIOObject *self) +{ + Py_RETURN_NONE; +} + +PyDoc_STRVAR(getval_doc, +"getvalue() -> bytes.\n" +"\n" +"Retrieve the entire contents of the BytesIO object."); + +static PyObject * +bytesio_getvalue(BytesIOObject *self) +{ + CHECK_CLOSED(self); + return PyBytes_FromStringAndSize(self->buf, self->string_size); +} + +PyDoc_STRVAR(isatty_doc, +"isatty() -> False.\n" +"\n" +"Always returns False since BytesIO objects are not connected\n" +"to a tty-like device."); + +static PyObject * +bytesio_isatty(BytesIOObject *self) +{ + CHECK_CLOSED(self); + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(tell_doc, +"tell() -> current file position, an integer\n"); + +static PyObject * +bytesio_tell(BytesIOObject *self) +{ + CHECK_CLOSED(self); + return PyLong_FromSsize_t(self->pos); +} + +PyDoc_STRVAR(read_doc, +"read([size]) -> read at most size bytes, returned as a string.\n" +"\n" +"If the size argument is negative, read until EOF is reached.\n" +"Return an empty string at EOF."); + +static PyObject * +bytesio_read(BytesIOObject *self, PyObject *args) +{ + Py_ssize_t size, n; + char *output; + PyObject *arg = Py_None; + + CHECK_CLOSED(self); + + if (!PyArg_ParseTuple(args, "|O:read", &arg)) + return NULL; + + if (PyLong_Check(arg)) { + size = PyLong_AsSsize_t(arg); + if (size == -1 && PyErr_Occurred()) + return NULL; + } + else if (arg == Py_None) { + /* Read until EOF is reached, by default. */ + size = -1; + } + else { + PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", + Py_TYPE(arg)->tp_name); + return NULL; + } + + /* adjust invalid sizes */ + n = self->string_size - self->pos; + if (size < 0 || size > n) { + size = n; + if (size < 0) + size = 0; + } + + assert(self->buf != NULL); + output = self->buf + self->pos; + self->pos += size; + + return PyBytes_FromStringAndSize(output, size); +} + + +PyDoc_STRVAR(read1_doc, +"read1(size) -> read at most size bytes, returned as a string.\n" +"\n" +"If the size argument is negative or omitted, read until EOF is reached.\n" +"Return an empty string at EOF."); + +static PyObject * +bytesio_read1(BytesIOObject *self, PyObject *n) +{ + PyObject *arg, *res; + + arg = PyTuple_Pack(1, n); + if (arg == NULL) + return NULL; + res = bytesio_read(self, arg); + Py_DECREF(arg); + return res; +} + +PyDoc_STRVAR(readline_doc, +"readline([size]) -> next line from the file, as a string.\n" +"\n" +"Retain newline. A non-negative size argument limits the maximum\n" +"number of bytes to return (an incomplete line may be returned then).\n" +"Return an empty string at EOF.\n"); + +static PyObject * +bytesio_readline(BytesIOObject *self, PyObject *args) +{ + Py_ssize_t size, n; + char *output; + PyObject *arg = Py_None; + + CHECK_CLOSED(self); + + if (!PyArg_ParseTuple(args, "|O:readline", &arg)) + return NULL; + + if (PyLong_Check(arg)) { + size = PyLong_AsSsize_t(arg); + if (size == -1 && PyErr_Occurred()) + return NULL; + } + else if (arg == Py_None) { + /* No size limit, by default. */ + size = -1; + } + else { + PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", + Py_TYPE(arg)->tp_name); + return NULL; + } + + n = get_line(self, &output); + + if (size >= 0 && size < n) { + size = n - size; + n -= size; + self->pos -= size; + } + + return PyBytes_FromStringAndSize(output, n); +} + +PyDoc_STRVAR(readlines_doc, +"readlines([size]) -> list of strings, each a line from the file.\n" +"\n" +"Call readline() repeatedly and return a list of the lines so read.\n" +"The optional size argument, if given, is an approximate bound on the\n" +"total number of bytes in the lines returned.\n"); + +static PyObject * +bytesio_readlines(BytesIOObject *self, PyObject *args) +{ + Py_ssize_t maxsize, size, n; + PyObject *result, *line; + char *output; + PyObject *arg = Py_None; + + CHECK_CLOSED(self); + + if (!PyArg_ParseTuple(args, "|O:readlines", &arg)) + return NULL; + + if (PyLong_Check(arg)) { + maxsize = PyLong_AsSsize_t(arg); + if (maxsize == -1 && PyErr_Occurred()) + return NULL; + } + else if (arg == Py_None) { + /* No size limit, by default. */ + maxsize = -1; + } + else { + PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", + Py_TYPE(arg)->tp_name); + return NULL; + } + + size = 0; + result = PyList_New(0); + if (!result) + return NULL; + + while ((n = get_line(self, &output)) != 0) { + line = PyBytes_FromStringAndSize(output, n); + if (!line) + goto on_error; + if (PyList_Append(result, line) == -1) { + Py_DECREF(line); + goto on_error; + } + Py_DECREF(line); + size += n; + if (maxsize > 0 && size >= maxsize) + break; + } + return result; + + on_error: + Py_DECREF(result); + return NULL; +} + +PyDoc_STRVAR(readinto_doc, +"readinto(bytearray) -> int. Read up to len(b) bytes into b.\n" +"\n" +"Returns number of bytes read (0 for EOF), or None if the object\n" +"is set not to block as has no data to read."); + +static PyObject * +bytesio_readinto(BytesIOObject *self, PyObject *buffer) +{ + void *raw_buffer; + Py_ssize_t len; + + CHECK_CLOSED(self); + + if (PyObject_AsWriteBuffer(buffer, &raw_buffer, &len) == -1) + return NULL; + + if (self->pos + len > self->string_size) + len = self->string_size - self->pos; + + memcpy(raw_buffer, self->buf + self->pos, len); + assert(self->pos + len < PY_SSIZE_T_MAX); + assert(len >= 0); + self->pos += len; + + return PyLong_FromSsize_t(len); +} + +PyDoc_STRVAR(truncate_doc, +"truncate([size]) -> int. Truncate the file to at most size bytes.\n" +"\n" +"Size defaults to the current file position, as returned by tell().\n" +"Returns the new size. Imply an absolute seek to the position size."); + +static PyObject * +bytesio_truncate(BytesIOObject *self, PyObject *args) +{ + Py_ssize_t size; + PyObject *arg = Py_None; + + CHECK_CLOSED(self); + + if (!PyArg_ParseTuple(args, "|O:truncate", &arg)) + return NULL; + + if (PyLong_Check(arg)) { + size = PyLong_AsSsize_t(arg); + if (size == -1 && PyErr_Occurred()) + return NULL; + } + else if (arg == Py_None) { + /* Truncate to current position if no argument is passed. */ + size = self->pos; + } + else { + PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", + Py_TYPE(arg)->tp_name); + return NULL; + } + + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "negative size value %zd", size); + return NULL; + } + + if (size < self->string_size) { + self->string_size = size; + if (resize_buffer(self, size) < 0) + return NULL; + } + self->pos = size; + + return PyLong_FromSsize_t(size); +} + +static PyObject * +bytesio_iternext(BytesIOObject *self) +{ + char *next; + Py_ssize_t n; + + CHECK_CLOSED(self); + + n = get_line(self, &next); + + if (!next || n == 0) + return NULL; + + return PyBytes_FromStringAndSize(next, n); +} + +PyDoc_STRVAR(seek_doc, +"seek(pos, whence=0) -> int. Change stream position.\n" +"\n" +"Seek to byte offset pos relative to position indicated by whence:\n" +" 0 Start of stream (the default). pos should be >= 0;\n" +" 1 Current position - pos may be negative;\n" +" 2 End of stream - pos usually negative.\n" +"Returns the new absolute position."); + +static PyObject * +bytesio_seek(BytesIOObject *self, PyObject *args) +{ + Py_ssize_t pos; + int mode = 0; + + CHECK_CLOSED(self); + + if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &mode)) + return NULL; + + if (pos < 0 && mode == 0) { + PyErr_Format(PyExc_ValueError, + "negative seek value %zd", pos); + return NULL; + } + + /* mode 0: offset relative to beginning of the string. + mode 1: offset relative to current position. + mode 2: offset relative the end of the string. */ + if (mode == 1) { + if (pos > PY_SSIZE_T_MAX - self->pos) { + PyErr_SetString(PyExc_OverflowError, + "new position too large"); + return NULL; + } + pos += self->pos; + } + else if (mode == 2) { + if (pos > PY_SSIZE_T_MAX - self->string_size) { + PyErr_SetString(PyExc_OverflowError, + "new position too large"); + return NULL; + } + pos += self->string_size; + } + else if (mode != 0) { + PyErr_Format(PyExc_ValueError, + "invalid whence (%i, should be 0, 1 or 2)", mode); + return NULL; + } + + if (pos < 0) + pos = 0; + self->pos = pos; + + return PyLong_FromSsize_t(self->pos); +} + +PyDoc_STRVAR(write_doc, +"write(bytes) -> int. Write bytes to file.\n" +"\n" +"Return the number of bytes written."); + +static PyObject * +bytesio_write(BytesIOObject *self, PyObject *obj) +{ + Py_ssize_t n = 0; + Py_buffer buf; + PyObject *result = NULL; + + CHECK_CLOSED(self); + + if (PyObject_GetBuffer(obj, &buf, PyBUF_CONTIG_RO) < 0) + return NULL; + + if (buf.len != 0) + n = write_bytes(self, buf.buf, buf.len); + if (n >= 0) + result = PyLong_FromSsize_t(n); + + PyBuffer_Release(&buf); + return result; +} + +PyDoc_STRVAR(writelines_doc, +"writelines(sequence_of_strings) -> None. Write strings to the file.\n" +"\n" +"Note that newlines are not added. The sequence can be any iterable\n" +"object producing strings. This is equivalent to calling write() for\n" +"each string."); + +static PyObject * +bytesio_writelines(BytesIOObject *self, PyObject *v) +{ + PyObject *it, *item; + PyObject *ret; + + CHECK_CLOSED(self); + + it = PyObject_GetIter(v); + if (it == NULL) + return NULL; + + while ((item = PyIter_Next(it)) != NULL) { + ret = bytesio_write(self, item); + Py_DECREF(item); + if (ret == NULL) { + Py_DECREF(it); + return NULL; + } + Py_DECREF(ret); + } + Py_DECREF(it); + + /* See if PyIter_Next failed */ + if (PyErr_Occurred()) + return NULL; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(close_doc, +"close() -> None. Disable all I/O operations."); + +static PyObject * +bytesio_close(BytesIOObject *self) +{ + if (self->buf != NULL) { + PyMem_Free(self->buf); + self->buf = NULL; + } + Py_RETURN_NONE; +} + +static void +bytesio_dealloc(BytesIOObject *self) +{ + if (self->buf != NULL) { + PyMem_Free(self->buf); + self->buf = NULL; + } + Py_TYPE(self)->tp_clear((PyObject *)self); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + BytesIOObject *self; + + assert(type != NULL && type->tp_alloc != NULL); + self = (BytesIOObject *)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + + self->string_size = 0; + self->pos = 0; + self->buf_size = 0; + self->buf = (char *)PyMem_Malloc(0); + if (self->buf == NULL) { + Py_DECREF(self); + return PyErr_NoMemory(); + } + + return (PyObject *)self; +} + +static int +bytesio_init(BytesIOObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *initvalue = NULL; + + if (!PyArg_ParseTuple(args, "|O:BytesIO", &initvalue)) + return -1; + + /* In case, __init__ is called multiple times. */ + self->string_size = 0; + self->pos = 0; + + if (initvalue && initvalue != Py_None) { + PyObject *res; + res = bytesio_write(self, initvalue); + if (res == NULL) + return -1; + Py_DECREF(res); + self->pos = 0; + } + + return 0; +} + +static int +bytesio_traverse(BytesIOObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->dict); + Py_VISIT(self->weakreflist); + return 0; +} + +static int +bytesio_clear(BytesIOObject *self) +{ + Py_CLEAR(self->dict); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *)self); + return 0; +} + + +static PyGetSetDef bytesio_getsetlist[] = { + {"closed", (getter)bytesio_get_closed, NULL, + "True if the file is closed."}, + {0}, /* sentinel */ +}; + +static struct PyMethodDef bytesio_methods[] = { + {"readable", (PyCFunction)return_true, METH_NOARGS, NULL}, + {"seekable", (PyCFunction)return_true, METH_NOARGS, NULL}, + {"writable", (PyCFunction)return_true, METH_NOARGS, NULL}, + {"close", (PyCFunction)bytesio_close, METH_NOARGS, close_doc}, + {"flush", (PyCFunction)bytesio_flush, METH_NOARGS, flush_doc}, + {"isatty", (PyCFunction)bytesio_isatty, METH_NOARGS, isatty_doc}, + {"tell", (PyCFunction)bytesio_tell, METH_NOARGS, tell_doc}, + {"write", (PyCFunction)bytesio_write, METH_O, write_doc}, + {"writelines", (PyCFunction)bytesio_writelines, METH_O, writelines_doc}, + {"read1", (PyCFunction)bytesio_read1, METH_O, read1_doc}, + {"readinto", (PyCFunction)bytesio_readinto, METH_O, readinto_doc}, + {"readline", (PyCFunction)bytesio_readline, METH_VARARGS, readline_doc}, + {"readlines", (PyCFunction)bytesio_readlines, METH_VARARGS, readlines_doc}, + {"read", (PyCFunction)bytesio_read, METH_VARARGS, read_doc}, + {"getvalue", (PyCFunction)bytesio_getvalue, METH_VARARGS, getval_doc}, + {"seek", (PyCFunction)bytesio_seek, METH_VARARGS, seek_doc}, + {"truncate", (PyCFunction)bytesio_truncate, METH_VARARGS, truncate_doc}, + {NULL, NULL} /* sentinel */ +}; + +PyDoc_STRVAR(bytesio_doc, +"BytesIO([buffer]) -> object\n" +"\n" +"Create a buffered I/O implementation using an in-memory bytes\n" +"buffer, ready for reading and writing."); + +PyTypeObject PyBytesIO_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.BytesIO", /*tp_name*/ + sizeof(BytesIOObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)bytesio_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + bytesio_doc, /*tp_doc*/ + (traverseproc)bytesio_traverse, /*tp_traverse*/ + (inquiry)bytesio_clear, /*tp_clear*/ + 0, /*tp_richcompare*/ + offsetof(BytesIOObject, weakreflist), /*tp_weaklistoffset*/ + PyObject_SelfIter, /*tp_iter*/ + (iternextfunc)bytesio_iternext, /*tp_iternext*/ + bytesio_methods, /*tp_methods*/ + 0, /*tp_members*/ + bytesio_getsetlist, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + offsetof(BytesIOObject, dict), /*tp_dictoffset*/ + (initproc)bytesio_init, /*tp_init*/ + 0, /*tp_alloc*/ + bytesio_new, /*tp_new*/ +}; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c new file mode 100644 index 0000000..c0f5c90 --- /dev/null +++ b/Modules/_io/fileio.c @@ -0,0 +1,1034 @@ +/* Author: Daniel Stutzbach */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include +#include +#include +#include /* For offsetof */ +#include "_iomodule.h" + +/* + * Known likely problems: + * + * - Files larger then 2**32-1 + * - Files with unicode filenames + * - Passing numbers greater than 2**32-1 when an integer is expected + * - Making it work on Windows and other oddball platforms + * + * To Do: + * + * - autoconfify header file inclusion + */ + +#ifdef MS_WINDOWS +/* can simulate truncate with Win32 API functions; see file_truncate */ +#define HAVE_FTRUNCATE +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#if BUFSIZ < (8*1024) +#define SMALLCHUNK (8*1024) +#elif (BUFSIZ >= (2 << 25)) +#error "unreasonable BUFSIZ > 64MB defined" +#else +#define SMALLCHUNK BUFSIZ +#endif + +#if SIZEOF_INT < 4 +#define BIGCHUNK (512 * 32) +#else +#define BIGCHUNK (512 * 1024) +#endif + +typedef struct { + PyObject_HEAD + int fd; + unsigned readable : 1; + unsigned writable : 1; + int seekable : 2; /* -1 means unknown */ + int closefd : 1; + PyObject *weakreflist; + PyObject *dict; +} PyFileIOObject; + +PyTypeObject PyFileIO_Type; + +#define PyFileIO_Check(op) (PyObject_TypeCheck((op), &PyFileIO_Type)) + +int +_PyFileIO_closed(PyObject *self) +{ + return ((PyFileIOObject *)self)->fd < 0; +} + +static PyObject * +portable_lseek(int fd, PyObject *posobj, int whence); + +static PyObject *portable_lseek(int fd, PyObject *posobj, int whence); + +/* Returns 0 on success, -1 with exception set on failure. */ +static int +internal_close(PyFileIOObject *self) +{ + int err = 0; + int save_errno = 0; + if (self->fd >= 0) { + int fd = self->fd; + self->fd = -1; + /* fd is accessible and someone else may have closed it */ + if (_PyVerify_fd(fd)) { + Py_BEGIN_ALLOW_THREADS + err = close(fd); + if (err < 0) + save_errno = errno; + Py_END_ALLOW_THREADS + } else { + save_errno = errno; + err = -1; + } + } + if (err < 0) { + errno = save_errno; + PyErr_SetFromErrno(PyExc_IOError); + return -1; + } + return 0; +} + +static PyObject * +fileio_close(PyFileIOObject *self) +{ + if (!self->closefd) { + self->fd = -1; + Py_RETURN_NONE; + } + errno = internal_close(self); + if (errno < 0) + return NULL; + + return PyObject_CallMethod((PyObject*)&PyRawIOBase_Type, + "close", "O", self); +} + +static PyObject * +fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyFileIOObject *self; + + assert(type != NULL && type->tp_alloc != NULL); + + self = (PyFileIOObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->fd = -1; + self->readable = 0; + self->writable = 0; + self->seekable = -1; + self->closefd = 1; + self->weakreflist = NULL; + } + + return (PyObject *) self; +} + +/* On Unix, open will succeed for directories. + In Python, there should be no file objects referring to + directories, so we need a check. */ + +static int +dircheck(PyFileIOObject* self, const char *name) +{ +#if defined(HAVE_FSTAT) && defined(S_IFDIR) && defined(EISDIR) + struct stat buf; + if (self->fd < 0) + return 0; + if (fstat(self->fd, &buf) == 0 && S_ISDIR(buf.st_mode)) { + char *msg = strerror(EISDIR); + PyObject *exc; + if (internal_close(self)) + return -1; + + exc = PyObject_CallFunction(PyExc_IOError, "(iss)", + EISDIR, msg, name); + PyErr_SetObject(PyExc_IOError, exc); + Py_XDECREF(exc); + return -1; + } +#endif + return 0; +} + +static int +check_fd(int fd) +{ +#if defined(HAVE_FSTAT) + struct stat buf; + if (!_PyVerify_fd(fd) || (fstat(fd, &buf) < 0 && errno == EBADF)) { + PyObject *exc; + char *msg = strerror(EBADF); + exc = PyObject_CallFunction(PyExc_OSError, "(is)", + EBADF, msg); + PyErr_SetObject(PyExc_OSError, exc); + Py_XDECREF(exc); + return -1; + } +#endif + return 0; +} + + +static int +fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) +{ + PyFileIOObject *self = (PyFileIOObject *) oself; + static char *kwlist[] = {"file", "mode", "closefd", NULL}; + const char *name = NULL; + PyObject *nameobj, *stringobj = NULL; + char *mode = "r"; + char *s; +#ifdef MS_WINDOWS + Py_UNICODE *widename = NULL; +#endif + int ret = 0; + int rwa = 0, plus = 0, append = 0; + int flags = 0; + int fd = -1; + int closefd = 1; + + assert(PyFileIO_Check(oself)); + if (self->fd >= 0) { + /* Have to close the existing file first. */ + if (internal_close(self) < 0) + return -1; + } + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|si:fileio", + kwlist, &nameobj, &mode, &closefd)) + return -1; + + if (PyFloat_Check(nameobj)) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float"); + return -1; + } + + fd = PyLong_AsLong(nameobj); + if (fd < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "Negative filedescriptor"); + return -1; + } + PyErr_Clear(); + } + +#ifdef Py_WIN_WIDE_FILENAMES + if (GetVersion() < 0x80000000) { + /* On NT, so wide API available */ + if (PyUnicode_Check(nameobj)) + widename = PyUnicode_AS_UNICODE(nameobj); + } + if (widename == NULL) +#endif + if (fd < 0) + { + if (PyBytes_Check(nameobj) || PyByteArray_Check(nameobj)) { + Py_ssize_t namelen; + if (PyObject_AsCharBuffer(nameobj, &name, &namelen) < 0) + return -1; + } + else { + PyObject *u = PyUnicode_FromObject(nameobj); + + if (u == NULL) + return -1; + + stringobj = PyUnicode_AsEncodedString( + u, Py_FileSystemDefaultEncoding, NULL); + Py_DECREF(u); + if (stringobj == NULL) + return -1; + if (!PyBytes_Check(stringobj)) { + PyErr_SetString(PyExc_TypeError, + "encoder failed to return bytes"); + goto error; + } + name = PyBytes_AS_STRING(stringobj); + } + } + + s = mode; + while (*s) { + switch (*s++) { + case 'r': + if (rwa) { + bad_mode: + PyErr_SetString(PyExc_ValueError, + "Must have exactly one of read/write/append mode"); + goto error; + } + rwa = 1; + self->readable = 1; + break; + case 'w': + if (rwa) + goto bad_mode; + rwa = 1; + self->writable = 1; + flags |= O_CREAT | O_TRUNC; + break; + case 'a': + if (rwa) + goto bad_mode; + rwa = 1; + self->writable = 1; + flags |= O_CREAT; + append = 1; + break; + case 'b': + break; + case '+': + if (plus) + goto bad_mode; + self->readable = self->writable = 1; + plus = 1; + break; + default: + PyErr_Format(PyExc_ValueError, + "invalid mode: %.200s", mode); + goto error; + } + } + + if (!rwa) + goto bad_mode; + + if (self->readable && self->writable) + flags |= O_RDWR; + else if (self->readable) + flags |= O_RDONLY; + else + flags |= O_WRONLY; + +#ifdef O_BINARY + flags |= O_BINARY; +#endif + +#ifdef O_APPEND + if (append) + flags |= O_APPEND; +#endif + + if (fd >= 0) { + if (check_fd(fd)) + goto error; + self->fd = fd; + self->closefd = closefd; + } + else { + self->closefd = 1; + if (!closefd) { + PyErr_SetString(PyExc_ValueError, + "Cannot use closefd=False with file name"); + goto error; + } + + Py_BEGIN_ALLOW_THREADS + errno = 0; +#ifdef MS_WINDOWS + if (widename != NULL) + self->fd = _wopen(widename, flags, 0666); + else +#endif + self->fd = open(name, flags, 0666); + Py_END_ALLOW_THREADS + if (self->fd < 0) { +#ifdef MS_WINDOWS + if (widename != NULL) + PyErr_SetFromErrnoWithUnicodeFilename(PyExc_IOError, widename); + else +#endif + PyErr_SetFromErrnoWithFilename(PyExc_IOError, name); + goto error; + } + if(dircheck(self, name) < 0) + goto error; + } + + if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0) + goto error; + + if (append) { + /* For consistent behaviour, we explicitly seek to the + end of file (otherwise, it might be done only on the + first write()). */ + PyObject *pos = portable_lseek(self->fd, NULL, 2); + if (pos == NULL) + goto error; + Py_DECREF(pos); + } + + goto done; + + error: + ret = -1; + + done: + Py_CLEAR(stringobj); + return ret; +} + +static int +fileio_traverse(PyFileIOObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->dict); + return 0; +} + +static int +fileio_clear(PyFileIOObject *self) +{ + Py_CLEAR(self->dict); + return 0; +} + +static void +fileio_dealloc(PyFileIOObject *self) +{ + if (_PyIOBase_finalize((PyObject *) self) < 0) + return; + _PyObject_GC_UNTRACK(self); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + Py_CLEAR(self->dict); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyObject * +err_closed(void) +{ + PyErr_SetString(PyExc_ValueError, "I/O operation on closed file"); + return NULL; +} + +static PyObject * +err_mode(char *action) +{ + PyErr_Format(PyExc_ValueError, "File not open for %s", action); + return NULL; +} + +static PyObject * +fileio_fileno(PyFileIOObject *self) +{ + if (self->fd < 0) + return err_closed(); + return PyLong_FromLong((long) self->fd); +} + +static PyObject * +fileio_readable(PyFileIOObject *self) +{ + if (self->fd < 0) + return err_closed(); + return PyBool_FromLong((long) self->readable); +} + +static PyObject * +fileio_writable(PyFileIOObject *self) +{ + if (self->fd < 0) + return err_closed(); + return PyBool_FromLong((long) self->writable); +} + +static PyObject * +fileio_seekable(PyFileIOObject *self) +{ + if (self->fd < 0) + return err_closed(); + if (self->seekable < 0) { + PyObject *pos = portable_lseek(self->fd, NULL, SEEK_CUR); + if (pos == NULL) { + PyErr_Clear(); + self->seekable = 0; + } else { + Py_DECREF(pos); + self->seekable = 1; + } + } + return PyBool_FromLong((long) self->seekable); +} + +static PyObject * +fileio_readinto(PyFileIOObject *self, PyObject *args) +{ + Py_buffer pbuf; + Py_ssize_t n; + + if (self->fd < 0) + return err_closed(); + if (!self->readable) + return err_mode("reading"); + + if (!PyArg_ParseTuple(args, "w*", &pbuf)) + return NULL; + + if (_PyVerify_fd(self->fd)) { + Py_BEGIN_ALLOW_THREADS + errno = 0; + n = read(self->fd, pbuf.buf, pbuf.len); + Py_END_ALLOW_THREADS + } else + n = -1; + PyBuffer_Release(&pbuf); + if (n < 0) { + if (errno == EAGAIN) + Py_RETURN_NONE; + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } + + return PyLong_FromSsize_t(n); +} + +static size_t +new_buffersize(PyFileIOObject *self, size_t currentsize) +{ +#ifdef HAVE_FSTAT + off_t pos, end; + struct stat st; + if (fstat(self->fd, &st) == 0) { + end = st.st_size; + pos = lseek(self->fd, 0L, SEEK_CUR); + /* Files claiming a size smaller than SMALLCHUNK may + actually be streaming pseudo-files. In this case, we + apply the more aggressive algorithm below. + */ + if (end >= SMALLCHUNK && end >= pos && pos >= 0) { + /* Add 1 so if the file were to grow we'd notice. */ + return currentsize + end - pos + 1; + } + } +#endif + if (currentsize > SMALLCHUNK) { + /* Keep doubling until we reach BIGCHUNK; + then keep adding BIGCHUNK. */ + if (currentsize <= BIGCHUNK) + return currentsize + currentsize; + else + return currentsize + BIGCHUNK; + } + return currentsize + SMALLCHUNK; +} + +static PyObject * +fileio_readall(PyFileIOObject *self) +{ + PyObject *result; + Py_ssize_t total = 0; + int n; + + if (!_PyVerify_fd(self->fd)) + return PyErr_SetFromErrno(PyExc_IOError); + + result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); + if (result == NULL) + return NULL; + + while (1) { + size_t newsize = new_buffersize(self, total); + if (newsize > PY_SSIZE_T_MAX || newsize <= 0) { + PyErr_SetString(PyExc_OverflowError, + "unbounded read returned more bytes " + "than a Python string can hold "); + Py_DECREF(result); + return NULL; + } + + if (PyBytes_GET_SIZE(result) < newsize) { + if (_PyBytes_Resize(&result, newsize) < 0) { + if (total == 0) { + Py_DECREF(result); + return NULL; + } + PyErr_Clear(); + break; + } + } + Py_BEGIN_ALLOW_THREADS + errno = 0; + n = read(self->fd, + PyBytes_AS_STRING(result) + total, + newsize - total); + Py_END_ALLOW_THREADS + if (n == 0) + break; + if (n < 0) { + if (total > 0) + break; + if (errno == EAGAIN) { + Py_DECREF(result); + Py_RETURN_NONE; + } + Py_DECREF(result); + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } + total += n; + } + + if (PyBytes_GET_SIZE(result) > total) { + if (_PyBytes_Resize(&result, total) < 0) { + /* This should never happen, but just in case */ + Py_DECREF(result); + return NULL; + } + } + return result; +} + +static PyObject * +fileio_read(PyFileIOObject *self, PyObject *args) +{ + char *ptr; + Py_ssize_t n; + Py_ssize_t size = -1; + PyObject *bytes; + + if (self->fd < 0) + return err_closed(); + if (!self->readable) + return err_mode("reading"); + + if (!PyArg_ParseTuple(args, "|n", &size)) + return NULL; + + if (size < 0) { + return fileio_readall(self); + } + + bytes = PyBytes_FromStringAndSize(NULL, size); + if (bytes == NULL) + return NULL; + ptr = PyBytes_AS_STRING(bytes); + + if (_PyVerify_fd(self->fd)) { + Py_BEGIN_ALLOW_THREADS + errno = 0; + n = read(self->fd, ptr, size); + Py_END_ALLOW_THREADS + } else + n = -1; + + if (n < 0) { + Py_DECREF(bytes); + if (errno == EAGAIN) + Py_RETURN_NONE; + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } + + if (n != size) { + if (_PyBytes_Resize(&bytes, n) < 0) { + Py_DECREF(bytes); + return NULL; + } + } + + return (PyObject *) bytes; +} + +static PyObject * +fileio_write(PyFileIOObject *self, PyObject *args) +{ + Py_buffer pbuf; + Py_ssize_t n; + + if (self->fd < 0) + return err_closed(); + if (!self->writable) + return err_mode("writing"); + + if (!PyArg_ParseTuple(args, "s*", &pbuf)) + return NULL; + + if (_PyVerify_fd(self->fd)) { + Py_BEGIN_ALLOW_THREADS + errno = 0; + n = write(self->fd, pbuf.buf, pbuf.len); + Py_END_ALLOW_THREADS + } else + n = -1; + + PyBuffer_Release(&pbuf); + + if (n < 0) { + if (errno == EAGAIN) + Py_RETURN_NONE; + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } + + return PyLong_FromSsize_t(n); +} + +/* XXX Windows support below is likely incomplete */ + +/* Cribbed from posix_lseek() */ +static PyObject * +portable_lseek(int fd, PyObject *posobj, int whence) +{ + Py_off_t pos, res; + +#ifdef SEEK_SET + /* Turn 0, 1, 2 into SEEK_{SET,CUR,END} */ + switch (whence) { +#if SEEK_SET != 0 + case 0: whence = SEEK_SET; break; +#endif +#if SEEK_CUR != 1 + case 1: whence = SEEK_CUR; break; +#endif +#if SEEK_END != 2 + case 2: whence = SEEK_END; break; +#endif + } +#endif /* SEEK_SET */ + + if (posobj == NULL) + pos = 0; + else { + if(PyFloat_Check(posobj)) { + PyErr_SetString(PyExc_TypeError, "an integer is required"); + return NULL; + } +#if defined(HAVE_LARGEFILE_SUPPORT) + pos = PyLong_AsLongLong(posobj); +#else + pos = PyLong_AsLong(posobj); +#endif + if (PyErr_Occurred()) + return NULL; + } + + if (_PyVerify_fd(fd)) { + Py_BEGIN_ALLOW_THREADS +#if defined(MS_WIN64) || defined(MS_WINDOWS) + res = _lseeki64(fd, pos, whence); +#else + res = lseek(fd, pos, whence); +#endif + Py_END_ALLOW_THREADS + } else + res = -1; + if (res < 0) + return PyErr_SetFromErrno(PyExc_IOError); + +#if defined(HAVE_LARGEFILE_SUPPORT) + return PyLong_FromLongLong(res); +#else + return PyLong_FromLong(res); +#endif +} + +static PyObject * +fileio_seek(PyFileIOObject *self, PyObject *args) +{ + PyObject *posobj; + int whence = 0; + + if (self->fd < 0) + return err_closed(); + + if (!PyArg_ParseTuple(args, "O|i", &posobj, &whence)) + return NULL; + + return portable_lseek(self->fd, posobj, whence); +} + +static PyObject * +fileio_tell(PyFileIOObject *self, PyObject *args) +{ + if (self->fd < 0) + return err_closed(); + + return portable_lseek(self->fd, NULL, 1); +} + +#ifdef HAVE_FTRUNCATE +static PyObject * +fileio_truncate(PyFileIOObject *self, PyObject *args) +{ + PyObject *posobj = NULL; + Py_off_t pos; + int ret; + int fd; + + fd = self->fd; + if (fd < 0) + return err_closed(); + if (!self->writable) + return err_mode("writing"); + + if (!PyArg_ParseTuple(args, "|O", &posobj)) + return NULL; + + if (posobj == Py_None || posobj == NULL) { + /* Get the current position. */ + posobj = portable_lseek(fd, NULL, 1); + if (posobj == NULL) + return NULL; + } + else { + /* Move to the position to be truncated. */ + posobj = portable_lseek(fd, posobj, 0); + } + if (posobj == NULL) + return NULL; + +#if defined(HAVE_LARGEFILE_SUPPORT) + pos = PyLong_AsLongLong(posobj); +#else + pos = PyLong_AsLong(posobj); +#endif + if (pos == -1 && PyErr_Occurred()) + return NULL; + +#ifdef MS_WINDOWS + /* MS _chsize doesn't work if newsize doesn't fit in 32 bits, + so don't even try using it. */ + { + HANDLE hFile; + + /* Truncate. Note that this may grow the file! */ + Py_BEGIN_ALLOW_THREADS + errno = 0; + hFile = (HANDLE)_get_osfhandle(fd); + ret = hFile == (HANDLE)-1; + if (ret == 0) { + ret = SetEndOfFile(hFile) == 0; + if (ret) + errno = EACCES; + } + Py_END_ALLOW_THREADS + } +#else + Py_BEGIN_ALLOW_THREADS + errno = 0; + ret = ftruncate(fd, pos); + Py_END_ALLOW_THREADS +#endif /* !MS_WINDOWS */ + + if (ret != 0) { + PyErr_SetFromErrno(PyExc_IOError); + return NULL; + } + + return posobj; +} +#endif + +static char * +mode_string(PyFileIOObject *self) +{ + if (self->readable) { + if (self->writable) + return "rb+"; + else + return "rb"; + } + else + return "wb"; +} + +static PyObject * +fileio_repr(PyFileIOObject *self) +{ + if (self->fd < 0) + return PyUnicode_FromFormat("io.FileIO(-1)"); + + return PyUnicode_FromFormat("io.FileIO(%d, '%s')", + self->fd, mode_string(self)); +} + +static PyObject * +fileio_isatty(PyFileIOObject *self) +{ + long res; + + if (self->fd < 0) + return err_closed(); + Py_BEGIN_ALLOW_THREADS + res = isatty(self->fd); + Py_END_ALLOW_THREADS + return PyBool_FromLong(res); +} + + +PyDoc_STRVAR(fileio_doc, +"file(name: str[, mode: str]) -> file IO object\n" +"\n" +"Open a file. The mode can be 'r', 'w' or 'a' for reading (default),\n" +"writing or appending. The file will be created if it doesn't exist\n" +"when opened for writing or appending; it will be truncated when\n" +"opened for writing. Add a '+' to the mode to allow simultaneous\n" +"reading and writing."); + +PyDoc_STRVAR(read_doc, +"read(size: int) -> bytes. read at most size bytes, returned as bytes.\n" +"\n" +"Only makes one system call, so less data may be returned than requested\n" +"In non-blocking mode, returns None if no data is available.\n" +"On end-of-file, returns ''."); + +PyDoc_STRVAR(readall_doc, +"readall() -> bytes. read all data from the file, returned as bytes.\n" +"\n" +"In non-blocking mode, returns as much as is immediately available,\n" +"or None if no data is available. On end-of-file, returns ''."); + +PyDoc_STRVAR(write_doc, +"write(b: bytes) -> int. Write bytes b to file, return number written.\n" +"\n" +"Only makes one system call, so not all of the data may be written.\n" +"The number of bytes actually written is returned."); + +PyDoc_STRVAR(fileno_doc, +"fileno() -> int. \"file descriptor\".\n" +"\n" +"This is needed for lower-level file interfaces, such the fcntl module."); + +PyDoc_STRVAR(seek_doc, +"seek(offset: int[, whence: int]) -> None. Move to new file position.\n" +"\n" +"Argument offset is a byte count. Optional argument whence defaults to\n" +"0 (offset from start of file, offset should be >= 0); other values are 1\n" +"(move relative to current position, positive or negative), and 2 (move\n" +"relative to end of file, usually negative, although many platforms allow\n" +"seeking beyond the end of a file)." +"\n" +"Note that not all file objects are seekable."); + +#ifdef HAVE_FTRUNCATE +PyDoc_STRVAR(truncate_doc, +"truncate([size: int]) -> None. Truncate the file to at most size bytes.\n" +"\n" +"Size defaults to the current file position, as returned by tell()." +"The current file position is changed to the value of size."); +#endif + +PyDoc_STRVAR(tell_doc, +"tell() -> int. Current file position"); + +PyDoc_STRVAR(readinto_doc, +"readinto() -> Same as RawIOBase.readinto()."); + +PyDoc_STRVAR(close_doc, +"close() -> None. Close the file.\n" +"\n" +"A closed file cannot be used for further I/O operations. close() may be\n" +"called more than once without error. Changes the fileno to -1."); + +PyDoc_STRVAR(isatty_doc, +"isatty() -> bool. True if the file is connected to a tty device."); + +PyDoc_STRVAR(seekable_doc, +"seekable() -> bool. True if file supports random-access."); + +PyDoc_STRVAR(readable_doc, +"readable() -> bool. True if file was opened in a read mode."); + +PyDoc_STRVAR(writable_doc, +"writable() -> bool. True if file was opened in a write mode."); + +static PyMethodDef fileio_methods[] = { + {"read", (PyCFunction)fileio_read, METH_VARARGS, read_doc}, + {"readall", (PyCFunction)fileio_readall, METH_NOARGS, readall_doc}, + {"readinto", (PyCFunction)fileio_readinto, METH_VARARGS, readinto_doc}, + {"write", (PyCFunction)fileio_write, METH_VARARGS, write_doc}, + {"seek", (PyCFunction)fileio_seek, METH_VARARGS, seek_doc}, + {"tell", (PyCFunction)fileio_tell, METH_VARARGS, tell_doc}, +#ifdef HAVE_FTRUNCATE + {"truncate", (PyCFunction)fileio_truncate, METH_VARARGS, truncate_doc}, +#endif + {"close", (PyCFunction)fileio_close, METH_NOARGS, close_doc}, + {"seekable", (PyCFunction)fileio_seekable, METH_NOARGS, seekable_doc}, + {"readable", (PyCFunction)fileio_readable, METH_NOARGS, readable_doc}, + {"writable", (PyCFunction)fileio_writable, METH_NOARGS, writable_doc}, + {"fileno", (PyCFunction)fileio_fileno, METH_NOARGS, fileno_doc}, + {"isatty", (PyCFunction)fileio_isatty, METH_NOARGS, isatty_doc}, + {NULL, NULL} /* sentinel */ +}; + +/* 'closed' and 'mode' are attributes for backwards compatibility reasons. */ + +static PyObject * +get_closed(PyFileIOObject *self, void *closure) +{ + return PyBool_FromLong((long)(self->fd < 0)); +} + +static PyObject * +get_closefd(PyFileIOObject *self, void *closure) +{ + return PyBool_FromLong((long)(self->closefd)); +} + +static PyObject * +get_mode(PyFileIOObject *self, void *closure) +{ + return PyUnicode_FromString(mode_string(self)); +} + +static PyGetSetDef fileio_getsetlist[] = { + {"closed", (getter)get_closed, NULL, "True if the file is closed"}, + {"closefd", (getter)get_closefd, NULL, + "True if the file descriptor will be closed"}, + {"mode", (getter)get_mode, NULL, "String giving the file mode"}, + {0}, +}; + +PyTypeObject PyFileIO_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.FileIO", + sizeof(PyFileIOObject), + 0, + (destructor)fileio_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)fileio_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + fileio_doc, /* tp_doc */ + (traverseproc)fileio_traverse, /* tp_traverse */ + (inquiry)fileio_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(PyFileIOObject, weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + fileio_methods, /* tp_methods */ + 0, /* tp_members */ + fileio_getsetlist, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(PyFileIOObject, dict), /* tp_dictoffset */ + fileio_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + fileio_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c new file mode 100644 index 0000000..ef6d516 --- /dev/null +++ b/Modules/_io/iobase.c @@ -0,0 +1,894 @@ +/* + An implementation of the I/O abstract base classes hierarchy + as defined by PEP 3116 - "New I/O" + + Classes defined here: IOBase, RawIOBase. + + Written by Amaury Forgeot d'Arc and Antoine Pitrou +*/ + + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#include "_iomodule.h" + +/* + * IOBase class, an abstract class + */ + +typedef struct { + PyObject_HEAD + + PyObject *dict; + PyObject *weakreflist; +} IOBaseObject; + +PyDoc_STRVAR(IOBase_doc, + "The abstract base class for all I/O classes, acting on streams of\n" + "bytes. There is no public constructor.\n" + "\n" + "This class provides dummy implementations for many methods that\n" + "derived classes can override selectively; the default implementations\n" + "represent a file that cannot be read, written or seeked.\n" + "\n" + "Even though IOBase does not declare read, readinto, or write because\n" + "their signatures will vary, implementations and clients should\n" + "consider those methods part of the interface. Also, implementations\n" + "may raise a IOError when operations they do not support are called.\n" + "\n" + "The basic type used for binary data read from or written to a file is\n" + "bytes. bytearrays are accepted too, and in some cases (such as\n" + "readinto) needed. Text I/O classes work with str data.\n" + "\n" + "Note that calling any method (even inquiries) on a closed stream is\n" + "undefined. Implementations may raise IOError in this case.\n" + "\n" + "IOBase (and its subclasses) support the iterator protocol, meaning\n" + "that an IOBase object can be iterated over yielding the lines in a\n" + "stream.\n" + "\n" + "IOBase also supports the :keyword:`with` statement. In this example,\n" + "fp is closed after the suite of the with statment is complete:\n" + "\n" + "with open('spam.txt', 'r') as fp:\n" + " fp.write('Spam and eggs!')\n"); + +/* Use this macro whenever you want to check the internal `closed` status + of the IOBase object rather than the virtual `closed` attribute as returned + by whatever subclass. */ + +#define IS_CLOSED(self) \ + PyObject_HasAttrString(self, "__IOBase_closed") + +/* Internal methods */ +static PyObject * +IOBase_unsupported(const char *message) +{ + PyErr_SetString(IO_STATE->unsupported_operation, message); + return NULL; +} + +/* Positionning */ + +PyDoc_STRVAR(IOBase_seek_doc, + "Change stream position.\n" + "\n" + "Change the stream position to byte offset offset. offset is\n" + "interpreted relative to the position indicated by whence. Values\n" + "for whence are:\n" + "\n" + "* 0 -- start of stream (the default); offset should be zero or positive\n" + "* 1 -- current stream position; offset may be negative\n" + "* 2 -- end of stream; offset is usually negative\n" + "\n" + "Return the new absolute position."); + +static PyObject * +IOBase_seek(PyObject *self, PyObject *args) +{ + return IOBase_unsupported("seek"); +} + +PyDoc_STRVAR(IOBase_tell_doc, + "Return current stream position."); + +static PyObject * +IOBase_tell(PyObject *self, PyObject *args) +{ + return PyObject_CallMethod(self, "seek", "ii", 0, 1); +} + +PyDoc_STRVAR(IOBase_truncate_doc, + "Truncate file to size bytes.\n" + "\n" + "Size defaults to the current IO position as reported by tell(). Return\n" + "the new size."); + +static PyObject * +IOBase_truncate(PyObject *self, PyObject *args) +{ + return IOBase_unsupported("truncate"); +} + +/* Flush and close methods */ + +PyDoc_STRVAR(IOBase_flush_doc, + "Flush write buffers, if applicable.\n" + "\n" + "This is not implemented for read-only and non-blocking streams.\n"); + +static PyObject * +IOBase_flush(PyObject *self, PyObject *args) +{ + /* XXX Should this return the number of bytes written??? */ + if (IS_CLOSED(self)) { + PyErr_SetString(PyExc_ValueError, "I/O operation on closed file."); + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(IOBase_close_doc, + "Flush and close the IO object.\n" + "\n" + "This method has no effect if the file is already closed.\n"); + +static int +IOBase_closed(PyObject *self) +{ + PyObject *res; + int closed; + /* This gets the derived attribute, which is *not* __IOBase_closed + in most cases! */ + res = PyObject_GetAttr(self, _PyIO_str_closed); + if (res == NULL) + return 0; + closed = PyObject_IsTrue(res); + Py_DECREF(res); + return closed; +} + +static PyObject * +IOBase_closed_get(PyObject *self, void *context) +{ + return PyBool_FromLong(IS_CLOSED(self)); +} + +PyObject * +_PyIOBase_checkClosed(PyObject *self, PyObject *args) +{ + if (IOBase_closed(self)) { + PyErr_SetString(PyExc_ValueError, "I/O operation on closed file."); + return NULL; + } + if (args == Py_True) + return Py_None; + else + Py_RETURN_NONE; +} + +/* XXX: IOBase thinks it has to maintain its own internal state in + `__IOBase_closed` and call flush() by itself, but it is redundant with + whatever behaviour a non-trivial derived class will implement. */ + +static PyObject * +IOBase_close(PyObject *self, PyObject *args) +{ + PyObject *res; + + if (IS_CLOSED(self)) + Py_RETURN_NONE; + + res = PyObject_CallMethodObjArgs(self, _PyIO_str_flush, NULL); + PyObject_SetAttrString(self, "__IOBase_closed", Py_True); + if (res == NULL) { + /* If flush() fails, just give up */ + if (PyErr_ExceptionMatches(PyExc_IOError)) + PyErr_Clear(); + else + return NULL; + } + Py_XDECREF(res); + Py_RETURN_NONE; +} + +/* Finalization and garbage collection support */ + +int +_PyIOBase_finalize(PyObject *self) +{ + PyObject *res; + PyObject *tp, *v, *tb; + int closed = 1; + int is_zombie; + + /* If _PyIOBase_finalize() is called from a destructor, we need to + resurrect the object as calling close() can invoke arbitrary code. */ + is_zombie = (Py_REFCNT(self) == 0); + if (is_zombie) { + ++Py_REFCNT(self); + } + PyErr_Fetch(&tp, &v, &tb); + /* If `closed` doesn't exist or can't be evaluated as bool, then the + object is probably in an unusable state, so ignore. */ + res = PyObject_GetAttr(self, _PyIO_str_closed); + if (res == NULL) + PyErr_Clear(); + else { + closed = PyObject_IsTrue(res); + Py_DECREF(res); + if (closed == -1) + PyErr_Clear(); + } + if (closed == 0) { + res = PyObject_CallMethodObjArgs((PyObject *) self, _PyIO_str_close, + NULL); + /* Silencing I/O errors is bad, but printing spurious tracebacks is + equally as bad, and potentially more frequent (because of + shutdown issues). */ + if (res == NULL) + PyErr_Clear(); + else + Py_DECREF(res); + } + PyErr_Restore(tp, v, tb); + if (is_zombie) { + if (--Py_REFCNT(self) != 0) { + /* The object lives again. The following code is taken from + slot_tp_del in typeobject.c. */ + Py_ssize_t refcnt = Py_REFCNT(self); + _Py_NewReference(self); + Py_REFCNT(self) = refcnt; + /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so + * we need to undo that. */ + _Py_DEC_REFTOTAL; + /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object + * chain, so no more to do there. + * If COUNT_ALLOCS, the original decref bumped tp_frees, and + * _Py_NewReference bumped tp_allocs: both of those need to be + * undone. + */ +#ifdef COUNT_ALLOCS + --Py_TYPE(self)->tp_frees; + --Py_TYPE(self)->tp_allocs; +#endif + return -1; + } + } + return 0; +} + +static int +IOBase_traverse(IOBaseObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->dict); + return 0; +} + +static int +IOBase_clear(IOBaseObject *self) +{ + if (_PyIOBase_finalize((PyObject *) self) < 0) + return -1; + Py_CLEAR(self->dict); + return 0; +} + +/* Destructor */ + +static void +IOBase_dealloc(IOBaseObject *self) +{ + /* NOTE: since IOBaseObject has its own dict, Python-defined attributes + are still available here for close() to use. + However, if the derived class declares a __slots__, those slots are + already gone. + */ + if (_PyIOBase_finalize((PyObject *) self) < 0) { + /* When called from a heap type's dealloc, the type will be + decref'ed on return (see e.g. subtype_dealloc in typeobject.c). */ + if (PyType_HasFeature(Py_TYPE(self), Py_TPFLAGS_HEAPTYPE)) + Py_INCREF(Py_TYPE(self)); + return; + } + _PyObject_GC_UNTRACK(self); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + Py_CLEAR(self->dict); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +/* Inquiry methods */ + +PyDoc_STRVAR(IOBase_seekable_doc, + "Return whether object supports random access.\n" + "\n" + "If False, seek(), tell() and truncate() will raise IOError.\n" + "This method may need to do a test seek()."); + +static PyObject * +IOBase_seekable(PyObject *self, PyObject *args) +{ + Py_RETURN_FALSE; +} + +PyObject * +_PyIOBase_checkSeekable(PyObject *self, PyObject *args) +{ + PyObject *res = PyObject_CallMethodObjArgs(self, _PyIO_str_seekable, NULL); + if (res == NULL) + return NULL; + if (res != Py_True) { + Py_CLEAR(res); + PyErr_SetString(PyExc_IOError, "File or stream is not seekable."); + return NULL; + } + if (args == Py_True) { + Py_DECREF(res); + } + return res; +} + +PyDoc_STRVAR(IOBase_readable_doc, + "Return whether object was opened for reading.\n" + "\n" + "If False, read() will raise IOError."); + +static PyObject * +IOBase_readable(PyObject *self, PyObject *args) +{ + Py_RETURN_FALSE; +} + +/* May be called with any object */ +PyObject * +_PyIOBase_checkReadable(PyObject *self, PyObject *args) +{ + PyObject *res = PyObject_CallMethodObjArgs(self, _PyIO_str_readable, NULL); + if (res == NULL) + return NULL; + if (res != Py_True) { + Py_CLEAR(res); + PyErr_SetString(PyExc_IOError, "File or stream is not readable."); + return NULL; + } + if (args == Py_True) { + Py_DECREF(res); + } + return res; +} + +PyDoc_STRVAR(IOBase_writable_doc, + "Return whether object was opened for writing.\n" + "\n" + "If False, read() will raise IOError."); + +static PyObject * +IOBase_writable(PyObject *self, PyObject *args) +{ + Py_RETURN_FALSE; +} + +/* May be called with any object */ +PyObject * +_PyIOBase_checkWritable(PyObject *self, PyObject *args) +{ + PyObject *res = PyObject_CallMethodObjArgs(self, _PyIO_str_writable, NULL); + if (res == NULL) + return NULL; + if (res != Py_True) { + Py_CLEAR(res); + PyErr_SetString(PyExc_IOError, "File or stream is not writable."); + return NULL; + } + if (args == Py_True) { + Py_DECREF(res); + } + return res; +} + +/* Context manager */ + +static PyObject * +IOBase_enter(PyObject *self, PyObject *args) +{ + if (_PyIOBase_checkClosed(self, Py_True) == NULL) + return NULL; + + Py_INCREF(self); + return self; +} + +static PyObject * +IOBase_exit(PyObject *self, PyObject *args) +{ + return PyObject_CallMethodObjArgs(self, _PyIO_str_close, NULL); +} + +/* Lower-level APIs */ + +/* XXX Should these be present even if unimplemented? */ + +PyDoc_STRVAR(IOBase_fileno_doc, + "Returns underlying file descriptor if one exists.\n" + "\n" + "An IOError is raised if the IO object does not use a file descriptor.\n"); + +static PyObject * +IOBase_fileno(PyObject *self, PyObject *args) +{ + return IOBase_unsupported("fileno"); +} + +PyDoc_STRVAR(IOBase_isatty_doc, + "Return whether this is an 'interactive' stream.\n" + "\n" + "Return False if it can't be determined.\n"); + +static PyObject * +IOBase_isatty(PyObject *self, PyObject *args) +{ + if (_PyIOBase_checkClosed(self, Py_True) == NULL) + return NULL; + Py_RETURN_FALSE; +} + +/* Readline(s) and writelines */ + +PyDoc_STRVAR(IOBase_readline_doc, + "Read and return a line from the stream.\n" + "\n" + "If limit is specified, at most limit bytes will be read.\n" + "\n" + "The line terminator is always b'\n' for binary files; for text\n" + "files, the newlines argument to open can be used to select the line\n" + "terminator(s) recognized.\n"); + +static PyObject * +IOBase_readline(PyObject *self, PyObject *args) +{ + /* For backwards compatibility, a (slowish) readline(). */ + + Py_ssize_t limit = -1; + int has_peek = 0; + PyObject *buffer, *result; + Py_ssize_t old_size = -1; + + if (!PyArg_ParseTuple(args, "|n:readline", &limit)) { + return NULL; + } + + if (PyObject_HasAttrString(self, "peek")) + has_peek = 1; + + buffer = PyByteArray_FromStringAndSize(NULL, 0); + if (buffer == NULL) + return NULL; + + while (limit < 0 || Py_SIZE(buffer) < limit) { + Py_ssize_t nreadahead = 1; + PyObject *b; + + if (has_peek) { + PyObject *readahead = PyObject_CallMethod(self, "peek", "i", 1); + if (readahead == NULL) + goto fail; + if (!PyBytes_Check(readahead)) { + PyErr_Format(PyExc_IOError, + "peek() should have returned a bytes object, " + "not '%.200s'", Py_TYPE(readahead)->tp_name); + Py_DECREF(readahead); + goto fail; + } + if (PyBytes_GET_SIZE(readahead) > 0) { + Py_ssize_t n = 0; + const char *buf = PyBytes_AS_STRING(readahead); + if (limit >= 0) { + do { + if (n >= PyBytes_GET_SIZE(readahead) || n >= limit) + break; + if (buf[n++] == '\n') + break; + } while (1); + } + else { + do { + if (n >= PyBytes_GET_SIZE(readahead)) + break; + if (buf[n++] == '\n') + break; + } while (1); + } + nreadahead = n; + } + Py_DECREF(readahead); + } + + b = PyObject_CallMethod(self, "read", "n", nreadahead); + if (b == NULL) + goto fail; + if (!PyBytes_Check(b)) { + PyErr_Format(PyExc_IOError, + "read() should have returned a bytes object, " + "not '%.200s'", Py_TYPE(b)->tp_name); + Py_DECREF(b); + goto fail; + } + if (PyBytes_GET_SIZE(b) == 0) { + Py_DECREF(b); + break; + } + + old_size = PyByteArray_GET_SIZE(buffer); + PyByteArray_Resize(buffer, old_size + PyBytes_GET_SIZE(b)); + memcpy(PyByteArray_AS_STRING(buffer) + old_size, + PyBytes_AS_STRING(b), PyBytes_GET_SIZE(b)); + + Py_DECREF(b); + + if (PyByteArray_AS_STRING(buffer)[PyByteArray_GET_SIZE(buffer) - 1] == '\n') + break; + } + + result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(buffer), + PyByteArray_GET_SIZE(buffer)); + Py_DECREF(buffer); + return result; + fail: + Py_DECREF(buffer); + return NULL; +} + +static PyObject * +IOBase_iter(PyObject *self) +{ + if (_PyIOBase_checkClosed(self, Py_True) == NULL) + return NULL; + + Py_INCREF(self); + return self; +} + +static PyObject * +IOBase_iternext(PyObject *self) +{ + PyObject *line = PyObject_CallMethodObjArgs(self, _PyIO_str_readline, NULL); + + if (line == NULL) + return NULL; + + if (PyObject_Size(line) == 0) { + Py_DECREF(line); + return NULL; + } + + return line; +} + +PyDoc_STRVAR(IOBase_readlines_doc, + "Return a list of lines from the stream.\n" + "\n" + "hint can be specified to control the number of lines read: no more\n" + "lines will be read if the total size (in bytes/characters) of all\n" + "lines so far exceeds hint."); + +static PyObject * +IOBase_readlines(PyObject *self, PyObject *args) +{ + Py_ssize_t hint = -1, length = 0; + PyObject *hintobj = Py_None, *result; + + if (!PyArg_ParseTuple(args, "|O:readlines", &hintobj)) { + return NULL; + } + if (hintobj != Py_None) { + hint = PyNumber_AsSsize_t(hintobj, PyExc_ValueError); + if (hint == -1 && PyErr_Occurred()) + return NULL; + } + + result = PyList_New(0); + if (result == NULL) + return NULL; + + if (hint <= 0) { + /* XXX special-casing this made sense in the Python version in order + to remove the bytecode interpretation overhead, but it could + probably be removed here. */ + PyObject *ret = PyObject_CallMethod(result, "extend", "O", self); + if (ret == NULL) { + Py_DECREF(result); + return NULL; + } + Py_DECREF(ret); + return result; + } + + while (1) { + PyObject *line = PyIter_Next(self); + if (line == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(result); + return NULL; + } + else + break; /* StopIteration raised */ + } + + if (PyList_Append(result, line) < 0) { + Py_DECREF(line); + Py_DECREF(result); + return NULL; + } + length += PyObject_Size(line); + Py_DECREF(line); + + if (length > hint) + break; + } + return result; +} + +static PyObject * +IOBase_writelines(PyObject *self, PyObject *args) +{ + PyObject *lines, *iter, *res; + + if (!PyArg_ParseTuple(args, "O:writelines", &lines)) { + return NULL; + } + + if (_PyIOBase_checkClosed(self, Py_True) == NULL) + return NULL; + + iter = PyObject_GetIter(lines); + if (iter == NULL) + return NULL; + + while (1) { + PyObject *line = PyIter_Next(iter); + if (line == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(iter); + return NULL; + } + else + break; /* Stop Iteration */ + } + + res = PyObject_CallMethodObjArgs(self, _PyIO_str_write, line, NULL); + Py_DECREF(line); + if (res == NULL) { + Py_DECREF(iter); + return NULL; + } + Py_DECREF(res); + } + Py_DECREF(iter); + Py_RETURN_NONE; +} + +static PyMethodDef IOBase_methods[] = { + {"seek", IOBase_seek, METH_VARARGS, IOBase_seek_doc}, + {"tell", IOBase_tell, METH_NOARGS, IOBase_tell_doc}, + {"truncate", IOBase_truncate, METH_VARARGS, IOBase_truncate_doc}, + {"flush", IOBase_flush, METH_NOARGS, IOBase_flush_doc}, + {"close", IOBase_close, METH_NOARGS, IOBase_close_doc}, + + {"seekable", IOBase_seekable, METH_NOARGS, IOBase_seekable_doc}, + {"readable", IOBase_readable, METH_NOARGS, IOBase_readable_doc}, + {"writable", IOBase_writable, METH_NOARGS, IOBase_writable_doc}, + + {"_checkClosed", _PyIOBase_checkClosed, METH_NOARGS}, + {"_checkSeekable", _PyIOBase_checkSeekable, METH_NOARGS}, + {"_checkReadable", _PyIOBase_checkReadable, METH_NOARGS}, + {"_checkWritable", _PyIOBase_checkWritable, METH_NOARGS}, + + {"fileno", IOBase_fileno, METH_NOARGS, IOBase_fileno_doc}, + {"isatty", IOBase_isatty, METH_NOARGS, IOBase_isatty_doc}, + + {"__enter__", IOBase_enter, METH_NOARGS}, + {"__exit__", IOBase_exit, METH_VARARGS}, + + {"readline", IOBase_readline, METH_VARARGS, IOBase_readline_doc}, + {"readlines", IOBase_readlines, METH_VARARGS, IOBase_readlines_doc}, + {"writelines", IOBase_writelines, METH_VARARGS}, + + {NULL, NULL} +}; + +static PyGetSetDef IOBase_getset[] = { + {"closed", (getter)IOBase_closed_get, NULL, NULL}, + {0} +}; + + +PyTypeObject PyIOBase_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io._IOBase", /*tp_name*/ + sizeof(IOBaseObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)IOBase_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + IOBase_doc, /* tp_doc */ + (traverseproc)IOBase_traverse, /* tp_traverse */ + (inquiry)IOBase_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(IOBaseObject, weakreflist), /* tp_weaklistoffset */ + IOBase_iter, /* tp_iter */ + IOBase_iternext, /* tp_iternext */ + IOBase_methods, /* tp_methods */ + 0, /* tp_members */ + IOBase_getset, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(IOBaseObject, dict), /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + + +/* + * RawIOBase class, Inherits from IOBase. + */ +PyDoc_STRVAR(RawIOBase_doc, + "Base class for raw binary I/O."); + +/* + * The read() method is implemented by calling readinto(); derived classes + * that want to support read() only need to implement readinto() as a + * primitive operation. In general, readinto() can be more efficient than + * read(). + * + * (It would be tempting to also provide an implementation of readinto() in + * terms of read(), in case the latter is a more suitable primitive operation, + * but that would lead to nasty recursion in case a subclass doesn't implement + * either.) +*/ + +static PyObject * +RawIOBase_read(PyObject *self, PyObject *args) +{ + Py_ssize_t n = -1; + PyObject *b, *res; + + if (!PyArg_ParseTuple(args, "|n:read", &n)) { + return NULL; + } + + if (n < 0) + return PyObject_CallMethod(self, "readall", NULL); + + /* TODO: allocate a bytes object directly instead and manually construct + a writable memoryview pointing to it. */ + b = PyByteArray_FromStringAndSize(NULL, n); + if (b == NULL) + return NULL; + + res = PyObject_CallMethodObjArgs(self, _PyIO_str_readinto, b, NULL); + if (res == NULL) { + Py_DECREF(b); + return NULL; + } + + n = PyNumber_AsSsize_t(res, PyExc_ValueError); + Py_DECREF(res); + if (n == -1 && PyErr_Occurred()) { + Py_DECREF(b); + return NULL; + } + + res = PyBytes_FromStringAndSize(PyByteArray_AsString(b), n); + Py_DECREF(b); + return res; +} + + +PyDoc_STRVAR(RawIOBase_readall_doc, + "Read until EOF, using multiple read() call."); + +static PyObject * +RawIOBase_readall(PyObject *self, PyObject *args) +{ + int r; + PyObject *chunks = PyList_New(0); + PyObject *result; + + if (chunks == NULL) + return NULL; + + while (1) { + PyObject *data = PyObject_CallMethod(self, "read", + "i", DEFAULT_BUFFER_SIZE); + if (!data) { + Py_DECREF(chunks); + return NULL; + } + if (!PyBytes_Check(data)) { + Py_DECREF(chunks); + Py_DECREF(data); + PyErr_SetString(PyExc_TypeError, "read() should return bytes"); + return NULL; + } + if (PyBytes_GET_SIZE(data) == 0) { + /* EOF */ + Py_DECREF(data); + break; + } + r = PyList_Append(chunks, data); + Py_DECREF(data); + if (r < 0) { + Py_DECREF(chunks); + return NULL; + } + } + result = _PyBytes_Join(_PyIO_empty_bytes, chunks); + Py_DECREF(chunks); + return result; +} + +static PyMethodDef RawIOBase_methods[] = { + {"read", RawIOBase_read, METH_VARARGS}, + {"readall", RawIOBase_readall, METH_NOARGS, RawIOBase_readall_doc}, + {NULL, NULL} +}; + +PyTypeObject PyRawIOBase_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io._RawIOBase", /*tp_name*/ + 0, /*tp_basicsize*/ + 0, /*tp_itemsize*/ + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + RawIOBase_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + RawIOBase_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyIOBase_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c new file mode 100644 index 0000000..d75a604 --- /dev/null +++ b/Modules/_io/stringio.c @@ -0,0 +1,769 @@ +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#include "_iomodule.h" + +/* Implementation note: the buffer is always at least one character longer + than the enclosed string, for proper functioning of _PyIO_find_line_ending. +*/ + +typedef struct { + PyObject_HEAD + Py_UNICODE *buf; + Py_ssize_t pos; + Py_ssize_t string_size; + size_t buf_size; + + char ok; /* initialized? */ + char closed; + char readuniversal; + char readtranslate; + PyObject *decoder; + PyObject *readnl; + PyObject *writenl; + + PyObject *dict; + PyObject *weakreflist; +} StringIOObject; + +#define CHECK_INITIALIZED(self) \ + if (self->ok <= 0) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on uninitialized object"); \ + return NULL; \ + } + +#define CHECK_CLOSED(self) \ + if (self->closed) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on closed file"); \ + return NULL; \ + } + +PyDoc_STRVAR(stringio_doc, + "Text I/O implementation using an in-memory buffer.\n" + "\n" + "The initial_value argument sets the value of object. The newline\n" + "argument is like the one of TextIOWrapper's constructor."); + + +/* Internal routine for changing the size, in terms of characters, of the + buffer of StringIO objects. The caller should ensure that the 'size' + argument is non-negative. Returns 0 on success, -1 otherwise. */ +static int +resize_buffer(StringIOObject *self, size_t size) +{ + /* Here, unsigned types are used to avoid dealing with signed integer + overflow, which is undefined in C. */ + size_t alloc = self->buf_size; + Py_UNICODE *new_buf = NULL; + + assert(self->buf != NULL); + + /* Reserve one more char for line ending detection. */ + size = size + 1; + /* For simplicity, stay in the range of the signed type. Anyway, Python + doesn't allow strings to be longer than this. */ + if (size > PY_SSIZE_T_MAX) + goto overflow; + + if (size < alloc / 2) { + /* Major downsize; resize down to exact size. */ + alloc = size + 1; + } + else if (size < alloc) { + /* Within allocated size; quick exit */ + return 0; + } + else if (size <= alloc * 1.125) { + /* Moderate upsize; overallocate similar to list_resize() */ + alloc = size + (size >> 3) + (size < 9 ? 3 : 6); + } + else { + /* Major upsize; resize up to exact size */ + alloc = size + 1; + } + + if (alloc > ((size_t)-1) / sizeof(Py_UNICODE)) + goto overflow; + new_buf = (Py_UNICODE *)PyMem_Realloc(self->buf, + alloc * sizeof(Py_UNICODE)); + if (new_buf == NULL) { + PyErr_NoMemory(); + return -1; + } + self->buf_size = alloc; + self->buf = new_buf; + + return 0; + + overflow: + PyErr_SetString(PyExc_OverflowError, + "new buffer size too large"); + return -1; +} + +/* Internal routine for writing a whole PyUnicode object to the buffer of a + StringIO object. Returns 0 on success, or -1 on error. */ +static Py_ssize_t +write_str(StringIOObject *self, PyObject *obj) +{ + Py_UNICODE *str; + Py_ssize_t len; + PyObject *decoded = NULL; + assert(self->buf != NULL); + assert(self->pos >= 0); + + if (self->decoder != NULL) { + decoded = _PyIncrementalNewlineDecoder_decode( + self->decoder, obj, 1 /* always final */); + } + else { + decoded = obj; + Py_INCREF(decoded); + } + if (self->writenl) { + PyObject *translated = PyUnicode_Replace( + decoded, _PyIO_str_nl, self->writenl, -1); + Py_DECREF(decoded); + decoded = translated; + } + if (decoded == NULL) + return -1; + + assert(PyUnicode_Check(decoded)); + str = PyUnicode_AS_UNICODE(decoded); + len = PyUnicode_GET_SIZE(decoded); + + assert(len >= 0); + + /* This overflow check is not strictly necessary. However, it avoids us to + deal with funky things like comparing an unsigned and a signed + integer. */ + if (self->pos > PY_SSIZE_T_MAX - len) { + PyErr_SetString(PyExc_OverflowError, + "new position too large"); + goto fail; + } + if (self->pos + len > self->string_size) { + if (resize_buffer(self, self->pos + len) < 0) + goto fail; + } + + if (self->pos > self->string_size) { + /* In case of overseek, pad with null bytes the buffer region between + the end of stream and the current position. + + 0 lo string_size hi + | |<---used--->|<----------available----------->| + | | <--to pad-->|<---to write---> | + 0 buf positon + + */ + memset(self->buf + self->string_size, '\0', + (self->pos - self->string_size) * sizeof(Py_UNICODE)); + } + + /* Copy the data to the internal buffer, overwriting some of the + existing data if self->pos < self->string_size. */ + memcpy(self->buf + self->pos, str, len * sizeof(Py_UNICODE)); + self->pos += len; + + /* Set the new length of the internal string if it has changed. */ + if (self->string_size < self->pos) { + self->string_size = self->pos; + } + + Py_DECREF(decoded); + return 0; + +fail: + Py_XDECREF(decoded); + return -1; +} + +PyDoc_STRVAR(stringio_getvalue_doc, + "Retrieve the entire contents of the object."); + +static PyObject * +stringio_getvalue(StringIOObject *self) +{ + CHECK_INITIALIZED(self); + CHECK_CLOSED(self); + return PyUnicode_FromUnicode(self->buf, self->string_size); +} + +PyDoc_STRVAR(stringio_tell_doc, + "Tell the current file position."); + +static PyObject * +stringio_tell(StringIOObject *self) +{ + CHECK_INITIALIZED(self); + CHECK_CLOSED(self); + return PyLong_FromSsize_t(self->pos); +} + +PyDoc_STRVAR(stringio_read_doc, + "Read at most n characters, returned as a string.\n" + "\n" + "If the argument is negative or omitted, read until EOF\n" + "is reached. Return an empty string at EOF.\n"); + +static PyObject * +stringio_read(StringIOObject *self, PyObject *args) +{ + Py_ssize_t size, n; + Py_UNICODE *output; + PyObject *arg = Py_None; + + CHECK_INITIALIZED(self); + if (!PyArg_ParseTuple(args, "|O:read", &arg)) + return NULL; + CHECK_CLOSED(self); + + if (PyNumber_Check(arg)) { + size = PyNumber_AsSsize_t(arg, PyExc_OverflowError); + if (size == -1 && PyErr_Occurred()) + return NULL; + } + else if (arg == Py_None) { + /* Read until EOF is reached, by default. */ + size = -1; + } + else { + PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", + Py_TYPE(arg)->tp_name); + return NULL; + } + + /* adjust invalid sizes */ + n = self->string_size - self->pos; + if (size < 0 || size > n) { + size = n; + if (size < 0) + size = 0; + } + + output = self->buf + self->pos; + self->pos += size; + return PyUnicode_FromUnicode(output, size); +} + +/* Internal helper, used by stringio_readline and stringio_iternext */ +static PyObject * +_stringio_readline(StringIOObject *self, Py_ssize_t limit) +{ + Py_UNICODE *start, *end, old_char; + Py_ssize_t len, consumed; + + /* In case of overseek, return the empty string */ + if (self->pos >= self->string_size) + return PyUnicode_FromString(""); + + start = self->buf + self->pos; + if (limit < 0 || limit > self->string_size - self->pos) + limit = self->string_size - self->pos; + + end = start + limit; + old_char = *end; + *end = '\0'; + len = _PyIO_find_line_ending( + self->readtranslate, self->readuniversal, self->readnl, + start, end, &consumed); + *end = old_char; + /* If we haven't found any line ending, we just return everything + (`consumed` is ignored). */ + if (len < 0) + len = limit; + self->pos += len; + return PyUnicode_FromUnicode(start, len); +} + +PyDoc_STRVAR(stringio_readline_doc, + "Read until newline or EOF.\n" + "\n" + "Returns an empty string if EOF is hit immediately.\n"); + +static PyObject * +stringio_readline(StringIOObject *self, PyObject *args) +{ + PyObject *arg = Py_None; + Py_ssize_t limit = -1; + + CHECK_INITIALIZED(self); + if (!PyArg_ParseTuple(args, "|O:readline", &arg)) + return NULL; + CHECK_CLOSED(self); + + if (PyNumber_Check(arg)) { + limit = PyNumber_AsSsize_t(arg, PyExc_OverflowError); + if (limit == -1 && PyErr_Occurred()) + return NULL; + } + else if (arg != Py_None) { + PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", + Py_TYPE(arg)->tp_name); + return NULL; + } + return _stringio_readline(self, limit); +} + +static PyObject * +stringio_iternext(StringIOObject *self) +{ + PyObject *line; + + CHECK_INITIALIZED(self); + CHECK_CLOSED(self); + + if (Py_TYPE(self) == &PyStringIO_Type) { + /* Skip method call overhead for speed */ + line = _stringio_readline(self, -1); + } + else { + /* XXX is subclassing StringIO really supported? */ + line = PyObject_CallMethodObjArgs((PyObject *)self, + _PyIO_str_readline, NULL); + if (line && !PyUnicode_Check(line)) { + PyErr_Format(PyExc_IOError, + "readline() should have returned an str object, " + "not '%.200s'", Py_TYPE(line)->tp_name); + Py_DECREF(line); + return NULL; + } + } + + if (line == NULL) + return NULL; + + if (PyUnicode_GET_SIZE(line) == 0) { + /* Reached EOF */ + Py_DECREF(line); + return NULL; + } + + return line; +} + +PyDoc_STRVAR(stringio_truncate_doc, + "Truncate size to pos.\n" + "\n" + "The pos argument defaults to the current file position, as\n" + "returned by tell(). Imply an absolute seek to pos.\n" + "Returns the new absolute position.\n"); + +static PyObject * +stringio_truncate(StringIOObject *self, PyObject *args) +{ + Py_ssize_t size; + PyObject *arg = Py_None; + + CHECK_INITIALIZED(self); + if (!PyArg_ParseTuple(args, "|O:truncate", &arg)) + return NULL; + CHECK_CLOSED(self); + + if (PyNumber_Check(arg)) { + size = PyNumber_AsSsize_t(arg, PyExc_OverflowError); + if (size == -1 && PyErr_Occurred()) + return NULL; + } + else if (arg == Py_None) { + /* Truncate to current position if no argument is passed. */ + size = self->pos; + } + else { + PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", + Py_TYPE(arg)->tp_name); + return NULL; + } + + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "Negative size value %zd", size); + return NULL; + } + + if (size < self->string_size) { + if (resize_buffer(self, size) < 0) + return NULL; + self->string_size = size; + } + self->pos = size; + + return PyLong_FromSsize_t(size); +} + +PyDoc_STRVAR(stringio_seek_doc, + "Change stream position.\n" + "\n" + "Seek to character offset pos relative to position indicated by whence:\n" + " 0 Start of stream (the default). pos should be >= 0;\n" + " 1 Current position - pos must be 0;\n" + " 2 End of stream - pos must be 0.\n" + "Returns the new absolute position.\n"); + +static PyObject * +stringio_seek(StringIOObject *self, PyObject *args) +{ + Py_ssize_t pos; + int mode = 0; + + CHECK_INITIALIZED(self); + if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &mode)) + return NULL; + CHECK_CLOSED(self); + + if (mode != 0 && mode != 1 && mode != 2) { + PyErr_Format(PyExc_ValueError, + "Invalid whence (%i, should be 0, 1 or 2)", mode); + return NULL; + } + else if (pos < 0 && mode == 0) { + PyErr_Format(PyExc_ValueError, + "Negative seek position %zd", pos); + return NULL; + } + else if (mode != 0 && pos != 0) { + PyErr_SetString(PyExc_IOError, + "Can't do nonzero cur-relative seeks"); + return NULL; + } + + /* mode 0: offset relative to beginning of the string. + mode 1: no change to current position. + mode 2: change position to end of file. */ + if (mode == 1) { + pos = self->pos; + } + else if (mode == 2) { + pos = self->string_size; + } + + self->pos = pos; + + return PyLong_FromSsize_t(self->pos); +} + +PyDoc_STRVAR(stringio_write_doc, + "Write string to file.\n" + "\n" + "Returns the number of characters written, which is always equal to\n" + "the length of the string.\n"); + +static PyObject * +stringio_write(StringIOObject *self, PyObject *obj) +{ + Py_ssize_t size; + + CHECK_INITIALIZED(self); + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, "string argument expected, got '%s'", + Py_TYPE(obj)->tp_name); + return NULL; + } + CHECK_CLOSED(self); + size = PyUnicode_GET_SIZE(obj); + + if (size > 0 && write_str(self, obj) < 0) + return NULL; + + return PyLong_FromSsize_t(size); +} + +PyDoc_STRVAR(stringio_close_doc, + "Close the IO object. Attempting any further operation after the\n" + "object is closed will raise a ValueError.\n" + "\n" + "This method has no effect if the file is already closed.\n"); + +static PyObject * +stringio_close(StringIOObject *self) +{ + self->closed = 1; + /* Free up some memory */ + if (resize_buffer(self, 0) < 0) + return NULL; + Py_CLEAR(self->readnl); + Py_CLEAR(self->writenl); + Py_CLEAR(self->decoder); + Py_RETURN_NONE; +} + +static int +stringio_traverse(StringIOObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->dict); + return 0; +} + +static int +stringio_clear(StringIOObject *self) +{ + Py_CLEAR(self->dict); + return 0; +} + +static void +stringio_dealloc(StringIOObject *self) +{ + _PyObject_GC_UNTRACK(self); + Py_CLEAR(self->readnl); + Py_CLEAR(self->writenl); + Py_CLEAR(self->decoder); + if (self->buf) + PyMem_Free(self->buf); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +stringio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + StringIOObject *self; + + assert(type != NULL && type->tp_alloc != NULL); + self = (StringIOObject *)type->tp_alloc(type, 0); + if (self == NULL) + return NULL; + + self->string_size = 0; + self->pos = 0; + self->buf_size = 0; + self->buf = (Py_UNICODE *)PyMem_Malloc(0); + if (self->buf == NULL) { + Py_DECREF(self); + return PyErr_NoMemory(); + } + + return (PyObject *)self; +} + +static int +stringio_init(StringIOObject *self, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"initial_value", "newline", NULL}; + PyObject *value = NULL; + char *newline = "\n"; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oz:__init__", kwlist, + &value, &newline)) + return -1; + + if (newline && newline[0] != '\0' + && !(newline[0] == '\n' && newline[1] == '\0') + && !(newline[0] == '\r' && newline[1] == '\0') + && !(newline[0] == '\r' && newline[1] == '\n' && newline[2] == '\0')) { + PyErr_Format(PyExc_ValueError, + "illegal newline value: %s", newline); + return -1; + } + if (value && value != Py_None && !PyUnicode_Check(value)) { + PyErr_Format(PyExc_ValueError, + "initial_value must be str or None, not %.200s", + Py_TYPE(value)->tp_name); + return -1; + } + + self->ok = 0; + + Py_CLEAR(self->readnl); + Py_CLEAR(self->writenl); + Py_CLEAR(self->decoder); + + if (newline) { + self->readnl = PyUnicode_FromString(newline); + if (self->readnl == NULL) + return -1; + } + self->readuniversal = (newline == NULL || newline[0] == '\0'); + self->readtranslate = (newline == NULL); + /* If newline == "", we don't translate anything. + If newline == "\n" or newline == None, we translate to "\n", which is + a no-op. + (for newline == None, TextIOWrapper translates to os.sepline, but it + is pointless for StringIO) + */ + if (newline != NULL && newline[0] == '\r') { + self->writenl = self->readnl; + Py_INCREF(self->writenl); + } + + if (self->readuniversal) { + self->decoder = PyObject_CallFunction( + (PyObject *)&PyIncrementalNewlineDecoder_Type, + "Oi", Py_None, (int) self->readtranslate); + if (self->decoder == NULL) + return -1; + } + + /* Now everything is set up, resize buffer to size of initial value, + and copy it */ + self->string_size = 0; + if (value && value != Py_None) { + Py_ssize_t len = PyUnicode_GetSize(value); + /* This is a heuristic, for newline translation might change + the string length. */ + if (resize_buffer(self, len) < 0) + return -1; + self->pos = 0; + if (write_str(self, value) < 0) + return -1; + } + else { + if (resize_buffer(self, 0) < 0) + return -1; + } + self->pos = 0; + + self->closed = 0; + self->ok = 1; + return 0; +} + +/* Properties and pseudo-properties */ +static PyObject * +stringio_seekable(StringIOObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self); + Py_RETURN_TRUE; +} + +static PyObject * +stringio_readable(StringIOObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self); + Py_RETURN_TRUE; +} + +static PyObject * +stringio_writable(StringIOObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self); + Py_RETURN_TRUE; +} + +static PyObject * +stringio_buffer(StringIOObject *self, void *context) +{ + PyErr_SetString(IO_STATE->unsupported_operation, + "buffer attribute is unsupported on type StringIO"); + return NULL; +} + +static PyObject * +stringio_closed(StringIOObject *self, void *context) +{ + CHECK_INITIALIZED(self); + return PyBool_FromLong(self->closed); +} + +static PyObject * +stringio_encoding(StringIOObject *self, void *context) +{ + CHECK_INITIALIZED(self); + CHECK_CLOSED(self); + Py_RETURN_NONE; +} + +static PyObject * +stringio_errors(StringIOObject *self, void *context) +{ + CHECK_INITIALIZED(self); + CHECK_CLOSED(self); + return PyUnicode_FromString("strict"); +} + +static PyObject * +stringio_line_buffering(StringIOObject *self, void *context) +{ + CHECK_INITIALIZED(self); + CHECK_CLOSED(self); + Py_RETURN_FALSE; +} + +static PyObject * +stringio_newlines(StringIOObject *self, void *context) +{ + CHECK_INITIALIZED(self); + CHECK_CLOSED(self); + if (self->decoder == NULL) + Py_RETURN_NONE; + return PyObject_GetAttr(self->decoder, _PyIO_str_newlines); +} + +static struct PyMethodDef stringio_methods[] = { + {"close", (PyCFunction)stringio_close, METH_NOARGS, stringio_close_doc}, + {"getvalue", (PyCFunction)stringio_getvalue, METH_VARARGS, stringio_getvalue_doc}, + {"read", (PyCFunction)stringio_read, METH_VARARGS, stringio_read_doc}, + {"readline", (PyCFunction)stringio_readline, METH_VARARGS, stringio_readline_doc}, + {"tell", (PyCFunction)stringio_tell, METH_NOARGS, stringio_tell_doc}, + {"truncate", (PyCFunction)stringio_truncate, METH_VARARGS, stringio_truncate_doc}, + {"seek", (PyCFunction)stringio_seek, METH_VARARGS, stringio_seek_doc}, + {"write", (PyCFunction)stringio_write, METH_O, stringio_write_doc}, + + {"seekable", (PyCFunction)stringio_seekable, METH_NOARGS}, + {"readable", (PyCFunction)stringio_readable, METH_NOARGS}, + {"writable", (PyCFunction)stringio_writable, METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; + +static PyGetSetDef stringio_getset[] = { + {"closed", (getter)stringio_closed, NULL, NULL}, + {"newlines", (getter)stringio_newlines, NULL, NULL}, + /* (following comments straight off of the original Python wrapper:) + XXX Cruft to support the TextIOWrapper API. This would only + be meaningful if StringIO supported the buffer attribute. + Hopefully, a better solution, than adding these pseudo-attributes, + will be found. + */ + {"buffer", (getter)stringio_buffer, NULL, NULL}, + {"encoding", (getter)stringio_encoding, NULL, NULL}, + {"errors", (getter)stringio_errors, NULL, NULL}, + {"line_buffering", (getter)stringio_line_buffering, NULL, NULL}, + {0} +}; + +PyTypeObject PyStringIO_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.StringIO", /*tp_name*/ + sizeof(StringIOObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)stringio_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + stringio_doc, /*tp_doc*/ + (traverseproc)stringio_traverse, /*tp_traverse*/ + (inquiry)stringio_clear, /*tp_clear*/ + 0, /*tp_richcompare*/ + offsetof(StringIOObject, weakreflist), /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + (iternextfunc)stringio_iternext, /*tp_iternext*/ + stringio_methods, /*tp_methods*/ + 0, /*tp_members*/ + stringio_getset, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + offsetof(StringIOObject, dict), /*tp_dictoffset*/ + (initproc)stringio_init, /*tp_init*/ + 0, /*tp_alloc*/ + stringio_new, /*tp_new*/ +}; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c new file mode 100644 index 0000000..cc229a8 --- /dev/null +++ b/Modules/_io/textio.c @@ -0,0 +1,2420 @@ +/* + An implementation of Text I/O as defined by PEP 3116 - "New I/O" + + Classes defined here: TextIOBase, IncrementalNewlineDecoder, TextIOWrapper. + + Written by Amaury Forgeot d'Arc and Antoine Pitrou +*/ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#include "_iomodule.h" + +/* TextIOBase */ + +PyDoc_STRVAR(TextIOBase_doc, + "Base class for text I/O.\n" + "\n" + "This class provides a character and line based interface to stream\n" + "I/O. There is no readinto method because Python's character strings\n" + "are immutable. There is no public constructor.\n" + ); + +static PyObject * +_unsupported(const char *message) +{ + PyErr_SetString(IO_STATE->unsupported_operation, message); + return NULL; +} + +PyDoc_STRVAR(TextIOBase_read_doc, + "Read at most n characters from stream.\n" + "\n" + "Read from underlying buffer until we have n characters or we hit EOF.\n" + "If n is negative or omitted, read until EOF.\n" + ); + +static PyObject * +TextIOBase_read(PyObject *self, PyObject *args) +{ + return _unsupported("read"); +} + +PyDoc_STRVAR(TextIOBase_readline_doc, + "Read until newline or EOF.\n" + "\n" + "Returns an empty string if EOF is hit immediately.\n" + ); + +static PyObject * +TextIOBase_readline(PyObject *self, PyObject *args) +{ + return _unsupported("readline"); +} + +PyDoc_STRVAR(TextIOBase_write_doc, + "Write string to stream.\n" + "Returns the number of characters written (which is always equal to\n" + "the length of the string).\n" + ); + +static PyObject * +TextIOBase_write(PyObject *self, PyObject *args) +{ + return _unsupported("write"); +} + +PyDoc_STRVAR(TextIOBase_encoding_doc, + "Encoding of the text stream.\n" + "\n" + "Subclasses should override.\n" + ); + +static PyObject * +TextIOBase_encoding_get(PyObject *self, void *context) +{ + Py_RETURN_NONE; +} + +PyDoc_STRVAR(TextIOBase_newlines_doc, + "Line endings translated so far.\n" + "\n" + "Only line endings translated during reading are considered.\n" + "\n" + "Subclasses should override.\n" + ); + +static PyObject * +TextIOBase_newlines_get(PyObject *self, void *context) +{ + Py_RETURN_NONE; +} + + +static PyMethodDef TextIOBase_methods[] = { + {"read", TextIOBase_read, METH_VARARGS, TextIOBase_read_doc}, + {"readline", TextIOBase_readline, METH_VARARGS, TextIOBase_readline_doc}, + {"write", TextIOBase_write, METH_VARARGS, TextIOBase_write_doc}, + {NULL, NULL} +}; + +static PyGetSetDef TextIOBase_getset[] = { + {"encoding", (getter)TextIOBase_encoding_get, NULL, TextIOBase_encoding_doc}, + {"newlines", (getter)TextIOBase_newlines_get, NULL, TextIOBase_newlines_doc}, + {0} +}; + +PyTypeObject PyTextIOBase_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io._TextIOBase", /*tp_name*/ + 0, /*tp_basicsize*/ + 0, /*tp_itemsize*/ + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + TextIOBase_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + TextIOBase_methods, /* tp_methods */ + 0, /* tp_members */ + TextIOBase_getset, /* tp_getset */ + &PyIOBase_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; + + +/* IncrementalNewlineDecoder */ + +PyDoc_STRVAR(IncrementalNewlineDecoder_doc, + "Codec used when reading a file in universal newlines mode. It wraps\n" + "another incremental decoder, translating \\r\\n and \\r into \\n. It also\n" + "records the types of newlines encountered. When used with\n" + "translate=False, it ensures that the newline sequence is returned in\n" + "one piece. When used with decoder=None, it expects unicode strings as\n" + "decode input and translates newlines without first invoking an external\n" + "decoder.\n" + ); + +typedef struct { + PyObject_HEAD + PyObject *decoder; + PyObject *errors; + int pendingcr:1; + int translate:1; + unsigned int seennl:3; +} PyNewLineDecoderObject; + +static int +IncrementalNewlineDecoder_init(PyNewLineDecoderObject *self, + PyObject *args, PyObject *kwds) +{ + PyObject *decoder; + int translate; + PyObject *errors = NULL; + char *kwlist[] = {"decoder", "translate", "errors", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "Oi|O:IncrementalNewlineDecoder", + kwlist, &decoder, &translate, &errors)) + return -1; + + self->decoder = decoder; + Py_INCREF(decoder); + + if (errors == NULL) { + self->errors = PyUnicode_FromString("strict"); + if (self->errors == NULL) + return -1; + } + else { + Py_INCREF(errors); + self->errors = errors; + } + + self->translate = translate; + self->seennl = 0; + self->pendingcr = 0; + + return 0; +} + +static void +IncrementalNewlineDecoder_dealloc(PyNewLineDecoderObject *self) +{ + Py_CLEAR(self->decoder); + Py_CLEAR(self->errors); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +#define SEEN_CR 1 +#define SEEN_LF 2 +#define SEEN_CRLF 4 +#define SEEN_ALL (SEEN_CR | SEEN_LF | SEEN_CRLF) + +PyObject * +_PyIncrementalNewlineDecoder_decode(PyObject *_self, + PyObject *input, int final) +{ + PyObject *output; + Py_ssize_t output_len; + PyNewLineDecoderObject *self = (PyNewLineDecoderObject *) _self; + + if (self->decoder == NULL) { + PyErr_SetString(PyExc_ValueError, + "IncrementalNewlineDecoder.__init__ not called"); + return NULL; + } + + /* decode input (with the eventual \r from a previous pass) */ + if (self->decoder != Py_None) { + output = PyObject_CallMethodObjArgs(self->decoder, + _PyIO_str_decode, input, final ? Py_True : Py_False, NULL); + } + else { + output = input; + Py_INCREF(output); + } + + if (output == NULL) + return NULL; + + if (!PyUnicode_Check(output)) { + PyErr_SetString(PyExc_TypeError, + "decoder should return a string result"); + goto error; + } + + output_len = PyUnicode_GET_SIZE(output); + if (self->pendingcr && (final || output_len > 0)) { + Py_UNICODE *out; + PyObject *modified = PyUnicode_FromUnicode(NULL, output_len + 1); + if (modified == NULL) + goto error; + out = PyUnicode_AS_UNICODE(modified); + out[0] = '\r'; + memcpy(out + 1, PyUnicode_AS_UNICODE(output), + output_len * sizeof(Py_UNICODE)); + Py_DECREF(output); + output = modified; + self->pendingcr = 0; + output_len++; + } + + /* retain last \r even when not translating data: + * then readline() is sure to get \r\n in one pass + */ + if (!final) { + if (output_len > 0 + && PyUnicode_AS_UNICODE(output)[output_len - 1] == '\r') { + + if (Py_REFCNT(output) == 1) { + if (PyUnicode_Resize(&output, output_len - 1) < 0) + goto error; + } + else { + PyObject *modified = PyUnicode_FromUnicode( + PyUnicode_AS_UNICODE(output), + output_len - 1); + if (modified == NULL) + goto error; + Py_DECREF(output); + output = modified; + } + self->pendingcr = 1; + } + } + + /* Record which newlines are read and do newline translation if desired, + all in one pass. */ + { + Py_UNICODE *in_str; + Py_ssize_t len; + int seennl = self->seennl; + int only_lf = 0; + + in_str = PyUnicode_AS_UNICODE(output); + len = PyUnicode_GET_SIZE(output); + + if (len == 0) + return output; + + /* If, up to now, newlines are consistently \n, do a quick check + for the \r *byte* with the libc's optimized memchr. + */ + if (seennl == SEEN_LF || seennl == 0) { + only_lf = (memchr(in_str, '\r', len * sizeof(Py_UNICODE)) == NULL); + } + + if (only_lf) { + /* If not already seen, quick scan for a possible "\n" character. + (there's nothing else to be done, even when in translation mode) + */ + if (seennl == 0 && + memchr(in_str, '\n', len * sizeof(Py_UNICODE)) != NULL) { + Py_UNICODE *s, *end; + s = in_str; + end = in_str + len; + for (;;) { + Py_UNICODE c; + /* Fast loop for non-control characters */ + while (*s > '\n') + s++; + c = *s++; + if (c == '\n') { + seennl |= SEEN_LF; + break; + } + if (s > end) + break; + } + } + /* Finished: we have scanned for newlines, and none of them + need translating */ + } + else if (!self->translate) { + Py_UNICODE *s, *end; + /* We have already seen all newline types, no need to scan again */ + if (seennl == SEEN_ALL) + goto endscan; + s = in_str; + end = in_str + len; + for (;;) { + Py_UNICODE c; + /* Fast loop for non-control characters */ + while (*s > '\r') + s++; + c = *s++; + if (c == '\n') + seennl |= SEEN_LF; + else if (c == '\r') { + if (*s == '\n') { + seennl |= SEEN_CRLF; + s++; + } + else + seennl |= SEEN_CR; + } + if (s > end) + break; + if (seennl == SEEN_ALL) + break; + } + endscan: + ; + } + else { + PyObject *translated = NULL; + Py_UNICODE *out_str; + Py_UNICODE *in, *out, *end; + if (Py_REFCNT(output) != 1) { + /* We could try to optimize this so that we only do a copy + when there is something to translate. On the other hand, + most decoders should only output non-shared strings, i.e. + translation is done in place. */ + translated = PyUnicode_FromUnicode(NULL, len); + if (translated == NULL) + goto error; + assert(Py_REFCNT(translated) == 1); + memcpy(PyUnicode_AS_UNICODE(translated), + PyUnicode_AS_UNICODE(output), + len * sizeof(Py_UNICODE)); + } + else { + translated = output; + } + out_str = PyUnicode_AS_UNICODE(translated); + in = in_str; + out = out_str; + end = in_str + len; + for (;;) { + Py_UNICODE c; + /* Fast loop for non-control characters */ + while ((c = *in++) > '\r') + *out++ = c; + if (c == '\n') { + *out++ = c; + seennl |= SEEN_LF; + continue; + } + if (c == '\r') { + if (*in == '\n') { + in++; + seennl |= SEEN_CRLF; + } + else + seennl |= SEEN_CR; + *out++ = '\n'; + continue; + } + if (in > end) + break; + *out++ = c; + } + if (translated != output) { + Py_DECREF(output); + output = translated; + } + if (out - out_str != len) { + if (PyUnicode_Resize(&output, out - out_str) < 0) + goto error; + } + } + self->seennl |= seennl; + } + + return output; + + error: + Py_DECREF(output); + return NULL; +} + +static PyObject * +IncrementalNewlineDecoder_decode(PyNewLineDecoderObject *self, + PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"input", "final", NULL}; + PyObject *input; + int final = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|i:IncrementalNewlineDecoder", + kwlist, &input, &final)) + return NULL; + return _PyIncrementalNewlineDecoder_decode((PyObject *) self, input, final); +} + +static PyObject * +IncrementalNewlineDecoder_getstate(PyNewLineDecoderObject *self, PyObject *args) +{ + PyObject *buffer; + unsigned PY_LONG_LONG flag; + + if (self->decoder != Py_None) { + PyObject *state = PyObject_CallMethodObjArgs(self->decoder, + _PyIO_str_getstate, NULL); + if (state == NULL) + return NULL; + if (!PyArg_Parse(state, "(OK)", &buffer, &flag)) { + Py_DECREF(state); + return NULL; + } + Py_INCREF(buffer); + Py_DECREF(state); + } + else { + buffer = PyBytes_FromString(""); + flag = 0; + } + flag <<= 1; + if (self->pendingcr) + flag |= 1; + return Py_BuildValue("NK", buffer, flag); +} + +static PyObject * +IncrementalNewlineDecoder_setstate(PyNewLineDecoderObject *self, PyObject *state) +{ + PyObject *buffer; + unsigned PY_LONG_LONG flag; + + if (!PyArg_Parse(state, "(OK)", &buffer, &flag)) + return NULL; + + self->pendingcr = (int) flag & 1; + flag >>= 1; + + if (self->decoder != Py_None) + return PyObject_CallMethod(self->decoder, + "setstate", "((OK))", buffer, flag); + else + Py_RETURN_NONE; +} + +static PyObject * +IncrementalNewlineDecoder_reset(PyNewLineDecoderObject *self, PyObject *args) +{ + self->seennl = 0; + self->pendingcr = 0; + if (self->decoder != Py_None) + return PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_reset, NULL); + else + Py_RETURN_NONE; +} + +static PyObject * +IncrementalNewlineDecoder_newlines_get(PyNewLineDecoderObject *self, void *context) +{ + switch (self->seennl) { + case SEEN_CR: + return PyUnicode_FromString("\r"); + case SEEN_LF: + return PyUnicode_FromString("\n"); + case SEEN_CRLF: + return PyUnicode_FromString("\r\n"); + case SEEN_CR | SEEN_LF: + return Py_BuildValue("ss", "\r", "\n"); + case SEEN_CR | SEEN_CRLF: + return Py_BuildValue("ss", "\r", "\r\n"); + case SEEN_LF | SEEN_CRLF: + return Py_BuildValue("ss", "\n", "\r\n"); + case SEEN_CR | SEEN_LF | SEEN_CRLF: + return Py_BuildValue("sss", "\r", "\n", "\r\n"); + default: + Py_RETURN_NONE; + } + +} + + +static PyMethodDef IncrementalNewlineDecoder_methods[] = { + {"decode", (PyCFunction)IncrementalNewlineDecoder_decode, METH_VARARGS|METH_KEYWORDS}, + {"getstate", (PyCFunction)IncrementalNewlineDecoder_getstate, METH_NOARGS}, + {"setstate", (PyCFunction)IncrementalNewlineDecoder_setstate, METH_O}, + {"reset", (PyCFunction)IncrementalNewlineDecoder_reset, METH_NOARGS}, + {0} +}; + +static PyGetSetDef IncrementalNewlineDecoder_getset[] = { + {"newlines", (getter)IncrementalNewlineDecoder_newlines_get, NULL, NULL}, + {0} +}; + +PyTypeObject PyIncrementalNewlineDecoder_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.IncrementalNewlineDecoder", /*tp_name*/ + sizeof(PyNewLineDecoderObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)IncrementalNewlineDecoder_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare */ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + IncrementalNewlineDecoder_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /*tp_weaklistoffset*/ + 0, /* tp_iter */ + 0, /* tp_iternext */ + IncrementalNewlineDecoder_methods, /* tp_methods */ + 0, /* tp_members */ + IncrementalNewlineDecoder_getset, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)IncrementalNewlineDecoder_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + + +/* TextIOWrapper */ + +PyDoc_STRVAR(TextIOWrapper_doc, + "Character and line based layer over a BufferedIOBase object, buffer.\n" + "\n" + "encoding gives the name of the encoding that the stream will be\n" + "decoded or encoded with. It defaults to locale.getpreferredencoding.\n" + "\n" + "errors determines the strictness of encoding and decoding (see the\n" + "codecs.register) and defaults to \"strict\".\n" + "\n" + "newline can be None, '', '\\n', '\\r', or '\\r\\n'. It controls the\n" + "handling of line endings. If it is None, universal newlines is\n" + "enabled. With this enabled, on input, the lines endings '\\n', '\\r',\n" + "or '\\r\\n' are translated to '\\n' before being returned to the\n" + "caller. Conversely, on output, '\\n' is translated to the system\n" + "default line seperator, os.linesep. If newline is any other of its\n" + "legal values, that newline becomes the newline when the file is read\n" + "and it is returned untranslated. On output, '\\n' is converted to the\n" + "newline.\n" + "\n" + "If line_buffering is True, a call to flush is implied when a call to\n" + "write contains a newline character." + ); + +typedef PyObject * + (*encodefunc_t)(PyObject *, PyObject *); + +typedef struct +{ + PyObject_HEAD + int ok; /* initialized? */ + Py_ssize_t chunk_size; + PyObject *buffer; + PyObject *encoding; + PyObject *encoder; + PyObject *decoder; + PyObject *readnl; + PyObject *errors; + const char *writenl; /* utf-8 encoded, NULL stands for \n */ + char line_buffering; + char readuniversal; + char readtranslate; + char writetranslate; + char seekable; + char telling; + /* Specialized encoding func (see below) */ + encodefunc_t encodefunc; + + /* Reads and writes are internally buffered in order to speed things up. + However, any read will first flush the write buffer if itsn't empty. + + Please also note that text to be written is first encoded before being + buffered. This is necessary so that encoding errors are immediately + reported to the caller, but it unfortunately means that the + IncrementalEncoder (whose encode() method is always written in Python) + becomes a bottleneck for small writes. + */ + PyObject *decoded_chars; /* buffer for text returned from decoder */ + Py_ssize_t decoded_chars_used; /* offset into _decoded_chars for read() */ + PyObject *pending_bytes; /* list of bytes objects waiting to be + written, or NULL */ + Py_ssize_t pending_bytes_count; + PyObject *snapshot; + /* snapshot is either None, or a tuple (dec_flags, next_input) where + * dec_flags is the second (integer) item of the decoder state and + * next_input is the chunk of input bytes that comes next after the + * snapshot point. We use this to reconstruct decoder states in tell(). + */ + + /* Cache raw object if it's a FileIO object */ + PyObject *raw; + + PyObject *weakreflist; + PyObject *dict; +} PyTextIOWrapperObject; + + +/* A couple of specialized cases in order to bypass the slow incremental + encoding methods for the most popular encodings. */ + +static PyObject * +ascii_encode(PyTextIOWrapperObject *self, PyObject *text) +{ + return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(text), + PyUnicode_GET_SIZE(text), + PyBytes_AS_STRING(self->errors)); +} + +static PyObject * +utf16be_encode(PyTextIOWrapperObject *self, PyObject *text) +{ + return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(text), + PyUnicode_GET_SIZE(text), + PyBytes_AS_STRING(self->errors), 1); +} + +static PyObject * +utf16le_encode(PyTextIOWrapperObject *self, PyObject *text) +{ + return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(text), + PyUnicode_GET_SIZE(text), + PyBytes_AS_STRING(self->errors), -1); +} + +static PyObject * +utf16_encode(PyTextIOWrapperObject *self, PyObject *text) +{ + PyObject *res; + res = PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(text), + PyUnicode_GET_SIZE(text), + PyBytes_AS_STRING(self->errors), 0); + if (res == NULL) + return NULL; + /* Next writes will skip the BOM and use native byte ordering */ +#if defined(WORDS_BIGENDIAN) + self->encodefunc = (encodefunc_t) utf16be_encode; +#else + self->encodefunc = (encodefunc_t) utf16le_encode; +#endif + return res; +} + + +static PyObject * +utf8_encode(PyTextIOWrapperObject *self, PyObject *text) +{ + return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(text), + PyUnicode_GET_SIZE(text), + PyBytes_AS_STRING(self->errors)); +} + +static PyObject * +latin1_encode(PyTextIOWrapperObject *self, PyObject *text) +{ + return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(text), + PyUnicode_GET_SIZE(text), + PyBytes_AS_STRING(self->errors)); +} + +/* Map normalized encoding names onto the specialized encoding funcs */ + +typedef struct { + const char *name; + encodefunc_t encodefunc; +} encodefuncentry; + +static encodefuncentry encodefuncs[] = { + {"ascii", (encodefunc_t) ascii_encode}, + {"iso8859-1", (encodefunc_t) latin1_encode}, + {"utf-16-be", (encodefunc_t) utf16be_encode}, + {"utf-16-le", (encodefunc_t) utf16le_encode}, + {"utf-16", (encodefunc_t) utf16_encode}, + {"utf-8", (encodefunc_t) utf8_encode}, + {NULL, NULL} +}; + + +static int +TextIOWrapper_init(PyTextIOWrapperObject *self, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"buffer", "encoding", "errors", + "newline", "line_buffering", + NULL}; + PyObject *buffer, *raw; + char *encoding = NULL; + char *errors = NULL; + char *newline = NULL; + int line_buffering = 0; + _PyIO_State *state = IO_STATE; + + PyObject *res; + int r; + + self->ok = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|zzzi:fileio", + kwlist, &buffer, &encoding, &errors, + &newline, &line_buffering)) + return -1; + + if (newline && newline[0] != '\0' + && !(newline[0] == '\n' && newline[1] == '\0') + && !(newline[0] == '\r' && newline[1] == '\0') + && !(newline[0] == '\r' && newline[1] == '\n' && newline[2] == '\0')) { + PyErr_Format(PyExc_ValueError, + "illegal newline value: %s", newline); + return -1; + } + + Py_CLEAR(self->buffer); + Py_CLEAR(self->encoding); + Py_CLEAR(self->encoder); + Py_CLEAR(self->decoder); + Py_CLEAR(self->readnl); + Py_CLEAR(self->decoded_chars); + Py_CLEAR(self->pending_bytes); + Py_CLEAR(self->snapshot); + Py_CLEAR(self->errors); + Py_CLEAR(self->raw); + self->decoded_chars_used = 0; + self->pending_bytes_count = 0; + self->encodefunc = NULL; + + if (encoding == NULL) { + /* Try os.device_encoding(fileno) */ + PyObject *fileno; + fileno = PyObject_CallMethod(buffer, "fileno", NULL); + /* Ignore only AttributeError and UnsupportedOperation */ + if (fileno == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError) || + PyErr_ExceptionMatches(state->unsupported_operation)) { + PyErr_Clear(); + } + else { + goto error; + } + } + else { + self->encoding = PyObject_CallMethod(state->os_module, + "device_encoding", + "N", fileno); + if (self->encoding == NULL) + goto error; + else if (!PyUnicode_Check(self->encoding)) + Py_CLEAR(self->encoding); + } + } + if (encoding == NULL && self->encoding == NULL) { + if (state->locale_module == NULL) { + state->locale_module = PyImport_ImportModule("locale"); + if (state->locale_module == NULL) + goto catch_ImportError; + else + goto use_locale; + } + else { + use_locale: + self->encoding = PyObject_CallMethod( + state->locale_module, "getpreferredencoding", NULL); + if (self->encoding == NULL) { + catch_ImportError: + /* + Importing locale can raise a ImportError because of + _functools, and locale.getpreferredencoding can raise a + ImportError if _locale is not available. These will happen + during module building. + */ + if (PyErr_ExceptionMatches(PyExc_ImportError)) { + PyErr_Clear(); + self->encoding = PyUnicode_FromString("ascii"); + } + else + goto error; + } + else if (!PyUnicode_Check(self->encoding)) + Py_CLEAR(self->encoding); + } + } + if (self->encoding != NULL) + encoding = _PyUnicode_AsString(self->encoding); + else if (encoding != NULL) { + self->encoding = PyUnicode_FromString(encoding); + if (self->encoding == NULL) + goto error; + } + else { + PyErr_SetString(PyExc_IOError, + "could not determine default encoding"); + } + + if (errors == NULL) + errors = "strict"; + self->errors = PyBytes_FromString(errors); + if (self->errors == NULL) + goto error; + + self->chunk_size = 8192; + self->readuniversal = (newline == NULL || newline[0] == '\0'); + self->line_buffering = line_buffering; + self->readtranslate = (newline == NULL); + if (newline) { + self->readnl = PyUnicode_FromString(newline); + if (self->readnl == NULL) + return -1; + } + self->writetranslate = (newline == NULL || newline[0] != '\0'); + if (!self->readuniversal && self->readnl) { + self->writenl = _PyUnicode_AsString(self->readnl); + if (!strcmp(self->writenl, "\n")) + self->writenl = NULL; + } +#ifdef MS_WINDOWS + else + self->writenl = "\r\n"; +#endif + + /* Build the decoder object */ + res = PyObject_CallMethod(buffer, "readable", NULL); + if (res == NULL) + goto error; + r = PyObject_IsTrue(res); + Py_DECREF(res); + if (r == -1) + goto error; + if (r == 1) { + self->decoder = PyCodec_IncrementalDecoder( + encoding, errors); + if (self->decoder == NULL) + goto error; + + if (self->readuniversal) { + PyObject *incrementalDecoder = PyObject_CallFunction( + (PyObject *)&PyIncrementalNewlineDecoder_Type, + "Oi", self->decoder, (int)self->readtranslate); + if (incrementalDecoder == NULL) + goto error; + Py_CLEAR(self->decoder); + self->decoder = incrementalDecoder; + } + } + + /* Build the encoder object */ + res = PyObject_CallMethod(buffer, "writable", NULL); + if (res == NULL) + goto error; + r = PyObject_IsTrue(res); + Py_DECREF(res); + if (r == -1) + goto error; + if (r == 1) { + PyObject *ci; + self->encoder = PyCodec_IncrementalEncoder( + encoding, errors); + if (self->encoder == NULL) + goto error; + /* Get the normalized named of the codec */ + ci = _PyCodec_Lookup(encoding); + if (ci == NULL) + goto error; + res = PyObject_GetAttrString(ci, "name"); + Py_DECREF(ci); + if (res == NULL) + PyErr_Clear(); + else if (PyUnicode_Check(res)) { + encodefuncentry *e = encodefuncs; + while (e->name != NULL) { + if (!PyUnicode_CompareWithASCIIString(res, e->name)) { + self->encodefunc = e->encodefunc; + break; + } + e++; + } + } + Py_XDECREF(res); + } + + self->buffer = buffer; + Py_INCREF(buffer); + + if (Py_TYPE(buffer) == &PyBufferedReader_Type || + Py_TYPE(buffer) == &PyBufferedWriter_Type || + Py_TYPE(buffer) == &PyBufferedRandom_Type) { + raw = PyObject_GetAttrString(buffer, "raw"); + /* Cache the raw FileIO object to speed up 'closed' checks */ + if (raw == NULL) + PyErr_Clear(); + else if (Py_TYPE(raw) == &PyFileIO_Type) + self->raw = raw; + else + Py_DECREF(raw); + } + + res = PyObject_CallMethod(buffer, "seekable", NULL); + if (res == NULL) + goto error; + self->seekable = self->telling = PyObject_IsTrue(res); + Py_DECREF(res); + + self->ok = 1; + return 0; + + error: + return -1; +} + +static int +_TextIOWrapper_clear(PyTextIOWrapperObject *self) +{ + if (self->ok && _PyIOBase_finalize((PyObject *) self) < 0) + return -1; + self->ok = 0; + Py_CLEAR(self->buffer); + Py_CLEAR(self->encoding); + Py_CLEAR(self->encoder); + Py_CLEAR(self->decoder); + Py_CLEAR(self->readnl); + Py_CLEAR(self->decoded_chars); + Py_CLEAR(self->pending_bytes); + Py_CLEAR(self->snapshot); + Py_CLEAR(self->errors); + Py_CLEAR(self->raw); + return 0; +} + +static void +TextIOWrapper_dealloc(PyTextIOWrapperObject *self) +{ + if (_TextIOWrapper_clear(self) < 0) + return; + _PyObject_GC_UNTRACK(self); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *)self); + Py_CLEAR(self->dict); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int +TextIOWrapper_traverse(PyTextIOWrapperObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->buffer); + Py_VISIT(self->encoding); + Py_VISIT(self->encoder); + Py_VISIT(self->decoder); + Py_VISIT(self->readnl); + Py_VISIT(self->decoded_chars); + Py_VISIT(self->pending_bytes); + Py_VISIT(self->snapshot); + Py_VISIT(self->errors); + Py_VISIT(self->raw); + + Py_VISIT(self->dict); + return 0; +} + +static int +TextIOWrapper_clear(PyTextIOWrapperObject *self) +{ + if (_TextIOWrapper_clear(self) < 0) + return -1; + Py_CLEAR(self->dict); + return 0; +} + +static PyObject * +TextIOWrapper_closed_get(PyTextIOWrapperObject *self, void *context); + +/* This macro takes some shortcuts to make the common case faster. */ +#define CHECK_CLOSED(self) \ + do { \ + int r; \ + PyObject *_res; \ + if (Py_TYPE(self) == &PyTextIOWrapper_Type) { \ + if (self->raw != NULL) \ + r = _PyFileIO_closed(self->raw); \ + else { \ + _res = TextIOWrapper_closed_get(self, NULL); \ + if (_res == NULL) \ + return NULL; \ + r = PyObject_IsTrue(_res); \ + Py_DECREF(_res); \ + if (r < 0) \ + return NULL; \ + } \ + if (r > 0) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on closed file."); \ + return NULL; \ + } \ + } \ + else if (_PyIOBase_checkClosed((PyObject *)self, Py_True) == NULL) \ + return NULL; \ + } while (0) + +#define CHECK_INITIALIZED(self) \ + if (self->ok <= 0) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on uninitialized object"); \ + return NULL; \ + } + +#define CHECK_INITIALIZED_INT(self) \ + if (self->ok <= 0) { \ + PyErr_SetString(PyExc_ValueError, \ + "I/O operation on uninitialized object"); \ + return -1; \ + } + + +Py_LOCAL_INLINE(const Py_UNICODE *) +findchar(const Py_UNICODE *s, Py_ssize_t size, Py_UNICODE ch) +{ + /* like wcschr, but doesn't stop at NULL characters */ + while (size-- > 0) { + if (*s == ch) + return s; + s++; + } + return NULL; +} + +/* Flush the internal write buffer. This doesn't explicitly flush the + underlying buffered object, though. */ +static int +_TextIOWrapper_writeflush(PyTextIOWrapperObject *self) +{ + PyObject *b, *ret; + + if (self->pending_bytes == NULL) + return 0; + b = _PyBytes_Join(_PyIO_empty_bytes, self->pending_bytes); + if (b == NULL) + return -1; + ret = PyObject_CallMethodObjArgs(self->buffer, + _PyIO_str_write, b, NULL); + Py_DECREF(b); + if (ret == NULL) + return -1; + Py_DECREF(ret); + Py_CLEAR(self->pending_bytes); + self->pending_bytes_count = 0; + return 0; +} + +static PyObject * +TextIOWrapper_write(PyTextIOWrapperObject *self, PyObject *args) +{ + PyObject *ret; + PyObject *text; /* owned reference */ + PyObject *b; + Py_ssize_t textlen; + int haslf = 0; + int needflush = 0; + + CHECK_INITIALIZED(self); + + if (!PyArg_ParseTuple(args, "U:write", &text)) { + return NULL; + } + + CHECK_CLOSED(self); + + Py_INCREF(text); + + textlen = PyUnicode_GetSize(text); + + if ((self->writetranslate && self->writenl != NULL) || self->line_buffering) + if (findchar(PyUnicode_AS_UNICODE(text), + PyUnicode_GET_SIZE(text), '\n')) + haslf = 1; + + if (haslf && self->writetranslate && self->writenl != NULL) { + PyObject *newtext = PyObject_CallMethod( + text, "replace", "ss", "\n", self->writenl); + Py_DECREF(text); + if (newtext == NULL) + return NULL; + text = newtext; + } + + if (self->line_buffering && + (haslf || + findchar(PyUnicode_AS_UNICODE(text), + PyUnicode_GET_SIZE(text), '\r'))) + needflush = 1; + + /* XXX What if we were just reading? */ + if (self->encodefunc != NULL) + b = (*self->encodefunc)((PyObject *) self, text); + else + b = PyObject_CallMethodObjArgs(self->encoder, + _PyIO_str_encode, text, NULL); + Py_DECREF(text); + if (b == NULL) + return NULL; + + if (self->pending_bytes == NULL) { + self->pending_bytes = PyList_New(0); + if (self->pending_bytes == NULL) { + Py_DECREF(b); + return NULL; + } + self->pending_bytes_count = 0; + } + if (PyList_Append(self->pending_bytes, b) < 0) { + Py_DECREF(b); + return NULL; + } + self->pending_bytes_count += PyBytes_GET_SIZE(b); + Py_DECREF(b); + if (self->pending_bytes_count > self->chunk_size || needflush) { + if (_TextIOWrapper_writeflush(self) < 0) + return NULL; + } + + if (needflush) { + ret = PyObject_CallMethodObjArgs(self->buffer, _PyIO_str_flush, NULL); + if (ret == NULL) + return NULL; + Py_DECREF(ret); + } + + Py_CLEAR(self->snapshot); + + if (self->decoder) { + ret = PyObject_CallMethod(self->decoder, "reset", NULL); + if (ret == NULL) + return NULL; + Py_DECREF(ret); + } + + return PyLong_FromSsize_t(textlen); +} + +/* Steal a reference to chars and store it in the decoded_char buffer; + */ +static void +TextIOWrapper_set_decoded_chars(PyTextIOWrapperObject *self, PyObject *chars) +{ + Py_CLEAR(self->decoded_chars); + self->decoded_chars = chars; + self->decoded_chars_used = 0; +} + +static PyObject * +TextIOWrapper_get_decoded_chars(PyTextIOWrapperObject *self, Py_ssize_t n) +{ + PyObject *chars; + Py_ssize_t avail; + + if (self->decoded_chars == NULL) + return PyUnicode_FromStringAndSize(NULL, 0); + + avail = (PyUnicode_GET_SIZE(self->decoded_chars) + - self->decoded_chars_used); + + assert(avail >= 0); + + if (n < 0 || n > avail) + n = avail; + + if (self->decoded_chars_used > 0 || n < avail) { + chars = PyUnicode_FromUnicode( + PyUnicode_AS_UNICODE(self->decoded_chars) + + self->decoded_chars_used, n); + if (chars == NULL) + return NULL; + } + else { + chars = self->decoded_chars; + Py_INCREF(chars); + } + + self->decoded_chars_used += n; + return chars; +} + +/* Read and decode the next chunk of data from the BufferedReader. + */ +static int +TextIOWrapper_read_chunk(PyTextIOWrapperObject *self) +{ + PyObject *dec_buffer = NULL; + PyObject *dec_flags = NULL; + PyObject *input_chunk = NULL; + PyObject *decoded_chars, *chunk_size; + int eof; + + /* The return value is True unless EOF was reached. The decoded string is + * placed in self._decoded_chars (replacing its previous value). The + * entire input chunk is sent to the decoder, though some of it may remain + * buffered in the decoder, yet to be converted. + */ + + if (self->decoder == NULL) { + PyErr_SetString(PyExc_ValueError, "no decoder"); + return -1; + } + + if (self->telling) { + /* To prepare for tell(), we need to snapshot a point in the file + * where the decoder's input buffer is empty. + */ + + PyObject *state = PyObject_CallMethodObjArgs(self->decoder, + _PyIO_str_getstate, NULL); + if (state == NULL) + return -1; + /* Given this, we know there was a valid snapshot point + * len(dec_buffer) bytes ago with decoder state (b'', dec_flags). + */ + if (PyArg_Parse(state, "(OO)", &dec_buffer, &dec_flags) < 0) { + Py_DECREF(state); + return -1; + } + Py_INCREF(dec_buffer); + Py_INCREF(dec_flags); + Py_DECREF(state); + } + + /* Read a chunk, decode it, and put the result in self._decoded_chars. */ + chunk_size = PyLong_FromSsize_t(self->chunk_size); + if (chunk_size == NULL) + goto fail; + input_chunk = PyObject_CallMethodObjArgs(self->buffer, + _PyIO_str_read1, chunk_size, NULL); + Py_DECREF(chunk_size); + if (input_chunk == NULL) + goto fail; + assert(PyBytes_Check(input_chunk)); + + eof = (PyBytes_Size(input_chunk) == 0); + + if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type) { + decoded_chars = _PyIncrementalNewlineDecoder_decode( + self->decoder, input_chunk, eof); + } + else { + decoded_chars = PyObject_CallMethodObjArgs(self->decoder, + _PyIO_str_decode, input_chunk, eof ? Py_True : Py_False, NULL); + } + + /* TODO sanity check: isinstance(decoded_chars, unicode) */ + if (decoded_chars == NULL) + goto fail; + TextIOWrapper_set_decoded_chars(self, decoded_chars); + if (PyUnicode_GET_SIZE(decoded_chars) > 0) + eof = 0; + + if (self->telling) { + /* At the snapshot point, len(dec_buffer) bytes before the read, the + * next input to be decoded is dec_buffer + input_chunk. + */ + PyObject *next_input = PyNumber_Add(dec_buffer, input_chunk); + if (next_input == NULL) + goto fail; + assert (PyBytes_Check(next_input)); + Py_DECREF(dec_buffer); + Py_CLEAR(self->snapshot); + self->snapshot = Py_BuildValue("NN", dec_flags, next_input); + } + Py_DECREF(input_chunk); + + return (eof == 0); + + fail: + Py_XDECREF(dec_buffer); + Py_XDECREF(dec_flags); + Py_XDECREF(input_chunk); + return -1; +} + +static PyObject * +TextIOWrapper_read(PyTextIOWrapperObject *self, PyObject *args) +{ + Py_ssize_t n = -1; + PyObject *result = NULL, *chunks = NULL; + + CHECK_INITIALIZED(self); + + if (!PyArg_ParseTuple(args, "|n:read", &n)) + return NULL; + + CHECK_CLOSED(self); + + if (self->decoder == NULL) { + PyErr_SetString(PyExc_IOError, "not readable"); + return NULL; + } + + if (_TextIOWrapper_writeflush(self) < 0) + return NULL; + + if (n < 0) { + /* Read everything */ + PyObject *bytes = PyObject_CallMethod(self->buffer, "read", NULL); + PyObject *decoded; + if (bytes == NULL) + goto fail; + decoded = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_decode, + bytes, Py_True, NULL); + Py_DECREF(bytes); + if (decoded == NULL) + goto fail; + + result = TextIOWrapper_get_decoded_chars(self, -1); + + if (result == NULL) { + Py_DECREF(decoded); + return NULL; + } + + PyUnicode_AppendAndDel(&result, decoded); + if (result == NULL) + goto fail; + + Py_CLEAR(self->snapshot); + return result; + } + else { + int res = 1; + Py_ssize_t remaining = n; + + result = TextIOWrapper_get_decoded_chars(self, n); + if (result == NULL) + goto fail; + remaining -= PyUnicode_GET_SIZE(result); + + /* Keep reading chunks until we have n characters to return */ + while (remaining > 0) { + res = TextIOWrapper_read_chunk(self); + if (res < 0) + goto fail; + if (res == 0) /* EOF */ + break; + if (chunks == NULL) { + chunks = PyList_New(0); + if (chunks == NULL) + goto fail; + } + if (PyList_Append(chunks, result) < 0) + goto fail; + Py_DECREF(result); + result = TextIOWrapper_get_decoded_chars(self, remaining); + if (result == NULL) + goto fail; + remaining -= PyUnicode_GET_SIZE(result); + } + if (chunks != NULL) { + if (result != NULL && PyList_Append(chunks, result) < 0) + goto fail; + Py_CLEAR(result); + result = PyUnicode_Join(_PyIO_empty_str, chunks); + if (result == NULL) + goto fail; + Py_CLEAR(chunks); + } + return result; + } + fail: + Py_XDECREF(result); + Py_XDECREF(chunks); + return NULL; +} + + +/* NOTE: `end` must point to the real end of the Py_UNICODE storage, + that is to the NUL character. Otherwise the function will produce + incorrect results. */ +static Py_UNICODE * +find_control_char(Py_UNICODE *start, Py_UNICODE *end, Py_UNICODE ch) +{ + Py_UNICODE *s = start; + for (;;) { + while (*s > ch) + s++; + if (*s == ch) + return s; + if (s == end) + return NULL; + s++; + } +} + +Py_ssize_t +_PyIO_find_line_ending( + int translated, int universal, PyObject *readnl, + Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed) +{ + Py_ssize_t len = end - start; + + if (translated) { + /* Newlines are already translated, only search for \n */ + Py_UNICODE *pos = find_control_char(start, end, '\n'); + if (pos != NULL) + return pos - start + 1; + else { + *consumed = len; + return -1; + } + } + else if (universal) { + /* Universal newline search. Find any of \r, \r\n, \n + * The decoder ensures that \r\n are not split in two pieces + */ + Py_UNICODE *s = start; + for (;;) { + Py_UNICODE ch; + /* Fast path for non-control chars. The loop always ends + since the Py_UNICODE storage is NUL-terminated. */ + while (*s > '\r') + s++; + if (s >= end) { + *consumed = len; + return -1; + } + ch = *s++; + if (ch == '\n') + return s - start; + if (ch == '\r') { + if (*s == '\n') + return s - start + 1; + else + return s - start; + } + } + } + else { + /* Non-universal mode. */ + Py_ssize_t readnl_len = PyUnicode_GET_SIZE(readnl); + Py_UNICODE *nl = PyUnicode_AS_UNICODE(readnl); + if (readnl_len == 1) { + Py_UNICODE *pos = find_control_char(start, end, nl[0]); + if (pos != NULL) + return pos - start + 1; + *consumed = len; + return -1; + } + else { + Py_UNICODE *s = start; + Py_UNICODE *e = end - readnl_len + 1; + Py_UNICODE *pos; + if (e < s) + e = s; + while (s < e) { + Py_ssize_t i; + Py_UNICODE *pos = find_control_char(s, end, nl[0]); + if (pos == NULL || pos >= e) + break; + for (i = 1; i < readnl_len; i++) { + if (pos[i] != nl[i]) + break; + } + if (i == readnl_len) + return pos - start + readnl_len; + s = pos + 1; + } + pos = find_control_char(e, end, nl[0]); + if (pos == NULL) + *consumed = len; + else + *consumed = pos - start; + return -1; + } + } +} + +static PyObject * +_TextIOWrapper_readline(PyTextIOWrapperObject *self, Py_ssize_t limit) +{ + PyObject *line = NULL, *chunks = NULL, *remaining = NULL; + Py_ssize_t start, endpos, chunked, offset_to_buffer; + int res; + + CHECK_CLOSED(self); + + if (_TextIOWrapper_writeflush(self) < 0) + return NULL; + + chunked = 0; + + while (1) { + Py_UNICODE *ptr; + Py_ssize_t line_len; + Py_ssize_t consumed = 0; + + /* First, get some data if necessary */ + res = 1; + while (!self->decoded_chars || + !PyUnicode_GET_SIZE(self->decoded_chars)) { + res = TextIOWrapper_read_chunk(self); + if (res < 0) + goto error; + if (res == 0) + break; + } + if (res == 0) { + /* end of file */ + TextIOWrapper_set_decoded_chars(self, NULL); + Py_CLEAR(self->snapshot); + start = endpos = offset_to_buffer = 0; + break; + } + + if (remaining == NULL) { + line = self->decoded_chars; + start = self->decoded_chars_used; + offset_to_buffer = 0; + Py_INCREF(line); + } + else { + assert(self->decoded_chars_used == 0); + line = PyUnicode_Concat(remaining, self->decoded_chars); + start = 0; + offset_to_buffer = PyUnicode_GET_SIZE(remaining); + Py_CLEAR(remaining); + if (line == NULL) + goto error; + } + + ptr = PyUnicode_AS_UNICODE(line); + line_len = PyUnicode_GET_SIZE(line); + + endpos = _PyIO_find_line_ending( + self->readtranslate, self->readuniversal, self->readnl, + ptr + start, ptr + line_len, &consumed); + if (endpos >= 0) { + endpos += start; + if (limit >= 0 && (endpos - start) + chunked >= limit) + endpos = start + limit - chunked; + break; + } + + /* We can put aside up to `endpos` */ + endpos = consumed + start; + if (limit >= 0 && (endpos - start) + chunked >= limit) { + /* Didn't find line ending, but reached length limit */ + endpos = start + limit - chunked; + break; + } + + if (endpos > start) { + /* No line ending seen yet - put aside current data */ + PyObject *s; + if (chunks == NULL) { + chunks = PyList_New(0); + if (chunks == NULL) + goto error; + } + s = PyUnicode_FromUnicode(ptr + start, endpos - start); + if (s == NULL) + goto error; + if (PyList_Append(chunks, s) < 0) { + Py_DECREF(s); + goto error; + } + chunked += PyUnicode_GET_SIZE(s); + Py_DECREF(s); + } + /* There may be some remaining bytes we'll have to prepend to the + next chunk of data */ + if (endpos < line_len) { + remaining = PyUnicode_FromUnicode( + ptr + endpos, line_len - endpos); + if (remaining == NULL) + goto error; + } + Py_CLEAR(line); + /* We have consumed the buffer */ + TextIOWrapper_set_decoded_chars(self, NULL); + } + + if (line != NULL) { + /* Our line ends in the current buffer */ + self->decoded_chars_used = endpos - offset_to_buffer; + if (start > 0 || endpos < PyUnicode_GET_SIZE(line)) { + if (start == 0 && Py_REFCNT(line) == 1) { + if (PyUnicode_Resize(&line, endpos) < 0) + goto error; + } + else { + PyObject *s = PyUnicode_FromUnicode( + PyUnicode_AS_UNICODE(line) + start, endpos - start); + Py_CLEAR(line); + if (s == NULL) + goto error; + line = s; + } + } + } + if (remaining != NULL) { + if (chunks == NULL) { + chunks = PyList_New(0); + if (chunks == NULL) + goto error; + } + if (PyList_Append(chunks, remaining) < 0) + goto error; + Py_CLEAR(remaining); + } + if (chunks != NULL) { + if (line != NULL && PyList_Append(chunks, line) < 0) + goto error; + Py_CLEAR(line); + line = PyUnicode_Join(_PyIO_empty_str, chunks); + if (line == NULL) + goto error; + Py_DECREF(chunks); + } + if (line == NULL) + line = PyUnicode_FromStringAndSize(NULL, 0); + + return line; + + error: + Py_XDECREF(chunks); + Py_XDECREF(remaining); + Py_XDECREF(line); + return NULL; +} + +static PyObject * +TextIOWrapper_readline(PyTextIOWrapperObject *self, PyObject *args) +{ + Py_ssize_t limit = -1; + + CHECK_INITIALIZED(self); + if (!PyArg_ParseTuple(args, "|n:readline", &limit)) { + return NULL; + } + return _TextIOWrapper_readline(self, limit); +} + +/* Seek and Tell */ + +typedef struct { + Py_off_t start_pos; + int dec_flags; + int bytes_to_feed; + int chars_to_skip; + char need_eof; +} CookieStruct; + +/* + To speed up cookie packing/unpacking, we store the fields in a temporary + string and call _PyLong_FromByteArray() or _PyLong_AsByteArray (resp.). + The following macros define at which offsets in the intermediary byte + string the various CookieStruct fields will be stored. + */ + +#define COOKIE_BUF_LEN (sizeof(Py_off_t) + 3 * sizeof(int) + sizeof(char)) + +#if defined(WORDS_BIGENDIAN) + +# define IS_LITTLE_ENDIAN 0 + +/* We want the least significant byte of start_pos to also be the least + significant byte of the cookie, which means that in big-endian mode we + must copy the fields in reverse order. */ + +# define OFF_START_POS (sizeof(char) + 3 * sizeof(int)) +# define OFF_DEC_FLAGS (sizeof(char) + 2 * sizeof(int)) +# define OFF_BYTES_TO_FEED (sizeof(char) + sizeof(int)) +# define OFF_CHARS_TO_SKIP (sizeof(char)) +# define OFF_NEED_EOF 0 + +#else + +# define IS_LITTLE_ENDIAN 1 + +/* Little-endian mode: the least significant byte of start_pos will + naturally end up the least significant byte of the cookie. */ + +# define OFF_START_POS 0 +# define OFF_DEC_FLAGS (sizeof(Py_off_t)) +# define OFF_BYTES_TO_FEED (sizeof(Py_off_t) + sizeof(int)) +# define OFF_CHARS_TO_SKIP (sizeof(Py_off_t) + 2 * sizeof(int)) +# define OFF_NEED_EOF (sizeof(Py_off_t) + 3 * sizeof(int)) + +#endif + +static int +TextIOWrapper_parseCookie(CookieStruct *cookie, PyObject *cookieObj) +{ + unsigned char buffer[COOKIE_BUF_LEN]; + PyLongObject *cookieLong = (PyLongObject *)PyNumber_Long(cookieObj); + if (cookieLong == NULL) + return -1; + + if (_PyLong_AsByteArray(cookieLong, buffer, sizeof(buffer), + IS_LITTLE_ENDIAN, 0) < 0) { + Py_DECREF(cookieLong); + return -1; + } + Py_DECREF(cookieLong); + + memcpy(&cookie->start_pos, buffer + OFF_START_POS, sizeof(cookie->start_pos)); + memcpy(&cookie->dec_flags, buffer + OFF_DEC_FLAGS, sizeof(cookie->dec_flags)); + memcpy(&cookie->bytes_to_feed, buffer + OFF_BYTES_TO_FEED, sizeof(cookie->bytes_to_feed)); + memcpy(&cookie->chars_to_skip, buffer + OFF_CHARS_TO_SKIP, sizeof(cookie->chars_to_skip)); + memcpy(&cookie->need_eof, buffer + OFF_NEED_EOF, sizeof(cookie->need_eof)); + + return 0; +} + +static PyObject * +TextIOWrapper_buildCookie(CookieStruct *cookie) +{ + unsigned char buffer[COOKIE_BUF_LEN]; + + memcpy(buffer + OFF_START_POS, &cookie->start_pos, sizeof(cookie->start_pos)); + memcpy(buffer + OFF_DEC_FLAGS, &cookie->dec_flags, sizeof(cookie->dec_flags)); + memcpy(buffer + OFF_BYTES_TO_FEED, &cookie->bytes_to_feed, sizeof(cookie->bytes_to_feed)); + memcpy(buffer + OFF_CHARS_TO_SKIP, &cookie->chars_to_skip, sizeof(cookie->chars_to_skip)); + memcpy(buffer + OFF_NEED_EOF, &cookie->need_eof, sizeof(cookie->need_eof)); + + return _PyLong_FromByteArray(buffer, sizeof(buffer), IS_LITTLE_ENDIAN, 0); +} +#undef IS_LITTLE_ENDIAN + +static int +_TextIOWrapper_decoder_setstate(PyTextIOWrapperObject *self, + CookieStruct *cookie) +{ + PyObject *res; + /* When seeking to the start of the stream, we call decoder.reset() + rather than decoder.getstate(). + This is for a few decoders such as utf-16 for which the state value + at start is not (b"", 0) but e.g. (b"", 2) (meaning, in the case of + utf-16, that we are expecting a BOM). + */ + if (cookie->start_pos == 0 && cookie->dec_flags == 0) + res = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_reset, NULL); + else + res = PyObject_CallMethod(self->decoder, "setstate", + "((yi))", "", cookie->dec_flags); + if (res == NULL) + return -1; + Py_DECREF(res); + return 0; +} + +static PyObject * +TextIOWrapper_seek(PyTextIOWrapperObject *self, PyObject *args) +{ + PyObject *cookieObj, *posobj; + CookieStruct cookie; + int whence = 0; + static PyObject *zero = NULL; + PyObject *res; + int cmp; + + CHECK_INITIALIZED(self); + + if (zero == NULL) { + zero = PyLong_FromLong(0L); + if (zero == NULL) + return NULL; + } + + if (!PyArg_ParseTuple(args, "O|i:seek", &cookieObj, &whence)) + return NULL; + CHECK_CLOSED(self); + + Py_INCREF(cookieObj); + + if (!self->seekable) { + PyErr_SetString(PyExc_IOError, + "underlying stream is not seekable"); + goto fail; + } + + if (whence == 1) { + /* seek relative to current position */ + cmp = PyObject_RichCompareBool(cookieObj, zero, Py_EQ); + if (cmp < 0) + goto fail; + + if (cmp == 0) { + PyErr_SetString(PyExc_IOError, + "can't do nonzero cur-relative seeks"); + goto fail; + } + + /* Seeking to the current position should attempt to + * sync the underlying buffer with the current position. + */ + Py_DECREF(cookieObj); + cookieObj = PyObject_CallMethod((PyObject *)self, "tell", NULL); + if (cookieObj == NULL) + goto fail; + } + else if (whence == 2) { + /* seek relative to end of file */ + + cmp = PyObject_RichCompareBool(cookieObj, zero, Py_EQ); + if (cmp < 0) + goto fail; + + if (cmp == 0) { + PyErr_SetString(PyExc_IOError, + "can't do nonzero end-relative seeks"); + goto fail; + } + + res = PyObject_CallMethod((PyObject *)self, "flush", NULL); + if (res == NULL) + goto fail; + Py_DECREF(res); + + TextIOWrapper_set_decoded_chars(self, NULL); + Py_CLEAR(self->snapshot); + if (self->decoder) { + res = PyObject_CallMethod(self->decoder, "reset", NULL); + if (res == NULL) + goto fail; + Py_DECREF(res); + } + + res = PyObject_CallMethod(self->buffer, "seek", "ii", 0, 2); + Py_XDECREF(cookieObj); + return res; + } + else if (whence != 0) { + PyErr_Format(PyExc_ValueError, + "invalid whence (%d, should be 0, 1 or 2)", whence); + goto fail; + } + + cmp = PyObject_RichCompareBool(cookieObj, zero, Py_LT); + if (cmp < 0) + goto fail; + + if (cmp == 1) { + PyErr_Format(PyExc_ValueError, + "negative seek position %R", cookieObj); + goto fail; + } + + res = PyObject_CallMethodObjArgs((PyObject *)self, _PyIO_str_flush, NULL); + if (res == NULL) + goto fail; + Py_DECREF(res); + + /* The strategy of seek() is to go back to the safe start point + * and replay the effect of read(chars_to_skip) from there. + */ + if (TextIOWrapper_parseCookie(&cookie, cookieObj) < 0) + goto fail; + + /* Seek back to the safe start point. */ + posobj = PyLong_FromOff_t(cookie.start_pos); + if (posobj == NULL) + goto fail; + res = PyObject_CallMethodObjArgs(self->buffer, + _PyIO_str_seek, posobj, NULL); + Py_DECREF(posobj); + if (res == NULL) + goto fail; + Py_DECREF(res); + + TextIOWrapper_set_decoded_chars(self, NULL); + Py_CLEAR(self->snapshot); + + /* Restore the decoder to its state from the safe start point. */ + if (self->decoder) { + if (_TextIOWrapper_decoder_setstate(self, &cookie) < 0) + goto fail; + } + + if (cookie.chars_to_skip) { + /* Just like _read_chunk, feed the decoder and save a snapshot. */ + PyObject *input_chunk = PyObject_CallMethod( + self->buffer, "read", "i", cookie.bytes_to_feed); + PyObject *decoded; + + if (input_chunk == NULL) + goto fail; + + assert (PyBytes_Check(input_chunk)); + + self->snapshot = Py_BuildValue("iN", cookie.dec_flags, input_chunk); + if (self->snapshot == NULL) { + Py_DECREF(input_chunk); + goto fail; + } + + decoded = PyObject_CallMethod(self->decoder, "decode", + "Oi", input_chunk, (int)cookie.need_eof); + + if (decoded == NULL) + goto fail; + + TextIOWrapper_set_decoded_chars(self, decoded); + + /* Skip chars_to_skip of the decoded characters. */ + if (PyUnicode_GetSize(self->decoded_chars) < cookie.chars_to_skip) { + PyErr_SetString(PyExc_IOError, "can't restore logical file position"); + goto fail; + } + self->decoded_chars_used = cookie.chars_to_skip; + } + else { + self->snapshot = Py_BuildValue("iy", cookie.dec_flags, ""); + if (self->snapshot == NULL) + goto fail; + } + + return cookieObj; + fail: + Py_XDECREF(cookieObj); + return NULL; + +} + +static PyObject * +TextIOWrapper_tell(PyTextIOWrapperObject *self, PyObject *args) +{ + PyObject *res; + PyObject *posobj = NULL; + CookieStruct cookie = {0,0,0,0,0}; + PyObject *next_input; + Py_ssize_t chars_to_skip, chars_decoded; + PyObject *saved_state = NULL; + char *input, *input_end; + + CHECK_INITIALIZED(self); + CHECK_CLOSED(self); + + if (!self->seekable) { + PyErr_SetString(PyExc_IOError, + "underlying stream is not seekable"); + goto fail; + } + if (!self->telling) { + PyErr_SetString(PyExc_IOError, + "telling position disabled by next() call"); + goto fail; + } + + if (_TextIOWrapper_writeflush(self) < 0) + return NULL; + res = PyObject_CallMethod((PyObject *)self, "flush", NULL); + if (res == NULL) + goto fail; + Py_DECREF(res); + + posobj = PyObject_CallMethod(self->buffer, "tell", NULL); + if (posobj == NULL) + goto fail; + + if (self->decoder == NULL || self->snapshot == NULL) { + assert (self->decoded_chars == NULL || PyUnicode_GetSize(self->decoded_chars) == 0); + return posobj; + } + +#if defined(HAVE_LARGEFILE_SUPPORT) + cookie.start_pos = PyLong_AsLongLong(posobj); +#else + cookie.start_pos = PyLong_AsLong(posobj); +#endif + if (PyErr_Occurred()) + goto fail; + + /* Skip backward to the snapshot point (see _read_chunk). */ + if (!PyArg_Parse(self->snapshot, "(iO)", &cookie.dec_flags, &next_input)) + goto fail; + + assert (PyBytes_Check(next_input)); + + cookie.start_pos -= PyBytes_GET_SIZE(next_input); + + /* How many decoded characters have been used up since the snapshot? */ + if (self->decoded_chars_used == 0) { + /* We haven't moved from the snapshot point. */ + Py_DECREF(posobj); + return TextIOWrapper_buildCookie(&cookie); + } + + chars_to_skip = self->decoded_chars_used; + + /* Starting from the snapshot position, we will walk the decoder + * forward until it gives us enough decoded characters. + */ + saved_state = PyObject_CallMethodObjArgs(self->decoder, + _PyIO_str_getstate, NULL); + if (saved_state == NULL) + goto fail; + + /* Note our initial start point. */ + if (_TextIOWrapper_decoder_setstate(self, &cookie) < 0) + goto fail; + + /* Feed the decoder one byte at a time. As we go, note the + * nearest "safe start point" before the current location + * (a point where the decoder has nothing buffered, so seek() + * can safely start from there and advance to this location). + */ + chars_decoded = 0; + input = PyBytes_AS_STRING(next_input); + input_end = input + PyBytes_GET_SIZE(next_input); + while (input < input_end) { + PyObject *state; + char *dec_buffer; + Py_ssize_t dec_buffer_len; + int dec_flags; + + PyObject *decoded = PyObject_CallMethod( + self->decoder, "decode", "y#", input, 1); + if (decoded == NULL) + goto fail; + assert (PyUnicode_Check(decoded)); + chars_decoded += PyUnicode_GET_SIZE(decoded); + Py_DECREF(decoded); + + cookie.bytes_to_feed += 1; + + state = PyObject_CallMethodObjArgs(self->decoder, + _PyIO_str_getstate, NULL); + if (state == NULL) + goto fail; + if (!PyArg_Parse(state, "(y#i)", &dec_buffer, &dec_buffer_len, &dec_flags)) { + Py_DECREF(state); + goto fail; + } + Py_DECREF(state); + + if (dec_buffer_len == 0 && chars_decoded <= chars_to_skip) { + /* Decoder buffer is empty, so this is a safe start point. */ + cookie.start_pos += cookie.bytes_to_feed; + chars_to_skip -= chars_decoded; + cookie.dec_flags = dec_flags; + cookie.bytes_to_feed = 0; + chars_decoded = 0; + } + if (chars_decoded >= chars_to_skip) + break; + input++; + } + if (input == input_end) { + /* We didn't get enough decoded data; signal EOF to get more. */ + PyObject *decoded = PyObject_CallMethod( + self->decoder, "decode", "yi", "", /* final = */ 1); + if (decoded == NULL) + goto fail; + assert (PyUnicode_Check(decoded)); + chars_decoded += PyUnicode_GET_SIZE(decoded); + Py_DECREF(decoded); + cookie.need_eof = 1; + + if (chars_decoded < chars_to_skip) { + PyErr_SetString(PyExc_IOError, + "can't reconstruct logical file position"); + goto fail; + } + } + + /* finally */ + Py_XDECREF(posobj); + res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state); + Py_DECREF(saved_state); + if (res == NULL) + return NULL; + Py_DECREF(res); + + /* The returned cookie corresponds to the last safe start point. */ + cookie.chars_to_skip = Py_SAFE_DOWNCAST(chars_to_skip, Py_ssize_t, int); + return TextIOWrapper_buildCookie(&cookie); + + fail: + Py_XDECREF(posobj); + if (saved_state) { + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + + res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state); + Py_DECREF(saved_state); + if (res == NULL) + return NULL; + Py_DECREF(res); + + PyErr_Restore(type, value, traceback); + } + return NULL; +} + +static PyObject * +TextIOWrapper_truncate(PyTextIOWrapperObject *self, PyObject *args) +{ + PyObject *pos = Py_None; + PyObject *res; + + CHECK_INITIALIZED(self) + if (!PyArg_ParseTuple(args, "|O:truncate", &pos)) { + return NULL; + } + + res = PyObject_CallMethodObjArgs((PyObject *) self, _PyIO_str_flush, NULL); + if (res == NULL) + return NULL; + Py_DECREF(res); + + if (pos != Py_None) { + res = PyObject_CallMethodObjArgs((PyObject *) self, + _PyIO_str_seek, pos, NULL); + if (res == NULL) + return NULL; + Py_DECREF(res); + } + + return PyObject_CallMethodObjArgs(self->buffer, _PyIO_str_truncate, NULL); +} + +static PyObject * +TextIOWrapper_repr(PyTextIOWrapperObject *self) +{ + CHECK_INITIALIZED(self); + return PyUnicode_FromFormat("", self->encoding); +} + + +/* Inquiries */ + +static PyObject * +TextIOWrapper_fileno(PyTextIOWrapperObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self); + return PyObject_CallMethod(self->buffer, "fileno", NULL); +} + +static PyObject * +TextIOWrapper_seekable(PyTextIOWrapperObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self); + return PyObject_CallMethod(self->buffer, "seekable", NULL); +} + +static PyObject * +TextIOWrapper_readable(PyTextIOWrapperObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self); + return PyObject_CallMethod(self->buffer, "readable", NULL); +} + +static PyObject * +TextIOWrapper_writable(PyTextIOWrapperObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self); + return PyObject_CallMethod(self->buffer, "writable", NULL); +} + +static PyObject * +TextIOWrapper_isatty(PyTextIOWrapperObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self); + return PyObject_CallMethod(self->buffer, "isatty", NULL); +} + +static PyObject * +TextIOWrapper_flush(PyTextIOWrapperObject *self, PyObject *args) +{ + CHECK_INITIALIZED(self); + CHECK_CLOSED(self); + self->telling = self->seekable; + if (_TextIOWrapper_writeflush(self) < 0) + return NULL; + return PyObject_CallMethod(self->buffer, "flush", NULL); +} + +static PyObject * +TextIOWrapper_close(PyTextIOWrapperObject *self, PyObject *args) +{ + PyObject *res; + CHECK_INITIALIZED(self); + res = PyObject_CallMethod((PyObject *)self, "flush", NULL); + if (res == NULL) { + /* If flush() fails, just give up */ + PyErr_Clear(); + } + else + Py_DECREF(res); + + return PyObject_CallMethod(self->buffer, "close", NULL); +} + +static PyObject * +TextIOWrapper_iternext(PyTextIOWrapperObject *self) +{ + PyObject *line; + + CHECK_INITIALIZED(self); + + self->telling = 0; + if (Py_TYPE(self) == &PyTextIOWrapper_Type) { + /* Skip method call overhead for speed */ + line = _TextIOWrapper_readline(self, -1); + } + else { + line = PyObject_CallMethodObjArgs((PyObject *)self, + _PyIO_str_readline, NULL); + if (line && !PyUnicode_Check(line)) { + PyErr_Format(PyExc_IOError, + "readline() should have returned an str object, " + "not '%.200s'", Py_TYPE(line)->tp_name); + Py_DECREF(line); + return NULL; + } + } + + if (line == NULL) + return NULL; + + if (PyUnicode_GET_SIZE(line) == 0) { + /* Reached EOF or would have blocked */ + Py_DECREF(line); + Py_CLEAR(self->snapshot); + self->telling = self->seekable; + return NULL; + } + + return line; +} + +static PyObject * +TextIOWrapper_name_get(PyTextIOWrapperObject *self, void *context) +{ + CHECK_INITIALIZED(self); + return PyObject_GetAttrString(self->buffer, "name"); +} + +static PyObject * +TextIOWrapper_closed_get(PyTextIOWrapperObject *self, void *context) +{ + CHECK_INITIALIZED(self); + return PyObject_GetAttr(self->buffer, _PyIO_str_closed); +} + +static PyObject * +TextIOWrapper_newlines_get(PyTextIOWrapperObject *self, void *context) +{ + PyObject *res; + CHECK_INITIALIZED(self); + if (self->decoder == NULL) + Py_RETURN_NONE; + res = PyObject_GetAttr(self->decoder, _PyIO_str_newlines); + if (res == NULL) { + PyErr_Clear(); + Py_RETURN_NONE; + } + return res; +} + +static PyObject * +TextIOWrapper_chunk_size_get(PyTextIOWrapperObject *self, void *context) +{ + CHECK_INITIALIZED(self); + return PyLong_FromSsize_t(self->chunk_size); +} + +static int +TextIOWrapper_chunk_size_set(PyTextIOWrapperObject *self, + PyObject *arg, void *context) +{ + Py_ssize_t n; + CHECK_INITIALIZED_INT(self); + n = PyNumber_AsSsize_t(arg, PyExc_TypeError); + if (n == -1 && PyErr_Occurred()) + return -1; + if (n <= 0) { + PyErr_SetString(PyExc_ValueError, + "a strictly positive integer is required"); + return -1; + } + self->chunk_size = n; + return 0; +} + +static PyMethodDef TextIOWrapper_methods[] = { + {"write", (PyCFunction)TextIOWrapper_write, METH_VARARGS}, + {"read", (PyCFunction)TextIOWrapper_read, METH_VARARGS}, + {"readline", (PyCFunction)TextIOWrapper_readline, METH_VARARGS}, + {"flush", (PyCFunction)TextIOWrapper_flush, METH_NOARGS}, + {"close", (PyCFunction)TextIOWrapper_close, METH_NOARGS}, + + {"fileno", (PyCFunction)TextIOWrapper_fileno, METH_NOARGS}, + {"seekable", (PyCFunction)TextIOWrapper_seekable, METH_NOARGS}, + {"readable", (PyCFunction)TextIOWrapper_readable, METH_NOARGS}, + {"writable", (PyCFunction)TextIOWrapper_writable, METH_NOARGS}, + {"isatty", (PyCFunction)TextIOWrapper_isatty, METH_NOARGS}, + + {"seek", (PyCFunction)TextIOWrapper_seek, METH_VARARGS}, + {"tell", (PyCFunction)TextIOWrapper_tell, METH_NOARGS}, + {"truncate", (PyCFunction)TextIOWrapper_truncate, METH_VARARGS}, + {NULL, NULL} +}; + +static PyMemberDef TextIOWrapper_members[] = { + {"encoding", T_OBJECT, offsetof(PyTextIOWrapperObject, encoding), READONLY}, + {"buffer", T_OBJECT, offsetof(PyTextIOWrapperObject, buffer), READONLY}, + {"line_buffering", T_BOOL, offsetof(PyTextIOWrapperObject, line_buffering), READONLY}, + {NULL} +}; + +static PyGetSetDef TextIOWrapper_getset[] = { + {"name", (getter)TextIOWrapper_name_get, NULL, NULL}, + {"closed", (getter)TextIOWrapper_closed_get, NULL, NULL}, +/* {"mode", (getter)TextIOWrapper_mode_get, NULL, NULL}, +*/ + {"newlines", (getter)TextIOWrapper_newlines_get, NULL, NULL}, + {"_CHUNK_SIZE", (getter)TextIOWrapper_chunk_size_get, + (setter)TextIOWrapper_chunk_size_set, NULL}, + {0} +}; + +PyTypeObject PyTextIOWrapper_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_io.TextIOWrapper", /*tp_name*/ + sizeof(PyTextIOWrapperObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)TextIOWrapper_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tps_etattr*/ + 0, /*tp_compare */ + (reprfunc)TextIOWrapper_repr,/*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + TextIOWrapper_doc, /* tp_doc */ + (traverseproc)TextIOWrapper_traverse, /* tp_traverse */ + (inquiry)TextIOWrapper_clear, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(PyTextIOWrapperObject, weakreflist), /*tp_weaklistoffset*/ + 0, /* tp_iter */ + (iternextfunc)TextIOWrapper_iternext, /* tp_iternext */ + TextIOWrapper_methods, /* tp_methods */ + TextIOWrapper_members, /* tp_members */ + TextIOWrapper_getset, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(PyTextIOWrapperObject, dict), /*tp_dictoffset*/ + (initproc)TextIOWrapper_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; diff --git a/Modules/_iobase.c b/Modules/_iobase.c deleted file mode 100644 index ef6d516..0000000 --- a/Modules/_iobase.c +++ /dev/null @@ -1,894 +0,0 @@ -/* - An implementation of the I/O abstract base classes hierarchy - as defined by PEP 3116 - "New I/O" - - Classes defined here: IOBase, RawIOBase. - - Written by Amaury Forgeot d'Arc and Antoine Pitrou -*/ - - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#include "_iomodule.h" - -/* - * IOBase class, an abstract class - */ - -typedef struct { - PyObject_HEAD - - PyObject *dict; - PyObject *weakreflist; -} IOBaseObject; - -PyDoc_STRVAR(IOBase_doc, - "The abstract base class for all I/O classes, acting on streams of\n" - "bytes. There is no public constructor.\n" - "\n" - "This class provides dummy implementations for many methods that\n" - "derived classes can override selectively; the default implementations\n" - "represent a file that cannot be read, written or seeked.\n" - "\n" - "Even though IOBase does not declare read, readinto, or write because\n" - "their signatures will vary, implementations and clients should\n" - "consider those methods part of the interface. Also, implementations\n" - "may raise a IOError when operations they do not support are called.\n" - "\n" - "The basic type used for binary data read from or written to a file is\n" - "bytes. bytearrays are accepted too, and in some cases (such as\n" - "readinto) needed. Text I/O classes work with str data.\n" - "\n" - "Note that calling any method (even inquiries) on a closed stream is\n" - "undefined. Implementations may raise IOError in this case.\n" - "\n" - "IOBase (and its subclasses) support the iterator protocol, meaning\n" - "that an IOBase object can be iterated over yielding the lines in a\n" - "stream.\n" - "\n" - "IOBase also supports the :keyword:`with` statement. In this example,\n" - "fp is closed after the suite of the with statment is complete:\n" - "\n" - "with open('spam.txt', 'r') as fp:\n" - " fp.write('Spam and eggs!')\n"); - -/* Use this macro whenever you want to check the internal `closed` status - of the IOBase object rather than the virtual `closed` attribute as returned - by whatever subclass. */ - -#define IS_CLOSED(self) \ - PyObject_HasAttrString(self, "__IOBase_closed") - -/* Internal methods */ -static PyObject * -IOBase_unsupported(const char *message) -{ - PyErr_SetString(IO_STATE->unsupported_operation, message); - return NULL; -} - -/* Positionning */ - -PyDoc_STRVAR(IOBase_seek_doc, - "Change stream position.\n" - "\n" - "Change the stream position to byte offset offset. offset is\n" - "interpreted relative to the position indicated by whence. Values\n" - "for whence are:\n" - "\n" - "* 0 -- start of stream (the default); offset should be zero or positive\n" - "* 1 -- current stream position; offset may be negative\n" - "* 2 -- end of stream; offset is usually negative\n" - "\n" - "Return the new absolute position."); - -static PyObject * -IOBase_seek(PyObject *self, PyObject *args) -{ - return IOBase_unsupported("seek"); -} - -PyDoc_STRVAR(IOBase_tell_doc, - "Return current stream position."); - -static PyObject * -IOBase_tell(PyObject *self, PyObject *args) -{ - return PyObject_CallMethod(self, "seek", "ii", 0, 1); -} - -PyDoc_STRVAR(IOBase_truncate_doc, - "Truncate file to size bytes.\n" - "\n" - "Size defaults to the current IO position as reported by tell(). Return\n" - "the new size."); - -static PyObject * -IOBase_truncate(PyObject *self, PyObject *args) -{ - return IOBase_unsupported("truncate"); -} - -/* Flush and close methods */ - -PyDoc_STRVAR(IOBase_flush_doc, - "Flush write buffers, if applicable.\n" - "\n" - "This is not implemented for read-only and non-blocking streams.\n"); - -static PyObject * -IOBase_flush(PyObject *self, PyObject *args) -{ - /* XXX Should this return the number of bytes written??? */ - if (IS_CLOSED(self)) { - PyErr_SetString(PyExc_ValueError, "I/O operation on closed file."); - return NULL; - } - Py_RETURN_NONE; -} - -PyDoc_STRVAR(IOBase_close_doc, - "Flush and close the IO object.\n" - "\n" - "This method has no effect if the file is already closed.\n"); - -static int -IOBase_closed(PyObject *self) -{ - PyObject *res; - int closed; - /* This gets the derived attribute, which is *not* __IOBase_closed - in most cases! */ - res = PyObject_GetAttr(self, _PyIO_str_closed); - if (res == NULL) - return 0; - closed = PyObject_IsTrue(res); - Py_DECREF(res); - return closed; -} - -static PyObject * -IOBase_closed_get(PyObject *self, void *context) -{ - return PyBool_FromLong(IS_CLOSED(self)); -} - -PyObject * -_PyIOBase_checkClosed(PyObject *self, PyObject *args) -{ - if (IOBase_closed(self)) { - PyErr_SetString(PyExc_ValueError, "I/O operation on closed file."); - return NULL; - } - if (args == Py_True) - return Py_None; - else - Py_RETURN_NONE; -} - -/* XXX: IOBase thinks it has to maintain its own internal state in - `__IOBase_closed` and call flush() by itself, but it is redundant with - whatever behaviour a non-trivial derived class will implement. */ - -static PyObject * -IOBase_close(PyObject *self, PyObject *args) -{ - PyObject *res; - - if (IS_CLOSED(self)) - Py_RETURN_NONE; - - res = PyObject_CallMethodObjArgs(self, _PyIO_str_flush, NULL); - PyObject_SetAttrString(self, "__IOBase_closed", Py_True); - if (res == NULL) { - /* If flush() fails, just give up */ - if (PyErr_ExceptionMatches(PyExc_IOError)) - PyErr_Clear(); - else - return NULL; - } - Py_XDECREF(res); - Py_RETURN_NONE; -} - -/* Finalization and garbage collection support */ - -int -_PyIOBase_finalize(PyObject *self) -{ - PyObject *res; - PyObject *tp, *v, *tb; - int closed = 1; - int is_zombie; - - /* If _PyIOBase_finalize() is called from a destructor, we need to - resurrect the object as calling close() can invoke arbitrary code. */ - is_zombie = (Py_REFCNT(self) == 0); - if (is_zombie) { - ++Py_REFCNT(self); - } - PyErr_Fetch(&tp, &v, &tb); - /* If `closed` doesn't exist or can't be evaluated as bool, then the - object is probably in an unusable state, so ignore. */ - res = PyObject_GetAttr(self, _PyIO_str_closed); - if (res == NULL) - PyErr_Clear(); - else { - closed = PyObject_IsTrue(res); - Py_DECREF(res); - if (closed == -1) - PyErr_Clear(); - } - if (closed == 0) { - res = PyObject_CallMethodObjArgs((PyObject *) self, _PyIO_str_close, - NULL); - /* Silencing I/O errors is bad, but printing spurious tracebacks is - equally as bad, and potentially more frequent (because of - shutdown issues). */ - if (res == NULL) - PyErr_Clear(); - else - Py_DECREF(res); - } - PyErr_Restore(tp, v, tb); - if (is_zombie) { - if (--Py_REFCNT(self) != 0) { - /* The object lives again. The following code is taken from - slot_tp_del in typeobject.c. */ - Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReference(self); - Py_REFCNT(self) = refcnt; - /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so - * we need to undo that. */ - _Py_DEC_REFTOTAL; - /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object - * chain, so no more to do there. - * If COUNT_ALLOCS, the original decref bumped tp_frees, and - * _Py_NewReference bumped tp_allocs: both of those need to be - * undone. - */ -#ifdef COUNT_ALLOCS - --Py_TYPE(self)->tp_frees; - --Py_TYPE(self)->tp_allocs; -#endif - return -1; - } - } - return 0; -} - -static int -IOBase_traverse(IOBaseObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->dict); - return 0; -} - -static int -IOBase_clear(IOBaseObject *self) -{ - if (_PyIOBase_finalize((PyObject *) self) < 0) - return -1; - Py_CLEAR(self->dict); - return 0; -} - -/* Destructor */ - -static void -IOBase_dealloc(IOBaseObject *self) -{ - /* NOTE: since IOBaseObject has its own dict, Python-defined attributes - are still available here for close() to use. - However, if the derived class declares a __slots__, those slots are - already gone. - */ - if (_PyIOBase_finalize((PyObject *) self) < 0) { - /* When called from a heap type's dealloc, the type will be - decref'ed on return (see e.g. subtype_dealloc in typeobject.c). */ - if (PyType_HasFeature(Py_TYPE(self), Py_TPFLAGS_HEAPTYPE)) - Py_INCREF(Py_TYPE(self)); - return; - } - _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -/* Inquiry methods */ - -PyDoc_STRVAR(IOBase_seekable_doc, - "Return whether object supports random access.\n" - "\n" - "If False, seek(), tell() and truncate() will raise IOError.\n" - "This method may need to do a test seek()."); - -static PyObject * -IOBase_seekable(PyObject *self, PyObject *args) -{ - Py_RETURN_FALSE; -} - -PyObject * -_PyIOBase_checkSeekable(PyObject *self, PyObject *args) -{ - PyObject *res = PyObject_CallMethodObjArgs(self, _PyIO_str_seekable, NULL); - if (res == NULL) - return NULL; - if (res != Py_True) { - Py_CLEAR(res); - PyErr_SetString(PyExc_IOError, "File or stream is not seekable."); - return NULL; - } - if (args == Py_True) { - Py_DECREF(res); - } - return res; -} - -PyDoc_STRVAR(IOBase_readable_doc, - "Return whether object was opened for reading.\n" - "\n" - "If False, read() will raise IOError."); - -static PyObject * -IOBase_readable(PyObject *self, PyObject *args) -{ - Py_RETURN_FALSE; -} - -/* May be called with any object */ -PyObject * -_PyIOBase_checkReadable(PyObject *self, PyObject *args) -{ - PyObject *res = PyObject_CallMethodObjArgs(self, _PyIO_str_readable, NULL); - if (res == NULL) - return NULL; - if (res != Py_True) { - Py_CLEAR(res); - PyErr_SetString(PyExc_IOError, "File or stream is not readable."); - return NULL; - } - if (args == Py_True) { - Py_DECREF(res); - } - return res; -} - -PyDoc_STRVAR(IOBase_writable_doc, - "Return whether object was opened for writing.\n" - "\n" - "If False, read() will raise IOError."); - -static PyObject * -IOBase_writable(PyObject *self, PyObject *args) -{ - Py_RETURN_FALSE; -} - -/* May be called with any object */ -PyObject * -_PyIOBase_checkWritable(PyObject *self, PyObject *args) -{ - PyObject *res = PyObject_CallMethodObjArgs(self, _PyIO_str_writable, NULL); - if (res == NULL) - return NULL; - if (res != Py_True) { - Py_CLEAR(res); - PyErr_SetString(PyExc_IOError, "File or stream is not writable."); - return NULL; - } - if (args == Py_True) { - Py_DECREF(res); - } - return res; -} - -/* Context manager */ - -static PyObject * -IOBase_enter(PyObject *self, PyObject *args) -{ - if (_PyIOBase_checkClosed(self, Py_True) == NULL) - return NULL; - - Py_INCREF(self); - return self; -} - -static PyObject * -IOBase_exit(PyObject *self, PyObject *args) -{ - return PyObject_CallMethodObjArgs(self, _PyIO_str_close, NULL); -} - -/* Lower-level APIs */ - -/* XXX Should these be present even if unimplemented? */ - -PyDoc_STRVAR(IOBase_fileno_doc, - "Returns underlying file descriptor if one exists.\n" - "\n" - "An IOError is raised if the IO object does not use a file descriptor.\n"); - -static PyObject * -IOBase_fileno(PyObject *self, PyObject *args) -{ - return IOBase_unsupported("fileno"); -} - -PyDoc_STRVAR(IOBase_isatty_doc, - "Return whether this is an 'interactive' stream.\n" - "\n" - "Return False if it can't be determined.\n"); - -static PyObject * -IOBase_isatty(PyObject *self, PyObject *args) -{ - if (_PyIOBase_checkClosed(self, Py_True) == NULL) - return NULL; - Py_RETURN_FALSE; -} - -/* Readline(s) and writelines */ - -PyDoc_STRVAR(IOBase_readline_doc, - "Read and return a line from the stream.\n" - "\n" - "If limit is specified, at most limit bytes will be read.\n" - "\n" - "The line terminator is always b'\n' for binary files; for text\n" - "files, the newlines argument to open can be used to select the line\n" - "terminator(s) recognized.\n"); - -static PyObject * -IOBase_readline(PyObject *self, PyObject *args) -{ - /* For backwards compatibility, a (slowish) readline(). */ - - Py_ssize_t limit = -1; - int has_peek = 0; - PyObject *buffer, *result; - Py_ssize_t old_size = -1; - - if (!PyArg_ParseTuple(args, "|n:readline", &limit)) { - return NULL; - } - - if (PyObject_HasAttrString(self, "peek")) - has_peek = 1; - - buffer = PyByteArray_FromStringAndSize(NULL, 0); - if (buffer == NULL) - return NULL; - - while (limit < 0 || Py_SIZE(buffer) < limit) { - Py_ssize_t nreadahead = 1; - PyObject *b; - - if (has_peek) { - PyObject *readahead = PyObject_CallMethod(self, "peek", "i", 1); - if (readahead == NULL) - goto fail; - if (!PyBytes_Check(readahead)) { - PyErr_Format(PyExc_IOError, - "peek() should have returned a bytes object, " - "not '%.200s'", Py_TYPE(readahead)->tp_name); - Py_DECREF(readahead); - goto fail; - } - if (PyBytes_GET_SIZE(readahead) > 0) { - Py_ssize_t n = 0; - const char *buf = PyBytes_AS_STRING(readahead); - if (limit >= 0) { - do { - if (n >= PyBytes_GET_SIZE(readahead) || n >= limit) - break; - if (buf[n++] == '\n') - break; - } while (1); - } - else { - do { - if (n >= PyBytes_GET_SIZE(readahead)) - break; - if (buf[n++] == '\n') - break; - } while (1); - } - nreadahead = n; - } - Py_DECREF(readahead); - } - - b = PyObject_CallMethod(self, "read", "n", nreadahead); - if (b == NULL) - goto fail; - if (!PyBytes_Check(b)) { - PyErr_Format(PyExc_IOError, - "read() should have returned a bytes object, " - "not '%.200s'", Py_TYPE(b)->tp_name); - Py_DECREF(b); - goto fail; - } - if (PyBytes_GET_SIZE(b) == 0) { - Py_DECREF(b); - break; - } - - old_size = PyByteArray_GET_SIZE(buffer); - PyByteArray_Resize(buffer, old_size + PyBytes_GET_SIZE(b)); - memcpy(PyByteArray_AS_STRING(buffer) + old_size, - PyBytes_AS_STRING(b), PyBytes_GET_SIZE(b)); - - Py_DECREF(b); - - if (PyByteArray_AS_STRING(buffer)[PyByteArray_GET_SIZE(buffer) - 1] == '\n') - break; - } - - result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(buffer), - PyByteArray_GET_SIZE(buffer)); - Py_DECREF(buffer); - return result; - fail: - Py_DECREF(buffer); - return NULL; -} - -static PyObject * -IOBase_iter(PyObject *self) -{ - if (_PyIOBase_checkClosed(self, Py_True) == NULL) - return NULL; - - Py_INCREF(self); - return self; -} - -static PyObject * -IOBase_iternext(PyObject *self) -{ - PyObject *line = PyObject_CallMethodObjArgs(self, _PyIO_str_readline, NULL); - - if (line == NULL) - return NULL; - - if (PyObject_Size(line) == 0) { - Py_DECREF(line); - return NULL; - } - - return line; -} - -PyDoc_STRVAR(IOBase_readlines_doc, - "Return a list of lines from the stream.\n" - "\n" - "hint can be specified to control the number of lines read: no more\n" - "lines will be read if the total size (in bytes/characters) of all\n" - "lines so far exceeds hint."); - -static PyObject * -IOBase_readlines(PyObject *self, PyObject *args) -{ - Py_ssize_t hint = -1, length = 0; - PyObject *hintobj = Py_None, *result; - - if (!PyArg_ParseTuple(args, "|O:readlines", &hintobj)) { - return NULL; - } - if (hintobj != Py_None) { - hint = PyNumber_AsSsize_t(hintobj, PyExc_ValueError); - if (hint == -1 && PyErr_Occurred()) - return NULL; - } - - result = PyList_New(0); - if (result == NULL) - return NULL; - - if (hint <= 0) { - /* XXX special-casing this made sense in the Python version in order - to remove the bytecode interpretation overhead, but it could - probably be removed here. */ - PyObject *ret = PyObject_CallMethod(result, "extend", "O", self); - if (ret == NULL) { - Py_DECREF(result); - return NULL; - } - Py_DECREF(ret); - return result; - } - - while (1) { - PyObject *line = PyIter_Next(self); - if (line == NULL) { - if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; - } - else - break; /* StopIteration raised */ - } - - if (PyList_Append(result, line) < 0) { - Py_DECREF(line); - Py_DECREF(result); - return NULL; - } - length += PyObject_Size(line); - Py_DECREF(line); - - if (length > hint) - break; - } - return result; -} - -static PyObject * -IOBase_writelines(PyObject *self, PyObject *args) -{ - PyObject *lines, *iter, *res; - - if (!PyArg_ParseTuple(args, "O:writelines", &lines)) { - return NULL; - } - - if (_PyIOBase_checkClosed(self, Py_True) == NULL) - return NULL; - - iter = PyObject_GetIter(lines); - if (iter == NULL) - return NULL; - - while (1) { - PyObject *line = PyIter_Next(iter); - if (line == NULL) { - if (PyErr_Occurred()) { - Py_DECREF(iter); - return NULL; - } - else - break; /* Stop Iteration */ - } - - res = PyObject_CallMethodObjArgs(self, _PyIO_str_write, line, NULL); - Py_DECREF(line); - if (res == NULL) { - Py_DECREF(iter); - return NULL; - } - Py_DECREF(res); - } - Py_DECREF(iter); - Py_RETURN_NONE; -} - -static PyMethodDef IOBase_methods[] = { - {"seek", IOBase_seek, METH_VARARGS, IOBase_seek_doc}, - {"tell", IOBase_tell, METH_NOARGS, IOBase_tell_doc}, - {"truncate", IOBase_truncate, METH_VARARGS, IOBase_truncate_doc}, - {"flush", IOBase_flush, METH_NOARGS, IOBase_flush_doc}, - {"close", IOBase_close, METH_NOARGS, IOBase_close_doc}, - - {"seekable", IOBase_seekable, METH_NOARGS, IOBase_seekable_doc}, - {"readable", IOBase_readable, METH_NOARGS, IOBase_readable_doc}, - {"writable", IOBase_writable, METH_NOARGS, IOBase_writable_doc}, - - {"_checkClosed", _PyIOBase_checkClosed, METH_NOARGS}, - {"_checkSeekable", _PyIOBase_checkSeekable, METH_NOARGS}, - {"_checkReadable", _PyIOBase_checkReadable, METH_NOARGS}, - {"_checkWritable", _PyIOBase_checkWritable, METH_NOARGS}, - - {"fileno", IOBase_fileno, METH_NOARGS, IOBase_fileno_doc}, - {"isatty", IOBase_isatty, METH_NOARGS, IOBase_isatty_doc}, - - {"__enter__", IOBase_enter, METH_NOARGS}, - {"__exit__", IOBase_exit, METH_VARARGS}, - - {"readline", IOBase_readline, METH_VARARGS, IOBase_readline_doc}, - {"readlines", IOBase_readlines, METH_VARARGS, IOBase_readlines_doc}, - {"writelines", IOBase_writelines, METH_VARARGS}, - - {NULL, NULL} -}; - -static PyGetSetDef IOBase_getset[] = { - {"closed", (getter)IOBase_closed_get, NULL, NULL}, - {0} -}; - - -PyTypeObject PyIOBase_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._IOBase", /*tp_name*/ - sizeof(IOBaseObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)IOBase_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - IOBase_doc, /* tp_doc */ - (traverseproc)IOBase_traverse, /* tp_traverse */ - (inquiry)IOBase_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(IOBaseObject, weakreflist), /* tp_weaklistoffset */ - IOBase_iter, /* tp_iter */ - IOBase_iternext, /* tp_iternext */ - IOBase_methods, /* tp_methods */ - 0, /* tp_members */ - IOBase_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(IOBaseObject, dict), /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ -}; - - -/* - * RawIOBase class, Inherits from IOBase. - */ -PyDoc_STRVAR(RawIOBase_doc, - "Base class for raw binary I/O."); - -/* - * The read() method is implemented by calling readinto(); derived classes - * that want to support read() only need to implement readinto() as a - * primitive operation. In general, readinto() can be more efficient than - * read(). - * - * (It would be tempting to also provide an implementation of readinto() in - * terms of read(), in case the latter is a more suitable primitive operation, - * but that would lead to nasty recursion in case a subclass doesn't implement - * either.) -*/ - -static PyObject * -RawIOBase_read(PyObject *self, PyObject *args) -{ - Py_ssize_t n = -1; - PyObject *b, *res; - - if (!PyArg_ParseTuple(args, "|n:read", &n)) { - return NULL; - } - - if (n < 0) - return PyObject_CallMethod(self, "readall", NULL); - - /* TODO: allocate a bytes object directly instead and manually construct - a writable memoryview pointing to it. */ - b = PyByteArray_FromStringAndSize(NULL, n); - if (b == NULL) - return NULL; - - res = PyObject_CallMethodObjArgs(self, _PyIO_str_readinto, b, NULL); - if (res == NULL) { - Py_DECREF(b); - return NULL; - } - - n = PyNumber_AsSsize_t(res, PyExc_ValueError); - Py_DECREF(res); - if (n == -1 && PyErr_Occurred()) { - Py_DECREF(b); - return NULL; - } - - res = PyBytes_FromStringAndSize(PyByteArray_AsString(b), n); - Py_DECREF(b); - return res; -} - - -PyDoc_STRVAR(RawIOBase_readall_doc, - "Read until EOF, using multiple read() call."); - -static PyObject * -RawIOBase_readall(PyObject *self, PyObject *args) -{ - int r; - PyObject *chunks = PyList_New(0); - PyObject *result; - - if (chunks == NULL) - return NULL; - - while (1) { - PyObject *data = PyObject_CallMethod(self, "read", - "i", DEFAULT_BUFFER_SIZE); - if (!data) { - Py_DECREF(chunks); - return NULL; - } - if (!PyBytes_Check(data)) { - Py_DECREF(chunks); - Py_DECREF(data); - PyErr_SetString(PyExc_TypeError, "read() should return bytes"); - return NULL; - } - if (PyBytes_GET_SIZE(data) == 0) { - /* EOF */ - Py_DECREF(data); - break; - } - r = PyList_Append(chunks, data); - Py_DECREF(data); - if (r < 0) { - Py_DECREF(chunks); - return NULL; - } - } - result = _PyBytes_Join(_PyIO_empty_bytes, chunks); - Py_DECREF(chunks); - return result; -} - -static PyMethodDef RawIOBase_methods[] = { - {"read", RawIOBase_read, METH_VARARGS}, - {"readall", RawIOBase_readall, METH_NOARGS, RawIOBase_readall_doc}, - {NULL, NULL} -}; - -PyTypeObject PyRawIOBase_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._RawIOBase", /*tp_name*/ - 0, /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - RawIOBase_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - RawIOBase_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyIOBase_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; diff --git a/Modules/_iomodule.h b/Modules/_iomodule.h deleted file mode 100644 index a44f127..0000000 --- a/Modules/_iomodule.h +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Declarations shared between the different parts of the io module - */ - -/* ABCs */ -extern PyTypeObject PyIOBase_Type; -extern PyTypeObject PyRawIOBase_Type; -extern PyTypeObject PyBufferedIOBase_Type; -extern PyTypeObject PyTextIOBase_Type; - -/* Concrete classes */ -extern PyTypeObject PyFileIO_Type; -extern PyTypeObject PyBytesIO_Type; -extern PyTypeObject PyStringIO_Type; -extern PyTypeObject PyBufferedReader_Type; -extern PyTypeObject PyBufferedWriter_Type; -extern PyTypeObject PyBufferedRWPair_Type; -extern PyTypeObject PyBufferedRandom_Type; -extern PyTypeObject PyTextIOWrapper_Type; -extern PyTypeObject PyIncrementalNewlineDecoder_Type; - -/* These functions are used as METH_NOARGS methods, are normally called - * with args=NULL, and return a new reference. - * BUT when args=Py_True is passed, they return a borrowed reference. - */ -extern PyObject* _PyIOBase_checkReadable(PyObject *self, PyObject *args); -extern PyObject* _PyIOBase_checkWritable(PyObject *self, PyObject *args); -extern PyObject* _PyIOBase_checkSeekable(PyObject *self, PyObject *args); -extern PyObject* _PyIOBase_checkClosed(PyObject *self, PyObject *args); - -/* Helper for finalization. - This function will revive an object ready to be deallocated and try to - close() it. It returns 0 if the object can be destroyed, or -1 if it - is alive again. */ -extern int _PyIOBase_finalize(PyObject *self); - -/* Returns true if the given FileIO object is closed. - Doesn't check the argument type, so be careful! */ -extern int _PyFileIO_closed(PyObject *self); - -/* Shortcut to the core of the IncrementalNewlineDecoder.decode method */ -extern PyObject *_PyIncrementalNewlineDecoder_decode( - PyObject *self, PyObject *input, int final); - -/* Finds the first line ending between `start` and `end`. - If found, returns the index after the line ending and doesn't touch - `*consumed`. - If not found, returns -1 and sets `*consumed` to the number of characters - which can be safely put aside until another search. - - NOTE: for performance reasons, `end` must point to a NUL character ('\0'). - Otherwise, the function will scan further and return garbage. */ -extern Py_ssize_t _PyIO_find_line_ending( - int translated, int universal, PyObject *readnl, - Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed); - - -#define DEFAULT_BUFFER_SIZE (8 * 1024) /* bytes */ - -typedef struct { - PyException_HEAD - PyObject *myerrno; - PyObject *strerror; - PyObject *filename; /* Not used, but part of the IOError object */ - Py_ssize_t written; -} PyBlockingIOErrorObject; -PyAPI_DATA(PyObject *) PyExc_BlockingIOError; - -/* - * Offset type for positioning. - */ - -#if defined(MS_WIN64) || defined(MS_WINDOWS) - -/* Windows uses long long for offsets */ -typedef PY_LONG_LONG Py_off_t; -# define PyLong_AsOff_t PyLong_AsLongLong -# define PyLong_FromOff_t PyLong_FromLongLong -# define PY_OFF_T_MAX PY_LLONG_MAX -# define PY_OFF_T_MIN PY_LLONG_MIN - -#else - -/* Other platforms use off_t */ -typedef off_t Py_off_t; -#if (SIZEOF_OFF_T == SIZEOF_SIZE_T) -# define PyLong_AsOff_t PyLong_AsSsize_t -# define PyLong_FromOff_t PyLong_FromSsize_t -# define PY_OFF_T_MAX PY_SSIZE_T_MAX -# define PY_OFF_T_MIN PY_SSIZE_T_MIN -#elif (SIZEOF_OFF_T == SIZEOF_LONG_LONG) -# define PyLong_AsOff_t PyLong_AsLongLong -# define PyLong_FromOff_t PyLong_FromLongLong -# define PY_OFF_T_MAX PY_LLONG_MAX -# define PY_OFF_T_MIN PY_LLONG_MIN -#elif (SIZEOF_OFF_T == SIZEOF_LONG) -# define PyLong_AsOff_t PyLong_AsLong -# define PyLong_FromOff_t PyLong_FromLong -# define PY_OFF_T_MAX LONG_MAX -# define PY_OFF_T_MIN LONG_MIN -#else -# error off_t does not match either size_t, long, or long long! -#endif - -#endif - -extern Py_off_t PyNumber_AsOff_t(PyObject *item, PyObject *err); - -/* Implementation details */ - -/* IO module structure */ - -extern PyModuleDef _PyIO_Module; - -typedef struct { - int initialized; - PyObject *os_module; - PyObject *locale_module; - - PyObject *unsupported_operation; -} _PyIO_State; - -#define IO_MOD_STATE(mod) ((_PyIO_State *)PyModule_GetState(mod)) -#define IO_STATE IO_MOD_STATE(PyState_FindModule(&_PyIO_Module)) - -extern PyObject *_PyIO_str_close; -extern PyObject *_PyIO_str_closed; -extern PyObject *_PyIO_str_decode; -extern PyObject *_PyIO_str_encode; -extern PyObject *_PyIO_str_fileno; -extern PyObject *_PyIO_str_flush; -extern PyObject *_PyIO_str_getstate; -extern PyObject *_PyIO_str_isatty; -extern PyObject *_PyIO_str_newlines; -extern PyObject *_PyIO_str_nl; -extern PyObject *_PyIO_str_read; -extern PyObject *_PyIO_str_read1; -extern PyObject *_PyIO_str_readable; -extern PyObject *_PyIO_str_readinto; -extern PyObject *_PyIO_str_readline; -extern PyObject *_PyIO_str_reset; -extern PyObject *_PyIO_str_seek; -extern PyObject *_PyIO_str_seekable; -extern PyObject *_PyIO_str_tell; -extern PyObject *_PyIO_str_truncate; -extern PyObject *_PyIO_str_writable; -extern PyObject *_PyIO_str_write; - -extern PyObject *_PyIO_empty_str; -extern PyObject *_PyIO_empty_bytes; diff --git a/Modules/_stringio.c b/Modules/_stringio.c deleted file mode 100644 index d75a604..0000000 --- a/Modules/_stringio.c +++ /dev/null @@ -1,769 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#include "_iomodule.h" - -/* Implementation note: the buffer is always at least one character longer - than the enclosed string, for proper functioning of _PyIO_find_line_ending. -*/ - -typedef struct { - PyObject_HEAD - Py_UNICODE *buf; - Py_ssize_t pos; - Py_ssize_t string_size; - size_t buf_size; - - char ok; /* initialized? */ - char closed; - char readuniversal; - char readtranslate; - PyObject *decoder; - PyObject *readnl; - PyObject *writenl; - - PyObject *dict; - PyObject *weakreflist; -} StringIOObject; - -#define CHECK_INITIALIZED(self) \ - if (self->ok <= 0) { \ - PyErr_SetString(PyExc_ValueError, \ - "I/O operation on uninitialized object"); \ - return NULL; \ - } - -#define CHECK_CLOSED(self) \ - if (self->closed) { \ - PyErr_SetString(PyExc_ValueError, \ - "I/O operation on closed file"); \ - return NULL; \ - } - -PyDoc_STRVAR(stringio_doc, - "Text I/O implementation using an in-memory buffer.\n" - "\n" - "The initial_value argument sets the value of object. The newline\n" - "argument is like the one of TextIOWrapper's constructor."); - - -/* Internal routine for changing the size, in terms of characters, of the - buffer of StringIO objects. The caller should ensure that the 'size' - argument is non-negative. Returns 0 on success, -1 otherwise. */ -static int -resize_buffer(StringIOObject *self, size_t size) -{ - /* Here, unsigned types are used to avoid dealing with signed integer - overflow, which is undefined in C. */ - size_t alloc = self->buf_size; - Py_UNICODE *new_buf = NULL; - - assert(self->buf != NULL); - - /* Reserve one more char for line ending detection. */ - size = size + 1; - /* For simplicity, stay in the range of the signed type. Anyway, Python - doesn't allow strings to be longer than this. */ - if (size > PY_SSIZE_T_MAX) - goto overflow; - - if (size < alloc / 2) { - /* Major downsize; resize down to exact size. */ - alloc = size + 1; - } - else if (size < alloc) { - /* Within allocated size; quick exit */ - return 0; - } - else if (size <= alloc * 1.125) { - /* Moderate upsize; overallocate similar to list_resize() */ - alloc = size + (size >> 3) + (size < 9 ? 3 : 6); - } - else { - /* Major upsize; resize up to exact size */ - alloc = size + 1; - } - - if (alloc > ((size_t)-1) / sizeof(Py_UNICODE)) - goto overflow; - new_buf = (Py_UNICODE *)PyMem_Realloc(self->buf, - alloc * sizeof(Py_UNICODE)); - if (new_buf == NULL) { - PyErr_NoMemory(); - return -1; - } - self->buf_size = alloc; - self->buf = new_buf; - - return 0; - - overflow: - PyErr_SetString(PyExc_OverflowError, - "new buffer size too large"); - return -1; -} - -/* Internal routine for writing a whole PyUnicode object to the buffer of a - StringIO object. Returns 0 on success, or -1 on error. */ -static Py_ssize_t -write_str(StringIOObject *self, PyObject *obj) -{ - Py_UNICODE *str; - Py_ssize_t len; - PyObject *decoded = NULL; - assert(self->buf != NULL); - assert(self->pos >= 0); - - if (self->decoder != NULL) { - decoded = _PyIncrementalNewlineDecoder_decode( - self->decoder, obj, 1 /* always final */); - } - else { - decoded = obj; - Py_INCREF(decoded); - } - if (self->writenl) { - PyObject *translated = PyUnicode_Replace( - decoded, _PyIO_str_nl, self->writenl, -1); - Py_DECREF(decoded); - decoded = translated; - } - if (decoded == NULL) - return -1; - - assert(PyUnicode_Check(decoded)); - str = PyUnicode_AS_UNICODE(decoded); - len = PyUnicode_GET_SIZE(decoded); - - assert(len >= 0); - - /* This overflow check is not strictly necessary. However, it avoids us to - deal with funky things like comparing an unsigned and a signed - integer. */ - if (self->pos > PY_SSIZE_T_MAX - len) { - PyErr_SetString(PyExc_OverflowError, - "new position too large"); - goto fail; - } - if (self->pos + len > self->string_size) { - if (resize_buffer(self, self->pos + len) < 0) - goto fail; - } - - if (self->pos > self->string_size) { - /* In case of overseek, pad with null bytes the buffer region between - the end of stream and the current position. - - 0 lo string_size hi - | |<---used--->|<----------available----------->| - | | <--to pad-->|<---to write---> | - 0 buf positon - - */ - memset(self->buf + self->string_size, '\0', - (self->pos - self->string_size) * sizeof(Py_UNICODE)); - } - - /* Copy the data to the internal buffer, overwriting some of the - existing data if self->pos < self->string_size. */ - memcpy(self->buf + self->pos, str, len * sizeof(Py_UNICODE)); - self->pos += len; - - /* Set the new length of the internal string if it has changed. */ - if (self->string_size < self->pos) { - self->string_size = self->pos; - } - - Py_DECREF(decoded); - return 0; - -fail: - Py_XDECREF(decoded); - return -1; -} - -PyDoc_STRVAR(stringio_getvalue_doc, - "Retrieve the entire contents of the object."); - -static PyObject * -stringio_getvalue(StringIOObject *self) -{ - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - return PyUnicode_FromUnicode(self->buf, self->string_size); -} - -PyDoc_STRVAR(stringio_tell_doc, - "Tell the current file position."); - -static PyObject * -stringio_tell(StringIOObject *self) -{ - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - return PyLong_FromSsize_t(self->pos); -} - -PyDoc_STRVAR(stringio_read_doc, - "Read at most n characters, returned as a string.\n" - "\n" - "If the argument is negative or omitted, read until EOF\n" - "is reached. Return an empty string at EOF.\n"); - -static PyObject * -stringio_read(StringIOObject *self, PyObject *args) -{ - Py_ssize_t size, n; - Py_UNICODE *output; - PyObject *arg = Py_None; - - CHECK_INITIALIZED(self); - if (!PyArg_ParseTuple(args, "|O:read", &arg)) - return NULL; - CHECK_CLOSED(self); - - if (PyNumber_Check(arg)) { - size = PyNumber_AsSsize_t(arg, PyExc_OverflowError); - if (size == -1 && PyErr_Occurred()) - return NULL; - } - else if (arg == Py_None) { - /* Read until EOF is reached, by default. */ - size = -1; - } - else { - PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", - Py_TYPE(arg)->tp_name); - return NULL; - } - - /* adjust invalid sizes */ - n = self->string_size - self->pos; - if (size < 0 || size > n) { - size = n; - if (size < 0) - size = 0; - } - - output = self->buf + self->pos; - self->pos += size; - return PyUnicode_FromUnicode(output, size); -} - -/* Internal helper, used by stringio_readline and stringio_iternext */ -static PyObject * -_stringio_readline(StringIOObject *self, Py_ssize_t limit) -{ - Py_UNICODE *start, *end, old_char; - Py_ssize_t len, consumed; - - /* In case of overseek, return the empty string */ - if (self->pos >= self->string_size) - return PyUnicode_FromString(""); - - start = self->buf + self->pos; - if (limit < 0 || limit > self->string_size - self->pos) - limit = self->string_size - self->pos; - - end = start + limit; - old_char = *end; - *end = '\0'; - len = _PyIO_find_line_ending( - self->readtranslate, self->readuniversal, self->readnl, - start, end, &consumed); - *end = old_char; - /* If we haven't found any line ending, we just return everything - (`consumed` is ignored). */ - if (len < 0) - len = limit; - self->pos += len; - return PyUnicode_FromUnicode(start, len); -} - -PyDoc_STRVAR(stringio_readline_doc, - "Read until newline or EOF.\n" - "\n" - "Returns an empty string if EOF is hit immediately.\n"); - -static PyObject * -stringio_readline(StringIOObject *self, PyObject *args) -{ - PyObject *arg = Py_None; - Py_ssize_t limit = -1; - - CHECK_INITIALIZED(self); - if (!PyArg_ParseTuple(args, "|O:readline", &arg)) - return NULL; - CHECK_CLOSED(self); - - if (PyNumber_Check(arg)) { - limit = PyNumber_AsSsize_t(arg, PyExc_OverflowError); - if (limit == -1 && PyErr_Occurred()) - return NULL; - } - else if (arg != Py_None) { - PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", - Py_TYPE(arg)->tp_name); - return NULL; - } - return _stringio_readline(self, limit); -} - -static PyObject * -stringio_iternext(StringIOObject *self) -{ - PyObject *line; - - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - - if (Py_TYPE(self) == &PyStringIO_Type) { - /* Skip method call overhead for speed */ - line = _stringio_readline(self, -1); - } - else { - /* XXX is subclassing StringIO really supported? */ - line = PyObject_CallMethodObjArgs((PyObject *)self, - _PyIO_str_readline, NULL); - if (line && !PyUnicode_Check(line)) { - PyErr_Format(PyExc_IOError, - "readline() should have returned an str object, " - "not '%.200s'", Py_TYPE(line)->tp_name); - Py_DECREF(line); - return NULL; - } - } - - if (line == NULL) - return NULL; - - if (PyUnicode_GET_SIZE(line) == 0) { - /* Reached EOF */ - Py_DECREF(line); - return NULL; - } - - return line; -} - -PyDoc_STRVAR(stringio_truncate_doc, - "Truncate size to pos.\n" - "\n" - "The pos argument defaults to the current file position, as\n" - "returned by tell(). Imply an absolute seek to pos.\n" - "Returns the new absolute position.\n"); - -static PyObject * -stringio_truncate(StringIOObject *self, PyObject *args) -{ - Py_ssize_t size; - PyObject *arg = Py_None; - - CHECK_INITIALIZED(self); - if (!PyArg_ParseTuple(args, "|O:truncate", &arg)) - return NULL; - CHECK_CLOSED(self); - - if (PyNumber_Check(arg)) { - size = PyNumber_AsSsize_t(arg, PyExc_OverflowError); - if (size == -1 && PyErr_Occurred()) - return NULL; - } - else if (arg == Py_None) { - /* Truncate to current position if no argument is passed. */ - size = self->pos; - } - else { - PyErr_Format(PyExc_TypeError, "integer argument expected, got '%s'", - Py_TYPE(arg)->tp_name); - return NULL; - } - - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "Negative size value %zd", size); - return NULL; - } - - if (size < self->string_size) { - if (resize_buffer(self, size) < 0) - return NULL; - self->string_size = size; - } - self->pos = size; - - return PyLong_FromSsize_t(size); -} - -PyDoc_STRVAR(stringio_seek_doc, - "Change stream position.\n" - "\n" - "Seek to character offset pos relative to position indicated by whence:\n" - " 0 Start of stream (the default). pos should be >= 0;\n" - " 1 Current position - pos must be 0;\n" - " 2 End of stream - pos must be 0.\n" - "Returns the new absolute position.\n"); - -static PyObject * -stringio_seek(StringIOObject *self, PyObject *args) -{ - Py_ssize_t pos; - int mode = 0; - - CHECK_INITIALIZED(self); - if (!PyArg_ParseTuple(args, "n|i:seek", &pos, &mode)) - return NULL; - CHECK_CLOSED(self); - - if (mode != 0 && mode != 1 && mode != 2) { - PyErr_Format(PyExc_ValueError, - "Invalid whence (%i, should be 0, 1 or 2)", mode); - return NULL; - } - else if (pos < 0 && mode == 0) { - PyErr_Format(PyExc_ValueError, - "Negative seek position %zd", pos); - return NULL; - } - else if (mode != 0 && pos != 0) { - PyErr_SetString(PyExc_IOError, - "Can't do nonzero cur-relative seeks"); - return NULL; - } - - /* mode 0: offset relative to beginning of the string. - mode 1: no change to current position. - mode 2: change position to end of file. */ - if (mode == 1) { - pos = self->pos; - } - else if (mode == 2) { - pos = self->string_size; - } - - self->pos = pos; - - return PyLong_FromSsize_t(self->pos); -} - -PyDoc_STRVAR(stringio_write_doc, - "Write string to file.\n" - "\n" - "Returns the number of characters written, which is always equal to\n" - "the length of the string.\n"); - -static PyObject * -stringio_write(StringIOObject *self, PyObject *obj) -{ - Py_ssize_t size; - - CHECK_INITIALIZED(self); - if (!PyUnicode_Check(obj)) { - PyErr_Format(PyExc_TypeError, "string argument expected, got '%s'", - Py_TYPE(obj)->tp_name); - return NULL; - } - CHECK_CLOSED(self); - size = PyUnicode_GET_SIZE(obj); - - if (size > 0 && write_str(self, obj) < 0) - return NULL; - - return PyLong_FromSsize_t(size); -} - -PyDoc_STRVAR(stringio_close_doc, - "Close the IO object. Attempting any further operation after the\n" - "object is closed will raise a ValueError.\n" - "\n" - "This method has no effect if the file is already closed.\n"); - -static PyObject * -stringio_close(StringIOObject *self) -{ - self->closed = 1; - /* Free up some memory */ - if (resize_buffer(self, 0) < 0) - return NULL; - Py_CLEAR(self->readnl); - Py_CLEAR(self->writenl); - Py_CLEAR(self->decoder); - Py_RETURN_NONE; -} - -static int -stringio_traverse(StringIOObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->dict); - return 0; -} - -static int -stringio_clear(StringIOObject *self) -{ - Py_CLEAR(self->dict); - return 0; -} - -static void -stringio_dealloc(StringIOObject *self) -{ - _PyObject_GC_UNTRACK(self); - Py_CLEAR(self->readnl); - Py_CLEAR(self->writenl); - Py_CLEAR(self->decoder); - if (self->buf) - PyMem_Free(self->buf); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - Py_TYPE(self)->tp_free(self); -} - -static PyObject * -stringio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - StringIOObject *self; - - assert(type != NULL && type->tp_alloc != NULL); - self = (StringIOObject *)type->tp_alloc(type, 0); - if (self == NULL) - return NULL; - - self->string_size = 0; - self->pos = 0; - self->buf_size = 0; - self->buf = (Py_UNICODE *)PyMem_Malloc(0); - if (self->buf == NULL) { - Py_DECREF(self); - return PyErr_NoMemory(); - } - - return (PyObject *)self; -} - -static int -stringio_init(StringIOObject *self, PyObject *args, PyObject *kwds) -{ - char *kwlist[] = {"initial_value", "newline", NULL}; - PyObject *value = NULL; - char *newline = "\n"; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oz:__init__", kwlist, - &value, &newline)) - return -1; - - if (newline && newline[0] != '\0' - && !(newline[0] == '\n' && newline[1] == '\0') - && !(newline[0] == '\r' && newline[1] == '\0') - && !(newline[0] == '\r' && newline[1] == '\n' && newline[2] == '\0')) { - PyErr_Format(PyExc_ValueError, - "illegal newline value: %s", newline); - return -1; - } - if (value && value != Py_None && !PyUnicode_Check(value)) { - PyErr_Format(PyExc_ValueError, - "initial_value must be str or None, not %.200s", - Py_TYPE(value)->tp_name); - return -1; - } - - self->ok = 0; - - Py_CLEAR(self->readnl); - Py_CLEAR(self->writenl); - Py_CLEAR(self->decoder); - - if (newline) { - self->readnl = PyUnicode_FromString(newline); - if (self->readnl == NULL) - return -1; - } - self->readuniversal = (newline == NULL || newline[0] == '\0'); - self->readtranslate = (newline == NULL); - /* If newline == "", we don't translate anything. - If newline == "\n" or newline == None, we translate to "\n", which is - a no-op. - (for newline == None, TextIOWrapper translates to os.sepline, but it - is pointless for StringIO) - */ - if (newline != NULL && newline[0] == '\r') { - self->writenl = self->readnl; - Py_INCREF(self->writenl); - } - - if (self->readuniversal) { - self->decoder = PyObject_CallFunction( - (PyObject *)&PyIncrementalNewlineDecoder_Type, - "Oi", Py_None, (int) self->readtranslate); - if (self->decoder == NULL) - return -1; - } - - /* Now everything is set up, resize buffer to size of initial value, - and copy it */ - self->string_size = 0; - if (value && value != Py_None) { - Py_ssize_t len = PyUnicode_GetSize(value); - /* This is a heuristic, for newline translation might change - the string length. */ - if (resize_buffer(self, len) < 0) - return -1; - self->pos = 0; - if (write_str(self, value) < 0) - return -1; - } - else { - if (resize_buffer(self, 0) < 0) - return -1; - } - self->pos = 0; - - self->closed = 0; - self->ok = 1; - return 0; -} - -/* Properties and pseudo-properties */ -static PyObject * -stringio_seekable(StringIOObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self); - Py_RETURN_TRUE; -} - -static PyObject * -stringio_readable(StringIOObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self); - Py_RETURN_TRUE; -} - -static PyObject * -stringio_writable(StringIOObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self); - Py_RETURN_TRUE; -} - -static PyObject * -stringio_buffer(StringIOObject *self, void *context) -{ - PyErr_SetString(IO_STATE->unsupported_operation, - "buffer attribute is unsupported on type StringIO"); - return NULL; -} - -static PyObject * -stringio_closed(StringIOObject *self, void *context) -{ - CHECK_INITIALIZED(self); - return PyBool_FromLong(self->closed); -} - -static PyObject * -stringio_encoding(StringIOObject *self, void *context) -{ - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - Py_RETURN_NONE; -} - -static PyObject * -stringio_errors(StringIOObject *self, void *context) -{ - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - return PyUnicode_FromString("strict"); -} - -static PyObject * -stringio_line_buffering(StringIOObject *self, void *context) -{ - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - Py_RETURN_FALSE; -} - -static PyObject * -stringio_newlines(StringIOObject *self, void *context) -{ - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - if (self->decoder == NULL) - Py_RETURN_NONE; - return PyObject_GetAttr(self->decoder, _PyIO_str_newlines); -} - -static struct PyMethodDef stringio_methods[] = { - {"close", (PyCFunction)stringio_close, METH_NOARGS, stringio_close_doc}, - {"getvalue", (PyCFunction)stringio_getvalue, METH_VARARGS, stringio_getvalue_doc}, - {"read", (PyCFunction)stringio_read, METH_VARARGS, stringio_read_doc}, - {"readline", (PyCFunction)stringio_readline, METH_VARARGS, stringio_readline_doc}, - {"tell", (PyCFunction)stringio_tell, METH_NOARGS, stringio_tell_doc}, - {"truncate", (PyCFunction)stringio_truncate, METH_VARARGS, stringio_truncate_doc}, - {"seek", (PyCFunction)stringio_seek, METH_VARARGS, stringio_seek_doc}, - {"write", (PyCFunction)stringio_write, METH_O, stringio_write_doc}, - - {"seekable", (PyCFunction)stringio_seekable, METH_NOARGS}, - {"readable", (PyCFunction)stringio_readable, METH_NOARGS}, - {"writable", (PyCFunction)stringio_writable, METH_NOARGS}, - {NULL, NULL} /* sentinel */ -}; - -static PyGetSetDef stringio_getset[] = { - {"closed", (getter)stringio_closed, NULL, NULL}, - {"newlines", (getter)stringio_newlines, NULL, NULL}, - /* (following comments straight off of the original Python wrapper:) - XXX Cruft to support the TextIOWrapper API. This would only - be meaningful if StringIO supported the buffer attribute. - Hopefully, a better solution, than adding these pseudo-attributes, - will be found. - */ - {"buffer", (getter)stringio_buffer, NULL, NULL}, - {"encoding", (getter)stringio_encoding, NULL, NULL}, - {"errors", (getter)stringio_errors, NULL, NULL}, - {"line_buffering", (getter)stringio_line_buffering, NULL, NULL}, - {0} -}; - -PyTypeObject PyStringIO_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.StringIO", /*tp_name*/ - sizeof(StringIOObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)stringio_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - stringio_doc, /*tp_doc*/ - (traverseproc)stringio_traverse, /*tp_traverse*/ - (inquiry)stringio_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - offsetof(StringIOObject, weakreflist), /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - (iternextfunc)stringio_iternext, /*tp_iternext*/ - stringio_methods, /*tp_methods*/ - 0, /*tp_members*/ - stringio_getset, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - offsetof(StringIOObject, dict), /*tp_dictoffset*/ - (initproc)stringio_init, /*tp_init*/ - 0, /*tp_alloc*/ - stringio_new, /*tp_new*/ -}; diff --git a/Modules/_textio.c b/Modules/_textio.c deleted file mode 100644 index cc229a8..0000000 --- a/Modules/_textio.c +++ /dev/null @@ -1,2420 +0,0 @@ -/* - An implementation of Text I/O as defined by PEP 3116 - "New I/O" - - Classes defined here: TextIOBase, IncrementalNewlineDecoder, TextIOWrapper. - - Written by Amaury Forgeot d'Arc and Antoine Pitrou -*/ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#include "_iomodule.h" - -/* TextIOBase */ - -PyDoc_STRVAR(TextIOBase_doc, - "Base class for text I/O.\n" - "\n" - "This class provides a character and line based interface to stream\n" - "I/O. There is no readinto method because Python's character strings\n" - "are immutable. There is no public constructor.\n" - ); - -static PyObject * -_unsupported(const char *message) -{ - PyErr_SetString(IO_STATE->unsupported_operation, message); - return NULL; -} - -PyDoc_STRVAR(TextIOBase_read_doc, - "Read at most n characters from stream.\n" - "\n" - "Read from underlying buffer until we have n characters or we hit EOF.\n" - "If n is negative or omitted, read until EOF.\n" - ); - -static PyObject * -TextIOBase_read(PyObject *self, PyObject *args) -{ - return _unsupported("read"); -} - -PyDoc_STRVAR(TextIOBase_readline_doc, - "Read until newline or EOF.\n" - "\n" - "Returns an empty string if EOF is hit immediately.\n" - ); - -static PyObject * -TextIOBase_readline(PyObject *self, PyObject *args) -{ - return _unsupported("readline"); -} - -PyDoc_STRVAR(TextIOBase_write_doc, - "Write string to stream.\n" - "Returns the number of characters written (which is always equal to\n" - "the length of the string).\n" - ); - -static PyObject * -TextIOBase_write(PyObject *self, PyObject *args) -{ - return _unsupported("write"); -} - -PyDoc_STRVAR(TextIOBase_encoding_doc, - "Encoding of the text stream.\n" - "\n" - "Subclasses should override.\n" - ); - -static PyObject * -TextIOBase_encoding_get(PyObject *self, void *context) -{ - Py_RETURN_NONE; -} - -PyDoc_STRVAR(TextIOBase_newlines_doc, - "Line endings translated so far.\n" - "\n" - "Only line endings translated during reading are considered.\n" - "\n" - "Subclasses should override.\n" - ); - -static PyObject * -TextIOBase_newlines_get(PyObject *self, void *context) -{ - Py_RETURN_NONE; -} - - -static PyMethodDef TextIOBase_methods[] = { - {"read", TextIOBase_read, METH_VARARGS, TextIOBase_read_doc}, - {"readline", TextIOBase_readline, METH_VARARGS, TextIOBase_readline_doc}, - {"write", TextIOBase_write, METH_VARARGS, TextIOBase_write_doc}, - {NULL, NULL} -}; - -static PyGetSetDef TextIOBase_getset[] = { - {"encoding", (getter)TextIOBase_encoding_get, NULL, TextIOBase_encoding_doc}, - {"newlines", (getter)TextIOBase_newlines_get, NULL, TextIOBase_newlines_doc}, - {0} -}; - -PyTypeObject PyTextIOBase_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io._TextIOBase", /*tp_name*/ - 0, /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - TextIOBase_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - TextIOBase_methods, /* tp_methods */ - 0, /* tp_members */ - TextIOBase_getset, /* tp_getset */ - &PyIOBase_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - - -/* IncrementalNewlineDecoder */ - -PyDoc_STRVAR(IncrementalNewlineDecoder_doc, - "Codec used when reading a file in universal newlines mode. It wraps\n" - "another incremental decoder, translating \\r\\n and \\r into \\n. It also\n" - "records the types of newlines encountered. When used with\n" - "translate=False, it ensures that the newline sequence is returned in\n" - "one piece. When used with decoder=None, it expects unicode strings as\n" - "decode input and translates newlines without first invoking an external\n" - "decoder.\n" - ); - -typedef struct { - PyObject_HEAD - PyObject *decoder; - PyObject *errors; - int pendingcr:1; - int translate:1; - unsigned int seennl:3; -} PyNewLineDecoderObject; - -static int -IncrementalNewlineDecoder_init(PyNewLineDecoderObject *self, - PyObject *args, PyObject *kwds) -{ - PyObject *decoder; - int translate; - PyObject *errors = NULL; - char *kwlist[] = {"decoder", "translate", "errors", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "Oi|O:IncrementalNewlineDecoder", - kwlist, &decoder, &translate, &errors)) - return -1; - - self->decoder = decoder; - Py_INCREF(decoder); - - if (errors == NULL) { - self->errors = PyUnicode_FromString("strict"); - if (self->errors == NULL) - return -1; - } - else { - Py_INCREF(errors); - self->errors = errors; - } - - self->translate = translate; - self->seennl = 0; - self->pendingcr = 0; - - return 0; -} - -static void -IncrementalNewlineDecoder_dealloc(PyNewLineDecoderObject *self) -{ - Py_CLEAR(self->decoder); - Py_CLEAR(self->errors); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -#define SEEN_CR 1 -#define SEEN_LF 2 -#define SEEN_CRLF 4 -#define SEEN_ALL (SEEN_CR | SEEN_LF | SEEN_CRLF) - -PyObject * -_PyIncrementalNewlineDecoder_decode(PyObject *_self, - PyObject *input, int final) -{ - PyObject *output; - Py_ssize_t output_len; - PyNewLineDecoderObject *self = (PyNewLineDecoderObject *) _self; - - if (self->decoder == NULL) { - PyErr_SetString(PyExc_ValueError, - "IncrementalNewlineDecoder.__init__ not called"); - return NULL; - } - - /* decode input (with the eventual \r from a previous pass) */ - if (self->decoder != Py_None) { - output = PyObject_CallMethodObjArgs(self->decoder, - _PyIO_str_decode, input, final ? Py_True : Py_False, NULL); - } - else { - output = input; - Py_INCREF(output); - } - - if (output == NULL) - return NULL; - - if (!PyUnicode_Check(output)) { - PyErr_SetString(PyExc_TypeError, - "decoder should return a string result"); - goto error; - } - - output_len = PyUnicode_GET_SIZE(output); - if (self->pendingcr && (final || output_len > 0)) { - Py_UNICODE *out; - PyObject *modified = PyUnicode_FromUnicode(NULL, output_len + 1); - if (modified == NULL) - goto error; - out = PyUnicode_AS_UNICODE(modified); - out[0] = '\r'; - memcpy(out + 1, PyUnicode_AS_UNICODE(output), - output_len * sizeof(Py_UNICODE)); - Py_DECREF(output); - output = modified; - self->pendingcr = 0; - output_len++; - } - - /* retain last \r even when not translating data: - * then readline() is sure to get \r\n in one pass - */ - if (!final) { - if (output_len > 0 - && PyUnicode_AS_UNICODE(output)[output_len - 1] == '\r') { - - if (Py_REFCNT(output) == 1) { - if (PyUnicode_Resize(&output, output_len - 1) < 0) - goto error; - } - else { - PyObject *modified = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(output), - output_len - 1); - if (modified == NULL) - goto error; - Py_DECREF(output); - output = modified; - } - self->pendingcr = 1; - } - } - - /* Record which newlines are read and do newline translation if desired, - all in one pass. */ - { - Py_UNICODE *in_str; - Py_ssize_t len; - int seennl = self->seennl; - int only_lf = 0; - - in_str = PyUnicode_AS_UNICODE(output); - len = PyUnicode_GET_SIZE(output); - - if (len == 0) - return output; - - /* If, up to now, newlines are consistently \n, do a quick check - for the \r *byte* with the libc's optimized memchr. - */ - if (seennl == SEEN_LF || seennl == 0) { - only_lf = (memchr(in_str, '\r', len * sizeof(Py_UNICODE)) == NULL); - } - - if (only_lf) { - /* If not already seen, quick scan for a possible "\n" character. - (there's nothing else to be done, even when in translation mode) - */ - if (seennl == 0 && - memchr(in_str, '\n', len * sizeof(Py_UNICODE)) != NULL) { - Py_UNICODE *s, *end; - s = in_str; - end = in_str + len; - for (;;) { - Py_UNICODE c; - /* Fast loop for non-control characters */ - while (*s > '\n') - s++; - c = *s++; - if (c == '\n') { - seennl |= SEEN_LF; - break; - } - if (s > end) - break; - } - } - /* Finished: we have scanned for newlines, and none of them - need translating */ - } - else if (!self->translate) { - Py_UNICODE *s, *end; - /* We have already seen all newline types, no need to scan again */ - if (seennl == SEEN_ALL) - goto endscan; - s = in_str; - end = in_str + len; - for (;;) { - Py_UNICODE c; - /* Fast loop for non-control characters */ - while (*s > '\r') - s++; - c = *s++; - if (c == '\n') - seennl |= SEEN_LF; - else if (c == '\r') { - if (*s == '\n') { - seennl |= SEEN_CRLF; - s++; - } - else - seennl |= SEEN_CR; - } - if (s > end) - break; - if (seennl == SEEN_ALL) - break; - } - endscan: - ; - } - else { - PyObject *translated = NULL; - Py_UNICODE *out_str; - Py_UNICODE *in, *out, *end; - if (Py_REFCNT(output) != 1) { - /* We could try to optimize this so that we only do a copy - when there is something to translate. On the other hand, - most decoders should only output non-shared strings, i.e. - translation is done in place. */ - translated = PyUnicode_FromUnicode(NULL, len); - if (translated == NULL) - goto error; - assert(Py_REFCNT(translated) == 1); - memcpy(PyUnicode_AS_UNICODE(translated), - PyUnicode_AS_UNICODE(output), - len * sizeof(Py_UNICODE)); - } - else { - translated = output; - } - out_str = PyUnicode_AS_UNICODE(translated); - in = in_str; - out = out_str; - end = in_str + len; - for (;;) { - Py_UNICODE c; - /* Fast loop for non-control characters */ - while ((c = *in++) > '\r') - *out++ = c; - if (c == '\n') { - *out++ = c; - seennl |= SEEN_LF; - continue; - } - if (c == '\r') { - if (*in == '\n') { - in++; - seennl |= SEEN_CRLF; - } - else - seennl |= SEEN_CR; - *out++ = '\n'; - continue; - } - if (in > end) - break; - *out++ = c; - } - if (translated != output) { - Py_DECREF(output); - output = translated; - } - if (out - out_str != len) { - if (PyUnicode_Resize(&output, out - out_str) < 0) - goto error; - } - } - self->seennl |= seennl; - } - - return output; - - error: - Py_DECREF(output); - return NULL; -} - -static PyObject * -IncrementalNewlineDecoder_decode(PyNewLineDecoderObject *self, - PyObject *args, PyObject *kwds) -{ - char *kwlist[] = {"input", "final", NULL}; - PyObject *input; - int final = 0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|i:IncrementalNewlineDecoder", - kwlist, &input, &final)) - return NULL; - return _PyIncrementalNewlineDecoder_decode((PyObject *) self, input, final); -} - -static PyObject * -IncrementalNewlineDecoder_getstate(PyNewLineDecoderObject *self, PyObject *args) -{ - PyObject *buffer; - unsigned PY_LONG_LONG flag; - - if (self->decoder != Py_None) { - PyObject *state = PyObject_CallMethodObjArgs(self->decoder, - _PyIO_str_getstate, NULL); - if (state == NULL) - return NULL; - if (!PyArg_Parse(state, "(OK)", &buffer, &flag)) { - Py_DECREF(state); - return NULL; - } - Py_INCREF(buffer); - Py_DECREF(state); - } - else { - buffer = PyBytes_FromString(""); - flag = 0; - } - flag <<= 1; - if (self->pendingcr) - flag |= 1; - return Py_BuildValue("NK", buffer, flag); -} - -static PyObject * -IncrementalNewlineDecoder_setstate(PyNewLineDecoderObject *self, PyObject *state) -{ - PyObject *buffer; - unsigned PY_LONG_LONG flag; - - if (!PyArg_Parse(state, "(OK)", &buffer, &flag)) - return NULL; - - self->pendingcr = (int) flag & 1; - flag >>= 1; - - if (self->decoder != Py_None) - return PyObject_CallMethod(self->decoder, - "setstate", "((OK))", buffer, flag); - else - Py_RETURN_NONE; -} - -static PyObject * -IncrementalNewlineDecoder_reset(PyNewLineDecoderObject *self, PyObject *args) -{ - self->seennl = 0; - self->pendingcr = 0; - if (self->decoder != Py_None) - return PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_reset, NULL); - else - Py_RETURN_NONE; -} - -static PyObject * -IncrementalNewlineDecoder_newlines_get(PyNewLineDecoderObject *self, void *context) -{ - switch (self->seennl) { - case SEEN_CR: - return PyUnicode_FromString("\r"); - case SEEN_LF: - return PyUnicode_FromString("\n"); - case SEEN_CRLF: - return PyUnicode_FromString("\r\n"); - case SEEN_CR | SEEN_LF: - return Py_BuildValue("ss", "\r", "\n"); - case SEEN_CR | SEEN_CRLF: - return Py_BuildValue("ss", "\r", "\r\n"); - case SEEN_LF | SEEN_CRLF: - return Py_BuildValue("ss", "\n", "\r\n"); - case SEEN_CR | SEEN_LF | SEEN_CRLF: - return Py_BuildValue("sss", "\r", "\n", "\r\n"); - default: - Py_RETURN_NONE; - } - -} - - -static PyMethodDef IncrementalNewlineDecoder_methods[] = { - {"decode", (PyCFunction)IncrementalNewlineDecoder_decode, METH_VARARGS|METH_KEYWORDS}, - {"getstate", (PyCFunction)IncrementalNewlineDecoder_getstate, METH_NOARGS}, - {"setstate", (PyCFunction)IncrementalNewlineDecoder_setstate, METH_O}, - {"reset", (PyCFunction)IncrementalNewlineDecoder_reset, METH_NOARGS}, - {0} -}; - -static PyGetSetDef IncrementalNewlineDecoder_getset[] = { - {"newlines", (getter)IncrementalNewlineDecoder_newlines_get, NULL, NULL}, - {0} -}; - -PyTypeObject PyIncrementalNewlineDecoder_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.IncrementalNewlineDecoder", /*tp_name*/ - sizeof(PyNewLineDecoderObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)IncrementalNewlineDecoder_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - IncrementalNewlineDecoder_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /*tp_weaklistoffset*/ - 0, /* tp_iter */ - 0, /* tp_iternext */ - IncrementalNewlineDecoder_methods, /* tp_methods */ - 0, /* tp_members */ - IncrementalNewlineDecoder_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)IncrementalNewlineDecoder_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ -}; - - -/* TextIOWrapper */ - -PyDoc_STRVAR(TextIOWrapper_doc, - "Character and line based layer over a BufferedIOBase object, buffer.\n" - "\n" - "encoding gives the name of the encoding that the stream will be\n" - "decoded or encoded with. It defaults to locale.getpreferredencoding.\n" - "\n" - "errors determines the strictness of encoding and decoding (see the\n" - "codecs.register) and defaults to \"strict\".\n" - "\n" - "newline can be None, '', '\\n', '\\r', or '\\r\\n'. It controls the\n" - "handling of line endings. If it is None, universal newlines is\n" - "enabled. With this enabled, on input, the lines endings '\\n', '\\r',\n" - "or '\\r\\n' are translated to '\\n' before being returned to the\n" - "caller. Conversely, on output, '\\n' is translated to the system\n" - "default line seperator, os.linesep. If newline is any other of its\n" - "legal values, that newline becomes the newline when the file is read\n" - "and it is returned untranslated. On output, '\\n' is converted to the\n" - "newline.\n" - "\n" - "If line_buffering is True, a call to flush is implied when a call to\n" - "write contains a newline character." - ); - -typedef PyObject * - (*encodefunc_t)(PyObject *, PyObject *); - -typedef struct -{ - PyObject_HEAD - int ok; /* initialized? */ - Py_ssize_t chunk_size; - PyObject *buffer; - PyObject *encoding; - PyObject *encoder; - PyObject *decoder; - PyObject *readnl; - PyObject *errors; - const char *writenl; /* utf-8 encoded, NULL stands for \n */ - char line_buffering; - char readuniversal; - char readtranslate; - char writetranslate; - char seekable; - char telling; - /* Specialized encoding func (see below) */ - encodefunc_t encodefunc; - - /* Reads and writes are internally buffered in order to speed things up. - However, any read will first flush the write buffer if itsn't empty. - - Please also note that text to be written is first encoded before being - buffered. This is necessary so that encoding errors are immediately - reported to the caller, but it unfortunately means that the - IncrementalEncoder (whose encode() method is always written in Python) - becomes a bottleneck for small writes. - */ - PyObject *decoded_chars; /* buffer for text returned from decoder */ - Py_ssize_t decoded_chars_used; /* offset into _decoded_chars for read() */ - PyObject *pending_bytes; /* list of bytes objects waiting to be - written, or NULL */ - Py_ssize_t pending_bytes_count; - PyObject *snapshot; - /* snapshot is either None, or a tuple (dec_flags, next_input) where - * dec_flags is the second (integer) item of the decoder state and - * next_input is the chunk of input bytes that comes next after the - * snapshot point. We use this to reconstruct decoder states in tell(). - */ - - /* Cache raw object if it's a FileIO object */ - PyObject *raw; - - PyObject *weakreflist; - PyObject *dict; -} PyTextIOWrapperObject; - - -/* A couple of specialized cases in order to bypass the slow incremental - encoding methods for the most popular encodings. */ - -static PyObject * -ascii_encode(PyTextIOWrapperObject *self, PyObject *text) -{ - return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors)); -} - -static PyObject * -utf16be_encode(PyTextIOWrapperObject *self, PyObject *text) -{ - return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors), 1); -} - -static PyObject * -utf16le_encode(PyTextIOWrapperObject *self, PyObject *text) -{ - return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors), -1); -} - -static PyObject * -utf16_encode(PyTextIOWrapperObject *self, PyObject *text) -{ - PyObject *res; - res = PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors), 0); - if (res == NULL) - return NULL; - /* Next writes will skip the BOM and use native byte ordering */ -#if defined(WORDS_BIGENDIAN) - self->encodefunc = (encodefunc_t) utf16be_encode; -#else - self->encodefunc = (encodefunc_t) utf16le_encode; -#endif - return res; -} - - -static PyObject * -utf8_encode(PyTextIOWrapperObject *self, PyObject *text) -{ - return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors)); -} - -static PyObject * -latin1_encode(PyTextIOWrapperObject *self, PyObject *text) -{ - return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors)); -} - -/* Map normalized encoding names onto the specialized encoding funcs */ - -typedef struct { - const char *name; - encodefunc_t encodefunc; -} encodefuncentry; - -static encodefuncentry encodefuncs[] = { - {"ascii", (encodefunc_t) ascii_encode}, - {"iso8859-1", (encodefunc_t) latin1_encode}, - {"utf-16-be", (encodefunc_t) utf16be_encode}, - {"utf-16-le", (encodefunc_t) utf16le_encode}, - {"utf-16", (encodefunc_t) utf16_encode}, - {"utf-8", (encodefunc_t) utf8_encode}, - {NULL, NULL} -}; - - -static int -TextIOWrapper_init(PyTextIOWrapperObject *self, PyObject *args, PyObject *kwds) -{ - char *kwlist[] = {"buffer", "encoding", "errors", - "newline", "line_buffering", - NULL}; - PyObject *buffer, *raw; - char *encoding = NULL; - char *errors = NULL; - char *newline = NULL; - int line_buffering = 0; - _PyIO_State *state = IO_STATE; - - PyObject *res; - int r; - - self->ok = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|zzzi:fileio", - kwlist, &buffer, &encoding, &errors, - &newline, &line_buffering)) - return -1; - - if (newline && newline[0] != '\0' - && !(newline[0] == '\n' && newline[1] == '\0') - && !(newline[0] == '\r' && newline[1] == '\0') - && !(newline[0] == '\r' && newline[1] == '\n' && newline[2] == '\0')) { - PyErr_Format(PyExc_ValueError, - "illegal newline value: %s", newline); - return -1; - } - - Py_CLEAR(self->buffer); - Py_CLEAR(self->encoding); - Py_CLEAR(self->encoder); - Py_CLEAR(self->decoder); - Py_CLEAR(self->readnl); - Py_CLEAR(self->decoded_chars); - Py_CLEAR(self->pending_bytes); - Py_CLEAR(self->snapshot); - Py_CLEAR(self->errors); - Py_CLEAR(self->raw); - self->decoded_chars_used = 0; - self->pending_bytes_count = 0; - self->encodefunc = NULL; - - if (encoding == NULL) { - /* Try os.device_encoding(fileno) */ - PyObject *fileno; - fileno = PyObject_CallMethod(buffer, "fileno", NULL); - /* Ignore only AttributeError and UnsupportedOperation */ - if (fileno == NULL) { - if (PyErr_ExceptionMatches(PyExc_AttributeError) || - PyErr_ExceptionMatches(state->unsupported_operation)) { - PyErr_Clear(); - } - else { - goto error; - } - } - else { - self->encoding = PyObject_CallMethod(state->os_module, - "device_encoding", - "N", fileno); - if (self->encoding == NULL) - goto error; - else if (!PyUnicode_Check(self->encoding)) - Py_CLEAR(self->encoding); - } - } - if (encoding == NULL && self->encoding == NULL) { - if (state->locale_module == NULL) { - state->locale_module = PyImport_ImportModule("locale"); - if (state->locale_module == NULL) - goto catch_ImportError; - else - goto use_locale; - } - else { - use_locale: - self->encoding = PyObject_CallMethod( - state->locale_module, "getpreferredencoding", NULL); - if (self->encoding == NULL) { - catch_ImportError: - /* - Importing locale can raise a ImportError because of - _functools, and locale.getpreferredencoding can raise a - ImportError if _locale is not available. These will happen - during module building. - */ - if (PyErr_ExceptionMatches(PyExc_ImportError)) { - PyErr_Clear(); - self->encoding = PyUnicode_FromString("ascii"); - } - else - goto error; - } - else if (!PyUnicode_Check(self->encoding)) - Py_CLEAR(self->encoding); - } - } - if (self->encoding != NULL) - encoding = _PyUnicode_AsString(self->encoding); - else if (encoding != NULL) { - self->encoding = PyUnicode_FromString(encoding); - if (self->encoding == NULL) - goto error; - } - else { - PyErr_SetString(PyExc_IOError, - "could not determine default encoding"); - } - - if (errors == NULL) - errors = "strict"; - self->errors = PyBytes_FromString(errors); - if (self->errors == NULL) - goto error; - - self->chunk_size = 8192; - self->readuniversal = (newline == NULL || newline[0] == '\0'); - self->line_buffering = line_buffering; - self->readtranslate = (newline == NULL); - if (newline) { - self->readnl = PyUnicode_FromString(newline); - if (self->readnl == NULL) - return -1; - } - self->writetranslate = (newline == NULL || newline[0] != '\0'); - if (!self->readuniversal && self->readnl) { - self->writenl = _PyUnicode_AsString(self->readnl); - if (!strcmp(self->writenl, "\n")) - self->writenl = NULL; - } -#ifdef MS_WINDOWS - else - self->writenl = "\r\n"; -#endif - - /* Build the decoder object */ - res = PyObject_CallMethod(buffer, "readable", NULL); - if (res == NULL) - goto error; - r = PyObject_IsTrue(res); - Py_DECREF(res); - if (r == -1) - goto error; - if (r == 1) { - self->decoder = PyCodec_IncrementalDecoder( - encoding, errors); - if (self->decoder == NULL) - goto error; - - if (self->readuniversal) { - PyObject *incrementalDecoder = PyObject_CallFunction( - (PyObject *)&PyIncrementalNewlineDecoder_Type, - "Oi", self->decoder, (int)self->readtranslate); - if (incrementalDecoder == NULL) - goto error; - Py_CLEAR(self->decoder); - self->decoder = incrementalDecoder; - } - } - - /* Build the encoder object */ - res = PyObject_CallMethod(buffer, "writable", NULL); - if (res == NULL) - goto error; - r = PyObject_IsTrue(res); - Py_DECREF(res); - if (r == -1) - goto error; - if (r == 1) { - PyObject *ci; - self->encoder = PyCodec_IncrementalEncoder( - encoding, errors); - if (self->encoder == NULL) - goto error; - /* Get the normalized named of the codec */ - ci = _PyCodec_Lookup(encoding); - if (ci == NULL) - goto error; - res = PyObject_GetAttrString(ci, "name"); - Py_DECREF(ci); - if (res == NULL) - PyErr_Clear(); - else if (PyUnicode_Check(res)) { - encodefuncentry *e = encodefuncs; - while (e->name != NULL) { - if (!PyUnicode_CompareWithASCIIString(res, e->name)) { - self->encodefunc = e->encodefunc; - break; - } - e++; - } - } - Py_XDECREF(res); - } - - self->buffer = buffer; - Py_INCREF(buffer); - - if (Py_TYPE(buffer) == &PyBufferedReader_Type || - Py_TYPE(buffer) == &PyBufferedWriter_Type || - Py_TYPE(buffer) == &PyBufferedRandom_Type) { - raw = PyObject_GetAttrString(buffer, "raw"); - /* Cache the raw FileIO object to speed up 'closed' checks */ - if (raw == NULL) - PyErr_Clear(); - else if (Py_TYPE(raw) == &PyFileIO_Type) - self->raw = raw; - else - Py_DECREF(raw); - } - - res = PyObject_CallMethod(buffer, "seekable", NULL); - if (res == NULL) - goto error; - self->seekable = self->telling = PyObject_IsTrue(res); - Py_DECREF(res); - - self->ok = 1; - return 0; - - error: - return -1; -} - -static int -_TextIOWrapper_clear(PyTextIOWrapperObject *self) -{ - if (self->ok && _PyIOBase_finalize((PyObject *) self) < 0) - return -1; - self->ok = 0; - Py_CLEAR(self->buffer); - Py_CLEAR(self->encoding); - Py_CLEAR(self->encoder); - Py_CLEAR(self->decoder); - Py_CLEAR(self->readnl); - Py_CLEAR(self->decoded_chars); - Py_CLEAR(self->pending_bytes); - Py_CLEAR(self->snapshot); - Py_CLEAR(self->errors); - Py_CLEAR(self->raw); - return 0; -} - -static void -TextIOWrapper_dealloc(PyTextIOWrapperObject *self) -{ - if (_TextIOWrapper_clear(self) < 0) - return; - _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)self); - Py_CLEAR(self->dict); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static int -TextIOWrapper_traverse(PyTextIOWrapperObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->buffer); - Py_VISIT(self->encoding); - Py_VISIT(self->encoder); - Py_VISIT(self->decoder); - Py_VISIT(self->readnl); - Py_VISIT(self->decoded_chars); - Py_VISIT(self->pending_bytes); - Py_VISIT(self->snapshot); - Py_VISIT(self->errors); - Py_VISIT(self->raw); - - Py_VISIT(self->dict); - return 0; -} - -static int -TextIOWrapper_clear(PyTextIOWrapperObject *self) -{ - if (_TextIOWrapper_clear(self) < 0) - return -1; - Py_CLEAR(self->dict); - return 0; -} - -static PyObject * -TextIOWrapper_closed_get(PyTextIOWrapperObject *self, void *context); - -/* This macro takes some shortcuts to make the common case faster. */ -#define CHECK_CLOSED(self) \ - do { \ - int r; \ - PyObject *_res; \ - if (Py_TYPE(self) == &PyTextIOWrapper_Type) { \ - if (self->raw != NULL) \ - r = _PyFileIO_closed(self->raw); \ - else { \ - _res = TextIOWrapper_closed_get(self, NULL); \ - if (_res == NULL) \ - return NULL; \ - r = PyObject_IsTrue(_res); \ - Py_DECREF(_res); \ - if (r < 0) \ - return NULL; \ - } \ - if (r > 0) { \ - PyErr_SetString(PyExc_ValueError, \ - "I/O operation on closed file."); \ - return NULL; \ - } \ - } \ - else if (_PyIOBase_checkClosed((PyObject *)self, Py_True) == NULL) \ - return NULL; \ - } while (0) - -#define CHECK_INITIALIZED(self) \ - if (self->ok <= 0) { \ - PyErr_SetString(PyExc_ValueError, \ - "I/O operation on uninitialized object"); \ - return NULL; \ - } - -#define CHECK_INITIALIZED_INT(self) \ - if (self->ok <= 0) { \ - PyErr_SetString(PyExc_ValueError, \ - "I/O operation on uninitialized object"); \ - return -1; \ - } - - -Py_LOCAL_INLINE(const Py_UNICODE *) -findchar(const Py_UNICODE *s, Py_ssize_t size, Py_UNICODE ch) -{ - /* like wcschr, but doesn't stop at NULL characters */ - while (size-- > 0) { - if (*s == ch) - return s; - s++; - } - return NULL; -} - -/* Flush the internal write buffer. This doesn't explicitly flush the - underlying buffered object, though. */ -static int -_TextIOWrapper_writeflush(PyTextIOWrapperObject *self) -{ - PyObject *b, *ret; - - if (self->pending_bytes == NULL) - return 0; - b = _PyBytes_Join(_PyIO_empty_bytes, self->pending_bytes); - if (b == NULL) - return -1; - ret = PyObject_CallMethodObjArgs(self->buffer, - _PyIO_str_write, b, NULL); - Py_DECREF(b); - if (ret == NULL) - return -1; - Py_DECREF(ret); - Py_CLEAR(self->pending_bytes); - self->pending_bytes_count = 0; - return 0; -} - -static PyObject * -TextIOWrapper_write(PyTextIOWrapperObject *self, PyObject *args) -{ - PyObject *ret; - PyObject *text; /* owned reference */ - PyObject *b; - Py_ssize_t textlen; - int haslf = 0; - int needflush = 0; - - CHECK_INITIALIZED(self); - - if (!PyArg_ParseTuple(args, "U:write", &text)) { - return NULL; - } - - CHECK_CLOSED(self); - - Py_INCREF(text); - - textlen = PyUnicode_GetSize(text); - - if ((self->writetranslate && self->writenl != NULL) || self->line_buffering) - if (findchar(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), '\n')) - haslf = 1; - - if (haslf && self->writetranslate && self->writenl != NULL) { - PyObject *newtext = PyObject_CallMethod( - text, "replace", "ss", "\n", self->writenl); - Py_DECREF(text); - if (newtext == NULL) - return NULL; - text = newtext; - } - - if (self->line_buffering && - (haslf || - findchar(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), '\r'))) - needflush = 1; - - /* XXX What if we were just reading? */ - if (self->encodefunc != NULL) - b = (*self->encodefunc)((PyObject *) self, text); - else - b = PyObject_CallMethodObjArgs(self->encoder, - _PyIO_str_encode, text, NULL); - Py_DECREF(text); - if (b == NULL) - return NULL; - - if (self->pending_bytes == NULL) { - self->pending_bytes = PyList_New(0); - if (self->pending_bytes == NULL) { - Py_DECREF(b); - return NULL; - } - self->pending_bytes_count = 0; - } - if (PyList_Append(self->pending_bytes, b) < 0) { - Py_DECREF(b); - return NULL; - } - self->pending_bytes_count += PyBytes_GET_SIZE(b); - Py_DECREF(b); - if (self->pending_bytes_count > self->chunk_size || needflush) { - if (_TextIOWrapper_writeflush(self) < 0) - return NULL; - } - - if (needflush) { - ret = PyObject_CallMethodObjArgs(self->buffer, _PyIO_str_flush, NULL); - if (ret == NULL) - return NULL; - Py_DECREF(ret); - } - - Py_CLEAR(self->snapshot); - - if (self->decoder) { - ret = PyObject_CallMethod(self->decoder, "reset", NULL); - if (ret == NULL) - return NULL; - Py_DECREF(ret); - } - - return PyLong_FromSsize_t(textlen); -} - -/* Steal a reference to chars and store it in the decoded_char buffer; - */ -static void -TextIOWrapper_set_decoded_chars(PyTextIOWrapperObject *self, PyObject *chars) -{ - Py_CLEAR(self->decoded_chars); - self->decoded_chars = chars; - self->decoded_chars_used = 0; -} - -static PyObject * -TextIOWrapper_get_decoded_chars(PyTextIOWrapperObject *self, Py_ssize_t n) -{ - PyObject *chars; - Py_ssize_t avail; - - if (self->decoded_chars == NULL) - return PyUnicode_FromStringAndSize(NULL, 0); - - avail = (PyUnicode_GET_SIZE(self->decoded_chars) - - self->decoded_chars_used); - - assert(avail >= 0); - - if (n < 0 || n > avail) - n = avail; - - if (self->decoded_chars_used > 0 || n < avail) { - chars = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(self->decoded_chars) - + self->decoded_chars_used, n); - if (chars == NULL) - return NULL; - } - else { - chars = self->decoded_chars; - Py_INCREF(chars); - } - - self->decoded_chars_used += n; - return chars; -} - -/* Read and decode the next chunk of data from the BufferedReader. - */ -static int -TextIOWrapper_read_chunk(PyTextIOWrapperObject *self) -{ - PyObject *dec_buffer = NULL; - PyObject *dec_flags = NULL; - PyObject *input_chunk = NULL; - PyObject *decoded_chars, *chunk_size; - int eof; - - /* The return value is True unless EOF was reached. The decoded string is - * placed in self._decoded_chars (replacing its previous value). The - * entire input chunk is sent to the decoder, though some of it may remain - * buffered in the decoder, yet to be converted. - */ - - if (self->decoder == NULL) { - PyErr_SetString(PyExc_ValueError, "no decoder"); - return -1; - } - - if (self->telling) { - /* To prepare for tell(), we need to snapshot a point in the file - * where the decoder's input buffer is empty. - */ - - PyObject *state = PyObject_CallMethodObjArgs(self->decoder, - _PyIO_str_getstate, NULL); - if (state == NULL) - return -1; - /* Given this, we know there was a valid snapshot point - * len(dec_buffer) bytes ago with decoder state (b'', dec_flags). - */ - if (PyArg_Parse(state, "(OO)", &dec_buffer, &dec_flags) < 0) { - Py_DECREF(state); - return -1; - } - Py_INCREF(dec_buffer); - Py_INCREF(dec_flags); - Py_DECREF(state); - } - - /* Read a chunk, decode it, and put the result in self._decoded_chars. */ - chunk_size = PyLong_FromSsize_t(self->chunk_size); - if (chunk_size == NULL) - goto fail; - input_chunk = PyObject_CallMethodObjArgs(self->buffer, - _PyIO_str_read1, chunk_size, NULL); - Py_DECREF(chunk_size); - if (input_chunk == NULL) - goto fail; - assert(PyBytes_Check(input_chunk)); - - eof = (PyBytes_Size(input_chunk) == 0); - - if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type) { - decoded_chars = _PyIncrementalNewlineDecoder_decode( - self->decoder, input_chunk, eof); - } - else { - decoded_chars = PyObject_CallMethodObjArgs(self->decoder, - _PyIO_str_decode, input_chunk, eof ? Py_True : Py_False, NULL); - } - - /* TODO sanity check: isinstance(decoded_chars, unicode) */ - if (decoded_chars == NULL) - goto fail; - TextIOWrapper_set_decoded_chars(self, decoded_chars); - if (PyUnicode_GET_SIZE(decoded_chars) > 0) - eof = 0; - - if (self->telling) { - /* At the snapshot point, len(dec_buffer) bytes before the read, the - * next input to be decoded is dec_buffer + input_chunk. - */ - PyObject *next_input = PyNumber_Add(dec_buffer, input_chunk); - if (next_input == NULL) - goto fail; - assert (PyBytes_Check(next_input)); - Py_DECREF(dec_buffer); - Py_CLEAR(self->snapshot); - self->snapshot = Py_BuildValue("NN", dec_flags, next_input); - } - Py_DECREF(input_chunk); - - return (eof == 0); - - fail: - Py_XDECREF(dec_buffer); - Py_XDECREF(dec_flags); - Py_XDECREF(input_chunk); - return -1; -} - -static PyObject * -TextIOWrapper_read(PyTextIOWrapperObject *self, PyObject *args) -{ - Py_ssize_t n = -1; - PyObject *result = NULL, *chunks = NULL; - - CHECK_INITIALIZED(self); - - if (!PyArg_ParseTuple(args, "|n:read", &n)) - return NULL; - - CHECK_CLOSED(self); - - if (self->decoder == NULL) { - PyErr_SetString(PyExc_IOError, "not readable"); - return NULL; - } - - if (_TextIOWrapper_writeflush(self) < 0) - return NULL; - - if (n < 0) { - /* Read everything */ - PyObject *bytes = PyObject_CallMethod(self->buffer, "read", NULL); - PyObject *decoded; - if (bytes == NULL) - goto fail; - decoded = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_decode, - bytes, Py_True, NULL); - Py_DECREF(bytes); - if (decoded == NULL) - goto fail; - - result = TextIOWrapper_get_decoded_chars(self, -1); - - if (result == NULL) { - Py_DECREF(decoded); - return NULL; - } - - PyUnicode_AppendAndDel(&result, decoded); - if (result == NULL) - goto fail; - - Py_CLEAR(self->snapshot); - return result; - } - else { - int res = 1; - Py_ssize_t remaining = n; - - result = TextIOWrapper_get_decoded_chars(self, n); - if (result == NULL) - goto fail; - remaining -= PyUnicode_GET_SIZE(result); - - /* Keep reading chunks until we have n characters to return */ - while (remaining > 0) { - res = TextIOWrapper_read_chunk(self); - if (res < 0) - goto fail; - if (res == 0) /* EOF */ - break; - if (chunks == NULL) { - chunks = PyList_New(0); - if (chunks == NULL) - goto fail; - } - if (PyList_Append(chunks, result) < 0) - goto fail; - Py_DECREF(result); - result = TextIOWrapper_get_decoded_chars(self, remaining); - if (result == NULL) - goto fail; - remaining -= PyUnicode_GET_SIZE(result); - } - if (chunks != NULL) { - if (result != NULL && PyList_Append(chunks, result) < 0) - goto fail; - Py_CLEAR(result); - result = PyUnicode_Join(_PyIO_empty_str, chunks); - if (result == NULL) - goto fail; - Py_CLEAR(chunks); - } - return result; - } - fail: - Py_XDECREF(result); - Py_XDECREF(chunks); - return NULL; -} - - -/* NOTE: `end` must point to the real end of the Py_UNICODE storage, - that is to the NUL character. Otherwise the function will produce - incorrect results. */ -static Py_UNICODE * -find_control_char(Py_UNICODE *start, Py_UNICODE *end, Py_UNICODE ch) -{ - Py_UNICODE *s = start; - for (;;) { - while (*s > ch) - s++; - if (*s == ch) - return s; - if (s == end) - return NULL; - s++; - } -} - -Py_ssize_t -_PyIO_find_line_ending( - int translated, int universal, PyObject *readnl, - Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed) -{ - Py_ssize_t len = end - start; - - if (translated) { - /* Newlines are already translated, only search for \n */ - Py_UNICODE *pos = find_control_char(start, end, '\n'); - if (pos != NULL) - return pos - start + 1; - else { - *consumed = len; - return -1; - } - } - else if (universal) { - /* Universal newline search. Find any of \r, \r\n, \n - * The decoder ensures that \r\n are not split in two pieces - */ - Py_UNICODE *s = start; - for (;;) { - Py_UNICODE ch; - /* Fast path for non-control chars. The loop always ends - since the Py_UNICODE storage is NUL-terminated. */ - while (*s > '\r') - s++; - if (s >= end) { - *consumed = len; - return -1; - } - ch = *s++; - if (ch == '\n') - return s - start; - if (ch == '\r') { - if (*s == '\n') - return s - start + 1; - else - return s - start; - } - } - } - else { - /* Non-universal mode. */ - Py_ssize_t readnl_len = PyUnicode_GET_SIZE(readnl); - Py_UNICODE *nl = PyUnicode_AS_UNICODE(readnl); - if (readnl_len == 1) { - Py_UNICODE *pos = find_control_char(start, end, nl[0]); - if (pos != NULL) - return pos - start + 1; - *consumed = len; - return -1; - } - else { - Py_UNICODE *s = start; - Py_UNICODE *e = end - readnl_len + 1; - Py_UNICODE *pos; - if (e < s) - e = s; - while (s < e) { - Py_ssize_t i; - Py_UNICODE *pos = find_control_char(s, end, nl[0]); - if (pos == NULL || pos >= e) - break; - for (i = 1; i < readnl_len; i++) { - if (pos[i] != nl[i]) - break; - } - if (i == readnl_len) - return pos - start + readnl_len; - s = pos + 1; - } - pos = find_control_char(e, end, nl[0]); - if (pos == NULL) - *consumed = len; - else - *consumed = pos - start; - return -1; - } - } -} - -static PyObject * -_TextIOWrapper_readline(PyTextIOWrapperObject *self, Py_ssize_t limit) -{ - PyObject *line = NULL, *chunks = NULL, *remaining = NULL; - Py_ssize_t start, endpos, chunked, offset_to_buffer; - int res; - - CHECK_CLOSED(self); - - if (_TextIOWrapper_writeflush(self) < 0) - return NULL; - - chunked = 0; - - while (1) { - Py_UNICODE *ptr; - Py_ssize_t line_len; - Py_ssize_t consumed = 0; - - /* First, get some data if necessary */ - res = 1; - while (!self->decoded_chars || - !PyUnicode_GET_SIZE(self->decoded_chars)) { - res = TextIOWrapper_read_chunk(self); - if (res < 0) - goto error; - if (res == 0) - break; - } - if (res == 0) { - /* end of file */ - TextIOWrapper_set_decoded_chars(self, NULL); - Py_CLEAR(self->snapshot); - start = endpos = offset_to_buffer = 0; - break; - } - - if (remaining == NULL) { - line = self->decoded_chars; - start = self->decoded_chars_used; - offset_to_buffer = 0; - Py_INCREF(line); - } - else { - assert(self->decoded_chars_used == 0); - line = PyUnicode_Concat(remaining, self->decoded_chars); - start = 0; - offset_to_buffer = PyUnicode_GET_SIZE(remaining); - Py_CLEAR(remaining); - if (line == NULL) - goto error; - } - - ptr = PyUnicode_AS_UNICODE(line); - line_len = PyUnicode_GET_SIZE(line); - - endpos = _PyIO_find_line_ending( - self->readtranslate, self->readuniversal, self->readnl, - ptr + start, ptr + line_len, &consumed); - if (endpos >= 0) { - endpos += start; - if (limit >= 0 && (endpos - start) + chunked >= limit) - endpos = start + limit - chunked; - break; - } - - /* We can put aside up to `endpos` */ - endpos = consumed + start; - if (limit >= 0 && (endpos - start) + chunked >= limit) { - /* Didn't find line ending, but reached length limit */ - endpos = start + limit - chunked; - break; - } - - if (endpos > start) { - /* No line ending seen yet - put aside current data */ - PyObject *s; - if (chunks == NULL) { - chunks = PyList_New(0); - if (chunks == NULL) - goto error; - } - s = PyUnicode_FromUnicode(ptr + start, endpos - start); - if (s == NULL) - goto error; - if (PyList_Append(chunks, s) < 0) { - Py_DECREF(s); - goto error; - } - chunked += PyUnicode_GET_SIZE(s); - Py_DECREF(s); - } - /* There may be some remaining bytes we'll have to prepend to the - next chunk of data */ - if (endpos < line_len) { - remaining = PyUnicode_FromUnicode( - ptr + endpos, line_len - endpos); - if (remaining == NULL) - goto error; - } - Py_CLEAR(line); - /* We have consumed the buffer */ - TextIOWrapper_set_decoded_chars(self, NULL); - } - - if (line != NULL) { - /* Our line ends in the current buffer */ - self->decoded_chars_used = endpos - offset_to_buffer; - if (start > 0 || endpos < PyUnicode_GET_SIZE(line)) { - if (start == 0 && Py_REFCNT(line) == 1) { - if (PyUnicode_Resize(&line, endpos) < 0) - goto error; - } - else { - PyObject *s = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(line) + start, endpos - start); - Py_CLEAR(line); - if (s == NULL) - goto error; - line = s; - } - } - } - if (remaining != NULL) { - if (chunks == NULL) { - chunks = PyList_New(0); - if (chunks == NULL) - goto error; - } - if (PyList_Append(chunks, remaining) < 0) - goto error; - Py_CLEAR(remaining); - } - if (chunks != NULL) { - if (line != NULL && PyList_Append(chunks, line) < 0) - goto error; - Py_CLEAR(line); - line = PyUnicode_Join(_PyIO_empty_str, chunks); - if (line == NULL) - goto error; - Py_DECREF(chunks); - } - if (line == NULL) - line = PyUnicode_FromStringAndSize(NULL, 0); - - return line; - - error: - Py_XDECREF(chunks); - Py_XDECREF(remaining); - Py_XDECREF(line); - return NULL; -} - -static PyObject * -TextIOWrapper_readline(PyTextIOWrapperObject *self, PyObject *args) -{ - Py_ssize_t limit = -1; - - CHECK_INITIALIZED(self); - if (!PyArg_ParseTuple(args, "|n:readline", &limit)) { - return NULL; - } - return _TextIOWrapper_readline(self, limit); -} - -/* Seek and Tell */ - -typedef struct { - Py_off_t start_pos; - int dec_flags; - int bytes_to_feed; - int chars_to_skip; - char need_eof; -} CookieStruct; - -/* - To speed up cookie packing/unpacking, we store the fields in a temporary - string and call _PyLong_FromByteArray() or _PyLong_AsByteArray (resp.). - The following macros define at which offsets in the intermediary byte - string the various CookieStruct fields will be stored. - */ - -#define COOKIE_BUF_LEN (sizeof(Py_off_t) + 3 * sizeof(int) + sizeof(char)) - -#if defined(WORDS_BIGENDIAN) - -# define IS_LITTLE_ENDIAN 0 - -/* We want the least significant byte of start_pos to also be the least - significant byte of the cookie, which means that in big-endian mode we - must copy the fields in reverse order. */ - -# define OFF_START_POS (sizeof(char) + 3 * sizeof(int)) -# define OFF_DEC_FLAGS (sizeof(char) + 2 * sizeof(int)) -# define OFF_BYTES_TO_FEED (sizeof(char) + sizeof(int)) -# define OFF_CHARS_TO_SKIP (sizeof(char)) -# define OFF_NEED_EOF 0 - -#else - -# define IS_LITTLE_ENDIAN 1 - -/* Little-endian mode: the least significant byte of start_pos will - naturally end up the least significant byte of the cookie. */ - -# define OFF_START_POS 0 -# define OFF_DEC_FLAGS (sizeof(Py_off_t)) -# define OFF_BYTES_TO_FEED (sizeof(Py_off_t) + sizeof(int)) -# define OFF_CHARS_TO_SKIP (sizeof(Py_off_t) + 2 * sizeof(int)) -# define OFF_NEED_EOF (sizeof(Py_off_t) + 3 * sizeof(int)) - -#endif - -static int -TextIOWrapper_parseCookie(CookieStruct *cookie, PyObject *cookieObj) -{ - unsigned char buffer[COOKIE_BUF_LEN]; - PyLongObject *cookieLong = (PyLongObject *)PyNumber_Long(cookieObj); - if (cookieLong == NULL) - return -1; - - if (_PyLong_AsByteArray(cookieLong, buffer, sizeof(buffer), - IS_LITTLE_ENDIAN, 0) < 0) { - Py_DECREF(cookieLong); - return -1; - } - Py_DECREF(cookieLong); - - memcpy(&cookie->start_pos, buffer + OFF_START_POS, sizeof(cookie->start_pos)); - memcpy(&cookie->dec_flags, buffer + OFF_DEC_FLAGS, sizeof(cookie->dec_flags)); - memcpy(&cookie->bytes_to_feed, buffer + OFF_BYTES_TO_FEED, sizeof(cookie->bytes_to_feed)); - memcpy(&cookie->chars_to_skip, buffer + OFF_CHARS_TO_SKIP, sizeof(cookie->chars_to_skip)); - memcpy(&cookie->need_eof, buffer + OFF_NEED_EOF, sizeof(cookie->need_eof)); - - return 0; -} - -static PyObject * -TextIOWrapper_buildCookie(CookieStruct *cookie) -{ - unsigned char buffer[COOKIE_BUF_LEN]; - - memcpy(buffer + OFF_START_POS, &cookie->start_pos, sizeof(cookie->start_pos)); - memcpy(buffer + OFF_DEC_FLAGS, &cookie->dec_flags, sizeof(cookie->dec_flags)); - memcpy(buffer + OFF_BYTES_TO_FEED, &cookie->bytes_to_feed, sizeof(cookie->bytes_to_feed)); - memcpy(buffer + OFF_CHARS_TO_SKIP, &cookie->chars_to_skip, sizeof(cookie->chars_to_skip)); - memcpy(buffer + OFF_NEED_EOF, &cookie->need_eof, sizeof(cookie->need_eof)); - - return _PyLong_FromByteArray(buffer, sizeof(buffer), IS_LITTLE_ENDIAN, 0); -} -#undef IS_LITTLE_ENDIAN - -static int -_TextIOWrapper_decoder_setstate(PyTextIOWrapperObject *self, - CookieStruct *cookie) -{ - PyObject *res; - /* When seeking to the start of the stream, we call decoder.reset() - rather than decoder.getstate(). - This is for a few decoders such as utf-16 for which the state value - at start is not (b"", 0) but e.g. (b"", 2) (meaning, in the case of - utf-16, that we are expecting a BOM). - */ - if (cookie->start_pos == 0 && cookie->dec_flags == 0) - res = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_reset, NULL); - else - res = PyObject_CallMethod(self->decoder, "setstate", - "((yi))", "", cookie->dec_flags); - if (res == NULL) - return -1; - Py_DECREF(res); - return 0; -} - -static PyObject * -TextIOWrapper_seek(PyTextIOWrapperObject *self, PyObject *args) -{ - PyObject *cookieObj, *posobj; - CookieStruct cookie; - int whence = 0; - static PyObject *zero = NULL; - PyObject *res; - int cmp; - - CHECK_INITIALIZED(self); - - if (zero == NULL) { - zero = PyLong_FromLong(0L); - if (zero == NULL) - return NULL; - } - - if (!PyArg_ParseTuple(args, "O|i:seek", &cookieObj, &whence)) - return NULL; - CHECK_CLOSED(self); - - Py_INCREF(cookieObj); - - if (!self->seekable) { - PyErr_SetString(PyExc_IOError, - "underlying stream is not seekable"); - goto fail; - } - - if (whence == 1) { - /* seek relative to current position */ - cmp = PyObject_RichCompareBool(cookieObj, zero, Py_EQ); - if (cmp < 0) - goto fail; - - if (cmp == 0) { - PyErr_SetString(PyExc_IOError, - "can't do nonzero cur-relative seeks"); - goto fail; - } - - /* Seeking to the current position should attempt to - * sync the underlying buffer with the current position. - */ - Py_DECREF(cookieObj); - cookieObj = PyObject_CallMethod((PyObject *)self, "tell", NULL); - if (cookieObj == NULL) - goto fail; - } - else if (whence == 2) { - /* seek relative to end of file */ - - cmp = PyObject_RichCompareBool(cookieObj, zero, Py_EQ); - if (cmp < 0) - goto fail; - - if (cmp == 0) { - PyErr_SetString(PyExc_IOError, - "can't do nonzero end-relative seeks"); - goto fail; - } - - res = PyObject_CallMethod((PyObject *)self, "flush", NULL); - if (res == NULL) - goto fail; - Py_DECREF(res); - - TextIOWrapper_set_decoded_chars(self, NULL); - Py_CLEAR(self->snapshot); - if (self->decoder) { - res = PyObject_CallMethod(self->decoder, "reset", NULL); - if (res == NULL) - goto fail; - Py_DECREF(res); - } - - res = PyObject_CallMethod(self->buffer, "seek", "ii", 0, 2); - Py_XDECREF(cookieObj); - return res; - } - else if (whence != 0) { - PyErr_Format(PyExc_ValueError, - "invalid whence (%d, should be 0, 1 or 2)", whence); - goto fail; - } - - cmp = PyObject_RichCompareBool(cookieObj, zero, Py_LT); - if (cmp < 0) - goto fail; - - if (cmp == 1) { - PyErr_Format(PyExc_ValueError, - "negative seek position %R", cookieObj); - goto fail; - } - - res = PyObject_CallMethodObjArgs((PyObject *)self, _PyIO_str_flush, NULL); - if (res == NULL) - goto fail; - Py_DECREF(res); - - /* The strategy of seek() is to go back to the safe start point - * and replay the effect of read(chars_to_skip) from there. - */ - if (TextIOWrapper_parseCookie(&cookie, cookieObj) < 0) - goto fail; - - /* Seek back to the safe start point. */ - posobj = PyLong_FromOff_t(cookie.start_pos); - if (posobj == NULL) - goto fail; - res = PyObject_CallMethodObjArgs(self->buffer, - _PyIO_str_seek, posobj, NULL); - Py_DECREF(posobj); - if (res == NULL) - goto fail; - Py_DECREF(res); - - TextIOWrapper_set_decoded_chars(self, NULL); - Py_CLEAR(self->snapshot); - - /* Restore the decoder to its state from the safe start point. */ - if (self->decoder) { - if (_TextIOWrapper_decoder_setstate(self, &cookie) < 0) - goto fail; - } - - if (cookie.chars_to_skip) { - /* Just like _read_chunk, feed the decoder and save a snapshot. */ - PyObject *input_chunk = PyObject_CallMethod( - self->buffer, "read", "i", cookie.bytes_to_feed); - PyObject *decoded; - - if (input_chunk == NULL) - goto fail; - - assert (PyBytes_Check(input_chunk)); - - self->snapshot = Py_BuildValue("iN", cookie.dec_flags, input_chunk); - if (self->snapshot == NULL) { - Py_DECREF(input_chunk); - goto fail; - } - - decoded = PyObject_CallMethod(self->decoder, "decode", - "Oi", input_chunk, (int)cookie.need_eof); - - if (decoded == NULL) - goto fail; - - TextIOWrapper_set_decoded_chars(self, decoded); - - /* Skip chars_to_skip of the decoded characters. */ - if (PyUnicode_GetSize(self->decoded_chars) < cookie.chars_to_skip) { - PyErr_SetString(PyExc_IOError, "can't restore logical file position"); - goto fail; - } - self->decoded_chars_used = cookie.chars_to_skip; - } - else { - self->snapshot = Py_BuildValue("iy", cookie.dec_flags, ""); - if (self->snapshot == NULL) - goto fail; - } - - return cookieObj; - fail: - Py_XDECREF(cookieObj); - return NULL; - -} - -static PyObject * -TextIOWrapper_tell(PyTextIOWrapperObject *self, PyObject *args) -{ - PyObject *res; - PyObject *posobj = NULL; - CookieStruct cookie = {0,0,0,0,0}; - PyObject *next_input; - Py_ssize_t chars_to_skip, chars_decoded; - PyObject *saved_state = NULL; - char *input, *input_end; - - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - - if (!self->seekable) { - PyErr_SetString(PyExc_IOError, - "underlying stream is not seekable"); - goto fail; - } - if (!self->telling) { - PyErr_SetString(PyExc_IOError, - "telling position disabled by next() call"); - goto fail; - } - - if (_TextIOWrapper_writeflush(self) < 0) - return NULL; - res = PyObject_CallMethod((PyObject *)self, "flush", NULL); - if (res == NULL) - goto fail; - Py_DECREF(res); - - posobj = PyObject_CallMethod(self->buffer, "tell", NULL); - if (posobj == NULL) - goto fail; - - if (self->decoder == NULL || self->snapshot == NULL) { - assert (self->decoded_chars == NULL || PyUnicode_GetSize(self->decoded_chars) == 0); - return posobj; - } - -#if defined(HAVE_LARGEFILE_SUPPORT) - cookie.start_pos = PyLong_AsLongLong(posobj); -#else - cookie.start_pos = PyLong_AsLong(posobj); -#endif - if (PyErr_Occurred()) - goto fail; - - /* Skip backward to the snapshot point (see _read_chunk). */ - if (!PyArg_Parse(self->snapshot, "(iO)", &cookie.dec_flags, &next_input)) - goto fail; - - assert (PyBytes_Check(next_input)); - - cookie.start_pos -= PyBytes_GET_SIZE(next_input); - - /* How many decoded characters have been used up since the snapshot? */ - if (self->decoded_chars_used == 0) { - /* We haven't moved from the snapshot point. */ - Py_DECREF(posobj); - return TextIOWrapper_buildCookie(&cookie); - } - - chars_to_skip = self->decoded_chars_used; - - /* Starting from the snapshot position, we will walk the decoder - * forward until it gives us enough decoded characters. - */ - saved_state = PyObject_CallMethodObjArgs(self->decoder, - _PyIO_str_getstate, NULL); - if (saved_state == NULL) - goto fail; - - /* Note our initial start point. */ - if (_TextIOWrapper_decoder_setstate(self, &cookie) < 0) - goto fail; - - /* Feed the decoder one byte at a time. As we go, note the - * nearest "safe start point" before the current location - * (a point where the decoder has nothing buffered, so seek() - * can safely start from there and advance to this location). - */ - chars_decoded = 0; - input = PyBytes_AS_STRING(next_input); - input_end = input + PyBytes_GET_SIZE(next_input); - while (input < input_end) { - PyObject *state; - char *dec_buffer; - Py_ssize_t dec_buffer_len; - int dec_flags; - - PyObject *decoded = PyObject_CallMethod( - self->decoder, "decode", "y#", input, 1); - if (decoded == NULL) - goto fail; - assert (PyUnicode_Check(decoded)); - chars_decoded += PyUnicode_GET_SIZE(decoded); - Py_DECREF(decoded); - - cookie.bytes_to_feed += 1; - - state = PyObject_CallMethodObjArgs(self->decoder, - _PyIO_str_getstate, NULL); - if (state == NULL) - goto fail; - if (!PyArg_Parse(state, "(y#i)", &dec_buffer, &dec_buffer_len, &dec_flags)) { - Py_DECREF(state); - goto fail; - } - Py_DECREF(state); - - if (dec_buffer_len == 0 && chars_decoded <= chars_to_skip) { - /* Decoder buffer is empty, so this is a safe start point. */ - cookie.start_pos += cookie.bytes_to_feed; - chars_to_skip -= chars_decoded; - cookie.dec_flags = dec_flags; - cookie.bytes_to_feed = 0; - chars_decoded = 0; - } - if (chars_decoded >= chars_to_skip) - break; - input++; - } - if (input == input_end) { - /* We didn't get enough decoded data; signal EOF to get more. */ - PyObject *decoded = PyObject_CallMethod( - self->decoder, "decode", "yi", "", /* final = */ 1); - if (decoded == NULL) - goto fail; - assert (PyUnicode_Check(decoded)); - chars_decoded += PyUnicode_GET_SIZE(decoded); - Py_DECREF(decoded); - cookie.need_eof = 1; - - if (chars_decoded < chars_to_skip) { - PyErr_SetString(PyExc_IOError, - "can't reconstruct logical file position"); - goto fail; - } - } - - /* finally */ - Py_XDECREF(posobj); - res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state); - Py_DECREF(saved_state); - if (res == NULL) - return NULL; - Py_DECREF(res); - - /* The returned cookie corresponds to the last safe start point. */ - cookie.chars_to_skip = Py_SAFE_DOWNCAST(chars_to_skip, Py_ssize_t, int); - return TextIOWrapper_buildCookie(&cookie); - - fail: - Py_XDECREF(posobj); - if (saved_state) { - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - - res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state); - Py_DECREF(saved_state); - if (res == NULL) - return NULL; - Py_DECREF(res); - - PyErr_Restore(type, value, traceback); - } - return NULL; -} - -static PyObject * -TextIOWrapper_truncate(PyTextIOWrapperObject *self, PyObject *args) -{ - PyObject *pos = Py_None; - PyObject *res; - - CHECK_INITIALIZED(self) - if (!PyArg_ParseTuple(args, "|O:truncate", &pos)) { - return NULL; - } - - res = PyObject_CallMethodObjArgs((PyObject *) self, _PyIO_str_flush, NULL); - if (res == NULL) - return NULL; - Py_DECREF(res); - - if (pos != Py_None) { - res = PyObject_CallMethodObjArgs((PyObject *) self, - _PyIO_str_seek, pos, NULL); - if (res == NULL) - return NULL; - Py_DECREF(res); - } - - return PyObject_CallMethodObjArgs(self->buffer, _PyIO_str_truncate, NULL); -} - -static PyObject * -TextIOWrapper_repr(PyTextIOWrapperObject *self) -{ - CHECK_INITIALIZED(self); - return PyUnicode_FromFormat("", self->encoding); -} - - -/* Inquiries */ - -static PyObject * -TextIOWrapper_fileno(PyTextIOWrapperObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "fileno", NULL); -} - -static PyObject * -TextIOWrapper_seekable(PyTextIOWrapperObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "seekable", NULL); -} - -static PyObject * -TextIOWrapper_readable(PyTextIOWrapperObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "readable", NULL); -} - -static PyObject * -TextIOWrapper_writable(PyTextIOWrapperObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "writable", NULL); -} - -static PyObject * -TextIOWrapper_isatty(PyTextIOWrapperObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "isatty", NULL); -} - -static PyObject * -TextIOWrapper_flush(PyTextIOWrapperObject *self, PyObject *args) -{ - CHECK_INITIALIZED(self); - CHECK_CLOSED(self); - self->telling = self->seekable; - if (_TextIOWrapper_writeflush(self) < 0) - return NULL; - return PyObject_CallMethod(self->buffer, "flush", NULL); -} - -static PyObject * -TextIOWrapper_close(PyTextIOWrapperObject *self, PyObject *args) -{ - PyObject *res; - CHECK_INITIALIZED(self); - res = PyObject_CallMethod((PyObject *)self, "flush", NULL); - if (res == NULL) { - /* If flush() fails, just give up */ - PyErr_Clear(); - } - else - Py_DECREF(res); - - return PyObject_CallMethod(self->buffer, "close", NULL); -} - -static PyObject * -TextIOWrapper_iternext(PyTextIOWrapperObject *self) -{ - PyObject *line; - - CHECK_INITIALIZED(self); - - self->telling = 0; - if (Py_TYPE(self) == &PyTextIOWrapper_Type) { - /* Skip method call overhead for speed */ - line = _TextIOWrapper_readline(self, -1); - } - else { - line = PyObject_CallMethodObjArgs((PyObject *)self, - _PyIO_str_readline, NULL); - if (line && !PyUnicode_Check(line)) { - PyErr_Format(PyExc_IOError, - "readline() should have returned an str object, " - "not '%.200s'", Py_TYPE(line)->tp_name); - Py_DECREF(line); - return NULL; - } - } - - if (line == NULL) - return NULL; - - if (PyUnicode_GET_SIZE(line) == 0) { - /* Reached EOF or would have blocked */ - Py_DECREF(line); - Py_CLEAR(self->snapshot); - self->telling = self->seekable; - return NULL; - } - - return line; -} - -static PyObject * -TextIOWrapper_name_get(PyTextIOWrapperObject *self, void *context) -{ - CHECK_INITIALIZED(self); - return PyObject_GetAttrString(self->buffer, "name"); -} - -static PyObject * -TextIOWrapper_closed_get(PyTextIOWrapperObject *self, void *context) -{ - CHECK_INITIALIZED(self); - return PyObject_GetAttr(self->buffer, _PyIO_str_closed); -} - -static PyObject * -TextIOWrapper_newlines_get(PyTextIOWrapperObject *self, void *context) -{ - PyObject *res; - CHECK_INITIALIZED(self); - if (self->decoder == NULL) - Py_RETURN_NONE; - res = PyObject_GetAttr(self->decoder, _PyIO_str_newlines); - if (res == NULL) { - PyErr_Clear(); - Py_RETURN_NONE; - } - return res; -} - -static PyObject * -TextIOWrapper_chunk_size_get(PyTextIOWrapperObject *self, void *context) -{ - CHECK_INITIALIZED(self); - return PyLong_FromSsize_t(self->chunk_size); -} - -static int -TextIOWrapper_chunk_size_set(PyTextIOWrapperObject *self, - PyObject *arg, void *context) -{ - Py_ssize_t n; - CHECK_INITIALIZED_INT(self); - n = PyNumber_AsSsize_t(arg, PyExc_TypeError); - if (n == -1 && PyErr_Occurred()) - return -1; - if (n <= 0) { - PyErr_SetString(PyExc_ValueError, - "a strictly positive integer is required"); - return -1; - } - self->chunk_size = n; - return 0; -} - -static PyMethodDef TextIOWrapper_methods[] = { - {"write", (PyCFunction)TextIOWrapper_write, METH_VARARGS}, - {"read", (PyCFunction)TextIOWrapper_read, METH_VARARGS}, - {"readline", (PyCFunction)TextIOWrapper_readline, METH_VARARGS}, - {"flush", (PyCFunction)TextIOWrapper_flush, METH_NOARGS}, - {"close", (PyCFunction)TextIOWrapper_close, METH_NOARGS}, - - {"fileno", (PyCFunction)TextIOWrapper_fileno, METH_NOARGS}, - {"seekable", (PyCFunction)TextIOWrapper_seekable, METH_NOARGS}, - {"readable", (PyCFunction)TextIOWrapper_readable, METH_NOARGS}, - {"writable", (PyCFunction)TextIOWrapper_writable, METH_NOARGS}, - {"isatty", (PyCFunction)TextIOWrapper_isatty, METH_NOARGS}, - - {"seek", (PyCFunction)TextIOWrapper_seek, METH_VARARGS}, - {"tell", (PyCFunction)TextIOWrapper_tell, METH_NOARGS}, - {"truncate", (PyCFunction)TextIOWrapper_truncate, METH_VARARGS}, - {NULL, NULL} -}; - -static PyMemberDef TextIOWrapper_members[] = { - {"encoding", T_OBJECT, offsetof(PyTextIOWrapperObject, encoding), READONLY}, - {"buffer", T_OBJECT, offsetof(PyTextIOWrapperObject, buffer), READONLY}, - {"line_buffering", T_BOOL, offsetof(PyTextIOWrapperObject, line_buffering), READONLY}, - {NULL} -}; - -static PyGetSetDef TextIOWrapper_getset[] = { - {"name", (getter)TextIOWrapper_name_get, NULL, NULL}, - {"closed", (getter)TextIOWrapper_closed_get, NULL, NULL}, -/* {"mode", (getter)TextIOWrapper_mode_get, NULL, NULL}, -*/ - {"newlines", (getter)TextIOWrapper_newlines_get, NULL, NULL}, - {"_CHUNK_SIZE", (getter)TextIOWrapper_chunk_size_get, - (setter)TextIOWrapper_chunk_size_set, NULL}, - {0} -}; - -PyTypeObject PyTextIOWrapper_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_io.TextIOWrapper", /*tp_name*/ - sizeof(PyTextIOWrapperObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)TextIOWrapper_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tps_etattr*/ - 0, /*tp_compare */ - (reprfunc)TextIOWrapper_repr,/*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ - TextIOWrapper_doc, /* tp_doc */ - (traverseproc)TextIOWrapper_traverse, /* tp_traverse */ - (inquiry)TextIOWrapper_clear, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(PyTextIOWrapperObject, weakreflist), /*tp_weaklistoffset*/ - 0, /* tp_iter */ - (iternextfunc)TextIOWrapper_iternext, /* tp_iternext */ - TextIOWrapper_methods, /* tp_methods */ - TextIOWrapper_members, /* tp_members */ - TextIOWrapper_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - offsetof(PyTextIOWrapperObject, dict), /*tp_dictoffset*/ - (initproc)TextIOWrapper_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ -}; diff --git a/Modules/io.c b/Modules/io.c deleted file mode 100644 index 58b5798..0000000 --- a/Modules/io.c +++ /dev/null @@ -1,760 +0,0 @@ -/* - An implementation of the new I/O lib as defined by PEP 3116 - "New I/O" - - Classes defined here: UnsupportedOperation, BlockingIOError. - Functions defined here: open(). - - Mostly written by Amaury Forgeot d'Arc -*/ - -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#include "_iomodule.h" - -#ifdef HAVE_SYS_TYPES_H -#include -#endif /* HAVE_SYS_TYPES_H */ - -#ifdef HAVE_SYS_STAT_H -#include -#endif /* HAVE_SYS_STAT_H */ - - -/* Various interned strings */ - -PyObject *_PyIO_str_close; -PyObject *_PyIO_str_closed; -PyObject *_PyIO_str_decode; -PyObject *_PyIO_str_encode; -PyObject *_PyIO_str_fileno; -PyObject *_PyIO_str_flush; -PyObject *_PyIO_str_getstate; -PyObject *_PyIO_str_isatty; -PyObject *_PyIO_str_newlines; -PyObject *_PyIO_str_nl; -PyObject *_PyIO_str_read; -PyObject *_PyIO_str_read1; -PyObject *_PyIO_str_readable; -PyObject *_PyIO_str_readinto; -PyObject *_PyIO_str_readline; -PyObject *_PyIO_str_reset; -PyObject *_PyIO_str_seek; -PyObject *_PyIO_str_seekable; -PyObject *_PyIO_str_tell; -PyObject *_PyIO_str_truncate; -PyObject *_PyIO_str_writable; -PyObject *_PyIO_str_write; - -PyObject *_PyIO_empty_str; -PyObject *_PyIO_empty_bytes; - - -PyDoc_STRVAR(module_doc, -"The io module provides the Python interfaces to stream handling. The\n" -"builtin open function is defined in this module.\n" -"\n" -"At the top of the I/O hierarchy is the abstract base class IOBase. It\n" -"defines the basic interface to a stream. Note, however, that there is no\n" -"seperation between reading and writing to streams; implementations are\n" -"allowed to throw an IOError if they do not support a given operation.\n" -"\n" -"Extending IOBase is RawIOBase which deals simply with the reading and\n" -"writing of raw bytes to a stream. FileIO subc lasses RawIOBase to provide\n" -"an interface to OS files.\n" -"\n" -"BufferedIOBase deals with buffering on a raw byte stream (RawIOBase). Its\n" -"subclasses, BufferedWriter, BufferedReader, and BufferedRWPair buffer\n" -"streams that are readable, writable, and both respectively.\n" -"BufferedRandom provides a buffered interface to random access\n" -"streams. BytesIO is a simple stream of in-memory bytes.\n" -"\n" -"Another IOBase subclass, TextIOBase, deals with the encoding and decoding\n" -"of streams into text. TextIOWrapper, which extends it, is a buffered text\n" -"interface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO\n" -"is a in-memory stream for text.\n" -"\n" -"Argument names are not part of the specification, and only the arguments\n" -"of open() are intended to be used as keyword arguments.\n" -"\n" -"data:\n" -"\n" -"DEFAULT_BUFFER_SIZE\n" -"\n" -" An int containing the default buffer size used by the module's buffered\n" -" I/O classes. open() uses the file's blksize (as obtained by os.stat) if\n" -" possible.\n" - ); - - -/* - * BlockingIOError extends IOError - */ - -static int -BlockingIOError_init(PyBlockingIOErrorObject *self, PyObject *args, - PyObject *kwds) -{ - PyObject *myerrno = NULL, *strerror = NULL; - PyObject *baseargs = NULL; - Py_ssize_t written = 0; - - assert(PyTuple_Check(args)); - - self->written = 0; - if (!PyArg_ParseTuple(args, "OO|n:BlockingIOError", - &myerrno, &strerror, &written)) - return -1; - - baseargs = PyTuple_Pack(2, myerrno, strerror); - if (baseargs == NULL) - return -1; - /* This will take care of initializing of myerrno and strerror members */ - if (((PyTypeObject *)PyExc_IOError)->tp_init( - (PyObject *)self, baseargs, kwds) == -1) { - Py_DECREF(baseargs); - return -1; - } - Py_DECREF(baseargs); - - self->written = written; - return 0; -} - -static PyMemberDef BlockingIOError_members[] = { - {"characters_written", T_PYSSIZET, offsetof(PyBlockingIOErrorObject, written), 0}, - {NULL} /* Sentinel */ -}; - -static PyTypeObject _PyExc_BlockingIOError = { - PyVarObject_HEAD_INIT(NULL, 0) - "BlockingIOError", /*tp_name*/ - sizeof(PyBlockingIOErrorObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - PyDoc_STR("Exception raised when I/O would block " - "on a non-blocking I/O stream"), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - BlockingIOError_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)BlockingIOError_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; -PyObject *PyExc_BlockingIOError = (PyObject *)&_PyExc_BlockingIOError; - - -/* - * The main open() function - */ -PyDoc_STRVAR(open_doc, -"Open file and return a stream. Raise IOError upon failure.\n" -"\n" -"file is either a text or byte string giving the name (and the path\n" -"if the file isn't in the current working directory) of the file to\n" -"be opened or an integer file descriptor of the file to be\n" -"wrapped. (If a file descriptor is given, it is closed when the\n" -"returned I/O object is closed, unless closefd is set to False.)\n" -"\n" -"mode is an optional string that specifies the mode in which the file\n" -"is opened. It defaults to 'r' which means open for reading in text\n" -"mode. Other common values are 'w' for writing (truncating the file if\n" -"it already exists), and 'a' for appending (which on some Unix systems,\n" -"means that all writes append to the end of the file regardless of the\n" -"current seek position). In text mode, if encoding is not specified the\n" -"encoding used is platform dependent. (For reading and writing raw\n" -"bytes use binary mode and leave encoding unspecified.) The available\n" -"modes are:\n" -"\n" -"========= ===============================================================\n" -"Character Meaning\n" -"--------- ---------------------------------------------------------------\n" -"'r' open for reading (default)\n" -"'w' open for writing, truncating the file first\n" -"'a' open for writing, appending to the end of the file if it exists\n" -"'b' binary mode\n" -"'t' text mode (default)\n" -"'+' open a disk file for updating (reading and writing)\n" -"'U' universal newline mode (for backwards compatibility; unneeded\n" -" for new code)\n" -"========= ===============================================================\n" -"\n" -"The default mode is 'rt' (open for reading text). For binary random\n" -"access, the mode 'w+b' opens and truncates the file to 0 bytes, while\n" -"'r+b' opens the file without truncation.\n" -"\n" -"Python distinguishes between files opened in binary and text modes,\n" -"even when the underlying operating system doesn't. Files opened in\n" -"binary mode (appending 'b' to the mode argument) return contents as\n" -"bytes objects without any decoding. In text mode (the default, or when\n" -"'t' is appended to the mode argument), the contents of the file are\n" -"returned as strings, the bytes having been first decoded using a\n" -"platform-dependent encoding or using the specified encoding if given.\n" -"\n" -"buffering is an optional integer used to set the buffering policy. By\n" -"default full buffering is on. Pass 0 to switch buffering off (only\n" -"allowed in binary mode), 1 to set line buffering, and an integer > 1\n" -"for full buffering.\n" -"\n" -"encoding is the name of the encoding used to decode or encode the\n" -"file. This should only be used in text mode. The default encoding is\n" -"platform dependent, but any encoding supported by Python can be\n" -"passed. See the codecs module for the list of supported encodings.\n" -"\n" -"errors is an optional string that specifies how encoding errors are to\n" -"be handled---this argument should not be used in binary mode. Pass\n" -"'strict' to raise a ValueError exception if there is an encoding error\n" -"(the default of None has the same effect), or pass 'ignore' to ignore\n" -"errors. (Note that ignoring encoding errors can lead to data loss.)\n" -"See the documentation for codecs.register for a list of the permitted\n" -"encoding error strings.\n" -"\n" -"newline controls how universal newlines works (it only applies to text\n" -"mode). It can be None, '', '\\n', '\\r', and '\\r\\n'. It works as\n" -"follows:\n" -"\n" -"* On input, if newline is None, universal newlines mode is\n" -" enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n" -" these are translated into '\\n' before being returned to the\n" -" caller. If it is '', universal newline mode is enabled, but line\n" -" endings are returned to the caller untranslated. If it has any of\n" -" the other legal values, input lines are only terminated by the given\n" -" string, and the line ending is returned to the caller untranslated.\n" -"\n" -"* On output, if newline is None, any '\\n' characters written are\n" -" translated to the system default line separator, os.linesep. If\n" -" newline is '', no translation takes place. If newline is any of the\n" -" other legal values, any '\\n' characters written are translated to\n" -" the given string.\n" -"\n" -"If closefd is False, the underlying file descriptor will be kept open\n" -"when the file is closed. This does not work when a file name is given\n" -"and must be True in that case.\n" -"\n" -"open() returns a file object whose type depends on the mode, and\n" -"through which the standard file operations such as reading and writing\n" -"are performed. When open() is used to open a file in a text mode ('w',\n" -"'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open\n" -"a file in a binary mode, the returned class varies: in read binary\n" -"mode, it returns a BufferedReader; in write binary and append binary\n" -"modes, it returns a BufferedWriter, and in read/write mode, it returns\n" -"a BufferedRandom.\n" -"\n" -"It is also possible to use a string or bytearray as a file for both\n" -"reading and writing. For strings StringIO can be used like a file\n" -"opened in a text mode, and for bytes a BytesIO can be used like a file\n" -"opened in a binary mode.\n" - ); - -static PyObject * -io_open(PyObject *self, PyObject *args, PyObject *kwds) -{ - char *kwlist[] = {"file", "mode", "buffering", - "encoding", "errors", "newline", - "closefd", NULL}; - PyObject *file; - char *mode = "r"; - int buffering = -1, closefd = 1; - char *encoding = NULL, *errors = NULL, *newline = NULL; - unsigned i; - - int reading = 0, writing = 0, appending = 0, updating = 0; - int text = 0, binary = 0, universal = 0; - - char rawmode[5], *m; - int line_buffering, isatty; - - PyObject *raw, *modeobj = NULL, *buffer = NULL, *wrapper = NULL; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|sizzzi:open", kwlist, - &file, &mode, &buffering, - &encoding, &errors, &newline, - &closefd)) { - return NULL; - } - - if (!PyUnicode_Check(file) && - !PyBytes_Check(file) && - !PyNumber_Check(file)) { - PyErr_Format(PyExc_TypeError, "invalid file: %R", file); - return NULL; - } - - /* Decode mode */ - for (i = 0; i < strlen(mode); i++) { - char c = mode[i]; - - switch (c) { - case 'r': - reading = 1; - break; - case 'w': - writing = 1; - break; - case 'a': - appending = 1; - break; - case '+': - updating = 1; - break; - case 't': - text = 1; - break; - case 'b': - binary = 1; - break; - case 'U': - universal = 1; - reading = 1; - break; - default: - goto invalid_mode; - } - - /* c must not be duplicated */ - if (strchr(mode+i+1, c)) { - invalid_mode: - PyErr_Format(PyExc_ValueError, "invalid mode: '%s'", mode); - return NULL; - } - - } - - m = rawmode; - if (reading) *(m++) = 'r'; - if (writing) *(m++) = 'w'; - if (appending) *(m++) = 'a'; - if (updating) *(m++) = '+'; - *m = '\0'; - - /* Parameters validation */ - if (universal) { - if (writing || appending) { - PyErr_SetString(PyExc_ValueError, - "can't use U and writing mode at once"); - return NULL; - } - reading = 1; - } - - if (text && binary) { - PyErr_SetString(PyExc_ValueError, - "can't have text and binary mode at once"); - return NULL; - } - - if (reading + writing + appending > 1) { - PyErr_SetString(PyExc_ValueError, - "must have exactly one of read/write/append mode"); - return NULL; - } - - if (binary && encoding != NULL) { - PyErr_SetString(PyExc_ValueError, - "binary mode doesn't take an encoding argument"); - return NULL; - } - - if (binary && errors != NULL) { - PyErr_SetString(PyExc_ValueError, - "binary mode doesn't take an errors argument"); - return NULL; - } - - if (binary && newline != NULL) { - PyErr_SetString(PyExc_ValueError, - "binary mode doesn't take a newline argument"); - return NULL; - } - - /* Create the Raw file stream */ - raw = PyObject_CallFunction((PyObject *)&PyFileIO_Type, - "Osi", file, rawmode, closefd); - if (raw == NULL) - return NULL; - - modeobj = PyUnicode_FromString(mode); - if (modeobj == NULL) - goto error; - - /* buffering */ - { - PyObject *res = PyObject_CallMethod(raw, "isatty", NULL); - if (res == NULL) - goto error; - isatty = PyLong_AsLong(res); - Py_DECREF(res); - if (isatty == -1 && PyErr_Occurred()) - goto error; - } - - if (buffering == 1 || (buffering < 0 && isatty)) { - buffering = -1; - line_buffering = 1; - } - else - line_buffering = 0; - - if (buffering < 0) { - buffering = DEFAULT_BUFFER_SIZE; -#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE - { - struct stat st; - long fileno; - PyObject *res = PyObject_CallMethod(raw, "fileno", NULL); - if (res == NULL) - goto error; - - fileno = PyLong_AsLong(res); - Py_DECREF(res); - if (fileno == -1 && PyErr_Occurred()) - goto error; - - if (fstat(fileno, &st) >= 0) - buffering = st.st_blksize; - } -#endif - } - if (buffering < 0) { - PyErr_SetString(PyExc_ValueError, - "invalid buffering size"); - goto error; - } - - /* if not buffering, returns the raw file object */ - if (buffering == 0) { - if (!binary) { - PyErr_SetString(PyExc_ValueError, - "can't have unbuffered text I/O"); - goto error; - } - - Py_DECREF(modeobj); - return raw; - } - - /* wraps into a buffered file */ - { - PyObject *Buffered_class; - - if (updating) - Buffered_class = (PyObject *)&PyBufferedRandom_Type; - else if (writing || appending) - Buffered_class = (PyObject *)&PyBufferedWriter_Type; - else if (reading) - Buffered_class = (PyObject *)&PyBufferedReader_Type; - else { - PyErr_Format(PyExc_ValueError, - "unknown mode: '%s'", mode); - goto error; - } - - buffer = PyObject_CallFunction(Buffered_class, "Oi", raw, buffering); - } - Py_CLEAR(raw); - if (buffer == NULL) - goto error; - - - /* if binary, returns the buffered file */ - if (binary) { - Py_DECREF(modeobj); - return buffer; - } - - /* wraps into a TextIOWrapper */ - wrapper = PyObject_CallFunction((PyObject *)&PyTextIOWrapper_Type, - "Osssi", - buffer, - encoding, errors, newline, - line_buffering); - Py_CLEAR(buffer); - if (wrapper == NULL) - goto error; - - if (PyObject_SetAttrString(wrapper, "mode", modeobj) < 0) - goto error; - Py_DECREF(modeobj); - return wrapper; - - error: - Py_XDECREF(raw); - Py_XDECREF(modeobj); - Py_XDECREF(buffer); - Py_XDECREF(wrapper); - return NULL; -} - -/* - * Private helpers for the io module. - */ - -Py_off_t -PyNumber_AsOff_t(PyObject *item, PyObject *err) -{ - Py_off_t result; - PyObject *runerr; - PyObject *value = PyNumber_Index(item); - if (value == NULL) - return -1; - - /* We're done if PyLong_AsSsize_t() returns without error. */ - result = PyLong_AsOff_t(value); - if (result != -1 || !(runerr = PyErr_Occurred())) - goto finish; - - /* Error handling code -- only manage OverflowError differently */ - if (!PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) - goto finish; - - PyErr_Clear(); - /* If no error-handling desired then the default clipping - is sufficient. - */ - if (!err) { - assert(PyLong_Check(value)); - /* Whether or not it is less than or equal to - zero is determined by the sign of ob_size - */ - if (_PyLong_Sign(value) < 0) - result = PY_OFF_T_MIN; - else - result = PY_OFF_T_MAX; - } - else { - /* Otherwise replace the error with caller's error object. */ - PyErr_Format(err, - "cannot fit '%.200s' into an offset-sized integer", - item->ob_type->tp_name); - } - - finish: - Py_DECREF(value); - return result; -} - -static int -iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { - _PyIO_State *state = IO_MOD_STATE(mod); - if (!state->initialized) - return 0; - Py_VISIT(state->os_module); - if (state->locale_module != NULL) { - Py_VISIT(state->locale_module); - } - Py_VISIT(state->unsupported_operation); - return 0; -} - -static int -iomodule_clear(PyObject *mod) { - _PyIO_State *state = IO_MOD_STATE(mod); - if (!state->initialized) - return 0; - Py_CLEAR(state->os_module); - if (state->locale_module != NULL) - Py_CLEAR(state->locale_module); - Py_CLEAR(state->unsupported_operation); - return 0; -} - -static void -iomodule_free(PyObject *mod) { - iomodule_clear(mod); -} - -/* - * Module definition - */ - -static PyMethodDef module_methods[] = { - {"open", (PyCFunction)io_open, METH_VARARGS|METH_KEYWORDS, open_doc}, - {NULL, NULL} -}; - -struct PyModuleDef _PyIO_Module = { - PyModuleDef_HEAD_INIT, - "io", - module_doc, - sizeof(_PyIO_State), - module_methods, - NULL, - iomodule_traverse, - iomodule_clear, - (freefunc)iomodule_free, -}; - -PyMODINIT_FUNC -PyInit__io(void) -{ - PyObject *m = PyModule_Create(&_PyIO_Module); - _PyIO_State *state = NULL; - if (m == NULL) - return NULL; - state = IO_MOD_STATE(m); - state->initialized = 0; - - /* put os in the module state */ - state->os_module = PyImport_ImportModule("os"); - if (state->os_module == NULL) - goto fail; - -#define ADD_TYPE(type, name) \ - if (PyType_Ready(type) < 0) \ - goto fail; \ - Py_INCREF(type); \ - if (PyModule_AddObject(m, name, (PyObject *)type) < 0) { \ - Py_DECREF(type); \ - goto fail; \ - } - - /* DEFAULT_BUFFER_SIZE */ - if (PyModule_AddIntMacro(m, DEFAULT_BUFFER_SIZE) < 0) - goto fail; - - /* UnsupportedOperation inherits from ValueError and IOError */ - state->unsupported_operation = PyObject_CallFunction( - (PyObject *)&PyType_Type, "s(OO){}", - "UnsupportedOperation", PyExc_ValueError, PyExc_IOError); - if (state->unsupported_operation == NULL) - goto fail; - Py_INCREF(state->unsupported_operation); - if (PyModule_AddObject(m, "UnsupportedOperation", - state->unsupported_operation) < 0) - goto fail; - - /* BlockingIOError */ - _PyExc_BlockingIOError.tp_base = (PyTypeObject *) PyExc_IOError; - ADD_TYPE(&_PyExc_BlockingIOError, "BlockingIOError"); - - /* Concrete base types of the IO ABCs. - (the ABCs themselves are declared through inheritance in io.py) - */ - ADD_TYPE(&PyIOBase_Type, "_IOBase"); - ADD_TYPE(&PyRawIOBase_Type, "_RawIOBase"); - ADD_TYPE(&PyBufferedIOBase_Type, "_BufferedIOBase"); - ADD_TYPE(&PyTextIOBase_Type, "_TextIOBase"); - - /* Implementation of concrete IO objects. */ - /* FileIO */ - PyFileIO_Type.tp_base = &PyRawIOBase_Type; - ADD_TYPE(&PyFileIO_Type, "FileIO"); - - /* BytesIO */ - PyBytesIO_Type.tp_base = &PyBufferedIOBase_Type; - ADD_TYPE(&PyBytesIO_Type, "BytesIO"); - - /* StringIO */ - PyStringIO_Type.tp_base = &PyTextIOBase_Type; - ADD_TYPE(&PyStringIO_Type, "StringIO"); - - /* BufferedReader */ - PyBufferedReader_Type.tp_base = &PyBufferedIOBase_Type; - ADD_TYPE(&PyBufferedReader_Type, "BufferedReader"); - - /* BufferedWriter */ - PyBufferedWriter_Type.tp_base = &PyBufferedIOBase_Type; - ADD_TYPE(&PyBufferedWriter_Type, "BufferedWriter"); - - /* BufferedRWPair */ - PyBufferedRWPair_Type.tp_base = &PyBufferedIOBase_Type; - ADD_TYPE(&PyBufferedRWPair_Type, "BufferedRWPair"); - - /* BufferedRandom */ - PyBufferedRandom_Type.tp_base = &PyBufferedIOBase_Type; - ADD_TYPE(&PyBufferedRandom_Type, "BufferedRandom"); - - /* TextIOWrapper */ - PyTextIOWrapper_Type.tp_base = &PyTextIOBase_Type; - ADD_TYPE(&PyTextIOWrapper_Type, "TextIOWrapper"); - - /* IncrementalNewlineDecoder */ - ADD_TYPE(&PyIncrementalNewlineDecoder_Type, "IncrementalNewlineDecoder"); - - /* Interned strings */ - if (!(_PyIO_str_close = PyUnicode_InternFromString("close"))) - goto fail; - if (!(_PyIO_str_closed = PyUnicode_InternFromString("closed"))) - goto fail; - if (!(_PyIO_str_decode = PyUnicode_InternFromString("decode"))) - goto fail; - if (!(_PyIO_str_encode = PyUnicode_InternFromString("encode"))) - goto fail; - if (!(_PyIO_str_fileno = PyUnicode_InternFromString("fileno"))) - goto fail; - if (!(_PyIO_str_flush = PyUnicode_InternFromString("flush"))) - goto fail; - if (!(_PyIO_str_getstate = PyUnicode_InternFromString("getstate"))) - goto fail; - if (!(_PyIO_str_isatty = PyUnicode_InternFromString("isatty"))) - goto fail; - if (!(_PyIO_str_newlines = PyUnicode_InternFromString("newlines"))) - goto fail; - if (!(_PyIO_str_nl = PyUnicode_InternFromString("\n"))) - goto fail; - if (!(_PyIO_str_read = PyUnicode_InternFromString("read"))) - goto fail; - if (!(_PyIO_str_read1 = PyUnicode_InternFromString("read1"))) - goto fail; - if (!(_PyIO_str_readable = PyUnicode_InternFromString("readable"))) - goto fail; - if (!(_PyIO_str_readinto = PyUnicode_InternFromString("readinto"))) - goto fail; - if (!(_PyIO_str_readline = PyUnicode_InternFromString("readline"))) - goto fail; - if (!(_PyIO_str_reset = PyUnicode_InternFromString("reset"))) - goto fail; - if (!(_PyIO_str_seek = PyUnicode_InternFromString("seek"))) - goto fail; - if (!(_PyIO_str_seekable = PyUnicode_InternFromString("seekable"))) - goto fail; - if (!(_PyIO_str_tell = PyUnicode_InternFromString("tell"))) - goto fail; - if (!(_PyIO_str_truncate = PyUnicode_InternFromString("truncate"))) - goto fail; - if (!(_PyIO_str_write = PyUnicode_InternFromString("write"))) - goto fail; - if (!(_PyIO_str_writable = PyUnicode_InternFromString("writable"))) - goto fail; - - if (!(_PyIO_empty_str = PyUnicode_FromStringAndSize(NULL, 0))) - goto fail; - if (!(_PyIO_empty_bytes = PyBytes_FromStringAndSize(NULL, 0))) - goto fail; - - state->initialized = 1; - - return m; - - fail: - Py_XDECREF(state->os_module); - Py_XDECREF(state->unsupported_operation); - Py_DECREF(m); - return NULL; -} diff --git a/PC/VC6/pythoncore.dsp b/PC/VC6/pythoncore.dsp index 3803c03..5640682 100644 --- a/PC/VC6/pythoncore.dsp +++ b/PC/VC6/pythoncore.dsp @@ -97,14 +97,6 @@ SOURCE=..\..\Modules\_bisectmodule.c # End Source File # Begin Source File -SOURCE=..\..\Modules\_bufferedio.c -# End Source File -# Begin Source File - -SOURCE=..\..\Modules\_bytesio.c -# End Source File -# Begin Source File - SOURCE=..\..\Modules\cjkcodecs\_codecs_cn.c # End Source File # Begin Source File @@ -141,19 +133,39 @@ SOURCE=..\..\Modules\_csv.c # End Source File # Begin Source File -SOURCE=..\..\Modules\_fileio.c +SOURCE=..\..\Modules\_functoolsmodule.c # End Source File # Begin Source File -SOURCE=..\..\Modules\_functoolsmodule.c +SOURCE=..\..\Modules\_heapqmodule.c # End Source File # Begin Source File -SOURCE=..\..\Modules\_heapqmodule.c +SOURCE=..\..\Modules\_io\bytesio.c +# End Source File +# Begin Source File + +SOURCE=..\..\Modules\_io\stringio.c +# End Source File +# Begin Source File + +SOURCE=..\..\Modules\_io\fileio.c +# End Source File +# Begin Source File + +SOURCE="..\..\Modules\_io\bufferedio.c" +# End Source File +# Begin Source File + +SOURCE=..\..\Modules\_io\iobase.c # End Source File # Begin Source File -SOURCE=..\..\Modules\_iobase.c +SOURCE=..\..\Modules\_io\textio.c +# End Source File +# Begin Source File + +SOURCE=..\..\Modules\_io\_iomodule.c # End Source File # Begin Source File @@ -181,10 +193,6 @@ SOURCE=..\..\Modules\_sre.c # End Source File # Begin Source File -SOURCE=..\..\Modules\_stringio.c -# End Source File -# Begin Source File - SOURCE=..\..\Modules\_struct.c # End Source File # Begin Source File @@ -193,10 +201,6 @@ SOURCE=..\..\PC\_subprocess.c # End Source File # Begin Source File -SOURCE=..\..\Modules\_textio.c -# End Source File -# Begin Source File - SOURCE=..\..\Modules\_threadmodule.c # End Source File # Begin Source File @@ -475,10 +479,6 @@ SOURCE=..\..\Modules\zlib\inftrees.c # End Source File # Begin Source File -SOURCE=..\..\Modules\io.c -# End Source File -# Begin Source File - SOURCE=..\..\Objects\iterobject.c # End Source File # Begin Source File diff --git a/PC/VS7.1/pythoncore.vcproj b/PC/VS7.1/pythoncore.vcproj index a5df0d1..e24e573 100644 --- a/PC/VS7.1/pythoncore.vcproj +++ b/PC/VS7.1/pythoncore.vcproj @@ -274,6 +274,32 @@ + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -1043,10 +1051,6 @@ > - - diff --git a/PCbuild/pythoncore.vcproj b/PCbuild/pythoncore.vcproj index dc64bfa..f8ba1f1 100644 --- a/PCbuild/pythoncore.vcproj +++ b/PCbuild/pythoncore.vcproj @@ -975,10 +975,6 @@ > - - @@ -991,14 +987,6 @@ > - - - - @@ -1007,14 +995,6 @@ > - - - - @@ -1039,18 +1019,10 @@ > - - - - @@ -1091,10 +1063,6 @@ > - - @@ -1171,6 +1139,42 @@ > + + + + + + + + + + + + + + + + + +