summaryrefslogtreecommitdiffstats
path: root/Modules/_io
diff options
context:
space:
mode:
Diffstat (limited to 'Modules/_io')
-rw-r--r--Modules/_io/_iomodule.c3
-rw-r--r--Modules/_io/_iomodule.h3
-rw-r--r--Modules/_io/bufferedio.c147
-rw-r--r--Modules/_io/bytesio.c2
-rw-r--r--Modules/_io/fileio.c47
-rw-r--r--Modules/_io/stringio.c73
-rw-r--r--Modules/_io/textio.c498
7 files changed, 482 insertions, 291 deletions
diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c
index 44bdac6..6f5bd48 100644
--- a/Modules/_io/_iomodule.c
+++ b/Modules/_io/_iomodule.c
@@ -36,6 +36,7 @@ PyObject *_PyIO_str_nl;
PyObject *_PyIO_str_read;
PyObject *_PyIO_str_read1;
PyObject *_PyIO_str_readable;
+PyObject *_PyIO_str_readall;
PyObject *_PyIO_str_readinto;
PyObject *_PyIO_str_readline;
PyObject *_PyIO_str_reset;
@@ -767,6 +768,8 @@ PyInit__io(void)
goto fail;
if (!(_PyIO_str_readable = PyUnicode_InternFromString("readable")))
goto fail;
+ if (!(_PyIO_str_readall = PyUnicode_InternFromString("readall")))
+ goto fail;
if (!(_PyIO_str_readinto = PyUnicode_InternFromString("readinto")))
goto fail;
if (!(_PyIO_str_readline = PyUnicode_InternFromString("readline")))
diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h
index 925e4f2..4e97dd1 100644
--- a/Modules/_io/_iomodule.h
+++ b/Modules/_io/_iomodule.h
@@ -55,7 +55,7 @@ extern PyObject *_PyIncrementalNewlineDecoder_decode(
Otherwise, the function will scan further and return garbage. */
extern Py_ssize_t _PyIO_find_line_ending(
int translated, int universal, PyObject *readnl,
- Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed);
+ int kind, char *start, char *end, Py_ssize_t *consumed);
#define DEFAULT_BUFFER_SIZE (8 * 1024) /* bytes */
@@ -155,6 +155,7 @@ extern PyObject *_PyIO_str_nl;
extern PyObject *_PyIO_str_read;
extern PyObject *_PyIO_str_read1;
extern PyObject *_PyIO_str_readable;
+extern PyObject *_PyIO_str_readall;
extern PyObject *_PyIO_str_readinto;
extern PyObject *_PyIO_str_readline;
extern PyObject *_PyIO_str_reset;
diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c
index c979ac2..86f7412 100644
--- a/Modules/_io/bufferedio.c
+++ b/Modules/_io/bufferedio.c
@@ -1,9 +1,9 @@
/*
An implementation of Buffered I/O as defined by PEP 3116 - "New I/O"
-
+
Classes defined here: BufferedIOBase, BufferedReader, BufferedWriter,
BufferedRandom.
-
+
Written by Amaury Forgeot d'Arc and Antoine Pitrou
*/
@@ -198,7 +198,7 @@ typedef struct {
int readable;
int writable;
int deallocating;
-
+
/* True if this is a vanilla Buffered object (rather than a user derived
class) *and* the raw stream is a vanilla FileIO object. */
int fast_closed_checks;
@@ -237,7 +237,7 @@ typedef struct {
/*
Implementation notes:
-
+
* BufferedReader, BufferedWriter and BufferedRandom try to share most
methods (this is helped by the members `readable` and `writable`, which
are initialized in the respective constructors)
@@ -255,7 +255,7 @@ typedef struct {
NOTE: we should try to maintain block alignment of reads and writes to the
raw stream (according to the buffer size), but for now it is only done
in read() and friends.
-
+
*/
/* These macros protect the buffered object against concurrent operations. */
@@ -589,14 +589,15 @@ _bufferedreader_reset_buf(buffered *self);
static void
_bufferedwriter_reset_buf(buffered *self);
static PyObject *
-_bufferedreader_peek_unlocked(buffered *self, Py_ssize_t);
+_bufferedreader_peek_unlocked(buffered *self);
static PyObject *
_bufferedreader_read_all(buffered *self);
static PyObject *
_bufferedreader_read_fast(buffered *self, Py_ssize_t);
static PyObject *
_bufferedreader_read_generic(buffered *self, Py_ssize_t);
-
+static Py_ssize_t
+_bufferedreader_raw_read(buffered *self, char *start, Py_ssize_t len);
/*
* Helpers
@@ -635,7 +636,7 @@ _buffered_raw_tell(buffered *self)
if (!PyErr_Occurred())
PyErr_Format(PyExc_IOError,
"Raw stream returned invalid position %" PY_PRIdOFF,
- (PY_OFF_T_COMPAT)n);
+ (PY_OFF_T_COMPAT)n);
return -1;
}
self->abs_pos = n;
@@ -668,7 +669,7 @@ _buffered_raw_seek(buffered *self, Py_off_t target, int whence)
if (!PyErr_Occurred())
PyErr_Format(PyExc_IOError,
"Raw stream returned invalid position %" PY_PRIdOFF,
- (PY_OFF_T_COMPAT)n);
+ (PY_OFF_T_COMPAT)n);
return -1;
}
self->abs_pos = n;
@@ -809,7 +810,7 @@ buffered_peek(buffered *self, PyObject *args)
goto end;
Py_CLEAR(res);
}
- res = _bufferedreader_peek_unlocked(self, n);
+ res = _bufferedreader_peek_unlocked(self);
end:
LEAVE_BUFFERED(self)
@@ -875,7 +876,7 @@ buffered_read1(buffered *self, PyObject *args)
if (!ENTER_BUFFERED(self))
return NULL;
-
+
/* Return up to n bytes. If at least one byte is buffered, we
only return buffered bytes. Otherwise, we do one raw read. */
@@ -924,10 +925,78 @@ end:
static PyObject *
buffered_readinto(buffered *self, PyObject *args)
{
+ Py_buffer buf;
+ Py_ssize_t n, written = 0, remaining;
+ PyObject *res = NULL;
+
CHECK_INITIALIZED(self)
-
- /* TODO: use raw.readinto() (or a direct copy from our buffer) instead! */
- return bufferediobase_readinto((PyObject *)self, args);
+
+ if (!PyArg_ParseTuple(args, "w*:readinto", &buf))
+ return NULL;
+
+ n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t);
+ if (n > 0) {
+ if (n >= buf.len) {
+ memcpy(buf.buf, self->buffer + self->pos, buf.len);
+ self->pos += buf.len;
+ res = PyLong_FromSsize_t(buf.len);
+ goto end_unlocked;
+ }
+ memcpy(buf.buf, self->buffer + self->pos, n);
+ self->pos += n;
+ written = n;
+ }
+
+ if (!ENTER_BUFFERED(self))
+ goto end_unlocked;
+
+ if (self->writable) {
+ res = buffered_flush_and_rewind_unlocked(self);
+ if (res == NULL)
+ goto end;
+ Py_CLEAR(res);
+ }
+
+ _bufferedreader_reset_buf(self);
+ self->pos = 0;
+
+ for (remaining = buf.len - written;
+ remaining > 0;
+ written += n, remaining -= n) {
+ /* If remaining bytes is larger than internal buffer size, copy
+ * directly into caller's buffer. */
+ if (remaining > self->buffer_size) {
+ n = _bufferedreader_raw_read(self, (char *) buf.buf + written,
+ remaining);
+ }
+ else {
+ n = _bufferedreader_fill_buffer(self);
+ if (n > 0) {
+ if (n > remaining)
+ n = remaining;
+ memcpy((char *) buf.buf + written,
+ self->buffer + self->pos, n);
+ self->pos += n;
+ continue; /* short circuit */
+ }
+ }
+ if (n == 0 || (n == -2 && written > 0))
+ break;
+ if (n < 0) {
+ if (n == -2) {
+ Py_INCREF(Py_None);
+ res = Py_None;
+ }
+ goto end;
+ }
+ }
+ res = PyLong_FromSsize_t(written);
+
+end:
+ LEAVE_BUFFERED(self);
+end_unlocked:
+ PyBuffer_Release(&buf);
+ return res;
}
static PyObject *
@@ -1345,33 +1414,58 @@ static PyObject *
_bufferedreader_read_all(buffered *self)
{
Py_ssize_t current_size;
- PyObject *res, *data = NULL;
- PyObject *chunks = PyList_New(0);
-
- if (chunks == NULL)
- return NULL;
+ PyObject *res, *data = NULL, *chunk, *chunks;
/* First copy what we have in the current buffer. */
current_size = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t);
if (current_size) {
data = PyBytes_FromStringAndSize(
self->buffer + self->pos, current_size);
- if (data == NULL) {
- Py_DECREF(chunks);
+ if (data == NULL)
return NULL;
- }
self->pos += current_size;
}
/* We're going past the buffer's bounds, flush it */
if (self->writable) {
res = buffered_flush_and_rewind_unlocked(self);
- if (res == NULL) {
- Py_DECREF(chunks);
+ if (res == NULL)
return NULL;
- }
Py_CLEAR(res);
}
_bufferedreader_reset_buf(self);
+
+ if (PyObject_HasAttr(self->raw, _PyIO_str_readall)) {
+ chunk = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readall, NULL);
+ if (chunk == NULL)
+ return NULL;
+ if (chunk != Py_None && !PyBytes_Check(chunk)) {
+ Py_XDECREF(data);
+ Py_DECREF(chunk);
+ PyErr_SetString(PyExc_TypeError, "readall() should return bytes");
+ return NULL;
+ }
+ if (chunk == Py_None) {
+ if (current_size == 0)
+ return chunk;
+ else {
+ Py_DECREF(chunk);
+ return data;
+ }
+ }
+ else if (current_size) {
+ PyBytes_Concat(&data, chunk);
+ Py_DECREF(chunk);
+ if (data == NULL)
+ return NULL;
+ return data;
+ } else
+ return chunk;
+ }
+
+ chunks = PyList_New(0);
+ if (chunks == NULL)
+ return NULL;
+
while (1) {
if (data) {
if (PyList_Append(chunks, data) < 0) {
@@ -1533,7 +1627,7 @@ error:
}
static PyObject *
-_bufferedreader_peek_unlocked(buffered *self, Py_ssize_t n)
+_bufferedreader_peek_unlocked(buffered *self)
{
Py_ssize_t have, r;
@@ -1575,6 +1669,7 @@ static PyMethodDef bufferedreader_methods[] = {
{"read", (PyCFunction)buffered_read, METH_VARARGS},
{"peek", (PyCFunction)buffered_peek, METH_VARARGS},
{"read1", (PyCFunction)buffered_read1, METH_VARARGS},
+ {"readinto", (PyCFunction)buffered_readinto, METH_VARARGS},
{"readline", (PyCFunction)buffered_readline, METH_VARARGS},
{"seek", (PyCFunction)buffered_seek, METH_VARARGS},
{"tell", (PyCFunction)buffered_tell, METH_NOARGS},
diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c
index b40513f..65ec931 100644
--- a/Modules/_io/bytesio.c
+++ b/Modules/_io/bytesio.c
@@ -938,13 +938,11 @@ static int
bytesiobuf_getbuffer(bytesiobuf *obj, Py_buffer *view, int flags)
{
int ret;
- void *ptr;
bytesio *b = (bytesio *) obj->source;
if (view == NULL) {
b->exports++;
return 0;
}
- ptr = (void *) obj;
ret = PyBuffer_FillInfo(view, (PyObject*)obj, b->buf, b->string_size,
0, flags);
if (ret >= 0) {
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c
index b1d492b..2bf8933 100644
--- a/Modules/_io/fileio.c
+++ b/Modules/_io/fileio.c
@@ -259,9 +259,11 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
}
#ifdef MS_WINDOWS
- if (PyUnicode_Check(nameobj))
- widename = PyUnicode_AS_UNICODE(nameobj);
- if (widename == NULL)
+ if (PyUnicode_Check(nameobj)) {
+ widename = PyUnicode_AsUnicode(nameobj);
+ if (widename == NULL)
+ return -1;
+ } else
#endif
if (fd < 0)
{
@@ -378,7 +380,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
if (self->fd < 0) {
#ifdef MS_WINDOWS
if (widename != NULL)
- PyErr_SetFromErrnoWithUnicodeFilename(PyExc_IOError, widename);
+ PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, nameobj);
else
#endif
PyErr_SetFromErrnoWithFilename(PyExc_IOError, name);
@@ -547,14 +549,14 @@ fileio_readinto(fileio *self, PyObject *args)
}
static size_t
-new_buffersize(fileio *self, size_t currentsize)
+new_buffersize(fileio *self, size_t currentsize
+#ifdef HAVE_FSTAT
+ , off_t pos, off_t end
+#endif
+ )
{
#ifdef HAVE_FSTAT
- off_t pos, end;
- struct stat st;
- if (fstat(self->fd, &st) == 0) {
- end = st.st_size;
- pos = lseek(self->fd, 0L, SEEK_CUR);
+ if (end != (off_t)-1) {
/* Files claiming a size smaller than SMALLCHUNK may
actually be streaming pseudo-files. In this case, we
apply the more aggressive algorithm below.
@@ -579,9 +581,14 @@ new_buffersize(fileio *self, size_t currentsize)
static PyObject *
fileio_readall(fileio *self)
{
+#ifdef HAVE_FSTAT
+ struct stat st;
+ off_t pos, end;
+#endif
PyObject *result;
Py_ssize_t total = 0;
int n;
+ size_t newsize;
if (self->fd < 0)
return err_closed();
@@ -592,8 +599,23 @@ fileio_readall(fileio *self)
if (result == NULL)
return NULL;
+#ifdef HAVE_FSTAT
+#if defined(MS_WIN64) || defined(MS_WINDOWS)
+ pos = _lseeki64(self->fd, 0L, SEEK_CUR);
+#else
+ pos = lseek(self->fd, 0L, SEEK_CUR);
+#endif
+ if (fstat(self->fd, &st) == 0)
+ end = st.st_size;
+ else
+ end = (off_t)-1;
+#endif
while (1) {
- size_t newsize = new_buffersize(self, total);
+#ifdef HAVE_FSTAT
+ newsize = new_buffersize(self, total, pos, end);
+#else
+ newsize = new_buffersize(self, total);
+#endif
if (newsize > PY_SSIZE_T_MAX || newsize <= 0) {
PyErr_SetString(PyExc_OverflowError,
"unbounded read returned more bytes "
@@ -632,6 +654,9 @@ fileio_readall(fileio *self)
return NULL;
}
total += n;
+#ifdef HAVE_FSTAT
+ pos += n;
+#endif
}
if (PyBytes_GET_SIZE(result) > total) {
diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c
index c9d14b1..a4536b1 100644
--- a/Modules/_io/stringio.c
+++ b/Modules/_io/stringio.c
@@ -9,7 +9,7 @@
typedef struct {
PyObject_HEAD
- Py_UNICODE *buf;
+ Py_UCS4 *buf;
Py_ssize_t pos;
Py_ssize_t string_size;
size_t buf_size;
@@ -21,7 +21,7 @@ typedef struct {
PyObject *decoder;
PyObject *readnl;
PyObject *writenl;
-
+
PyObject *dict;
PyObject *weakreflist;
} stringio;
@@ -56,7 +56,7 @@ resize_buffer(stringio *self, size_t size)
/* Here, unsigned types are used to avoid dealing with signed integer
overflow, which is undefined in C. */
size_t alloc = self->buf_size;
- Py_UNICODE *new_buf = NULL;
+ Py_UCS4 *new_buf = NULL;
assert(self->buf != NULL);
@@ -84,10 +84,9 @@ resize_buffer(stringio *self, size_t size)
alloc = size + 1;
}
- if (alloc > ((size_t)-1) / sizeof(Py_UNICODE))
+ if (alloc > PY_SIZE_MAX / sizeof(Py_UCS4))
goto overflow;
- new_buf = (Py_UNICODE *)PyMem_Realloc(self->buf,
- alloc * sizeof(Py_UNICODE));
+ new_buf = (Py_UCS4 *)PyMem_Realloc(self->buf, alloc * sizeof(Py_UCS4));
if (new_buf == NULL) {
PyErr_NoMemory();
return -1;
@@ -108,9 +107,9 @@ resize_buffer(stringio *self, size_t size)
static Py_ssize_t
write_str(stringio *self, PyObject *obj)
{
- Py_UNICODE *str;
Py_ssize_t len;
PyObject *decoded = NULL;
+
assert(self->buf != NULL);
assert(self->pos >= 0);
@@ -132,8 +131,11 @@ write_str(stringio *self, PyObject *obj)
return -1;
assert(PyUnicode_Check(decoded));
- str = PyUnicode_AS_UNICODE(decoded);
- len = PyUnicode_GET_SIZE(decoded);
+ if (PyUnicode_READY(decoded)) {
+ Py_DECREF(decoded);
+ return -1;
+ }
+ len = PyUnicode_GET_LENGTH(decoded);
assert(len >= 0);
@@ -161,18 +163,21 @@ write_str(stringio *self, PyObject *obj)
*/
memset(self->buf + self->string_size, '\0',
- (self->pos - self->string_size) * sizeof(Py_UNICODE));
+ (self->pos - self->string_size) * sizeof(Py_UCS4));
}
/* Copy the data to the internal buffer, overwriting some of the
existing data if self->pos < self->string_size. */
- memcpy(self->buf + self->pos, str, len * sizeof(Py_UNICODE));
- self->pos += len;
+ if (!PyUnicode_AsUCS4(decoded,
+ self->buf + self->pos,
+ self->buf_size - self->pos,
+ 0))
+ goto fail;
/* Set the new length of the internal string if it has changed. */
- if (self->string_size < self->pos) {
+ self->pos += len;
+ if (self->string_size < self->pos)
self->string_size = self->pos;
- }
Py_DECREF(decoded);
return 0;
@@ -190,7 +195,8 @@ stringio_getvalue(stringio *self)
{
CHECK_INITIALIZED(self);
CHECK_CLOSED(self);
- return PyUnicode_FromUnicode(self->buf, self->string_size);
+ return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, self->buf,
+ self->string_size);
}
PyDoc_STRVAR(stringio_tell_doc,
@@ -214,7 +220,7 @@ static PyObject *
stringio_read(stringio *self, PyObject *args)
{
Py_ssize_t size, n;
- Py_UNICODE *output;
+ Py_UCS4 *output;
PyObject *arg = Py_None;
CHECK_INITIALIZED(self);
@@ -247,19 +253,19 @@ stringio_read(stringio *self, PyObject *args)
output = self->buf + self->pos;
self->pos += size;
- return PyUnicode_FromUnicode(output, size);
+ return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, output, size);
}
/* Internal helper, used by stringio_readline and stringio_iternext */
static PyObject *
_stringio_readline(stringio *self, Py_ssize_t limit)
{
- Py_UNICODE *start, *end, old_char;
+ Py_UCS4 *start, *end, old_char;
Py_ssize_t len, consumed;
/* In case of overseek, return the empty string */
if (self->pos >= self->string_size)
- return PyUnicode_FromString("");
+ return PyUnicode_New(0, 0);
start = self->buf + self->pos;
if (limit < 0 || limit > self->string_size - self->pos)
@@ -270,14 +276,14 @@ _stringio_readline(stringio *self, Py_ssize_t limit)
*end = '\0';
len = _PyIO_find_line_ending(
self->readtranslate, self->readuniversal, self->readnl,
- start, end, &consumed);
+ PyUnicode_4BYTE_KIND, (char*)start, (char*)end, &consumed);
*end = old_char;
/* If we haven't found any line ending, we just return everything
(`consumed` is ignored). */
if (len < 0)
len = limit;
self->pos += len;
- return PyUnicode_FromUnicode(start, len);
+ return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, start, len);
}
PyDoc_STRVAR(stringio_readline_doc,
@@ -462,8 +468,10 @@ stringio_write(stringio *self, PyObject *obj)
Py_TYPE(obj)->tp_name);
return NULL;
}
+ if (PyUnicode_READY(obj))
+ return NULL;
CHECK_CLOSED(self);
- size = PyUnicode_GET_SIZE(obj);
+ size = PyUnicode_GET_LENGTH(obj);
if (size > 0 && write_str(self, obj) < 0)
return NULL;
@@ -535,7 +543,7 @@ stringio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
/* tp_alloc initializes all the fields to zero. So we don't have to
initialize them here. */
- self->buf = (Py_UNICODE *)PyMem_Malloc(0);
+ self->buf = (Py_UCS4 *)PyMem_Malloc(0);
if (self->buf == NULL) {
Py_DECREF(self);
return PyErr_NoMemory();
@@ -747,11 +755,22 @@ stringio_setstate(stringio *self, PyObject *state)
once by __init__. So we do not take any chance and replace object's
buffer completely. */
{
- Py_UNICODE *buf = PyUnicode_AS_UNICODE(PyTuple_GET_ITEM(state, 0));
- Py_ssize_t bufsize = PyUnicode_GET_SIZE(PyTuple_GET_ITEM(state, 0));
- if (resize_buffer(self, bufsize) < 0)
+ PyObject *item;
+ Py_UCS4 *buf;
+ Py_ssize_t bufsize;
+
+ item = PyTuple_GET_ITEM(state, 0);
+ buf = PyUnicode_AsUCS4Copy(item);
+ if (buf == NULL)
return NULL;
- memcpy(self->buf, buf, bufsize * sizeof(Py_UNICODE));
+ bufsize = PyUnicode_GET_LENGTH(item);
+
+ if (resize_buffer(self, bufsize) < 0) {
+ PyMem_Free(buf);
+ return NULL;
+ }
+ memcpy(self->buf, buf, bufsize * sizeof(Py_UCS4));
+ PyMem_Free(buf);
self->string_size = bufsize;
}
diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c
index 9c5f441..880a5f0 100644
--- a/Modules/_io/textio.c
+++ b/Modules/_io/textio.c
@@ -274,18 +274,28 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
goto error;
}
- output_len = PyUnicode_GET_SIZE(output);
+ if (PyUnicode_READY(output) == -1)
+ goto error;
+
+ output_len = PyUnicode_GET_LENGTH(output);
if (self->pendingcr && (final || output_len > 0)) {
- Py_UNICODE *out;
- PyObject *modified = PyUnicode_FromUnicode(NULL, output_len + 1);
+ /* Prefix output with CR */
+ int kind;
+ PyObject *modified;
+ char *out;
+
+ modified = PyUnicode_New(output_len + 1,
+ PyUnicode_MAX_CHAR_VALUE(output));
if (modified == NULL)
goto error;
- out = PyUnicode_AS_UNICODE(modified);
- out[0] = '\r';
- memcpy(out + 1, PyUnicode_AS_UNICODE(output),
- output_len * sizeof(Py_UNICODE));
+ kind = PyUnicode_KIND(modified);
+ out = PyUnicode_DATA(modified);
+ PyUnicode_WRITE(kind, PyUnicode_DATA(modified), 0, '\r');
+ memcpy(out + PyUnicode_KIND_SIZE(kind, 1),
+ PyUnicode_DATA(output),
+ PyUnicode_KIND_SIZE(kind, output_len));
Py_DECREF(output);
- output = modified;
+ output = modified; /* output remains ready */
self->pendingcr = 0;
output_len++;
}
@@ -295,21 +305,13 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
*/
if (!final) {
if (output_len > 0
- && PyUnicode_AS_UNICODE(output)[output_len - 1] == '\r') {
-
- if (Py_REFCNT(output) == 1) {
- if (PyUnicode_Resize(&output, output_len - 1) < 0)
- goto error;
- }
- else {
- PyObject *modified = PyUnicode_FromUnicode(
- PyUnicode_AS_UNICODE(output),
- output_len - 1);
- if (modified == NULL)
- goto error;
- Py_DECREF(output);
- output = modified;
- }
+ && PyUnicode_READ_CHAR(output, output_len - 1) == '\r')
+ {
+ PyObject *modified = PyUnicode_Substring(output, 0, output_len -1);
+ if (modified == NULL)
+ goto error;
+ Py_DECREF(output);
+ output = modified;
self->pendingcr = 1;
}
}
@@ -317,13 +319,15 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
/* Record which newlines are read and do newline translation if desired,
all in one pass. */
{
- Py_UNICODE *in_str;
+ void *in_str;
Py_ssize_t len;
int seennl = self->seennl;
int only_lf = 0;
+ int kind;
- in_str = PyUnicode_AS_UNICODE(output);
- len = PyUnicode_GET_SIZE(output);
+ in_str = PyUnicode_DATA(output);
+ len = PyUnicode_GET_LENGTH(output);
+ kind = PyUnicode_KIND(output);
if (len == 0)
return output;
@@ -332,7 +336,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
for the \r *byte* with the libc's optimized memchr.
*/
if (seennl == SEEN_LF || seennl == 0) {
- only_lf = (memchr(in_str, '\r', len * sizeof(Py_UNICODE)) == NULL);
+ only_lf = (memchr(in_str, '\r', PyUnicode_KIND_SIZE(kind, len)) == NULL);
}
if (only_lf) {
@@ -340,21 +344,19 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
(there's nothing else to be done, even when in translation mode)
*/
if (seennl == 0 &&
- memchr(in_str, '\n', len * sizeof(Py_UNICODE)) != NULL) {
- Py_UNICODE *s, *end;
- s = in_str;
- end = in_str + len;
+ memchr(in_str, '\n', PyUnicode_KIND_SIZE(kind, len)) != NULL) {
+ Py_ssize_t i = 0;
for (;;) {
- Py_UNICODE c;
+ Py_UCS4 c;
/* Fast loop for non-control characters */
- while (*s > '\n')
- s++;
- c = *s++;
+ while (PyUnicode_READ(kind, in_str, i) > '\n')
+ i++;
+ c = PyUnicode_READ(kind, in_str, i++);
if (c == '\n') {
seennl |= SEEN_LF;
break;
}
- if (s > end)
+ if (i >= len)
break;
}
}
@@ -362,29 +364,27 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
need translating */
}
else if (!self->translate) {
- Py_UNICODE *s, *end;
+ Py_ssize_t i = 0;
/* We have already seen all newline types, no need to scan again */
if (seennl == SEEN_ALL)
goto endscan;
- s = in_str;
- end = in_str + len;
for (;;) {
- Py_UNICODE c;
+ Py_UCS4 c;
/* Fast loop for non-control characters */
- while (*s > '\r')
- s++;
- c = *s++;
+ while (PyUnicode_READ(kind, in_str, i) > '\r')
+ i++;
+ c = PyUnicode_READ(kind, in_str, i++);
if (c == '\n')
seennl |= SEEN_LF;
else if (c == '\r') {
- if (*s == '\n') {
+ if (PyUnicode_READ(kind, in_str, i) == '\n') {
seennl |= SEEN_CRLF;
- s++;
+ i++;
}
else
seennl |= SEEN_CR;
}
- if (s > end)
+ if (i >= len)
break;
if (seennl == SEEN_ALL)
break;
@@ -393,61 +393,50 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
;
}
else {
- PyObject *translated = NULL;
- Py_UNICODE *out_str;
- Py_UNICODE *in, *out, *end;
- if (Py_REFCNT(output) != 1) {
- /* We could try to optimize this so that we only do a copy
- when there is something to translate. On the other hand,
- most decoders should only output non-shared strings, i.e.
- translation is done in place. */
- translated = PyUnicode_FromUnicode(NULL, len);
- if (translated == NULL)
- goto error;
- assert(Py_REFCNT(translated) == 1);
- memcpy(PyUnicode_AS_UNICODE(translated),
- PyUnicode_AS_UNICODE(output),
- len * sizeof(Py_UNICODE));
- }
- else {
- translated = output;
+ void *translated;
+ int kind = PyUnicode_KIND(output);
+ void *in_str = PyUnicode_DATA(output);
+ Py_ssize_t in, out;
+ /* XXX: Previous in-place translation here is disabled as
+ resizing is not possible anymore */
+ /* We could try to optimize this so that we only do a copy
+ when there is something to translate. On the other hand,
+ we already know there is a \r byte, so chances are high
+ that something needs to be done. */
+ translated = PyMem_Malloc(PyUnicode_KIND_SIZE(kind, len));
+ if (translated == NULL) {
+ PyErr_NoMemory();
+ goto error;
}
- out_str = PyUnicode_AS_UNICODE(translated);
- in = in_str;
- out = out_str;
- end = in_str + len;
+ in = out = 0;
for (;;) {
- Py_UNICODE c;
+ Py_UCS4 c;
/* Fast loop for non-control characters */
- while ((c = *in++) > '\r')
- *out++ = c;
+ while ((c = PyUnicode_READ(kind, in_str, in++)) > '\r')
+ PyUnicode_WRITE(kind, translated, out++, c);
if (c == '\n') {
- *out++ = c;
+ PyUnicode_WRITE(kind, translated, out++, c);
seennl |= SEEN_LF;
continue;
}
if (c == '\r') {
- if (*in == '\n') {
+ if (PyUnicode_READ(kind, in_str, in) == '\n') {
in++;
seennl |= SEEN_CRLF;
}
else
seennl |= SEEN_CR;
- *out++ = '\n';
+ PyUnicode_WRITE(kind, translated, out++, '\n');
continue;
}
- if (in > end)
+ if (in > len)
break;
- *out++ = c;
- }
- if (translated != output) {
- Py_DECREF(output);
- output = translated;
- }
- if (out - out_str != len) {
- if (PyUnicode_Resize(&output, out - out_str) < 0)
- goto error;
+ PyUnicode_WRITE(kind, translated, out++, c);
}
+ Py_DECREF(output);
+ output = PyUnicode_FromKindAndData(kind, translated, out);
+ if (!output)
+ goto error;
}
self->seennl |= seennl;
}
@@ -680,12 +669,16 @@ typedef struct
PyObject *pending_bytes; /* list of bytes objects waiting to be
written, or NULL */
Py_ssize_t pending_bytes_count;
- PyObject *snapshot;
+
/* snapshot is either None, or a tuple (dec_flags, next_input) where
* dec_flags is the second (integer) item of the decoder state and
* next_input is the chunk of input bytes that comes next after the
* snapshot point. We use this to reconstruct decoder states in tell().
*/
+ PyObject *snapshot;
+ /* Bytes-to-characters ratio for the current chunk. Serves as input for
+ the heuristic in tell(). */
+ double b2cratio;
/* Cache raw object if it's a FileIO object */
PyObject *raw;
@@ -701,9 +694,7 @@ typedef struct
static PyObject *
ascii_encode(textio *self, PyObject *text)
{
- return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(text),
- PyUnicode_GET_SIZE(text),
- PyBytes_AS_STRING(self->errors));
+ return _PyUnicode_AsASCIIString(text, PyBytes_AS_STRING(self->errors));
}
static PyObject *
@@ -773,17 +764,13 @@ utf32_encode(textio *self, PyObject *text)
static PyObject *
utf8_encode(textio *self, PyObject *text)
{
- return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(text),
- PyUnicode_GET_SIZE(text),
- PyBytes_AS_STRING(self->errors));
+ return _PyUnicode_AsUTF8String(text, PyBytes_AS_STRING(self->errors));
}
static PyObject *
latin1_encode(textio *self, PyObject *text)
{
- return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(text),
- PyUnicode_GET_SIZE(text),
- PyBytes_AS_STRING(self->errors));
+ return _PyUnicode_AsLatin1String(text, PyBytes_AS_STRING(self->errors));
}
/* Map normalized encoding names onto the specialized encoding funcs */
@@ -852,6 +839,7 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds)
self->decoded_chars_used = 0;
self->pending_bytes_count = 0;
self->encodefunc = NULL;
+ self->b2cratio = 0.0;
if (encoding == NULL) {
/* Try os.device_encoding(fileno) */
@@ -1208,18 +1196,6 @@ textiowrapper_detach(textio *self)
return buffer;
}
-Py_LOCAL_INLINE(const Py_UNICODE *)
-findchar(const Py_UNICODE *s, Py_ssize_t size, Py_UNICODE ch)
-{
- /* like wcschr, but doesn't stop at NULL characters */
- while (size-- > 0) {
- if (*s == ch)
- return s;
- s++;
- }
- return NULL;
-}
-
/* Flush the internal write buffer. This doesn't explicitly flush the
underlying buffered object, though. */
static int
@@ -1264,6 +1240,9 @@ textiowrapper_write(textio *self, PyObject *args)
return NULL;
}
+ if (PyUnicode_READY(text) == -1)
+ return NULL;
+
CHECK_CLOSED(self);
if (self->encoder == NULL)
@@ -1271,11 +1250,10 @@ textiowrapper_write(textio *self, PyObject *args)
Py_INCREF(text);
- textlen = PyUnicode_GetSize(text);
+ textlen = PyUnicode_GET_LENGTH(text);
if ((self->writetranslate && self->writenl != NULL) || self->line_buffering)
- if (findchar(PyUnicode_AS_UNICODE(text),
- PyUnicode_GET_SIZE(text), '\n'))
+ if (PyUnicode_FindChar(text, '\n', 0, PyUnicode_GET_LENGTH(text), 1) != -1)
haslf = 1;
if (haslf && self->writetranslate && self->writenl != NULL) {
@@ -1291,8 +1269,7 @@ textiowrapper_write(textio *self, PyObject *args)
needflush = 1;
else if (self->line_buffering &&
(haslf ||
- findchar(PyUnicode_AS_UNICODE(text),
- PyUnicode_GET_SIZE(text), '\r')))
+ PyUnicode_FindChar(text, '\r', 0, PyUnicode_GET_LENGTH(text), 1) != -1))
needflush = 1;
/* XXX What if we were just reading? */
@@ -1364,7 +1341,8 @@ textiowrapper_get_decoded_chars(textio *self, Py_ssize_t n)
if (self->decoded_chars == NULL)
return PyUnicode_FromStringAndSize(NULL, 0);
- avail = (PyUnicode_GET_SIZE(self->decoded_chars)
+ /* decoded_chars is guaranteed to be "ready". */
+ avail = (PyUnicode_GET_LENGTH(self->decoded_chars)
- self->decoded_chars_used);
assert(avail >= 0);
@@ -1373,9 +1351,9 @@ textiowrapper_get_decoded_chars(textio *self, Py_ssize_t n)
n = avail;
if (self->decoded_chars_used > 0 || n < avail) {
- chars = PyUnicode_FromUnicode(
- PyUnicode_AS_UNICODE(self->decoded_chars)
- + self->decoded_chars_used, n);
+ chars = PyUnicode_Substring(self->decoded_chars,
+ self->decoded_chars_used,
+ self->decoded_chars_used + n);
if (chars == NULL)
return NULL;
}
@@ -1397,6 +1375,7 @@ textiowrapper_read_chunk(textio *self)
PyObject *dec_flags = NULL;
PyObject *input_chunk = NULL;
PyObject *decoded_chars, *chunk_size;
+ Py_ssize_t nbytes, nchars;
int eof;
/* The return value is True unless EOF was reached. The decoded string is
@@ -1443,7 +1422,8 @@ textiowrapper_read_chunk(textio *self)
goto fail;
assert(PyBytes_Check(input_chunk));
- eof = (PyBytes_Size(input_chunk) == 0);
+ nbytes = PyBytes_Size(input_chunk);
+ eof = (nbytes == 0);
if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type) {
decoded_chars = _PyIncrementalNewlineDecoder_decode(
@@ -1457,8 +1437,15 @@ textiowrapper_read_chunk(textio *self)
/* TODO sanity check: isinstance(decoded_chars, unicode) */
if (decoded_chars == NULL)
goto fail;
+ if (PyUnicode_READY(decoded_chars) == -1)
+ goto fail;
textiowrapper_set_decoded_chars(self, decoded_chars);
- if (PyUnicode_GET_SIZE(decoded_chars) > 0)
+ nchars = PyUnicode_GET_LENGTH(decoded_chars);
+ if (nchars > 0)
+ self->b2cratio = (double) nbytes / nchars;
+ else
+ self->b2cratio = 0.0;
+ if (nchars > 0)
eof = 0;
if (self->telling) {
@@ -1509,8 +1496,13 @@ textiowrapper_read(textio *self, PyObject *args)
PyObject *decoded;
if (bytes == NULL)
goto fail;
- decoded = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_decode,
- bytes, Py_True, NULL);
+
+ if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type)
+ decoded = _PyIncrementalNewlineDecoder_decode(self->decoder,
+ bytes, 1);
+ else
+ decoded = PyObject_CallMethodObjArgs(
+ self->decoder, _PyIO_str_decode, bytes, Py_True, NULL);
Py_DECREF(bytes);
if (decoded == NULL)
goto fail;
@@ -1536,7 +1528,9 @@ textiowrapper_read(textio *self, PyObject *args)
result = textiowrapper_get_decoded_chars(self, n);
if (result == NULL)
goto fail;
- remaining -= PyUnicode_GET_SIZE(result);
+ if (PyUnicode_READY(result) == -1)
+ goto fail;
+ remaining -= PyUnicode_GET_LENGTH(result);
/* Keep reading chunks until we have n characters to return */
while (remaining > 0) {
@@ -1556,7 +1550,7 @@ textiowrapper_read(textio *self, PyObject *args)
result = textiowrapper_get_decoded_chars(self, remaining);
if (result == NULL)
goto fail;
- remaining -= PyUnicode_GET_SIZE(result);
+ remaining -= PyUnicode_GET_LENGTH(result);
}
if (chunks != NULL) {
if (result != NULL && PyList_Append(chunks, result) < 0)
@@ -1576,36 +1570,37 @@ textiowrapper_read(textio *self, PyObject *args)
}
-/* NOTE: `end` must point to the real end of the Py_UNICODE storage,
+/* NOTE: `end` must point to the real end of the Py_UCS4 storage,
that is to the NUL character. Otherwise the function will produce
incorrect results. */
-static Py_UNICODE *
-find_control_char(Py_UNICODE *start, Py_UNICODE *end, Py_UNICODE ch)
+static char *
+find_control_char(int kind, char *s, char *end, Py_UCS4 ch)
{
- Py_UNICODE *s = start;
+ int size = PyUnicode_KIND_SIZE(kind, 1);
for (;;) {
- while (*s > ch)
- s++;
- if (*s == ch)
+ while (PyUnicode_READ(kind, s, 0) > ch)
+ s += size;
+ if (PyUnicode_READ(kind, s, 0) == ch)
return s;
if (s == end)
return NULL;
- s++;
+ s += size;
}
}
Py_ssize_t
_PyIO_find_line_ending(
int translated, int universal, PyObject *readnl,
- Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed)
+ int kind, char *start, char *end, Py_ssize_t *consumed)
{
- Py_ssize_t len = end - start;
+ int size = PyUnicode_KIND_SIZE(kind, 1);
+ Py_ssize_t len = ((char*)end - (char*)start)/size;
if (translated) {
/* Newlines are already translated, only search for \n */
- Py_UNICODE *pos = find_control_char(start, end, '\n');
+ char *pos = find_control_char(kind, start, end, '\n');
if (pos != NULL)
- return pos - start + 1;
+ return (pos - start)/size + 1;
else {
*consumed = len;
return -1;
@@ -1615,63 +1610,66 @@ _PyIO_find_line_ending(
/* Universal newline search. Find any of \r, \r\n, \n
* The decoder ensures that \r\n are not split in two pieces
*/
- Py_UNICODE *s = start;
+ char *s = start;
for (;;) {
- Py_UNICODE ch;
+ Py_UCS4 ch;
/* Fast path for non-control chars. The loop always ends
- since the Py_UNICODE storage is NUL-terminated. */
- while (*s > '\r')
- s++;
+ since the Unicode string is NUL-terminated. */
+ while (PyUnicode_READ(kind, s, 0) > '\r')
+ s += size;
if (s >= end) {
*consumed = len;
return -1;
}
- ch = *s++;
+ ch = PyUnicode_READ(kind, s, 0);
+ s += size;
if (ch == '\n')
- return s - start;
+ return (s - start)/size;
if (ch == '\r') {
- if (*s == '\n')
- return s - start + 1;
+ if (PyUnicode_READ(kind, s, 0) == '\n')
+ return (s - start)/size + 1;
else
- return s - start;
+ return (s - start)/size;
}
}
}
else {
/* Non-universal mode. */
- Py_ssize_t readnl_len = PyUnicode_GET_SIZE(readnl);
- Py_UNICODE *nl = PyUnicode_AS_UNICODE(readnl);
+ Py_ssize_t readnl_len = PyUnicode_GET_LENGTH(readnl);
+ char *nl = PyUnicode_DATA(readnl);
+ /* Assume that readnl is an ASCII character. */
+ assert(PyUnicode_KIND(readnl) == PyUnicode_1BYTE_KIND);
if (readnl_len == 1) {
- Py_UNICODE *pos = find_control_char(start, end, nl[0]);
+ char *pos = find_control_char(kind, start, end, nl[0]);
if (pos != NULL)
- return pos - start + 1;
+ return (pos - start)/size + 1;
*consumed = len;
return -1;
}
else {
- Py_UNICODE *s = start;
- Py_UNICODE *e = end - readnl_len + 1;
- Py_UNICODE *pos;
+ char *s = start;
+ char *e = end - (readnl_len - 1)*size;
+ char *pos;
if (e < s)
e = s;
while (s < e) {
Py_ssize_t i;
- Py_UNICODE *pos = find_control_char(s, end, nl[0]);
+ char *pos = find_control_char(kind, s, end, nl[0]);
if (pos == NULL || pos >= e)
break;
for (i = 1; i < readnl_len; i++) {
- if (pos[i] != nl[i])
+ if (PyUnicode_READ(kind, pos, i) != nl[i])
break;
}
if (i == readnl_len)
- return pos - start + readnl_len;
- s = pos + 1;
+ return (pos - start)/size + readnl_len;
+ s = pos + size;
}
- pos = find_control_char(e, end, nl[0]);
+ pos = find_control_char(kind, e, end, nl[0]);
if (pos == NULL)
*consumed = len;
else
- *consumed = pos - start;
+ *consumed = (pos - start)/size;
return -1;
}
}
@@ -1692,14 +1690,15 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit)
chunked = 0;
while (1) {
- Py_UNICODE *ptr;
+ char *ptr;
Py_ssize_t line_len;
+ int kind;
Py_ssize_t consumed = 0;
/* First, get some data if necessary */
res = 1;
while (!self->decoded_chars ||
- !PyUnicode_GET_SIZE(self->decoded_chars)) {
+ !PyUnicode_GET_LENGTH(self->decoded_chars)) {
res = textiowrapper_read_chunk(self);
if (res < 0)
goto error;
@@ -1724,18 +1723,24 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit)
assert(self->decoded_chars_used == 0);
line = PyUnicode_Concat(remaining, self->decoded_chars);
start = 0;
- offset_to_buffer = PyUnicode_GET_SIZE(remaining);
+ offset_to_buffer = PyUnicode_GET_LENGTH(remaining);
Py_CLEAR(remaining);
if (line == NULL)
goto error;
+ if (PyUnicode_READY(line) == -1)
+ goto error;
}
- ptr = PyUnicode_AS_UNICODE(line);
- line_len = PyUnicode_GET_SIZE(line);
+ ptr = PyUnicode_DATA(line);
+ line_len = PyUnicode_GET_LENGTH(line);
+ kind = PyUnicode_KIND(line);
endpos = _PyIO_find_line_ending(
self->readtranslate, self->readuniversal, self->readnl,
- ptr + start, ptr + line_len, &consumed);
+ kind,
+ ptr + PyUnicode_KIND_SIZE(kind, start),
+ ptr + PyUnicode_KIND_SIZE(kind, line_len),
+ &consumed);
if (endpos >= 0) {
endpos += start;
if (limit >= 0 && (endpos - start) + chunked >= limit)
@@ -1759,21 +1764,20 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit)
if (chunks == NULL)
goto error;
}
- s = PyUnicode_FromUnicode(ptr + start, endpos - start);
+ s = PyUnicode_Substring(line, start, endpos);
if (s == NULL)
goto error;
if (PyList_Append(chunks, s) < 0) {
Py_DECREF(s);
goto error;
}
- chunked += PyUnicode_GET_SIZE(s);
+ chunked += PyUnicode_GET_LENGTH(s);
Py_DECREF(s);
}
/* There may be some remaining bytes we'll have to prepend to the
next chunk of data */
if (endpos < line_len) {
- remaining = PyUnicode_FromUnicode(
- ptr + endpos, line_len - endpos);
+ remaining = PyUnicode_Substring(line, endpos, line_len);
if (remaining == NULL)
goto error;
}
@@ -1785,19 +1789,12 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit)
if (line != NULL) {
/* Our line ends in the current buffer */
self->decoded_chars_used = endpos - offset_to_buffer;
- if (start > 0 || endpos < PyUnicode_GET_SIZE(line)) {
- if (start == 0 && Py_REFCNT(line) == 1) {
- if (PyUnicode_Resize(&line, endpos) < 0)
- goto error;
- }
- else {
- PyObject *s = PyUnicode_FromUnicode(
- PyUnicode_AS_UNICODE(line) + start, endpos - start);
- Py_CLEAR(line);
- if (s == NULL)
- goto error;
- line = s;
- }
+ if (start > 0 || endpos < PyUnicode_GET_LENGTH(line)) {
+ PyObject *s = PyUnicode_Substring(line, start, endpos);
+ Py_CLEAR(line);
+ if (s == NULL)
+ goto error;
+ line = s;
}
}
if (remaining != NULL) {
@@ -1811,16 +1808,20 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit)
Py_CLEAR(remaining);
}
if (chunks != NULL) {
- if (line != NULL && PyList_Append(chunks, line) < 0)
- goto error;
- Py_CLEAR(line);
+ if (line != NULL) {
+ if (PyList_Append(chunks, line) < 0)
+ goto error;
+ Py_DECREF(line);
+ }
line = PyUnicode_Join(_PyIO_empty_str, chunks);
if (line == NULL)
goto error;
- Py_DECREF(chunks);
+ Py_CLEAR(chunks);
+ }
+ if (line == NULL) {
+ Py_INCREF(_PyIO_empty_str);
+ line = _PyIO_empty_str;
}
- if (line == NULL)
- line = PyUnicode_FromStringAndSize(NULL, 0);
return line;
@@ -2111,6 +2112,10 @@ textiowrapper_seek(textio *self, PyObject *args)
if (decoded == NULL)
goto fail;
+ if (PyUnicode_READY(decoded) == -1) {
+ Py_DECREF(decoded);
+ goto fail;
+ }
textiowrapper_set_decoded_chars(self, decoded);
@@ -2147,8 +2152,12 @@ textiowrapper_tell(textio *self, PyObject *args)
cookie_type cookie = {0,0,0,0,0};
PyObject *next_input;
Py_ssize_t chars_to_skip, chars_decoded;
+ Py_ssize_t skip_bytes, skip_back;
PyObject *saved_state = NULL;
char *input, *input_end;
+ char *dec_buffer;
+ Py_ssize_t dec_buffer_len;
+ int dec_flags;
CHECK_INITIALIZED(self);
CHECK_CLOSED(self);
@@ -2184,6 +2193,7 @@ textiowrapper_tell(textio *self, PyObject *args)
#else
cookie.start_pos = PyLong_AsLong(posobj);
#endif
+ Py_DECREF(posobj);
if (PyErr_Occurred())
goto fail;
@@ -2198,57 +2208,99 @@ textiowrapper_tell(textio *self, PyObject *args)
/* How many decoded characters have been used up since the snapshot? */
if (self->decoded_chars_used == 0) {
/* We haven't moved from the snapshot point. */
- Py_DECREF(posobj);
return textiowrapper_build_cookie(&cookie);
}
chars_to_skip = self->decoded_chars_used;
- /* Starting from the snapshot position, we will walk the decoder
- * forward until it gives us enough decoded characters.
- */
+ /* Decoder state will be restored at the end */
saved_state = PyObject_CallMethodObjArgs(self->decoder,
_PyIO_str_getstate, NULL);
if (saved_state == NULL)
goto fail;
- /* Note our initial start point. */
- if (_textiowrapper_decoder_setstate(self, &cookie) < 0)
- goto fail;
+#define DECODER_GETSTATE() do { \
+ PyObject *_state = PyObject_CallMethodObjArgs(self->decoder, \
+ _PyIO_str_getstate, NULL); \
+ if (_state == NULL) \
+ goto fail; \
+ if (!PyArg_Parse(_state, "(y#i)", &dec_buffer, &dec_buffer_len, &dec_flags)) { \
+ Py_DECREF(_state); \
+ goto fail; \
+ } \
+ Py_DECREF(_state); \
+ } while (0)
+
+ /* TODO: replace assert with exception */
+#define DECODER_DECODE(start, len, res) do { \
+ PyObject *_decoded = PyObject_CallMethod( \
+ self->decoder, "decode", "y#", start, len); \
+ if (_decoded == NULL) \
+ goto fail; \
+ assert (PyUnicode_Check(_decoded)); \
+ res = PyUnicode_GET_LENGTH(_decoded); \
+ Py_DECREF(_decoded); \
+ } while (0)
+
+ /* Fast search for an acceptable start point, close to our
+ current pos */
+ skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip);
+ skip_back = 1;
+ assert(skip_back <= PyBytes_GET_SIZE(next_input));
+ input = PyBytes_AS_STRING(next_input);
+ while (skip_bytes > 0) {
+ /* Decode up to temptative start point */
+ if (_textiowrapper_decoder_setstate(self, &cookie) < 0)
+ goto fail;
+ DECODER_DECODE(input, skip_bytes, chars_decoded);
+ if (chars_decoded <= chars_to_skip) {
+ DECODER_GETSTATE();
+ if (dec_buffer_len == 0) {
+ /* Before pos and no bytes buffered in decoder => OK */
+ cookie.dec_flags = dec_flags;
+ chars_to_skip -= chars_decoded;
+ break;
+ }
+ /* Skip back by buffered amount and reset heuristic */
+ skip_bytes -= dec_buffer_len;
+ skip_back = 1;
+ }
+ else {
+ /* We're too far ahead, skip back a bit */
+ skip_bytes -= skip_back;
+ skip_back *= 2;
+ }
+ }
+ if (skip_bytes <= 0) {
+ skip_bytes = 0;
+ if (_textiowrapper_decoder_setstate(self, &cookie) < 0)
+ goto fail;
+ }
- /* Feed the decoder one byte at a time. As we go, note the
- * nearest "safe start point" before the current location
- * (a point where the decoder has nothing buffered, so seek()
+ /* Note our initial start point. */
+ cookie.start_pos += skip_bytes;
+ cookie.chars_to_skip = chars_to_skip;
+ if (chars_to_skip == 0)
+ goto finally;
+
+ /* We should be close to the desired position. Now feed the decoder one
+ * byte at a time until we reach the `chars_to_skip` target.
+ * As we go, note the nearest "safe start point" before the current
+ * location (a point where the decoder has nothing buffered, so seek()
* can safely start from there and advance to this location).
*/
chars_decoded = 0;
input = PyBytes_AS_STRING(next_input);
input_end = input + PyBytes_GET_SIZE(next_input);
+ input += skip_bytes;
while (input < input_end) {
- PyObject *state;
- char *dec_buffer;
- Py_ssize_t dec_buffer_len;
- int dec_flags;
-
- PyObject *decoded = PyObject_CallMethod(
- self->decoder, "decode", "y#", input, 1);
- if (decoded == NULL)
- goto fail;
- assert (PyUnicode_Check(decoded));
- chars_decoded += PyUnicode_GET_SIZE(decoded);
- Py_DECREF(decoded);
+ Py_ssize_t n;
+ DECODER_DECODE(input, 1, n);
+ /* We got n chars for 1 byte */
+ chars_decoded += n;
cookie.bytes_to_feed += 1;
-
- state = PyObject_CallMethodObjArgs(self->decoder,
- _PyIO_str_getstate, NULL);
- if (state == NULL)
- goto fail;
- if (!PyArg_Parse(state, "(y#i)", &dec_buffer, &dec_buffer_len, &dec_flags)) {
- Py_DECREF(state);
- goto fail;
- }
- Py_DECREF(state);
+ DECODER_GETSTATE();
if (dec_buffer_len == 0 && chars_decoded <= chars_to_skip) {
/* Decoder buffer is empty, so this is a safe start point. */
@@ -2269,7 +2321,7 @@ textiowrapper_tell(textio *self, PyObject *args)
if (decoded == NULL)
goto fail;
assert (PyUnicode_Check(decoded));
- chars_decoded += PyUnicode_GET_SIZE(decoded);
+ chars_decoded += PyUnicode_GET_LENGTH(decoded);
Py_DECREF(decoded);
cookie.need_eof = 1;
@@ -2280,8 +2332,7 @@ textiowrapper_tell(textio *self, PyObject *args)
}
}
- /* finally */
- Py_XDECREF(posobj);
+finally:
res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state);
Py_DECREF(saved_state);
if (res == NULL)
@@ -2292,8 +2343,7 @@ textiowrapper_tell(textio *self, PyObject *args)
cookie.chars_to_skip = Py_SAFE_DOWNCAST(chars_to_skip, Py_ssize_t, int);
return textiowrapper_build_cookie(&cookie);
- fail:
- Py_XDECREF(posobj);
+fail:
if (saved_state) {
PyObject *type, *value, *traceback;
PyErr_Fetch(&type, &value, &traceback);
@@ -2497,10 +2547,10 @@ textiowrapper_iternext(textio *self)
}
}
- if (line == NULL)
+ if (line == NULL || PyUnicode_READY(line) == -1)
return NULL;
- if (PyUnicode_GET_SIZE(line) == 0) {
+ if (PyUnicode_GET_LENGTH(line) == 0) {
/* Reached EOF or would have blocked */
Py_DECREF(line);
Py_CLEAR(self->snapshot);
generated by cgit v0.12 at 2024-12-11 15:54:14 (GMT)