summaryrefslogtreecommitdiffstats
path: root/Objects
diff options
context:
space:
mode:
Diffstat (limited to 'Objects')
-rw-r--r--Objects/abstract.c77
-rw-r--r--Objects/bytearrayobject.c74
-rw-r--r--Objects/bytesobject.c111
-rw-r--r--Objects/complexobject.c12
-rw-r--r--Objects/descrobject.c8
-rw-r--r--Objects/dictobject.c7
-rw-r--r--Objects/frameobject.c8
-rw-r--r--Objects/iterobject.c11
-rw-r--r--Objects/listobject.c2
-rw-r--r--Objects/longobject.c29
-rw-r--r--Objects/methodobject.c23
-rw-r--r--Objects/object.c20
-rw-r--r--Objects/obmalloc.c28
-rw-r--r--Objects/rangeobject.c205
-rw-r--r--Objects/sliceobject.c191
-rw-r--r--Objects/stringlib/codecs.h6
-rw-r--r--Objects/stringlib/join.h133
-rw-r--r--Objects/tupleobject.c3
-rw-r--r--Objects/typeobject.c23
-rw-r--r--Objects/unicodeobject.c3016
-rw-r--r--Objects/unicodetype_db.h13
21 files changed, 2047 insertions, 1953 deletions
diff --git a/Objects/abstract.c b/Objects/abstract.c
index a2737dd..4326cfa 100644
--- a/Objects/abstract.c
+++ b/Objects/abstract.c
@@ -64,49 +64,70 @@ PyObject_Length(PyObject *o)
}
#define PyObject_Length PyObject_Size
+int
+_PyObject_HasLen(PyObject *o) {
+ return (Py_TYPE(o)->tp_as_sequence && Py_TYPE(o)->tp_as_sequence->sq_length) ||
+ (Py_TYPE(o)->tp_as_mapping && Py_TYPE(o)->tp_as_mapping->mp_length);
+}
/* The length hint function returns a non-negative value from o.__len__()
- or o.__length_hint__(). If those methods aren't found or return a negative
- value, then the defaultvalue is returned. If one of the calls fails,
+ or o.__length_hint__(). If those methods aren't found the defaultvalue is
+ returned. If one of the calls fails with an exception other than TypeError
this function returns -1.
*/
Py_ssize_t
-_PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue)
+PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue)
{
+ PyObject *hint, *result;
+ Py_ssize_t res;
_Py_IDENTIFIER(__length_hint__);
- PyObject *ro, *hintmeth;
- Py_ssize_t rv;
-
- /* try o.__len__() */
- rv = PyObject_Size(o);
- if (rv >= 0)
- return rv;
- if (PyErr_Occurred()) {
- if (!PyErr_ExceptionMatches(PyExc_TypeError))
+ res = PyObject_Length(o);
+ if (res < 0 && PyErr_Occurred()) {
+ if (!PyErr_ExceptionMatches(PyExc_TypeError)) {
return -1;
+ }
PyErr_Clear();
}
-
- /* try o.__length_hint__() */
- hintmeth = _PyObject_LookupSpecial(o, &PyId___length_hint__);
- if (hintmeth == NULL) {
- if (PyErr_Occurred())
+ else {
+ return res;
+ }
+ hint = _PyObject_LookupSpecial(o, &PyId___length_hint__);
+ if (hint == NULL) {
+ if (PyErr_Occurred()) {
return -1;
- else
+ }
+ return defaultvalue;
+ }
+ result = PyObject_CallFunctionObjArgs(hint, NULL);
+ Py_DECREF(hint);
+ if (result == NULL) {
+ if (PyErr_ExceptionMatches(PyExc_TypeError)) {
+ PyErr_Clear();
return defaultvalue;
+ }
+ return -1;
}
- ro = PyObject_CallFunctionObjArgs(hintmeth, NULL);
- Py_DECREF(hintmeth);
- if (ro == NULL) {
- if (!PyErr_ExceptionMatches(PyExc_TypeError))
- return -1;
- PyErr_Clear();
+ else if (result == Py_NotImplemented) {
+ Py_DECREF(result);
return defaultvalue;
}
- rv = PyLong_Check(ro) ? PyLong_AsSsize_t(ro) : defaultvalue;
- Py_DECREF(ro);
- return rv;
+ if (!PyLong_Check(result)) {
+ PyErr_Format(PyExc_TypeError, "__length_hint__ must be an integer, not %.100s",
+ Py_TYPE(result)->tp_name);
+ Py_DECREF(result);
+ return -1;
+ }
+ res = PyLong_AsSsize_t(result);
+ Py_DECREF(result);
+ if (res < 0 && PyErr_Occurred()) {
+ return -1;
+ }
+ if (res < 0) {
+ PyErr_Format(PyExc_ValueError, "__length_hint__() should return >= 0");
+ return -1;
+ }
+ return res;
}
PyObject *
@@ -1687,7 +1708,7 @@ PySequence_Tuple(PyObject *v)
return NULL;
/* Guess result size and allocate space. */
- n = _PyObject_LengthHint(v, 10);
+ n = PyObject_LengthHint(v, 10);
if (n == -1)
goto Fail;
result = PyTuple_New(n);
diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
index 9f1cf0a..d1b70e5 100644
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -1038,6 +1038,7 @@ bytearray_dealloc(PyByteArrayObject *self)
#define FASTSEARCH fastsearch
#define STRINGLIB(F) stringlib_##F
#define STRINGLIB_CHAR char
+#define STRINGLIB_SIZEOF_CHAR 1
#define STRINGLIB_LEN PyByteArray_GET_SIZE
#define STRINGLIB_STR PyByteArray_AS_STRING
#define STRINGLIB_NEW PyByteArray_FromStringAndSize
@@ -1049,6 +1050,7 @@ bytearray_dealloc(PyByteArrayObject *self)
#include "stringlib/fastsearch.h"
#include "stringlib/count.h"
#include "stringlib/find.h"
+#include "stringlib/join.h"
#include "stringlib/partition.h"
#include "stringlib/split.h"
#include "stringlib/ctype.h"
@@ -2288,7 +2290,7 @@ bytearray_extend(PyByteArrayObject *self, PyObject *arg)
return NULL;
/* Try to determine the length of the argument. 32 is arbitrary. */
- buf_size = _PyObject_LengthHint(arg, 32);
+ buf_size = PyObject_LengthHint(arg, 32);
if (buf_size == -1) {
Py_DECREF(it);
return NULL;
@@ -2575,73 +2577,9 @@ Concatenate any number of bytes/bytearray objects, with B\n\
in between each pair, and return the result as a new bytearray.");
static PyObject *
-bytearray_join(PyByteArrayObject *self, PyObject *it)
-{
- PyObject *seq;
- Py_ssize_t mysize = Py_SIZE(self);
- Py_ssize_t i;
- Py_ssize_t n;
- PyObject **items;
- Py_ssize_t totalsize = 0;
- PyObject *result;
- char *dest;
-
- seq = PySequence_Fast(it, "can only join an iterable");
- if (seq == NULL)
- return NULL;
- n = PySequence_Fast_GET_SIZE(seq);
- items = PySequence_Fast_ITEMS(seq);
-
- /* Compute the total size, and check that they are all bytes */
- /* XXX Shouldn't we use _getbuffer() on these items instead? */
- for (i = 0; i < n; i++) {
- PyObject *obj = items[i];
- if (!PyByteArray_Check(obj) && !PyBytes_Check(obj)) {
- PyErr_Format(PyExc_TypeError,
- "can only join an iterable of bytes "
- "(item %ld has type '%.100s')",
- /* XXX %ld isn't right on Win64 */
- (long)i, Py_TYPE(obj)->tp_name);
- goto error;
- }
- if (i > 0)
- totalsize += mysize;
- totalsize += Py_SIZE(obj);
- if (totalsize < 0) {
- PyErr_NoMemory();
- goto error;
- }
- }
-
- /* Allocate the result, and copy the bytes */
- result = PyByteArray_FromStringAndSize(NULL, totalsize);
- if (result == NULL)
- goto error;
- dest = PyByteArray_AS_STRING(result);
- for (i = 0; i < n; i++) {
- PyObject *obj = items[i];
- Py_ssize_t size = Py_SIZE(obj);
- char *buf;
- if (PyByteArray_Check(obj))
- buf = PyByteArray_AS_STRING(obj);
- else
- buf = PyBytes_AS_STRING(obj);
- if (i) {
- memcpy(dest, self->ob_bytes, mysize);
- dest += mysize;
- }
- memcpy(dest, buf, size);
- dest += size;
- }
-
- /* Done */
- Py_DECREF(seq);
- return result;
-
- /* Error handling */
- error:
- Py_DECREF(seq);
- return NULL;
+bytearray_join(PyObject *self, PyObject *iterable)
+{
+ return stringlib_bytes_join(self, iterable);
}
PyDoc_STRVAR(splitlines__doc__,
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index bb9c6fb..20de5cc 100644
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -10,9 +10,18 @@
static Py_ssize_t
_getbuffer(PyObject *obj, Py_buffer *view)
{
- PyBufferProcs *buffer = Py_TYPE(obj)->tp_as_buffer;
-
- if (buffer == NULL || buffer->bf_getbuffer == NULL)
+ PyBufferProcs *bufferprocs;
+ if (PyBytes_CheckExact(obj)) {
+ /* Fast path, e.g. for .join() of many bytes objects */
+ Py_INCREF(obj);
+ view->obj = obj;
+ view->buf = PyBytes_AS_STRING(obj);
+ view->len = PyBytes_GET_SIZE(obj);
+ return view->len;
+ }
+
+ bufferprocs = Py_TYPE(obj)->tp_as_buffer;
+ if (bufferprocs == NULL || bufferprocs->bf_getbuffer == NULL)
{
PyErr_Format(PyExc_TypeError,
"Type %.100s doesn't support the buffer API",
@@ -20,7 +29,7 @@ _getbuffer(PyObject *obj, Py_buffer *view)
return -1;
}
- if (buffer->bf_getbuffer(obj, view, PyBUF_SIMPLE) < 0)
+ if (bufferprocs->bf_getbuffer(obj, view, PyBUF_SIMPLE) < 0)
return -1;
return view->len;
}
@@ -559,6 +568,7 @@ PyBytes_AsStringAndSize(register PyObject *obj,
#include "stringlib/fastsearch.h"
#include "stringlib/count.h"
#include "stringlib/find.h"
+#include "stringlib/join.h"
#include "stringlib/partition.h"
#include "stringlib/split.h"
#include "stringlib/ctype.h"
@@ -1111,94 +1121,9 @@ Concatenate any number of bytes objects, with B in between each pair.\n\
Example: b'.'.join([b'ab', b'pq', b'rs']) -> b'ab.pq.rs'.");
static PyObject *
-bytes_join(PyObject *self, PyObject *orig)
+bytes_join(PyObject *self, PyObject *iterable)
{
- char *sep = PyBytes_AS_STRING(self);
- const Py_ssize_t seplen = PyBytes_GET_SIZE(self);
- PyObject *res = NULL;
- char *p;
- Py_ssize_t seqlen = 0;
- size_t sz = 0;
- Py_ssize_t i;
- PyObject *seq, *item;
-
- seq = PySequence_Fast(orig, "");
- if (seq == NULL) {
- return NULL;
- }
-
- seqlen = PySequence_Size(seq);
- if (seqlen == 0) {
- Py_DECREF(seq);
- return PyBytes_FromString("");
- }
- if (seqlen == 1) {
- item = PySequence_Fast_GET_ITEM(seq, 0);
- if (PyBytes_CheckExact(item)) {
- Py_INCREF(item);
- Py_DECREF(seq);
- return item;
- }
- }
-
- /* There are at least two things to join, or else we have a subclass
- * of the builtin types in the sequence.
- * Do a pre-pass to figure out the total amount of space we'll
- * need (sz), and see whether all argument are bytes.
- */
- /* XXX Shouldn't we use _getbuffer() on these items instead? */
- for (i = 0; i < seqlen; i++) {
- const size_t old_sz = sz;
- item = PySequence_Fast_GET_ITEM(seq, i);
- if (!PyBytes_Check(item) && !PyByteArray_Check(item)) {
- PyErr_Format(PyExc_TypeError,
- "sequence item %zd: expected bytes,"
- " %.80s found",
- i, Py_TYPE(item)->tp_name);
- Py_DECREF(seq);
- return NULL;
- }
- sz += Py_SIZE(item);
- if (i != 0)
- sz += seplen;
- if (sz < old_sz || sz > PY_SSIZE_T_MAX) {
- PyErr_SetString(PyExc_OverflowError,
- "join() result is too long for bytes");
- Py_DECREF(seq);
- return NULL;
- }
- }
-
- /* Allocate result space. */
- res = PyBytes_FromStringAndSize((char*)NULL, sz);
- if (res == NULL) {
- Py_DECREF(seq);
- return NULL;
- }
-
- /* Catenate everything. */
- /* I'm not worried about a PyByteArray item growing because there's
- nowhere in this function where we release the GIL. */
- p = PyBytes_AS_STRING(res);
- for (i = 0; i < seqlen; ++i) {
- size_t n;
- char *q;
- if (i) {
- Py_MEMCPY(p, sep, seplen);
- p += seplen;
- }
- item = PySequence_Fast_GET_ITEM(seq, i);
- n = Py_SIZE(item);
- if (PyBytes_Check(item))
- q = PyBytes_AS_STRING(item);
- else
- q = PyByteArray_AS_STRING(item);
- Py_MEMCPY(p, q, n);
- p += n;
- }
-
- Py_DECREF(seq);
- return res;
+ return stringlib_bytes_join(self, iterable);
}
PyObject *
@@ -2315,8 +2240,6 @@ bytes_decode(PyObject *self, PyObject *args, PyObject *kwargs)
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ss:decode", kwlist, &encoding, &errors))
return NULL;
- if (encoding == NULL)
- encoding = PyUnicode_GetDefaultEncoding();
return PyUnicode_FromEncodedObject(self, encoding, errors);
}
@@ -2678,7 +2601,7 @@ PyBytes_FromObject(PyObject *x)
}
/* For iterator version, create a string object and resize as needed */
- size = _PyObject_LengthHint(x, 64);
+ size = PyObject_LengthHint(x, 64);
if (size == -1 && PyErr_Occurred())
return NULL;
/* Allocate an extra byte to prevent PyBytes_FromStringAndSize() from
diff --git a/Objects/complexobject.c b/Objects/complexobject.c
index 403c60c..355b063 100644
--- a/Objects/complexobject.c
+++ b/Objects/complexobject.c
@@ -271,6 +271,12 @@ try_complex_special_method(PyObject *op) {
if (f) {
PyObject *res = PyObject_CallFunctionObjArgs(f, NULL);
Py_DECREF(f);
+ if (res != NULL && !PyComplex_Check(res)) {
+ PyErr_SetString(PyExc_TypeError,
+ "__complex__ should return a complex object");
+ Py_DECREF(res);
+ return NULL;
+ }
return res;
}
return NULL;
@@ -296,12 +302,6 @@ PyComplex_AsCComplex(PyObject *op)
newop = try_complex_special_method(op);
if (newop) {
- if (!PyComplex_Check(newop)) {
- PyErr_SetString(PyExc_TypeError,
- "__complex__ should return a complex object");
- Py_DECREF(newop);
- return cv;
- }
cv = ((PyComplexObject *)newop)->cval;
Py_DECREF(newop);
return cv;
diff --git a/Objects/descrobject.c b/Objects/descrobject.c
index abcc002..3cf00d5 100644
--- a/Objects/descrobject.c
+++ b/Objects/descrobject.c
@@ -115,7 +115,7 @@ classmethod_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type)
((PyTypeObject *)type)->tp_name);
return NULL;
}
- return PyCFunction_New(descr->d_method, type);
+ return PyCFunction_NewEx(descr->d_method, type, NULL);
}
static PyObject *
@@ -125,7 +125,7 @@ method_get(PyMethodDescrObject *descr, PyObject *obj, PyObject *type)
if (descr_check((PyDescrObject *)descr, obj, &res))
return res;
- return PyCFunction_New(descr->d_method, obj);
+ return PyCFunction_NewEx(descr->d_method, obj, NULL);
}
static PyObject *
@@ -239,7 +239,7 @@ methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject *kwds)
return NULL;
}
- func = PyCFunction_New(descr->d_method, self);
+ func = PyCFunction_NewEx(descr->d_method, self, NULL);
if (func == NULL)
return NULL;
args = PyTuple_GetSlice(args, 1, argc);
@@ -292,7 +292,7 @@ classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args,
return NULL;
}
- func = PyCFunction_New(descr->d_method, self);
+ func = PyCFunction_NewEx(descr->d_method, self, NULL);
if (func == NULL)
return NULL;
args = PyTuple_GetSlice(args, 1, argc);
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index f4ad3dc..a3c6409 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -2114,13 +2114,18 @@ dict_equal(PyDictObject *a, PyDictObject *b)
if (aval != NULL) {
int cmp;
PyObject *bval;
+ PyObject **vaddr;
PyObject *key = ep->me_key;
/* temporarily bump aval's refcount to ensure it stays
alive until we're done with it */
Py_INCREF(aval);
/* ditto for key */
Py_INCREF(key);
- bval = PyDict_GetItemWithError((PyObject *)b, key);
+ /* reuse the known hash value */
+ if ((b->ma_keys->dk_lookup)(b, key, ep->me_hash, &vaddr) == NULL)
+ bval = NULL;
+ else
+ bval = *vaddr;
Py_DECREF(key);
if (bval == NULL) {
Py_DECREF(aval);
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index 808e595..6fff370 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -466,7 +466,7 @@ static int
frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
{
PyObject **fastlocals, **p;
- int i, slots;
+ Py_ssize_t i, slots;
Py_VISIT(f->f_back);
Py_VISIT(f->f_code);
@@ -496,7 +496,7 @@ static void
frame_clear(PyFrameObject *f)
{
PyObject **fastlocals, **p, **oldtop;
- int i, slots;
+ Py_ssize_t i, slots;
/* Before anything else, make sure that this frame is clearly marked
* as being defunct! Else, e.g., a generator reachable from this
@@ -848,7 +848,7 @@ PyFrame_FastToLocals(PyFrameObject *f)
PyObject *error_type, *error_value, *error_traceback;
PyCodeObject *co;
Py_ssize_t j;
- int ncells, nfreevars;
+ Py_ssize_t ncells, nfreevars;
if (f == NULL)
return;
locals = f->f_locals;
@@ -900,7 +900,7 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
PyObject *error_type, *error_value, *error_traceback;
PyCodeObject *co;
Py_ssize_t j;
- int ncells, nfreevars;
+ Py_ssize_t ncells, nfreevars;
if (f == NULL)
return;
locals = f->f_locals;
diff --git a/Objects/iterobject.c b/Objects/iterobject.c
index 3cfbeaf..9acd1b7 100644
--- a/Objects/iterobject.c
+++ b/Objects/iterobject.c
@@ -76,9 +76,14 @@ iter_len(seqiterobject *it)
Py_ssize_t seqsize, len;
if (it->it_seq) {
- seqsize = PySequence_Size(it->it_seq);
- if (seqsize == -1)
- return NULL;
+ if (_PyObject_HasLen(it->it_seq)) {
+ seqsize = PySequence_Size(it->it_seq);
+ if (seqsize == -1)
+ return NULL;
+ }
+ else {
+ Py_RETURN_NOTIMPLEMENTED;
+ }
len = seqsize - it->it_index;
if (len >= 0)
return PyLong_FromSsize_t(len);
diff --git a/Objects/listobject.c b/Objects/listobject.c
index 6e0d094..4cc34b5 100644
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -826,7 +826,7 @@ listextend(PyListObject *self, PyObject *b)
iternext = *it->ob_type->tp_iternext;
/* Guess a result list size. */
- n = _PyObject_LengthHint(b, 8);
+ n = PyObject_LengthHint(b, 8);
if (n == -1) {
Py_DECREF(it);
return NULL;
diff --git a/Objects/longobject.c b/Objects/longobject.c
index 32ccdc3..1a82b1c6 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -954,9 +954,6 @@ PyObject *
PyLong_FromVoidPtr(void *p)
{
#if SIZEOF_VOID_P <= SIZEOF_LONG
- /* special-case null pointer */
- if (!p)
- return PyLong_FromLong(0);
return PyLong_FromUnsignedLong((unsigned long)(Py_uintptr_t)p);
#else
@@ -966,9 +963,6 @@ PyLong_FromVoidPtr(void *p)
#if SIZEOF_LONG_LONG < SIZEOF_VOID_P
# error "PyLong_FromVoidPtr: sizeof(PY_LONG_LONG) < sizeof(void*)"
#endif
- /* special-case null pointer */
- if (!p)
- return PyLong_FromLong(0);
return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)(Py_uintptr_t)p);
#endif /* SIZEOF_VOID_P <= SIZEOF_LONG */
@@ -1014,7 +1008,6 @@ PyLong_AsVoidPtr(PyObject *vv)
* rewritten to use the newer PyLong_{As,From}ByteArray API.
*/
-#define IS_LITTLE_ENDIAN (int)*(unsigned char*)&one
#define PY_ABS_LLONG_MIN (0-(unsigned PY_LONG_LONG)PY_LLONG_MIN)
/* Create a new long int object from a C PY_LONG_LONG int. */
@@ -1167,7 +1160,6 @@ PyLong_AsLongLong(PyObject *vv)
{
PyLongObject *v;
PY_LONG_LONG bytes;
- int one = 1;
int res;
if (vv == NULL) {
@@ -1202,7 +1194,7 @@ PyLong_AsLongLong(PyObject *vv)
case 1: return v->ob_digit[0];
}
res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes,
- SIZEOF_LONG_LONG, IS_LITTLE_ENDIAN, 1);
+ SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 1);
/* Plan 9 can't handle PY_LONG_LONG in ? : expressions */
if (res < 0)
@@ -1219,7 +1211,6 @@ PyLong_AsUnsignedLongLong(PyObject *vv)
{
PyLongObject *v;
unsigned PY_LONG_LONG bytes;
- int one = 1;
int res;
if (vv == NULL) {
@@ -1238,7 +1229,7 @@ PyLong_AsUnsignedLongLong(PyObject *vv)
}
res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes,
- SIZEOF_LONG_LONG, IS_LITTLE_ENDIAN, 0);
+ SIZEOF_LONG_LONG, PY_LITTLE_ENDIAN, 0);
/* Plan 9 can't handle PY_LONG_LONG in ? : expressions */
if (res < 0)
@@ -1314,7 +1305,6 @@ PyLong_AsUnsignedLongLongMask(register PyObject *op)
return (unsigned PY_LONG_LONG)-1;
}
}
-#undef IS_LITTLE_ENDIAN
/* Get a C long long int from a long int object or any object that has an
__int__ method.
@@ -1676,7 +1666,6 @@ long_to_decimal_string_internal(PyObject *aa,
else \
p = (TYPE*)PyUnicode_DATA(str) + strlen; \
\
- *p = '\0'; \
/* pout[0] through pout[size-2] contribute exactly \
_PyLong_DECIMAL_SHIFT digits each */ \
for (i=0; i < size - 1; i++) { \
@@ -4101,7 +4090,7 @@ v_complement(digit *z, digit *a, Py_ssize_t m)
static PyObject *
long_bitwise(PyLongObject *a,
- int op, /* '&', '|', '^' */
+ char op, /* '&', '|', '^' */
PyLongObject *b)
{
int nega, negb, negz;
@@ -4276,8 +4265,7 @@ static PyObject *
long_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
PyObject *obase = NULL, *x = NULL;
- long base;
- int overflow;
+ Py_ssize_t base;
static char *kwlist[] = {"x", "base", 0};
if (type != &PyLong_Type)
@@ -4295,11 +4283,16 @@ long_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
}
if (obase == NULL)
return PyNumber_Long(x);
+ if (!PyLong_Check(obase)) {
+ PyErr_SetString(PyExc_TypeError,
+ "int() base must be an integer.");
+ return NULL;
+ }
- base = PyLong_AsLongAndOverflow(obase, &overflow);
+ base = PyNumber_AsSsize_t(obase, NULL);
if (base == -1 && PyErr_Occurred())
return NULL;
- if (overflow || (base != 0 && base < 2) || base > 36) {
+ if ((base != 0 && base < 2) || base > 36) {
PyErr_SetString(PyExc_ValueError,
"int() base must be >= 2 and <= 36");
return NULL;
diff --git a/Objects/methodobject.c b/Objects/methodobject.c
index 60df302..2d1f730 100644
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -13,6 +13,15 @@ static int numfree = 0;
#define PyCFunction_MAXFREELIST 256
#endif
+/* undefine macro trampoline to PyCFunction_NewEx */
+#undef PyCFunction_New
+
+PyObject *
+PyCFunction_New(PyMethodDef *ml, PyObject *self)
+{
+ return PyCFunction_NewEx(ml, self, NULL);
+}
+
PyObject *
PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module)
{
@@ -346,17 +355,3 @@ _PyCFunction_DebugMallocStats(FILE *out)
"free PyCFunctionObjects",
numfree, sizeof(PyCFunctionObject));
}
-
-/* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(),
- but it's part of the API so we need to keep a function around that
- existing C extensions can call.
-*/
-
-#undef PyCFunction_New
-PyAPI_FUNC(PyObject *) PyCFunction_New(PyMethodDef *, PyObject *);
-
-PyObject *
-PyCFunction_New(PyMethodDef *ml, PyObject *self)
-{
- return PyCFunction_NewEx(ml, self, NULL);
-}
diff --git a/Objects/object.c b/Objects/object.c
index 949e7dc..fd1fd25 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1524,12 +1524,21 @@ notimplemented_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
Py_RETURN_NOTIMPLEMENTED;
}
+static void
+notimplemented_dealloc(PyObject* ignore)
+{
+ /* This should never get called, but we also don't want to SEGV if
+ * we accidentally decref NotImplemented out of existence.
+ */
+ Py_FatalError("deallocating NotImplemented");
+}
+
static PyTypeObject PyNotImplemented_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"NotImplementedType",
0,
0,
- none_dealloc, /*tp_dealloc*/ /*never called*/
+ notimplemented_dealloc, /*tp_dealloc*/ /*never called*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
@@ -1699,15 +1708,6 @@ _Py_ReadyTypes(void)
if (PyType_Ready(&PyMemberDescr_Type) < 0)
Py_FatalError("Can't initialize member descriptor type");
- if (PyType_Ready(&PyFilter_Type) < 0)
- Py_FatalError("Can't initialize filter type");
-
- if (PyType_Ready(&PyMap_Type) < 0)
- Py_FatalError("Can't initialize map type");
-
- if (PyType_Ready(&PyZip_Type) < 0)
- Py_FatalError("Can't initialize zip type");
-
if (PyType_Ready(&_PyNamespace_Type) < 0)
Py_FatalError("Can't initialize namespace type");
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index 6225ebb..bbe2805 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -525,6 +525,15 @@ static size_t ntimes_arena_allocated = 0;
/* High water mark (max value ever seen) for narenas_currently_allocated. */
static size_t narenas_highwater = 0;
+static Py_ssize_t _Py_AllocatedBlocks = 0;
+
+Py_ssize_t
+_Py_GetAllocatedBlocks(void)
+{
+ return _Py_AllocatedBlocks;
+}
+
+
/* Allocate a new arena. If we run out of memory, return NULL. Else
* allocate a new arena, and return the address of an arena_object
* describing the new arena. It's expected that the caller will set
@@ -785,6 +794,8 @@ PyObject_Malloc(size_t nbytes)
if (nbytes > PY_SSIZE_T_MAX)
return NULL;
+ _Py_AllocatedBlocks++;
+
/*
* This implicitly redirects malloc(0).
*/
@@ -901,6 +912,7 @@ PyObject_Malloc(size_t nbytes)
* and free list are already initialized.
*/
bp = pool->freeblock;
+ assert(bp != NULL);
pool->freeblock = *(block **)bp;
UNLOCK();
return (void *)bp;
@@ -958,7 +970,12 @@ redirect:
*/
if (nbytes == 0)
nbytes = 1;
- return (void *)malloc(nbytes);
+ {
+ void *result = malloc(nbytes);
+ if (!result)
+ _Py_AllocatedBlocks--;
+ return result;
+ }
}
/* free */
@@ -978,6 +995,8 @@ PyObject_Free(void *p)
if (p == NULL) /* free(NULL) has no effect */
return;
+ _Py_AllocatedBlocks--;
+
#ifdef WITH_VALGRIND
if (UNLIKELY(running_on_valgrind > 0))
goto redirect;
@@ -1297,6 +1316,13 @@ PyObject_Free(void *p)
{
PyMem_FREE(p);
}
+
+Py_ssize_t
+_Py_GetAllocatedBlocks(void)
+{
+ return 0;
+}
+
#endif /* WITH_PYMALLOC */
#ifdef PYMALLOC_DEBUG
diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c
index 214b455..ba51fec 100644
--- a/Objects/rangeobject.c
+++ b/Objects/rangeobject.c
@@ -318,195 +318,6 @@ range_item(rangeobject *r, Py_ssize_t i)
return res;
}
-/* Additional helpers, since the standard slice helpers
- * all clip to PY_SSIZE_T_MAX
- */
-
-/* Replace _PyEval_SliceIndex */
-static PyObject *
-compute_slice_element(PyObject *obj)
-{
- PyObject *result = NULL;
- if (obj != NULL) {
- if (PyIndex_Check(obj)) {
- result = PyNumber_Index(obj);
- }
- else {
- PyErr_SetString(PyExc_TypeError,
- "slice indices must be integers or "
- "None or have an __index__ method");
- }
- }
- return result;
-}
-
-/* Replace PySlice_GetIndicesEx
- * Result indicates whether or not the slice is empty
- * (-1 = error, 0 = empty slice, 1 = slice contains elements)
- */
-static int
-compute_slice_indices(rangeobject *r, PySliceObject *slice,
- PyObject **start, PyObject **stop, PyObject **step)
-{
- int cmp_result, has_elements;
- Py_ssize_t clamped_step = 0;
- PyObject *zero = NULL, *one = NULL, *neg_one = NULL, *candidate = NULL;
- PyObject *tmp_start = NULL, *tmp_stop = NULL, *tmp_step = NULL;
- zero = PyLong_FromLong(0);
- if (zero == NULL) goto Fail;
- one = PyLong_FromLong(1);
- if (one == NULL) goto Fail;
- neg_one = PyLong_FromLong(-1);
- if (neg_one == NULL) goto Fail;
-
- /* Calculate step value */
- if (slice->step == Py_None) {
- clamped_step = 1;
- tmp_step = one;
- Py_INCREF(tmp_step);
- } else {
- if (!_PyEval_SliceIndex(slice->step, &clamped_step)) goto Fail;
- if (clamped_step == 0) {
- PyErr_SetString(PyExc_ValueError,
- "slice step cannot be zero");
- goto Fail;
- }
- tmp_step = compute_slice_element(slice->step);
- if (tmp_step == NULL) goto Fail;
- }
-
- /* Calculate start value */
- if (slice->start == Py_None) {
- if (clamped_step < 0) {
- tmp_start = PyNumber_Subtract(r->length, one);
- if (tmp_start == NULL) goto Fail;
- } else {
- tmp_start = zero;
- Py_INCREF(tmp_start);
- }
- } else {
- candidate = compute_slice_element(slice->start);
- if (candidate == NULL) goto Fail;
- cmp_result = PyObject_RichCompareBool(candidate, zero, Py_LT);
- if (cmp_result == -1) goto Fail;
- if (cmp_result) {
- /* candidate < 0 */
- tmp_start = PyNumber_Add(r->length, candidate);
- if (tmp_start == NULL) goto Fail;
- Py_CLEAR(candidate);
- } else {
- /* candidate >= 0 */
- tmp_start = candidate;
- candidate = NULL;
- }
- cmp_result = PyObject_RichCompareBool(tmp_start, zero, Py_LT);
- if (cmp_result == -1) goto Fail;
- if (cmp_result) {
- /* tmp_start < 0 */
- Py_CLEAR(tmp_start);
- if (clamped_step < 0) {
- tmp_start = neg_one;
- } else {
- tmp_start = zero;
- }
- Py_INCREF(tmp_start);
- } else {
- /* tmp_start >= 0 */
- cmp_result = PyObject_RichCompareBool(tmp_start, r->length, Py_GE);
- if (cmp_result == -1) goto Fail;
- if (cmp_result) {
- /* tmp_start >= r->length */
- Py_CLEAR(tmp_start);
- if (clamped_step < 0) {
- tmp_start = PyNumber_Subtract(r->length, one);
- if (tmp_start == NULL) goto Fail;
- } else {
- tmp_start = r->length;
- Py_INCREF(tmp_start);
- }
- }
- }
- }
-
- /* Calculate stop value */
- if (slice->stop == Py_None) {
- if (clamped_step < 0) {
- tmp_stop = neg_one;
- } else {
- tmp_stop = r->length;
- }
- Py_INCREF(tmp_stop);
- } else {
- candidate = compute_slice_element(slice->stop);
- if (candidate == NULL) goto Fail;
- cmp_result = PyObject_RichCompareBool(candidate, zero, Py_LT);
- if (cmp_result == -1) goto Fail;
- if (cmp_result) {
- /* candidate < 0 */
- tmp_stop = PyNumber_Add(r->length, candidate);
- if (tmp_stop == NULL) goto Fail;
- Py_CLEAR(candidate);
- } else {
- /* candidate >= 0 */
- tmp_stop = candidate;
- candidate = NULL;
- }
- cmp_result = PyObject_RichCompareBool(tmp_stop, zero, Py_LT);
- if (cmp_result == -1) goto Fail;
- if (cmp_result) {
- /* tmp_stop < 0 */
- Py_CLEAR(tmp_stop);
- if (clamped_step < 0) {
- tmp_stop = neg_one;
- } else {
- tmp_stop = zero;
- }
- Py_INCREF(tmp_stop);
- } else {
- /* tmp_stop >= 0 */
- cmp_result = PyObject_RichCompareBool(tmp_stop, r->length, Py_GE);
- if (cmp_result == -1) goto Fail;
- if (cmp_result) {
- /* tmp_stop >= r->length */
- Py_CLEAR(tmp_stop);
- if (clamped_step < 0) {
- tmp_stop = PyNumber_Subtract(r->length, one);
- if (tmp_stop == NULL) goto Fail;
- } else {
- tmp_stop = r->length;
- Py_INCREF(tmp_stop);
- }
- }
- }
- }
-
- /* Check if the slice is empty or not */
- if (clamped_step < 0) {
- has_elements = PyObject_RichCompareBool(tmp_start, tmp_stop, Py_GT);
- } else {
- has_elements = PyObject_RichCompareBool(tmp_start, tmp_stop, Py_LT);
- }
- if (has_elements == -1) goto Fail;
-
- *start = tmp_start;
- *stop = tmp_stop;
- *step = tmp_step;
- Py_DECREF(neg_one);
- Py_DECREF(one);
- Py_DECREF(zero);
- return has_elements;
-
- Fail:
- Py_XDECREF(tmp_start);
- Py_XDECREF(tmp_stop);
- Py_XDECREF(tmp_step);
- Py_XDECREF(candidate);
- Py_XDECREF(neg_one);
- Py_XDECREF(one);
- Py_XDECREF(zero);
- return -1;
-}
-
static PyObject *
compute_slice(rangeobject *r, PyObject *_slice)
{
@@ -514,10 +325,11 @@ compute_slice(rangeobject *r, PyObject *_slice)
rangeobject *result;
PyObject *start = NULL, *stop = NULL, *step = NULL;
PyObject *substart = NULL, *substop = NULL, *substep = NULL;
- int has_elements;
+ int error;
- has_elements = compute_slice_indices(r, slice, &start, &stop, &step);
- if (has_elements == -1) return NULL;
+ error = _PySlice_GetLongIndices(slice, r->length, &start, &stop, &step);
+ if (error == -1)
+ return NULL;
substep = PyNumber_Multiply(r->step, step);
if (substep == NULL) goto fail;
@@ -527,13 +339,8 @@ compute_slice(rangeobject *r, PyObject *_slice)
if (substart == NULL) goto fail;
Py_CLEAR(start);
- if (has_elements) {
- substop = compute_item(r, stop);
- if (substop == NULL) goto fail;
- } else {
- substop = substart;
- Py_INCREF(substop);
- }
+ substop = compute_item(r, stop);
+ if (substop == NULL) goto fail;
Py_CLEAR(stop);
result = make_range_object(Py_TYPE(r), substart, substop, substep);
diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c
index 1593335..52f1c89 100644
--- a/Objects/sliceobject.c
+++ b/Objects/sliceobject.c
@@ -299,23 +299,198 @@ static PyMemberDef slice_members[] = {
{0}
};
+/* Helper function to convert a slice argument to a PyLong, and raise TypeError
+ with a suitable message on failure. */
+
static PyObject*
-slice_indices(PySliceObject* self, PyObject* len)
+evaluate_slice_index(PyObject *v)
{
- Py_ssize_t ilen, start, stop, step, slicelength;
+ if (PyIndex_Check(v)) {
+ return PyNumber_Index(v);
+ }
+ else {
+ PyErr_SetString(PyExc_TypeError,
+ "slice indices must be integers or "
+ "None or have an __index__ method");
+ return NULL;
+ }
+}
- ilen = PyNumber_AsSsize_t(len, PyExc_OverflowError);
+/* Compute slice indices given a slice and length. Return -1 on failure. Used
+ by slice.indices and rangeobject slicing. Assumes that `len` is a
+ nonnegative instance of PyLong. */
- if (ilen == -1 && PyErr_Occurred()) {
- return NULL;
+int
+_PySlice_GetLongIndices(PySliceObject *self, PyObject *length,
+ PyObject **start_ptr, PyObject **stop_ptr,
+ PyObject **step_ptr)
+{
+ PyObject *start=NULL, *stop=NULL, *step=NULL;
+ PyObject *upper=NULL, *lower=NULL;
+ int step_is_negative, cmp_result;
+
+ /* Convert step to an integer; raise for zero step. */
+ if (self->step == Py_None) {
+ step = PyLong_FromLong(1L);
+ if (step == NULL)
+ goto error;
+ step_is_negative = 0;
+ }
+ else {
+ int step_sign;
+ step = evaluate_slice_index(self->step);
+ if (step == NULL)
+ goto error;
+ step_sign = _PyLong_Sign(step);
+ if (step_sign == 0) {
+ PyErr_SetString(PyExc_ValueError,
+ "slice step cannot be zero");
+ goto error;
+ }
+ step_is_negative = step_sign < 0;
+ }
+
+ /* Find lower and upper bounds for start and stop. */
+ if (step_is_negative) {
+ lower = PyLong_FromLong(-1L);
+ if (lower == NULL)
+ goto error;
+
+ upper = PyNumber_Add(length, lower);
+ if (upper == NULL)
+ goto error;
+ }
+ else {
+ lower = PyLong_FromLong(0L);
+ if (lower == NULL)
+ goto error;
+
+ upper = length;
+ Py_INCREF(upper);
}
- if (PySlice_GetIndicesEx((PyObject*)self, ilen, &start, &stop,
- &step, &slicelength) < 0) {
+ /* Compute start. */
+ if (self->start == Py_None) {
+ start = step_is_negative ? upper : lower;
+ Py_INCREF(start);
+ }
+ else {
+ start = evaluate_slice_index(self->start);
+ if (start == NULL)
+ goto error;
+
+ if (_PyLong_Sign(start) < 0) {
+ /* start += length */
+ PyObject *tmp = PyNumber_Add(start, length);
+ Py_DECREF(start);
+ start = tmp;
+ if (start == NULL)
+ goto error;
+
+ cmp_result = PyObject_RichCompareBool(start, lower, Py_LT);
+ if (cmp_result < 0)
+ goto error;
+ if (cmp_result) {
+ Py_INCREF(lower);
+ Py_DECREF(start);
+ start = lower;
+ }
+ }
+ else {
+ cmp_result = PyObject_RichCompareBool(start, upper, Py_GT);
+ if (cmp_result < 0)
+ goto error;
+ if (cmp_result) {
+ Py_INCREF(upper);
+ Py_DECREF(start);
+ start = upper;
+ }
+ }
+ }
+
+ /* Compute stop. */
+ if (self->stop == Py_None) {
+ stop = step_is_negative ? lower : upper;
+ Py_INCREF(stop);
+ }
+ else {
+ stop = evaluate_slice_index(self->stop);
+ if (stop == NULL)
+ goto error;
+
+ if (_PyLong_Sign(stop) < 0) {
+ /* stop += length */
+ PyObject *tmp = PyNumber_Add(stop, length);
+ Py_DECREF(stop);
+ stop = tmp;
+ if (stop == NULL)
+ goto error;
+
+ cmp_result = PyObject_RichCompareBool(stop, lower, Py_LT);
+ if (cmp_result < 0)
+ goto error;
+ if (cmp_result) {
+ Py_INCREF(lower);
+ Py_DECREF(stop);
+ stop = lower;
+ }
+ }
+ else {
+ cmp_result = PyObject_RichCompareBool(stop, upper, Py_GT);
+ if (cmp_result < 0)
+ goto error;
+ if (cmp_result) {
+ Py_INCREF(upper);
+ Py_DECREF(stop);
+ stop = upper;
+ }
+ }
+ }
+
+ *start_ptr = start;
+ *stop_ptr = stop;
+ *step_ptr = step;
+ Py_DECREF(upper);
+ Py_DECREF(lower);
+ return 0;
+
+ error:
+ *start_ptr = *stop_ptr = *step_ptr = NULL;
+ Py_XDECREF(start);
+ Py_XDECREF(stop);
+ Py_XDECREF(step);
+ Py_XDECREF(upper);
+ Py_XDECREF(lower);
+ return -1;
+}
+
+/* Implementation of slice.indices. */
+
+static PyObject*
+slice_indices(PySliceObject* self, PyObject* len)
+{
+ PyObject *start, *stop, *step;
+ PyObject *length;
+ int error;
+
+ /* Convert length to an integer if necessary; raise for negative length. */
+ length = PyNumber_Index(len);
+ if (length == NULL)
+ return NULL;
+
+ if (_PyLong_Sign(length) < 0) {
+ PyErr_SetString(PyExc_ValueError,
+ "length should not be negative");
+ Py_DECREF(length);
return NULL;
}
- return Py_BuildValue("(nnn)", start, stop, step);
+ error = _PySlice_GetLongIndices(self, length, &start, &stop, &step);
+ Py_DECREF(length);
+ if (error == -1)
+ return NULL;
+ else
+ return Py_BuildValue("(NNN)", start, stop, step);
}
PyDoc_STRVAR(slice_indices_doc,
diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h
index f353367..f855003 100644
--- a/Objects/stringlib/codecs.h
+++ b/Objects/stringlib/codecs.h
@@ -47,7 +47,7 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
unsigned long value = *(unsigned long *) _s;
if (value & ASCII_CHAR_MASK)
break;
-#ifdef BYTEORDER_IS_LITTLE_ENDIAN
+#if PY_LITTLE_ENDIAN
_p[0] = (STRINGLIB_CHAR)(value & 0xFFu);
_p[1] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu);
_p[2] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu);
@@ -486,7 +486,7 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e,
const unsigned char *q = *inptr;
STRINGLIB_CHAR *p = dest + *outpos;
/* Offsets from q for retrieving byte pairs in the right order. */
-#ifdef BYTEORDER_IS_LITTLE_ENDIAN
+#if PY_LITTLE_ENDIAN
int ihi = !!native_ordering, ilo = !native_ordering;
#else
int ihi = !native_ordering, ilo = !!native_ordering;
@@ -517,7 +517,7 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e,
block = SWAB(block);
#endif
}
-#ifdef BYTEORDER_IS_LITTLE_ENDIAN
+#if PY_LITTLE_ENDIAN
# if SIZEOF_LONG == 4
p[0] = (STRINGLIB_CHAR)(block & 0xFFFFu);
p[1] = (STRINGLIB_CHAR)(block >> 16);
diff --git a/Objects/stringlib/join.h b/Objects/stringlib/join.h
new file mode 100644
index 0000000..5568b31
--- /dev/null
+++ b/Objects/stringlib/join.h
@@ -0,0 +1,133 @@
+/* stringlib: bytes joining implementation */
+
+#if STRINGLIB_SIZEOF_CHAR != 1
+#error join.h only compatible with byte-wise strings
+#endif
+
+Py_LOCAL_INLINE(PyObject *)
+STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable)
+{
+ char *sepstr = STRINGLIB_STR(sep);
+ const Py_ssize_t seplen = STRINGLIB_LEN(sep);
+ PyObject *res = NULL;
+ char *p;
+ Py_ssize_t seqlen = 0;
+ Py_ssize_t sz = 0;
+ Py_ssize_t i, nbufs;
+ PyObject *seq, *item;
+ Py_buffer *buffers = NULL;
+#define NB_STATIC_BUFFERS 10
+ Py_buffer static_buffers[NB_STATIC_BUFFERS];
+
+ seq = PySequence_Fast(iterable, "can only join an iterable");
+ if (seq == NULL) {
+ return NULL;
+ }
+
+ seqlen = PySequence_Fast_GET_SIZE(seq);
+ if (seqlen == 0) {
+ Py_DECREF(seq);
+ return STRINGLIB_NEW(NULL, 0);
+ }
+#ifndef STRINGLIB_MUTABLE
+ if (seqlen == 1) {
+ item = PySequence_Fast_GET_ITEM(seq, 0);
+ if (STRINGLIB_CHECK_EXACT(item)) {
+ Py_INCREF(item);
+ Py_DECREF(seq);
+ return item;
+ }
+ }
+#endif
+ if (seqlen > NB_STATIC_BUFFERS) {
+ buffers = PyMem_NEW(Py_buffer, seqlen);
+ if (buffers == NULL) {
+ Py_DECREF(seq);
+ PyErr_NoMemory();
+ return NULL;
+ }
+ }
+ else {
+ buffers = static_buffers;
+ }
+
+ /* Here is the general case. Do a pre-pass to figure out the total
+ * amount of space we'll need (sz), and see whether all arguments are
+ * buffer-compatible.
+ */
+ for (i = 0, nbufs = 0; i < seqlen; i++) {
+ Py_ssize_t itemlen;
+ item = PySequence_Fast_GET_ITEM(seq, i);
+ if (_getbuffer(item, &buffers[i]) < 0) {
+ PyErr_Format(PyExc_TypeError,
+ "sequence item %zd: expected bytes, bytearray, "
+ "or an object with the buffer interface, %.80s found",
+ i, Py_TYPE(item)->tp_name);
+ goto error;
+ }
+ nbufs = i + 1; /* for error cleanup */
+ itemlen = buffers[i].len;
+ if (itemlen > PY_SSIZE_T_MAX - sz) {
+ PyErr_SetString(PyExc_OverflowError,
+ "join() result is too long");
+ goto error;
+ }
+ sz += itemlen;
+ if (i != 0) {
+ if (seplen > PY_SSIZE_T_MAX - sz) {
+ PyErr_SetString(PyExc_OverflowError,
+ "join() result is too long");
+ goto error;
+ }
+ sz += seplen;
+ }
+ if (seqlen != PySequence_Fast_GET_SIZE(seq)) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "sequence changed size during iteration");
+ goto error;
+ }
+ }
+
+ /* Allocate result space. */
+ res = STRINGLIB_NEW(NULL, sz);
+ if (res == NULL)
+ goto error;
+
+ /* Catenate everything. */
+ p = STRINGLIB_STR(res);
+ if (!seplen) {
+ /* fast path */
+ for (i = 0; i < nbufs; i++) {
+ Py_ssize_t n = buffers[i].len;
+ char *q = buffers[i].buf;
+ Py_MEMCPY(p, q, n);
+ p += n;
+ }
+ goto done;
+ }
+ for (i = 0; i < nbufs; i++) {
+ Py_ssize_t n;
+ char *q;
+ if (i) {
+ Py_MEMCPY(p, sepstr, seplen);
+ p += seplen;
+ }
+ n = buffers[i].len;
+ q = buffers[i].buf;
+ Py_MEMCPY(p, q, n);
+ p += n;
+ }
+ goto done;
+
+error:
+ res = NULL;
+done:
+ Py_DECREF(seq);
+ for (i = 0; i < nbufs; i++)
+ PyBuffer_Release(&buffers[i]);
+ if (buffers != static_buffers)
+ PyMem_FREE(buffers);
+ return res;
+}
+
+#undef NB_STATIC_BUFFERS
diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
index ec3f91b..0a95909 100644
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -322,6 +322,9 @@ error:
1082527, 1165049, 1082531, 1165057, 1247581, 1330103, 1082533,
1330111, 1412633, 1165069, 1247599, 1495177, 1577699
+
+ Tests have shown that it's not worth to cache the hash value, see
+ issue #9685.
*/
static Py_hash_t
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 413c7da..f71cad3 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1933,7 +1933,7 @@ type_init(PyObject *cls, PyObject *args, PyObject *kwds)
return res;
}
-long
+unsigned long
PyType_GetFlags(PyTypeObject *type)
{
return type->tp_flags;
@@ -3654,16 +3654,9 @@ object_format(PyObject *self, PyObject *args)
/* Issue 7994: If we're converting to a string, we
should reject format specifications */
if (PyUnicode_GET_LENGTH(format_spec) > 0) {
- if (PyErr_WarnEx(PyExc_DeprecationWarning,
- "object.__format__ with a non-empty format "
- "string is deprecated", 1) < 0) {
- goto done;
- }
- /* Eventually this will become an error:
- PyErr_Format(PyExc_TypeError,
+ PyErr_SetString(PyExc_TypeError,
"non-empty format string passed to object.__format__");
- goto done;
- */
+ goto done;
}
result = PyObject_Format(self_as_str, format_spec);
@@ -3818,7 +3811,7 @@ add_methods(PyTypeObject *type, PyMethodDef *meth)
descr = PyDescr_NewClassMethod(type, meth);
}
else if (meth->ml_flags & METH_STATIC) {
- PyObject *cfunc = PyCFunction_New(meth, (PyObject*)type);
+ PyObject *cfunc = PyCFunction_NewEx(meth, (PyObject*)type, NULL);
if (cfunc == NULL)
return -1;
descr = PyStaticMethod_New(cfunc);
@@ -4288,13 +4281,11 @@ PyType_Ready(PyTypeObject *type)
/* Warn for a type that implements tp_compare (now known as
tp_reserved) but not tp_richcompare. */
if (type->tp_reserved && !type->tp_richcompare) {
- int error;
- error = PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
+ PyErr_Format(PyExc_TypeError,
"Type %.100s defines tp_reserved (formerly tp_compare) "
"but not tp_richcompare. Comparisons may not behave as intended.",
type->tp_name);
- if (error == -1)
- goto error;
+ goto error;
}
/* All done -- set the ready flag */
@@ -4888,7 +4879,7 @@ add_tp_new_wrapper(PyTypeObject *type)
if (_PyDict_GetItemId(type->tp_dict, &PyId___new__) != NULL)
return 0;
- func = PyCFunction_New(tp_new_methoddef, (PyObject *)type);
+ func = PyCFunction_NewEx(tp_new_methoddef, (PyObject *)type, NULL);
if (func == NULL)
return -1;
if (_PyDict_SetItemId(type->tp_dict, &PyId___new__, func)) {
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index c96a91c..b4f4185 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -47,14 +47,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#include <windows.h>
#endif
-/* Endianness switches; defaults to little endian */
-
-#ifdef WORDS_BIGENDIAN
-# define BYTEORDER_IS_BIG_ENDIAN
-#else
-# define BYTEORDER_IS_LITTLE_ENDIAN
-#endif
-
/* --- Globals ------------------------------------------------------------
NOTE: In the interpreter's initialization phase, some globals are currently
@@ -432,8 +424,6 @@ unicode_result_wchar(PyObject *unicode)
#ifndef Py_DEBUG
Py_ssize_t len;
- assert(Py_REFCNT(unicode) == 1);
-
len = _PyUnicode_WSTR_LENGTH(unicode);
if (len == 0) {
Py_DECREF(unicode);
@@ -450,10 +440,12 @@ unicode_result_wchar(PyObject *unicode)
}
if (_PyUnicode_Ready(unicode) < 0) {
- Py_XDECREF(unicode);
+ Py_DECREF(unicode);
return NULL;
}
#else
+ assert(Py_REFCNT(unicode) == 1);
+
/* don't make the result ready in debug mode to ensure that the caller
makes the string ready before using it */
assert(_PyUnicode_CheckConsistency(unicode, 1));
@@ -659,6 +651,25 @@ Py_LOCAL_INLINE(Py_ssize_t) findchar(void *s, int kind,
}
}
+#ifdef Py_DEBUG
+/* Fill the data of an Unicode string with invalid characters to detect bugs
+ earlier.
+
+ _PyUnicode_CheckConsistency(str, 1) detects invalid characters, at least for
+ ASCII and UCS-4 strings. U+00FF is invalid in ASCII and U+FFFFFFFF is an
+ invalid character in Unicode 6.0. */
+static void
+unicode_fill_invalid(PyObject *unicode, Py_ssize_t old_length)
+{
+ int kind = PyUnicode_KIND(unicode);
+ Py_UCS1 *data = PyUnicode_1BYTE_DATA(unicode);
+ Py_ssize_t length = _PyUnicode_LENGTH(unicode);
+ if (length <= old_length)
+ return;
+ memset(data + old_length * kind, 0xff, (length - old_length) * kind);
+}
+#endif
+
static PyObject*
resize_compact(PyObject *unicode, Py_ssize_t length)
{
@@ -667,6 +678,10 @@ resize_compact(PyObject *unicode, Py_ssize_t length)
Py_ssize_t new_size;
int share_wstr;
PyObject *new_unicode;
+#ifdef Py_DEBUG
+ Py_ssize_t old_length = _PyUnicode_LENGTH(unicode);
+#endif
+
assert(unicode_modifiable(unicode));
assert(PyUnicode_IS_READY(unicode));
assert(PyUnicode_IS_COMPACT(unicode));
@@ -702,6 +717,9 @@ resize_compact(PyObject *unicode, Py_ssize_t length)
if (!PyUnicode_IS_ASCII(unicode))
_PyUnicode_WSTR_LENGTH(unicode) = length;
}
+#ifdef Py_DEBUG
+ unicode_fill_invalid(unicode, old_length);
+#endif
PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode),
length, 0);
assert(_PyUnicode_CheckConsistency(unicode, 0));
@@ -720,6 +738,9 @@ resize_inplace(PyObject *unicode, Py_ssize_t length)
Py_ssize_t char_size;
int share_wstr, share_utf8;
void *data;
+#ifdef Py_DEBUG
+ Py_ssize_t old_length = _PyUnicode_LENGTH(unicode);
+#endif
data = _PyUnicode_DATA_ANY(unicode);
char_size = PyUnicode_KIND(unicode);
@@ -755,6 +776,9 @@ resize_inplace(PyObject *unicode, Py_ssize_t length)
}
_PyUnicode_LENGTH(unicode) = length;
PyUnicode_WRITE(PyUnicode_KIND(unicode), data, length, 0);
+#ifdef Py_DEBUG
+ unicode_fill_invalid(unicode, old_length);
+#endif
if (share_wstr || _PyUnicode_WSTR(unicode) == NULL) {
assert(_PyUnicode_CheckConsistency(unicode, 0));
return 0;
@@ -807,8 +831,8 @@ resize_copy(PyObject *unicode, Py_ssize_t length)
return NULL;
copy_length = _PyUnicode_WSTR_LENGTH(unicode);
copy_length = Py_MIN(copy_length, length);
- Py_UNICODE_COPY(_PyUnicode_WSTR(w), _PyUnicode_WSTR(unicode),
- copy_length);
+ Py_MEMCPY(_PyUnicode_WSTR(w), _PyUnicode_WSTR(unicode),
+ copy_length * sizeof(wchar_t));
return w;
}
}
@@ -1079,11 +1103,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)
}
}
#ifdef Py_DEBUG
- /* Fill the data with invalid characters to detect bugs earlier.
- _PyUnicode_CheckConsistency(str, 1) detects invalid characters,
- at least for ASCII and UCS-4 strings. U+00FF is invalid in ASCII
- and U+FFFFFFFF is an invalid character in Unicode 6.0. */
- memset(data, 0xff, size * kind);
+ unicode_fill_invalid((PyObject*)unicode, 0);
#endif
assert(_PyUnicode_CheckConsistency((PyObject*)unicode, 0));
return obj;
@@ -1645,38 +1665,6 @@ PyUnicode_Resize(PyObject **p_unicode, Py_ssize_t length)
return unicode_resize(p_unicode, length);
}
-static int
-unicode_widen(PyObject **p_unicode, Py_ssize_t length,
- unsigned int maxchar)
-{
- PyObject *result;
- assert(PyUnicode_IS_READY(*p_unicode));
- assert(length <= PyUnicode_GET_LENGTH(*p_unicode));
- if (maxchar <= PyUnicode_MAX_CHAR_VALUE(*p_unicode))
- return 0;
- result = PyUnicode_New(PyUnicode_GET_LENGTH(*p_unicode),
- maxchar);
- if (result == NULL)
- return -1;
- _PyUnicode_FastCopyCharacters(result, 0, *p_unicode, 0, length);
- Py_DECREF(*p_unicode);
- *p_unicode = result;
- return 0;
-}
-
-static int
-unicode_putchar(PyObject **p_unicode, Py_ssize_t *pos,
- Py_UCS4 ch)
-{
- assert(ch <= MAX_UNICODE);
- if (unicode_widen(p_unicode, *pos, ch) < 0)
- return -1;
- PyUnicode_WRITE(PyUnicode_KIND(*p_unicode),
- PyUnicode_DATA(*p_unicode),
- (*pos)++, ch);
- return 0;
-}
-
/* Copy a ASCII or latin1 char* string into a Python Unicode string.
WARNING: The function doesn't copy the terminating null character and
@@ -1693,6 +1681,14 @@ unicode_write_cstr(PyObject *unicode, Py_ssize_t index,
switch (kind) {
case PyUnicode_1BYTE_KIND: {
assert(index + len <= PyUnicode_GET_LENGTH(unicode));
+#ifdef Py_DEBUG
+ if (PyUnicode_IS_ASCII(unicode)) {
+ Py_UCS4 maxchar = ucs1lib_find_max_char(
+ (const Py_UCS1*)str,
+ (const Py_UCS1*)str + len);
+ assert(maxchar < 128);
+ }
+#endif
memcpy((char *) data + index, str, len);
break;
}
@@ -2279,16 +2275,9 @@ PyUnicode_FromWideChar(register const wchar_t *w, Py_ssize_t size)
static void
makefmt(char *fmt, int longflag, int longlongflag, int size_tflag,
- int zeropad, int width, int precision, char c)
+ char c)
{
*fmt++ = '%';
- if (width) {
- if (zeropad)
- *fmt++ = '0';
- fmt += sprintf(fmt, "%d", width);
- }
- if (precision)
- fmt += sprintf(fmt, ".%d", precision);
if (longflag)
*fmt++ = 'l';
else if (longlongflag) {
@@ -2313,44 +2302,70 @@ makefmt(char *fmt, int longflag, int longlongflag, int size_tflag,
*fmt = '\0';
}
-/* helper for PyUnicode_FromFormatV() */
+/* maximum number of characters required for output of %lld or %p.
+ We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits,
+ plus 1 for the sign. 53/22 is an upper bound for log10(256). */
+#define MAX_LONG_LONG_CHARS (2 + (SIZEOF_LONG_LONG*53-1) / 22)
static const char*
-parse_format_flags(const char *f,
- int *p_width, int *p_precision,
- int *p_longflag, int *p_longlongflag, int *p_size_tflag)
+unicode_fromformat_arg(_PyUnicodeWriter *writer,
+ const char *f, va_list *vargs)
{
- int width, precision, longflag, longlongflag, size_tflag;
+ const char *p;
+ Py_ssize_t len;
+ int zeropad;
+ int width;
+ int precision;
+ int longflag;
+ int longlongflag;
+ int size_tflag;
+ int fill;
+
+ p = f;
+ f++;
+ zeropad = 0;
+ if (*f == '0') {
+ zeropad = 1;
+ f++;
+ }
/* parse the width.precision part, e.g. "%2.5s" => width=2, precision=5 */
- f++;
width = 0;
- while (Py_ISDIGIT((unsigned)*f))
- width = (width*10) + *f++ - '0';
+ while (Py_ISDIGIT((unsigned)*f)) {
+ if (width > (INT_MAX - ((int)*f - '0')) / 10) {
+ PyErr_SetString(PyExc_ValueError,
+ "width too big");
+ return NULL;
+ }
+ width = (width*10) + (*f - '0');
+ f++;
+ }
precision = 0;
if (*f == '.') {
f++;
- while (Py_ISDIGIT((unsigned)*f))
- precision = (precision*10) + *f++ - '0';
+ while (Py_ISDIGIT((unsigned)*f)) {
+ if (precision > (INT_MAX - ((int)*f - '0')) / 10) {
+ PyErr_SetString(PyExc_ValueError,
+ "precision too big");
+ return NULL;
+ }
+ precision = (precision*10) + (*f - '0');
+ f++;
+ }
if (*f == '%') {
/* "%.3%s" => f points to "3" */
f--;
}
}
if (*f == '\0') {
- /* bogus format "%.1" => go backward, f points to "1" */
+ /* bogus format "%.123" => go backward, f points to "3" */
f--;
}
- if (p_width != NULL)
- *p_width = width;
- if (p_precision != NULL)
- *p_precision = precision;
/* Handle %ld, %lu, %lld and %llu. */
longflag = 0;
longlongflag = 0;
size_tflag = 0;
-
if (*f == 'l') {
if (f[1] == 'd' || f[1] == 'u' || f[1] == 'i') {
longflag = 1;
@@ -2369,494 +2384,297 @@ parse_format_flags(const char *f,
size_tflag = 1;
++f;
}
- if (p_longflag != NULL)
- *p_longflag = longflag;
- if (p_longlongflag != NULL)
- *p_longlongflag = longlongflag;
- if (p_size_tflag != NULL)
- *p_size_tflag = size_tflag;
- return f;
-}
-/* maximum number of characters required for output of %ld. 21 characters
- allows for 64-bit integers (in decimal) and an optional sign. */
-#define MAX_LONG_CHARS 21
-/* maximum number of characters required for output of %lld.
- We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits,
- plus 1 for the sign. 53/22 is an upper bound for log10(256). */
-#define MAX_LONG_LONG_CHARS (2 + (SIZEOF_LONG_LONG*53-1) / 22)
+ if (f[1] == '\0')
+ writer->overallocate = 0;
-PyObject *
-PyUnicode_FromFormatV(const char *format, va_list vargs)
-{
- va_list count;
- Py_ssize_t callcount = 0;
- PyObject **callresults = NULL;
- PyObject **callresult = NULL;
- Py_ssize_t n = 0;
- int width = 0;
- int precision = 0;
- int zeropad;
- const char* f;
- PyObject *string;
- /* used by sprintf */
- char fmt[61]; /* should be enough for %0width.precisionlld */
- Py_UCS4 maxchar = 127; /* result is ASCII by default */
- Py_UCS4 argmaxchar;
- Py_ssize_t numbersize = 0;
- char *numberresults = NULL;
- char *numberresult = NULL;
- Py_ssize_t i;
- int kind;
- void *data;
+ switch (*f) {
+ case 'c':
+ {
+ int ordinal = va_arg(*vargs, int);
+ if (ordinal < 0 || ordinal > MAX_UNICODE) {
+ PyErr_SetString(PyExc_ValueError,
+ "character argument not in range(0x110000)");
+ return NULL;
+ }
+ if (_PyUnicodeWriter_Prepare(writer, 1, ordinal) == -1)
+ return NULL;
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ordinal);
+ writer->pos++;
+ break;
+ }
- Py_VA_COPY(count, vargs);
- /* step 1: count the number of %S/%R/%A/%s format specifications
- * (we call PyObject_Str()/PyObject_Repr()/PyObject_ASCII()/
- * PyUnicode_DecodeUTF8() for these objects once during step 3 and put the
- * result in an array)
- * also estimate a upper bound for all the number formats in the string,
- * numbers will be formatted in step 3 and be kept in a '\0'-separated
- * buffer before putting everything together. */
- for (f = format; *f; f++) {
- if (*f == '%') {
- int longlongflag;
- /* skip width or width.precision (eg. "1.2" of "%1.2f") */
- f = parse_format_flags(f, &width, NULL, NULL, &longlongflag, NULL);
- if (*f == 's' || *f=='S' || *f=='R' || *f=='A' || *f=='V')
- ++callcount;
+ case 'i':
+ case 'd':
+ case 'u':
+ case 'x':
+ {
+ /* used by sprintf */
+ char fmt[10]; /* should be enough for "%0lld\0" */
+ char buffer[MAX_LONG_LONG_CHARS];
- else if (*f == 'd' || *f=='u' || *f=='i' || *f=='x' || *f=='p') {
+ if (*f == 'u') {
+ makefmt(fmt, longflag, longlongflag, size_tflag, *f);
+
+ if (longflag)
+ len = sprintf(buffer, fmt,
+ va_arg(*vargs, unsigned long));
#ifdef HAVE_LONG_LONG
- if (longlongflag) {
- if (width < MAX_LONG_LONG_CHARS)
- width = MAX_LONG_LONG_CHARS;
- }
- else
+ else if (longlongflag)
+ len = sprintf(buffer, fmt,
+ va_arg(*vargs, unsigned PY_LONG_LONG));
+#endif
+ else if (size_tflag)
+ len = sprintf(buffer, fmt,
+ va_arg(*vargs, size_t));
+ else
+ len = sprintf(buffer, fmt,
+ va_arg(*vargs, unsigned int));
+ }
+ else if (*f == 'x') {
+ makefmt(fmt, 0, 0, 0, 'x');
+ len = sprintf(buffer, fmt, va_arg(*vargs, int));
+ }
+ else {
+ makefmt(fmt, longflag, longlongflag, size_tflag, *f);
+
+ if (longflag)
+ len = sprintf(buffer, fmt,
+ va_arg(*vargs, long));
+#ifdef HAVE_LONG_LONG
+ else if (longlongflag)
+ len = sprintf(buffer, fmt,
+ va_arg(*vargs, PY_LONG_LONG));
#endif
- /* MAX_LONG_CHARS is enough to hold a 64-bit integer,
- including sign. Decimal takes the most space. This
- isn't enough for octal. If a width is specified we
- need more (which we allocate later). */
- if (width < MAX_LONG_CHARS)
- width = MAX_LONG_CHARS;
-
- /* account for the size + '\0' to separate numbers
- inside of the numberresults buffer */
- numbersize += (width + 1);
+ else if (size_tflag)
+ len = sprintf(buffer, fmt,
+ va_arg(*vargs, Py_ssize_t));
+ else
+ len = sprintf(buffer, fmt,
+ va_arg(*vargs, int));
+ }
+ assert(len >= 0);
+
+ if (precision < len)
+ precision = len;
+ if (width > precision) {
+ Py_UCS4 fillchar;
+ fill = width - precision;
+ fillchar = zeropad?'0':' ';
+ if (_PyUnicodeWriter_Prepare(writer, fill, fillchar) == -1)
+ return NULL;
+ if (PyUnicode_Fill(writer->buffer, writer->pos, fill, fillchar) == -1)
+ return NULL;
+ writer->pos += fill;
+ }
+ if (precision > len) {
+ fill = precision - len;
+ if (_PyUnicodeWriter_Prepare(writer, fill, '0') == -1)
+ return NULL;
+ if (PyUnicode_Fill(writer->buffer, writer->pos, fill, '0') == -1)
+ return NULL;
+ writer->pos += fill;
+ }
+ if (_PyUnicodeWriter_WriteCstr(writer, buffer, len) == -1)
+ return NULL;
+ break;
+ }
+
+ case 'p':
+ {
+ char number[MAX_LONG_LONG_CHARS];
+
+ len = sprintf(number, "%p", va_arg(*vargs, void*));
+ assert(len >= 0);
+
+ /* %p is ill-defined: ensure leading 0x. */
+ if (number[1] == 'X')
+ number[1] = 'x';
+ else if (number[1] != 'x') {
+ memmove(number + 2, number,
+ strlen(number) + 1);
+ number[0] = '0';
+ number[1] = 'x';
+ len += 2;
+ }
+
+ if (_PyUnicodeWriter_WriteCstr(writer, number, len) == -1)
+ return NULL;
+ break;
+ }
+
+ case 's':
+ {
+ /* UTF-8 */
+ const char *s = va_arg(*vargs, const char*);
+ PyObject *str = PyUnicode_DecodeUTF8Stateful(s, strlen(s), "replace", NULL);
+ if (!str)
+ return NULL;
+ if (_PyUnicodeWriter_WriteStr(writer, str) == -1) {
+ Py_DECREF(str);
+ return NULL;
+ }
+ Py_DECREF(str);
+ break;
+ }
+
+ case 'U':
+ {
+ PyObject *obj = va_arg(*vargs, PyObject *);
+ assert(obj && _PyUnicode_CHECK(obj));
+
+ if (_PyUnicodeWriter_WriteStr(writer, obj) == -1)
+ return NULL;
+ break;
+ }
+
+ case 'V':
+ {
+ PyObject *obj = va_arg(*vargs, PyObject *);
+ const char *str = va_arg(*vargs, const char *);
+ PyObject *str_obj;
+ assert(obj || str);
+ if (obj) {
+ assert(_PyUnicode_CHECK(obj));
+ if (_PyUnicodeWriter_WriteStr(writer, obj) == -1)
+ return NULL;
+ }
+ else {
+ str_obj = PyUnicode_DecodeUTF8Stateful(str, strlen(str), "replace", NULL);
+ if (!str_obj)
+ return NULL;
+ if (_PyUnicodeWriter_WriteStr(writer, str_obj) == -1) {
+ Py_DECREF(str_obj);
+ return NULL;
}
+ Py_DECREF(str_obj);
}
- else if ((unsigned char)*f > 127) {
- PyErr_Format(PyExc_ValueError,
- "PyUnicode_FromFormatV() expects an ASCII-encoded format "
- "string, got a non-ASCII byte: 0x%02x",
- (unsigned char)*f);
+ break;
+ }
+
+ case 'S':
+ {
+ PyObject *obj = va_arg(*vargs, PyObject *);
+ PyObject *str;
+ assert(obj);
+ str = PyObject_Str(obj);
+ if (!str)
+ return NULL;
+ if (_PyUnicodeWriter_WriteStr(writer, str) == -1) {
+ Py_DECREF(str);
return NULL;
}
+ Py_DECREF(str);
+ break;
}
- /* step 2: allocate memory for the results of
- * PyObject_Str()/PyObject_Repr()/PyUnicode_DecodeUTF8() calls */
- if (callcount) {
- callresults = PyObject_Malloc(sizeof(PyObject *) * callcount);
- if (!callresults) {
- PyErr_NoMemory();
+
+ case 'R':
+ {
+ PyObject *obj = va_arg(*vargs, PyObject *);
+ PyObject *repr;
+ assert(obj);
+ repr = PyObject_Repr(obj);
+ if (!repr)
+ return NULL;
+ if (_PyUnicodeWriter_WriteStr(writer, repr) == -1) {
+ Py_DECREF(repr);
return NULL;
}
- callresult = callresults;
+ Py_DECREF(repr);
+ break;
}
- /* step 2.5: allocate memory for the results of formating numbers */
- if (numbersize) {
- numberresults = PyObject_Malloc(numbersize);
- if (!numberresults) {
- PyErr_NoMemory();
- goto fail;
+
+ case 'A':
+ {
+ PyObject *obj = va_arg(*vargs, PyObject *);
+ PyObject *ascii;
+ assert(obj);
+ ascii = PyObject_ASCII(obj);
+ if (!ascii)
+ return NULL;
+ if (_PyUnicodeWriter_WriteStr(writer, ascii) == -1) {
+ Py_DECREF(ascii);
+ return NULL;
}
- numberresult = numberresults;
+ Py_DECREF(ascii);
+ break;
}
- /* step 3: format numbers and figure out how large a buffer we need */
- for (f = format; *f; f++) {
- if (*f == '%') {
- const char* p;
- int longflag;
- int longlongflag;
- int size_tflag;
- int numprinted;
+ case '%':
+ if (_PyUnicodeWriter_Prepare(writer, 1, '%') == 1)
+ return NULL;
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '%');
+ writer->pos++;
+ break;
- p = f;
- zeropad = (f[1] == '0');
- f = parse_format_flags(f, &width, &precision,
- &longflag, &longlongflag, &size_tflag);
- switch (*f) {
- case 'c':
- {
- Py_UCS4 ordinal = va_arg(count, int);
- maxchar = MAX_MAXCHAR(maxchar, ordinal);
- n++;
- break;
- }
- case '%':
- n++;
- break;
- case 'i':
- case 'd':
- makefmt(fmt, longflag, longlongflag, size_tflag, zeropad,
- width, precision, *f);
- if (longflag)
- numprinted = sprintf(numberresult, fmt,
- va_arg(count, long));
-#ifdef HAVE_LONG_LONG
- else if (longlongflag)
- numprinted = sprintf(numberresult, fmt,
- va_arg(count, PY_LONG_LONG));
-#endif
- else if (size_tflag)
- numprinted = sprintf(numberresult, fmt,
- va_arg(count, Py_ssize_t));
- else
- numprinted = sprintf(numberresult, fmt,
- va_arg(count, int));
- n += numprinted;
- /* advance by +1 to skip over the '\0' */
- numberresult += (numprinted + 1);
- assert(*(numberresult - 1) == '\0');
- assert(*(numberresult - 2) != '\0');
- assert(numprinted >= 0);
- assert(numberresult <= numberresults + numbersize);
- break;
- case 'u':
- makefmt(fmt, longflag, longlongflag, size_tflag, zeropad,
- width, precision, 'u');
- if (longflag)
- numprinted = sprintf(numberresult, fmt,
- va_arg(count, unsigned long));
-#ifdef HAVE_LONG_LONG
- else if (longlongflag)
- numprinted = sprintf(numberresult, fmt,
- va_arg(count, unsigned PY_LONG_LONG));
-#endif
- else if (size_tflag)
- numprinted = sprintf(numberresult, fmt,
- va_arg(count, size_t));
- else
- numprinted = sprintf(numberresult, fmt,
- va_arg(count, unsigned int));
- n += numprinted;
- numberresult += (numprinted + 1);
- assert(*(numberresult - 1) == '\0');
- assert(*(numberresult - 2) != '\0');
- assert(numprinted >= 0);
- assert(numberresult <= numberresults + numbersize);
- break;
- case 'x':
- makefmt(fmt, 0, 0, 0, zeropad, width, precision, 'x');
- numprinted = sprintf(numberresult, fmt, va_arg(count, int));
- n += numprinted;
- numberresult += (numprinted + 1);
- assert(*(numberresult - 1) == '\0');
- assert(*(numberresult - 2) != '\0');
- assert(numprinted >= 0);
- assert(numberresult <= numberresults + numbersize);
- break;
- case 'p':
- numprinted = sprintf(numberresult, "%p", va_arg(count, void*));
- /* %p is ill-defined: ensure leading 0x. */
- if (numberresult[1] == 'X')
- numberresult[1] = 'x';
- else if (numberresult[1] != 'x') {
- memmove(numberresult + 2, numberresult,
- strlen(numberresult) + 1);
- numberresult[0] = '0';
- numberresult[1] = 'x';
- numprinted += 2;
- }
- n += numprinted;
- numberresult += (numprinted + 1);
- assert(*(numberresult - 1) == '\0');
- assert(*(numberresult - 2) != '\0');
- assert(numprinted >= 0);
- assert(numberresult <= numberresults + numbersize);
- break;
- case 's':
- {
- /* UTF-8 */
- const char *s = va_arg(count, const char*);
- PyObject *str = PyUnicode_DecodeUTF8Stateful(s, strlen(s), "replace", NULL);
- if (!str)
- goto fail;
- /* since PyUnicode_DecodeUTF8 returns already flexible
- unicode objects, there is no need to call ready on them */
- argmaxchar = PyUnicode_MAX_CHAR_VALUE(str);
- maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
- n += PyUnicode_GET_LENGTH(str);
- /* Remember the str and switch to the next slot */
- *callresult++ = str;
- break;
- }
- case 'U':
- {
- PyObject *obj = va_arg(count, PyObject *);
- assert(obj && _PyUnicode_CHECK(obj));
- if (PyUnicode_READY(obj) == -1)
- goto fail;
- argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj);
- maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
- n += PyUnicode_GET_LENGTH(obj);
- break;
- }
- case 'V':
- {
- PyObject *obj = va_arg(count, PyObject *);
- const char *str = va_arg(count, const char *);
- PyObject *str_obj;
- assert(obj || str);
- assert(!obj || _PyUnicode_CHECK(obj));
- if (obj) {
- if (PyUnicode_READY(obj) == -1)
- goto fail;
- argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj);
- maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
- n += PyUnicode_GET_LENGTH(obj);
- *callresult++ = NULL;
- }
- else {
- str_obj = PyUnicode_DecodeUTF8Stateful(str, strlen(str), "replace", NULL);
- if (!str_obj)
- goto fail;
- if (PyUnicode_READY(str_obj) == -1) {
- Py_DECREF(str_obj);
- goto fail;
- }
- argmaxchar = PyUnicode_MAX_CHAR_VALUE(str_obj);
- maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
- n += PyUnicode_GET_LENGTH(str_obj);
- *callresult++ = str_obj;
- }
- break;
- }
- case 'S':
- {
- PyObject *obj = va_arg(count, PyObject *);
- PyObject *str;
- assert(obj);
- str = PyObject_Str(obj);
- if (!str)
- goto fail;
- if (PyUnicode_READY(str) == -1) {
- Py_DECREF(str);
- goto fail;
- }
- argmaxchar = PyUnicode_MAX_CHAR_VALUE(str);
- maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
- n += PyUnicode_GET_LENGTH(str);
- /* Remember the str and switch to the next slot */
- *callresult++ = str;
- break;
- }
- case 'R':
- {
- PyObject *obj = va_arg(count, PyObject *);
- PyObject *repr;
- assert(obj);
- repr = PyObject_Repr(obj);
- if (!repr)
- goto fail;
- if (PyUnicode_READY(repr) == -1) {
- Py_DECREF(repr);
- goto fail;
- }
- argmaxchar = PyUnicode_MAX_CHAR_VALUE(repr);
- maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
- n += PyUnicode_GET_LENGTH(repr);
- /* Remember the repr and switch to the next slot */
- *callresult++ = repr;
- break;
- }
- case 'A':
- {
- PyObject *obj = va_arg(count, PyObject *);
- PyObject *ascii;
- assert(obj);
- ascii = PyObject_ASCII(obj);
- if (!ascii)
- goto fail;
- if (PyUnicode_READY(ascii) == -1) {
- Py_DECREF(ascii);
- goto fail;
- }
- argmaxchar = PyUnicode_MAX_CHAR_VALUE(ascii);
- maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
- n += PyUnicode_GET_LENGTH(ascii);
- /* Remember the repr and switch to the next slot */
- *callresult++ = ascii;
- break;
- }
- default:
- /* if we stumble upon an unknown
- formatting code, copy the rest of
- the format string to the output
- string. (we cannot just skip the
- code, since there's no way to know
- what's in the argument list) */
- n += strlen(p);
- goto expand;
- }
- } else
- n++;
- }
- expand:
- /* step 4: fill the buffer */
- /* Since we've analyzed how much space we need,
- we don't have to resize the string.
- There can be no errors beyond this point. */
- string = PyUnicode_New(n, maxchar);
- if (!string)
- goto fail;
- kind = PyUnicode_KIND(string);
- data = PyUnicode_DATA(string);
- callresult = callresults;
- numberresult = numberresults;
+ default:
+ /* if we stumble upon an unknown formatting code, copy the rest
+ of the format string to the output string. (we cannot just
+ skip the code, since there's no way to know what's in the
+ argument list) */
+ len = strlen(p);
+ if (_PyUnicodeWriter_WriteCstr(writer, p, len) == -1)
+ return NULL;
+ f = p+len;
+ return f;
+ }
+
+ f++;
+ return f;
+}
+
+PyObject *
+PyUnicode_FromFormatV(const char *format, va_list vargs)
+{
+ va_list vargs2;
+ const char *f;
+ _PyUnicodeWriter writer;
- for (i = 0, f = format; *f; f++) {
+ _PyUnicodeWriter_Init(&writer, strlen(format) + 100);
+
+ /* va_list may be an array (of 1 item) on some platforms (ex: AMD64).
+ Copy it to be able to pass a reference to a subfunction. */
+ Py_VA_COPY(vargs2, vargs);
+
+ for (f = format; *f; ) {
if (*f == '%') {
- const char* p;
+ f = unicode_fromformat_arg(&writer, f, &vargs2);
+ if (f == NULL)
+ goto fail;
+ }
+ else {
+ const char *p;
+ Py_ssize_t len;
p = f;
- f = parse_format_flags(f, NULL, NULL, NULL, NULL, NULL);
- /* checking for == because the last argument could be a empty
- string, which causes i to point to end, the assert at the end of
- the loop */
- assert(i <= PyUnicode_GET_LENGTH(string));
-
- switch (*f) {
- case 'c':
- {
- const int ordinal = va_arg(vargs, int);
- PyUnicode_WRITE(kind, data, i++, ordinal);
- break;
- }
- case 'i':
- case 'd':
- case 'u':
- case 'x':
- case 'p':
- {
- Py_ssize_t len;
- /* unused, since we already have the result */
- if (*f == 'p')
- (void) va_arg(vargs, void *);
- else
- (void) va_arg(vargs, int);
- /* extract the result from numberresults and append. */
- len = strlen(numberresult);
- unicode_write_cstr(string, i, numberresult, len);
- /* skip over the separating '\0' */
- i += len;
- numberresult += len;
- assert(*numberresult == '\0');
- numberresult++;
- assert(numberresult <= numberresults + numbersize);
- break;
- }
- case 's':
+ do
{
- /* unused, since we already have the result */
- Py_ssize_t size;
- (void) va_arg(vargs, char *);
- size = PyUnicode_GET_LENGTH(*callresult);
- assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string));
- _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size);
- i += size;
- /* We're done with the unicode()/repr() => forget it */
- Py_DECREF(*callresult);
- /* switch to next unicode()/repr() result */
- ++callresult;
- break;
- }
- case 'U':
- {
- PyObject *obj = va_arg(vargs, PyObject *);
- Py_ssize_t size;
- assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string));
- size = PyUnicode_GET_LENGTH(obj);
- _PyUnicode_FastCopyCharacters(string, i, obj, 0, size);
- i += size;
- break;
- }
- case 'V':
- {
- Py_ssize_t size;
- PyObject *obj = va_arg(vargs, PyObject *);
- va_arg(vargs, const char *);
- if (obj) {
- size = PyUnicode_GET_LENGTH(obj);
- assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string));
- _PyUnicode_FastCopyCharacters(string, i, obj, 0, size);
- i += size;
- } else {
- size = PyUnicode_GET_LENGTH(*callresult);
- assert(PyUnicode_KIND(*callresult) <=
- PyUnicode_KIND(string));
- _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size);
- i += size;
- Py_DECREF(*callresult);
+ if ((unsigned char)*p > 127) {
+ PyErr_Format(PyExc_ValueError,
+ "PyUnicode_FromFormatV() expects an ASCII-encoded format "
+ "string, got a non-ASCII byte: 0x%02x",
+ (unsigned char)*p);
+ return NULL;
}
- ++callresult;
- break;
+ p++;
}
- case 'S':
- case 'R':
- case 'A':
- {
- Py_ssize_t size = PyUnicode_GET_LENGTH(*callresult);
- /* unused, since we already have the result */
- (void) va_arg(vargs, PyObject *);
- assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string));
- _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size);
- i += size;
- /* We're done with the unicode()/repr() => forget it */
- Py_DECREF(*callresult);
- /* switch to next unicode()/repr() result */
- ++callresult;
- break;
- }
- case '%':
- PyUnicode_WRITE(kind, data, i++, '%');
- break;
- default:
- {
- Py_ssize_t len = strlen(p);
- unicode_write_cstr(string, i, p, len);
- i += len;
- assert(i == PyUnicode_GET_LENGTH(string));
- goto end;
- }
- }
- }
- else {
- assert(i < PyUnicode_GET_LENGTH(string));
- PyUnicode_WRITE(kind, data, i++, *f);
+ while (*p != '\0' && *p != '%');
+ len = p - f;
+
+ if (*p == '\0')
+ writer.overallocate = 0;
+ if (_PyUnicodeWriter_Prepare(&writer, len, 127) == -1)
+ goto fail;
+ unicode_write_cstr(writer.buffer, writer.pos, f, len);
+ writer.pos += len;
+
+ f = p;
}
}
- assert(i == PyUnicode_GET_LENGTH(string));
+ return _PyUnicodeWriter_Finish(&writer);
- end:
- if (callresults)
- PyObject_Free(callresults);
- if (numberresults)
- PyObject_Free(numberresults);
- return unicode_result(string);
fail:
- if (callresults) {
- PyObject **callresult2 = callresults;
- while (callresult2 < callresult) {
- Py_XDECREF(*callresult2);
- ++callresult2;
- }
- PyObject_Free(callresults);
- }
- if (numberresults)
- PyObject_Free(numberresults);
+ _PyUnicodeWriter_Dealloc(&writer);
return NULL;
}
@@ -3056,8 +2874,8 @@ PyUnicode_FromEncodedObject(register PyObject *obj,
1 on success. */
int
_Py_normalize_encoding(const char *encoding,
- char *lower,
- size_t lower_len)
+ char *lower,
+ size_t lower_len)
{
const char *e;
char *l;
@@ -3339,7 +3157,7 @@ PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
}
if (surrogateescape) {
- /* locale encoding with surrogateescape */
+ /* "surrogateescape" error handler */
char *str;
str = _Py_wchar2char(wstr, &error_pos);
@@ -3359,6 +3177,7 @@ PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
PyMem_Free(str);
}
else {
+ /* strict mode */
size_t len, len2;
len = wcstombs(NULL, wstr, 0);
@@ -3604,8 +3423,8 @@ PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
return NULL;
}
- if (surrogateescape)
- {
+ if (surrogateescape) {
+ /* "surrogateescape" error handler */
wstr = _Py_char2wchar(str, &wlen);
if (wstr == NULL) {
if (wlen == (size_t)-1)
@@ -3619,6 +3438,7 @@ PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
PyMem_Free(wstr);
}
else {
+ /* strict mode */
#ifndef HAVE_BROKEN_MBSTOWCS
wlen = mbstowcs(NULL, str, 0);
#else
@@ -3638,7 +3458,6 @@ PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
return PyErr_NoMemory();
}
- /* This shouldn't fail now */
wlen2 = mbstowcs(wstr, str, wlen+1);
if (wlen2 == (size_t)-1) {
if (wstr != smallbuf)
@@ -3732,18 +3551,20 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size)
int
-_PyUnicode_HasNULChars(PyObject* s)
+_PyUnicode_HasNULChars(PyObject* str)
{
- static PyObject *nul = NULL;
+ Py_ssize_t pos;
- if (nul == NULL)
- nul = PyUnicode_FromStringAndSize("\0", 1);
- if (nul == NULL)
+ if (PyUnicode_READY(str) == -1)
return -1;
- return PyUnicode_Contains(s, nul);
+ pos = findchar(PyUnicode_DATA(str), PyUnicode_KIND(str),
+ PyUnicode_GET_LENGTH(str), '\0', 1);
+ if (pos == -1)
+ return 0;
+ else
+ return 1;
}
-
int
PyUnicode_FSConverter(PyObject* arg, void* addr)
{
@@ -4082,6 +3903,7 @@ onError:
*exceptionObject = NULL;
}
+#ifdef HAVE_MBCS
/* error handling callback helper:
build arguments, call the callback and check the arguments,
if no exception occurred, copy the replacement to the output
@@ -4090,11 +3912,12 @@ onError:
*/
static int
-unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler,
- const char *encoding, const char *reason,
- const char **input, const char **inend, Py_ssize_t *startinpos,
- Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr,
- PyObject **output, Py_ssize_t *outpos)
+unicode_decode_call_errorhandler_wchar(
+ const char *errors, PyObject **errorHandler,
+ const char *encoding, const char *reason,
+ const char **input, const char **inend, Py_ssize_t *startinpos,
+ Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr,
+ PyObject **output, Py_ssize_t *outpos)
{
static char *argparse = "O!n;decoding error handler must return (str, int) tuple";
@@ -4105,12 +3928,11 @@ unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler,
Py_ssize_t requiredsize;
Py_ssize_t newpos;
PyObject *inputobj = NULL;
- int res = -1;
+ wchar_t *repwstr;
+ Py_ssize_t repwlen;
- if (_PyUnicode_KIND(*output) != PyUnicode_WCHAR_KIND)
- outsize = PyUnicode_GET_LENGTH(*output);
- else
- outsize = _PyUnicode_WSTR_LENGTH(*output);
+ assert (_PyUnicode_KIND(*output) == PyUnicode_WCHAR_KIND);
+ outsize = _PyUnicode_WSTR_LENGTH(*output);
if (*errorHandler == NULL) {
*errorHandler = PyCodec_LookupError(errors);
@@ -4135,8 +3957,6 @@ unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler,
}
if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos))
goto onError;
- if (PyUnicode_READY(repunicode) == -1)
- goto onError;
/* Copy back the bytes variables, which might have been modified by the
callback */
@@ -4160,54 +3980,112 @@ unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler,
goto onError;
}
- if (_PyUnicode_KIND(*output) != PyUnicode_WCHAR_KIND) {
- /* need more space? (at least enough for what we
- have+the replacement+the rest of the string (starting
- at the new input position), so we won't have to check space
- when there are no errors in the rest of the string) */
- Py_ssize_t replen = PyUnicode_GET_LENGTH(repunicode);
- requiredsize = *outpos + replen + insize-newpos;
- if (requiredsize > outsize) {
- if (requiredsize<2*outsize)
- requiredsize = 2*outsize;
- if (unicode_resize(output, requiredsize) < 0)
- goto onError;
- }
- if (unicode_widen(output, *outpos,
- PyUnicode_MAX_CHAR_VALUE(repunicode)) < 0)
+ repwstr = PyUnicode_AsUnicodeAndSize(repunicode, &repwlen);
+ if (repwstr == NULL)
+ goto onError;
+ /* need more space? (at least enough for what we
+ have+the replacement+the rest of the string (starting
+ at the new input position), so we won't have to check space
+ when there are no errors in the rest of the string) */
+ requiredsize = *outpos + repwlen + insize-newpos;
+ if (requiredsize > outsize) {
+ if (requiredsize < 2*outsize)
+ requiredsize = 2*outsize;
+ if (unicode_resize(output, requiredsize) < 0)
goto onError;
- _PyUnicode_FastCopyCharacters(*output, *outpos, repunicode, 0, replen);
- *outpos += replen;
}
- else {
- wchar_t *repwstr;
- Py_ssize_t repwlen;
- repwstr = PyUnicode_AsUnicodeAndSize(repunicode, &repwlen);
- if (repwstr == NULL)
+ wcsncpy(_PyUnicode_WSTR(*output) + *outpos, repwstr, repwlen);
+ *outpos += repwlen;
+
+ *endinpos = newpos;
+ *inptr = *input + newpos;
+
+ /* we made it! */
+ Py_XDECREF(restuple);
+ return 0;
+
+ onError:
+ Py_XDECREF(restuple);
+ return -1;
+}
+#endif /* HAVE_MBCS */
+
+static int
+unicode_decode_call_errorhandler_writer(
+ const char *errors, PyObject **errorHandler,
+ const char *encoding, const char *reason,
+ const char **input, const char **inend, Py_ssize_t *startinpos,
+ Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr,
+ _PyUnicodeWriter *writer /* PyObject **output, Py_ssize_t *outpos */)
+{
+ static char *argparse = "O!n;decoding error handler must return (str, int) tuple";
+
+ PyObject *restuple = NULL;
+ PyObject *repunicode = NULL;
+ Py_ssize_t insize;
+ Py_ssize_t newpos;
+ PyObject *inputobj = NULL;
+
+ if (*errorHandler == NULL) {
+ *errorHandler = PyCodec_LookupError(errors);
+ if (*errorHandler == NULL)
goto onError;
- /* need more space? (at least enough for what we
- have+the replacement+the rest of the string (starting
- at the new input position), so we won't have to check space
- when there are no errors in the rest of the string) */
- requiredsize = *outpos + repwlen + insize-newpos;
- if (requiredsize > outsize) {
- if (requiredsize < 2*outsize)
- requiredsize = 2*outsize;
- if (unicode_resize(output, requiredsize) < 0)
- goto onError;
- }
- wcsncpy(_PyUnicode_WSTR(*output) + *outpos, repwstr, repwlen);
- *outpos += repwlen;
}
+
+ make_decode_exception(exceptionObject,
+ encoding,
+ *input, *inend - *input,
+ *startinpos, *endinpos,
+ reason);
+ if (*exceptionObject == NULL)
+ goto onError;
+
+ restuple = PyObject_CallFunctionObjArgs(*errorHandler, *exceptionObject, NULL);
+ if (restuple == NULL)
+ goto onError;
+ if (!PyTuple_Check(restuple)) {
+ PyErr_SetString(PyExc_TypeError, &argparse[4]);
+ goto onError;
+ }
+ if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos))
+ goto onError;
+
+ /* Copy back the bytes variables, which might have been modified by the
+ callback */
+ inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject);
+ if (!inputobj)
+ goto onError;
+ if (!PyBytes_Check(inputobj)) {
+ PyErr_Format(PyExc_TypeError, "exception attribute object must be bytes");
+ }
+ *input = PyBytes_AS_STRING(inputobj);
+ insize = PyBytes_GET_SIZE(inputobj);
+ *inend = *input + insize;
+ /* we can DECREF safely, as the exception has another reference,
+ so the object won't go away. */
+ Py_DECREF(inputobj);
+
+ if (newpos<0)
+ newpos = insize+newpos;
+ if (newpos<0 || newpos>insize) {
+ PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", newpos);
+ goto onError;
+ }
+
+ writer->overallocate = 1;
+ if (_PyUnicodeWriter_WriteStr(writer, repunicode) == -1)
+ return
+
*endinpos = newpos;
*inptr = *input + newpos;
/* we made it! */
- res = 0;
+ Py_XDECREF(restuple);
+ return 0;
onError:
Py_XDECREF(restuple);
- return res;
+ return -1;
}
/* --- UTF-7 Codec -------------------------------------------------------- */
@@ -4315,9 +4193,8 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
const char *starts = s;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
- Py_ssize_t outpos;
const char *e;
- PyObject *unicode;
+ _PyUnicodeWriter writer;
const char *errmsg = "";
int inShift = 0;
Py_ssize_t shiftOutStart;
@@ -4327,17 +4204,18 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
- /* Start off assuming it's all ASCII. Widen later as necessary. */
- unicode = PyUnicode_New(size, 127);
- if (!unicode)
- return NULL;
if (size == 0) {
if (consumed)
*consumed = 0;
- return unicode;
+ _Py_RETURN_UNICODE_EMPTY();
}
- shiftOutStart = outpos = 0;
+ /* Start off assuming it's all ASCII. Widen later as necessary. */
+ _PyUnicodeWriter_Init(&writer, 0);
+ if (_PyUnicodeWriter_Prepare(&writer, size, 127) == -1)
+ goto onError;
+
+ shiftOutStart = 0;
e = s + size;
while (s < e) {
@@ -4359,14 +4237,18 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
/* expecting a second surrogate */
if (Py_UNICODE_IS_LOW_SURROGATE(outCh)) {
Py_UCS4 ch2 = Py_UNICODE_JOIN_SURROGATES(surrogate, outCh);
- if (unicode_putchar(&unicode, &outpos, ch2) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, ch2) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch2);
+ writer.pos++;
surrogate = 0;
continue;
}
else {
- if (unicode_putchar(&unicode, &outpos, surrogate) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, surrogate) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, surrogate);
+ writer.pos++;
surrogate = 0;
}
}
@@ -4375,8 +4257,10 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
surrogate = outCh;
}
else {
- if (unicode_putchar(&unicode, &outpos, outCh) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, outCh) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, outCh);
+ writer.pos++;
}
}
}
@@ -4384,8 +4268,10 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
inShift = 0;
s++;
if (surrogate) {
- if (unicode_putchar(&unicode, &outpos, surrogate) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, surrogate) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, surrogate);
+ writer.pos++;
surrogate = 0;
}
if (base64bits > 0) { /* left-over bits */
@@ -4405,8 +4291,10 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
if (ch != '-') {
/* '-' is absorbed; other terminating
characters are preserved */
- if (unicode_putchar(&unicode, &outpos, ch) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch);
+ writer.pos++;
}
}
}
@@ -4415,19 +4303,23 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
s++; /* consume '+' */
if (s < e && *s == '-') { /* '+-' encodes '+' */
s++;
- if (unicode_putchar(&unicode, &outpos, '+') < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, '+') == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, '+');
+ writer.pos++;
}
else { /* begin base64-encoded section */
inShift = 1;
- shiftOutStart = outpos;
+ shiftOutStart = writer.pos;
base64bits = 0;
}
}
else if (DECODE_DIRECT(ch)) { /* character decodes as itself */
- if (unicode_putchar(&unicode, &outpos, ch) < 0)
- goto onError;
s++;
+ if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1)
+ goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch);
+ writer.pos++;
}
else {
startinpos = s-starts;
@@ -4438,11 +4330,11 @@ PyUnicode_DecodeUTF7Stateful(const char *s,
continue;
utf7Error:
endinpos = s-starts;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"utf7", errmsg,
&starts, &e, &startinpos, &endinpos, &exc, &s,
- &unicode, &outpos))
+ &writer))
goto onError;
}
@@ -4454,11 +4346,11 @@ utf7Error:
(base64bits >= 6) ||
(base64bits > 0 && base64buffer != 0)) {
endinpos = size;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"utf7", "unterminated shift sequence",
&starts, &e, &startinpos, &endinpos, &exc, &s,
- &unicode, &outpos))
+ &writer))
goto onError;
if (s < e)
goto restart;
@@ -4468,7 +4360,7 @@ utf7Error:
/* return state */
if (consumed) {
if (inShift) {
- outpos = shiftOutStart; /* back off output */
+ writer.pos = shiftOutStart; /* back off output */
*consumed = startinpos;
}
else {
@@ -4476,17 +4368,14 @@ utf7Error:
}
}
- if (unicode_resize(&unicode, outpos) < 0)
- goto onError;
-
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- return unicode_result(unicode);
+ return _PyUnicodeWriter_Finish(&writer);
onError:
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- Py_DECREF(unicode);
+ _PyUnicodeWriter_Dealloc(&writer);
return NULL;
}
@@ -4569,7 +4458,7 @@ encode_char:
/* code first surrogate */
base64bits += 16;
- base64buffer = (base64buffer << 16) | 0xd800 | ((ch-0x10000) >> 10);
+ base64buffer = (base64buffer << 16) | Py_UNICODE_HIGH_SURROGATE(ch);
while (base64bits >= 6) {
*out++ = TO_BASE64(base64buffer >> (base64bits-6));
base64bits -= 6;
@@ -4712,10 +4601,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
const char *errors,
Py_ssize_t *consumed)
{
- PyObject *unicode;
+ _PyUnicodeWriter writer;
const char *starts = s;
const char *end = s + size;
- Py_ssize_t outpos;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
@@ -4736,29 +4624,25 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
return get_latin1_char((unsigned char)s[0]);
}
- unicode = PyUnicode_New(size, 127);
- if (!unicode)
- return NULL;
+ _PyUnicodeWriter_Init(&writer, 0);
+ if (_PyUnicodeWriter_Prepare(&writer, size, 127) == -1)
+ goto onError;
- outpos = ascii_decode(s, end, PyUnicode_1BYTE_DATA(unicode));
- s += outpos;
+ writer.pos = ascii_decode(s, end, writer.data);
+ s += writer.pos;
while (s < end) {
Py_UCS4 ch;
- int kind = PyUnicode_KIND(unicode);
+ int kind = writer.kind;
if (kind == PyUnicode_1BYTE_KIND) {
- if (PyUnicode_IS_ASCII(unicode))
- ch = asciilib_utf8_decode(&s, end,
- PyUnicode_1BYTE_DATA(unicode), &outpos);
+ if (PyUnicode_IS_ASCII(writer.buffer))
+ ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos);
else
- ch = ucs1lib_utf8_decode(&s, end,
- PyUnicode_1BYTE_DATA(unicode), &outpos);
+ ch = ucs1lib_utf8_decode(&s, end, writer.data, &writer.pos);
} else if (kind == PyUnicode_2BYTE_KIND) {
- ch = ucs2lib_utf8_decode(&s, end,
- PyUnicode_2BYTE_DATA(unicode), &outpos);
+ ch = ucs2lib_utf8_decode(&s, end, writer.data, &writer.pos);
} else {
assert(kind == PyUnicode_4BYTE_KIND);
- ch = ucs4lib_utf8_decode(&s, end,
- PyUnicode_4BYTE_DATA(unicode), &outpos);
+ ch = ucs4lib_utf8_decode(&s, end, writer.data, &writer.pos);
}
switch (ch) {
@@ -4782,35 +4666,33 @@ PyUnicode_DecodeUTF8Stateful(const char *s,
endinpos = startinpos + ch - 1;
break;
default:
- if (unicode_putchar(&unicode, &outpos, ch) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch);
+ writer.pos++;
continue;
}
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"utf-8", errmsg,
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &unicode, &outpos))
+ &writer))
goto onError;
}
End:
- if (unicode_resize(&unicode, outpos) < 0)
- goto onError;
-
if (consumed)
*consumed = s - starts;
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- assert(_PyUnicode_CheckConsistency(unicode, 1));
- return unicode;
+ return _PyUnicodeWriter_Finish(&writer);
onError:
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- Py_XDECREF(unicode);
+ _PyUnicodeWriter_Dealloc(&writer);
return NULL;
}
@@ -4956,17 +4838,10 @@ PyUnicode_DecodeUTF32Stateful(const char *s,
const char *starts = s;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
- Py_ssize_t outpos;
- PyObject *unicode;
+ _PyUnicodeWriter writer;
const unsigned char *q, *e;
- int bo = 0; /* assume native ordering by default */
+ int le, bo = 0; /* assume native ordering by default */
const char *errmsg = "";
- /* Offsets from q for retrieving bytes in the right order. */
-#ifdef BYTEORDER_IS_LITTLE_ENDIAN
- int iorder[] = {0, 1, 2, 3};
-#else
- int iorder[] = {3, 2, 1, 0};
-#endif
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
@@ -4980,107 +4855,107 @@ PyUnicode_DecodeUTF32Stateful(const char *s,
byte order setting accordingly. In native mode, the leading BOM
mark is skipped, in all other modes, it is copied to the output
stream as-is (giving a ZWNBSP character). */
- if (bo == 0) {
- if (size >= 4) {
- const Py_UCS4 bom = (q[iorder[3]] << 24) | (q[iorder[2]] << 16) |
- (q[iorder[1]] << 8) | q[iorder[0]];
-#ifdef BYTEORDER_IS_LITTLE_ENDIAN
- if (bom == 0x0000FEFF) {
- q += 4;
- bo = -1;
- }
- else if (bom == 0xFFFE0000) {
- q += 4;
- bo = 1;
- }
-#else
- if (bom == 0x0000FEFF) {
- q += 4;
- bo = 1;
- }
- else if (bom == 0xFFFE0000) {
- q += 4;
- bo = -1;
- }
-#endif
+ if (bo == 0 && size >= 4) {
+ Py_UCS4 bom = (q[3] << 24) | (q[2] << 16) | (q[1] << 8) | q[0];
+ if (bom == 0x0000FEFF) {
+ bo = -1;
+ q += 4;
+ }
+ else if (bom == 0xFFFE0000) {
+ bo = 1;
+ q += 4;
}
+ if (byteorder)
+ *byteorder = bo;
}
- if (bo == -1) {
- /* force LE */
- iorder[0] = 0;
- iorder[1] = 1;
- iorder[2] = 2;
- iorder[3] = 3;
- }
- else if (bo == 1) {
- /* force BE */
- iorder[0] = 3;
- iorder[1] = 2;
- iorder[2] = 1;
- iorder[3] = 0;
+ if (q == e) {
+ if (consumed)
+ *consumed = size;
+ _Py_RETURN_UNICODE_EMPTY();
}
- /* This might be one to much, because of a BOM */
- unicode = PyUnicode_New((size+3)/4, 127);
- if (!unicode)
- return NULL;
- if (size == 0)
- return unicode;
- outpos = 0;
+#ifdef WORDS_BIGENDIAN
+ le = bo < 0;
+#else
+ le = bo <= 0;
+#endif
- while (q < e) {
- Py_UCS4 ch;
- /* remaining bytes at the end? (size should be divisible by 4) */
- if (e-q<4) {
- if (consumed)
+ _PyUnicodeWriter_Init(&writer, 0);
+ if (_PyUnicodeWriter_Prepare(&writer, (e - q + 3) / 4, 127) == -1)
+ goto onError;
+
+ while (1) {
+ Py_UCS4 ch = 0;
+ Py_UCS4 maxch = PyUnicode_MAX_CHAR_VALUE(writer.buffer);
+
+ if (e - q >= 4) {
+ enum PyUnicode_Kind kind = writer.kind;
+ void *data = writer.data;
+ const unsigned char *last = e - 4;
+ Py_ssize_t pos = writer.pos;
+ if (le) {
+ do {
+ ch = (q[3] << 24) | (q[2] << 16) | (q[1] << 8) | q[0];
+ if (ch > maxch)
+ break;
+ PyUnicode_WRITE(kind, data, pos++, ch);
+ q += 4;
+ } while (q <= last);
+ }
+ else {
+ do {
+ ch = (q[0] << 24) | (q[1] << 16) | (q[2] << 8) | q[3];
+ if (ch > maxch)
+ break;
+ PyUnicode_WRITE(kind, data, pos++, ch);
+ q += 4;
+ } while (q <= last);
+ }
+ writer.pos = pos;
+ }
+
+ if (ch <= maxch) {
+ if (q == e || consumed)
break;
+ /* remaining bytes at the end? (size should be divisible by 4) */
errmsg = "truncated data";
- startinpos = ((const char *)q)-starts;
- endinpos = ((const char *)e)-starts;
- goto utf32Error;
- /* The remaining input chars are ignored if the callback
- chooses to skip the input */
+ startinpos = ((const char *)q) - starts;
+ endinpos = ((const char *)e) - starts;
}
- ch = (q[iorder[3]] << 24) | (q[iorder[2]] << 16) |
- (q[iorder[1]] << 8) | q[iorder[0]];
-
- if (ch >= 0x110000)
- {
+ else {
+ if (ch < 0x110000) {
+ if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1)
+ goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch);
+ writer.pos++;
+ q += 4;
+ continue;
+ }
errmsg = "codepoint not in range(0x110000)";
- startinpos = ((const char *)q)-starts;
- endinpos = startinpos+4;
- goto utf32Error;
+ startinpos = ((const char *)q) - starts;
+ endinpos = startinpos + 4;
}
- if (unicode_putchar(&unicode, &outpos, ch) < 0)
- goto onError;
- q += 4;
- continue;
- utf32Error:
- if (unicode_decode_call_errorhandler(
+
+ /* The remaining input chars are ignored if the callback
+ chooses to skip the input */
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"utf32", errmsg,
&starts, (const char **)&e, &startinpos, &endinpos, &exc, (const char **)&q,
- &unicode, &outpos))
+ &writer))
goto onError;
}
- if (byteorder)
- *byteorder = bo;
-
if (consumed)
*consumed = (const char *)q-starts;
- /* Adjust length */
- if (unicode_resize(&unicode, outpos) < 0)
- goto onError;
-
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- return unicode_result(unicode);
+ return _PyUnicodeWriter_Finish(&writer);
onError:
- Py_DECREF(unicode);
+ _PyUnicodeWriter_Dealloc(&writer);
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return NULL;
@@ -5098,7 +4973,7 @@ _PyUnicode_EncodeUTF32(PyObject *str,
unsigned char *p;
Py_ssize_t nsize, i;
/* Offsets from p for storing byte pairs in the right order. */
-#ifdef BYTEORDER_IS_LITTLE_ENDIAN
+#if PY_LITTLE_ENDIAN
int iorder[] = {0, 1, 2, 3};
#else
int iorder[] = {3, 2, 1, 0};
@@ -5201,8 +5076,7 @@ PyUnicode_DecodeUTF16Stateful(const char *s,
const char *starts = s;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
- Py_ssize_t outpos;
- PyObject *unicode;
+ _PyUnicodeWriter writer;
const unsigned char *q, *e;
int bo = 0; /* assume native ordering by default */
int native_ordering;
@@ -5240,7 +5114,7 @@ PyUnicode_DecodeUTF16Stateful(const char *s,
_Py_RETURN_UNICODE_EMPTY();
}
-#ifdef BYTEORDER_IS_LITTLE_ENDIAN
+#if PY_LITTLE_ENDIAN
native_ordering = bo <= 0;
#else
native_ordering = bo >= 0;
@@ -5248,32 +5122,31 @@ PyUnicode_DecodeUTF16Stateful(const char *s,
/* Note: size will always be longer than the resulting Unicode
character count */
- unicode = PyUnicode_New((e - q + 1) / 2, 127);
- if (!unicode)
- return NULL;
+ _PyUnicodeWriter_Init(&writer, 0);
+ if (_PyUnicodeWriter_Prepare(&writer, (e - q + 1) / 2, 127) == -1)
+ goto onError;
- outpos = 0;
while (1) {
Py_UCS4 ch = 0;
if (e - q >= 2) {
- int kind = PyUnicode_KIND(unicode);
+ int kind = writer.kind;
if (kind == PyUnicode_1BYTE_KIND) {
- if (PyUnicode_IS_ASCII(unicode))
+ if (PyUnicode_IS_ASCII(writer.buffer))
ch = asciilib_utf16_decode(&q, e,
- PyUnicode_1BYTE_DATA(unicode), &outpos,
+ (Py_UCS1*)writer.data, &writer.pos,
native_ordering);
else
ch = ucs1lib_utf16_decode(&q, e,
- PyUnicode_1BYTE_DATA(unicode), &outpos,
+ (Py_UCS1*)writer.data, &writer.pos,
native_ordering);
} else if (kind == PyUnicode_2BYTE_KIND) {
ch = ucs2lib_utf16_decode(&q, e,
- PyUnicode_2BYTE_DATA(unicode), &outpos,
+ (Py_UCS2*)writer.data, &writer.pos,
native_ordering);
} else {
assert(kind == PyUnicode_4BYTE_KIND);
ch = ucs4lib_utf16_decode(&q, e,
- PyUnicode_4BYTE_DATA(unicode), &outpos,
+ (Py_UCS4*)writer.data, &writer.pos,
native_ordering);
}
}
@@ -5309,12 +5182,14 @@ PyUnicode_DecodeUTF16Stateful(const char *s,
endinpos = startinpos + 2;
break;
default:
- if (unicode_putchar(&unicode, &outpos, ch) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch);
+ writer.pos++;
continue;
}
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors,
&errorHandler,
"utf16", errmsg,
@@ -5324,8 +5199,7 @@ PyUnicode_DecodeUTF16Stateful(const char *s,
&endinpos,
&exc,
(const char **)&q,
- &unicode,
- &outpos))
+ &writer))
goto onError;
}
@@ -5333,16 +5207,12 @@ End:
if (consumed)
*consumed = (const char *)q-starts;
- /* Adjust length */
- if (unicode_resize(&unicode, outpos) < 0)
- goto onError;
-
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- return unicode_result(unicode);
+ return _PyUnicodeWriter_Finish(&writer);
onError:
- Py_DECREF(unicode);
+ _PyUnicodeWriter_Dealloc(&writer);
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return NULL;
@@ -5360,7 +5230,7 @@ _PyUnicode_EncodeUTF16(PyObject *str,
unsigned short *out;
Py_ssize_t bytesize;
Py_ssize_t pairs;
-#ifdef WORDS_BIGENDIAN
+#if PY_BIG_ENDIAN
int native_ordering = byteorder >= 0;
#else
int native_ordering = byteorder <= 0;
@@ -5509,27 +5379,28 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
Py_ssize_t startinpos;
Py_ssize_t endinpos;
int j;
- PyObject *v;
+ _PyUnicodeWriter writer;
const char *end;
char* message;
Py_UCS4 chr = 0xffffffff; /* in case 'getcode' messes up */
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
Py_ssize_t len;
- Py_ssize_t i;
len = length_of_escaped_ascii_string(s, size);
+ if (len == 0)
+ _Py_RETURN_UNICODE_EMPTY();
/* After length_of_escaped_ascii_string() there are two alternatives,
either the string is pure ASCII with named escapes like \n, etc.
and we determined it's exact size (common case)
or it contains \x, \u, ... escape sequences. then we create a
legacy wchar string and resize it at the end of this function. */
- if (len >= 0) {
- v = PyUnicode_New(len, 127);
- if (!v)
+ _PyUnicodeWriter_Init(&writer, 0);
+ if (len > 0) {
+ if (_PyUnicodeWriter_Prepare(&writer, len, 127) == -1)
goto onError;
- assert(PyUnicode_KIND(v) == PyUnicode_1BYTE_KIND);
+ assert(writer.kind == PyUnicode_1BYTE_KIND);
}
else {
/* Escaped strings will always be longer than the resulting
@@ -5537,15 +5408,12 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
length after conversion to the true value.
(but if the error callback returns a long replacement string
we'll have to allocate more space) */
- v = PyUnicode_New(size, 127);
- if (!v)
+ if (_PyUnicodeWriter_Prepare(&writer, size, 127) == -1)
goto onError;
- len = size;
}
if (size == 0)
- return v;
- i = 0;
+ return _PyUnicodeWriter_Finish(&writer);
end = s + size;
while (s < end) {
@@ -5553,14 +5421,14 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
Py_UCS4 x;
int digits;
- /* The only case in which i == ascii_length is a backslash
- followed by a newline. */
- assert(i <= len);
-
/* Non-escape characters are interpreted as Unicode ordinals */
if (*s != '\\') {
- if (unicode_putchar(&v, &i, (unsigned char) *s++) < 0)
+ x = (unsigned char)*s;
+ s++;
+ if (_PyUnicodeWriter_Prepare(&writer, 1, x) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, x);
+ writer.pos++;
continue;
}
@@ -5573,16 +5441,18 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
/* The only case in which i == ascii_length is a backslash
followed by a newline. */
- assert(i < len || (i == len && c == '\n'));
+ assert(writer.pos < writer.size || (writer.pos == writer.size && c == '\n'));
switch (c) {
/* \x escapes */
-#define WRITECHAR(ch) \
- do { \
- if (unicode_putchar(&v, &i, ch) < 0) \
- goto onError; \
- }while(0)
+#define WRITECHAR(ch) \
+ do { \
+ if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1) \
+ goto onError; \
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch); \
+ writer.pos++; \
+ } while(0)
case '\n': break;
case '\\': WRITECHAR('\\'); break;
@@ -5632,11 +5502,11 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
chr = 0;
if (s+digits>end) {
endinpos = size;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"unicodeescape", "end of string in escape sequence",
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
+ &writer))
goto onError;
goto nextByte;
}
@@ -5644,13 +5514,12 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
c = (unsigned char) s[j];
if (!Py_ISXDIGIT(c)) {
endinpos = (s+j+1)-starts;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"unicodeescape", message,
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
+ &writer))
goto onError;
- len = PyUnicode_GET_LENGTH(v);
goto nextByte;
}
chr = (chr<<4) & ~0xF;
@@ -5672,11 +5541,11 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
WRITECHAR(chr);
} else {
endinpos = s-starts;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"unicodeescape", "illegal Unicode character",
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
+ &writer))
goto onError;
}
break;
@@ -5707,11 +5576,11 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
}
}
endinpos = s-starts;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"unicodeescape", message,
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
+ &writer))
goto onError;
break;
@@ -5720,11 +5589,11 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
message = "\\ at end of string";
s--;
endinpos = s-starts;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"unicodeescape", message,
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &i))
+ &writer))
goto onError;
}
else {
@@ -5738,24 +5607,22 @@ PyUnicode_DecodeUnicodeEscape(const char *s,
}
#undef WRITECHAR
- if (unicode_resize(&v, i) < 0)
- goto onError;
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- return unicode_result(v);
+ return _PyUnicodeWriter_Finish(&writer);
ucnhashError:
PyErr_SetString(
PyExc_UnicodeError,
"\\N escapes not supported (can't load unicodedata module)"
);
- Py_XDECREF(v);
+ _PyUnicodeWriter_Dealloc(&writer);
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return NULL;
onError:
- Py_XDECREF(v);
+ _PyUnicodeWriter_Dealloc(&writer);
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return NULL;
@@ -5908,23 +5775,23 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s,
const char *starts = s;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
- Py_ssize_t outpos;
- PyObject *v;
+ _PyUnicodeWriter writer;
const char *end;
const char *bs;
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
+ if (size == 0)
+ _Py_RETURN_UNICODE_EMPTY();
+
/* Escaped strings will always be longer than the resulting
Unicode string, so we start with size here and then reduce the
length after conversion to the true value. (But decoding error
handler might have to resize the string) */
- v = PyUnicode_New(size, 127);
- if (v == NULL)
+ _PyUnicodeWriter_Init(&writer, 1);
+ if (_PyUnicodeWriter_Prepare(&writer, size, 127) == -1)
goto onError;
- if (size == 0)
- return v;
- outpos = 0;
+
end = s + size;
while (s < end) {
unsigned char c;
@@ -5934,8 +5801,11 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s,
/* Non-escape characters are interpreted as Unicode ordinals */
if (*s != '\\') {
- if (unicode_putchar(&v, &outpos, (unsigned char)*s++) < 0)
+ x = (unsigned char)*s++;
+ if (_PyUnicodeWriter_Prepare(&writer, 1, x) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, x);
+ writer.pos++;
continue;
}
startinpos = s-starts;
@@ -5946,15 +5816,18 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s,
for (;s < end;) {
if (*s != '\\')
break;
- if (unicode_putchar(&v, &outpos, (unsigned char)*s++) < 0)
+ x = (unsigned char)*s++;
+ if (_PyUnicodeWriter_Prepare(&writer, 1, x) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, x);
+ writer.pos++;
}
if (((s - bs) & 1) == 0 ||
s >= end ||
(*s != 'u' && *s != 'U')) {
continue;
}
- outpos--;
+ writer.pos--;
count = *s=='u' ? 4 : 8;
s++;
@@ -5963,11 +5836,11 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s,
c = (unsigned char)*s;
if (!Py_ISXDIGIT(c)) {
endinpos = s-starts;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"rawunicodeescape", "truncated \\uXXXX",
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &outpos))
+ &writer))
goto onError;
goto nextByte;
}
@@ -5980,28 +5853,29 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s,
x += 10 + c - 'A';
}
if (x <= MAX_UNICODE) {
- if (unicode_putchar(&v, &outpos, x) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, x) == -1)
goto onError;
- } else {
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, x);
+ writer.pos++;
+ }
+ else {
endinpos = s-starts;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"rawunicodeescape", "\\Uxxxxxxxx out of range",
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &outpos))
+ &writer))
goto onError;
}
nextByte:
;
}
- if (unicode_resize(&v, outpos) < 0)
- goto onError;
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- return unicode_result(v);
+ return _PyUnicodeWriter_Finish(&writer);
onError:
- Py_XDECREF(v);
+ _PyUnicodeWriter_Dealloc(&writer);
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return NULL;
@@ -6101,8 +5975,7 @@ _PyUnicode_DecodeUnicodeInternal(const char *s,
const char *starts = s;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
- Py_ssize_t outpos;
- PyObject *v;
+ _PyUnicodeWriter writer;
const char *end;
const char *reason;
PyObject *errorHandler = NULL;
@@ -6113,13 +5986,13 @@ _PyUnicode_DecodeUnicodeInternal(const char *s,
1))
return NULL;
+ if (size == 0)
+ _Py_RETURN_UNICODE_EMPTY();
+
/* XXX overflow detection missing */
- v = PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE, 127);
- if (v == NULL)
+ _PyUnicodeWriter_Init(&writer, 0);
+ if (_PyUnicodeWriter_Prepare(&writer, (size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE, 127) == -1)
goto onError;
- if (PyUnicode_GET_LENGTH(v) == 0)
- return v;
- outpos = 0;
end = s + size;
while (s < end) {
@@ -6153,11 +6026,11 @@ _PyUnicode_DecodeUnicodeInternal(const char *s,
endinpos = s - starts + Py_UNICODE_SIZE;
reason = "illegal code point (> 0x10FFFF)";
}
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"unicode_internal", reason,
&starts, &end, &startinpos, &endinpos, &exc, &s,
- &v, &outpos))
+ &writer))
goto onError;
continue;
}
@@ -6177,18 +6050,18 @@ _PyUnicode_DecodeUnicodeInternal(const char *s,
}
#endif
- if (unicode_putchar(&v, &outpos, ch) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch);
+ writer.pos++;
}
- if (unicode_resize(&v, outpos) < 0)
- goto onError;
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- return unicode_result(v);
+ return _PyUnicodeWriter_Finish(&writer);
onError:
- Py_XDECREF(v);
+ _PyUnicodeWriter_Dealloc(&writer);
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return NULL;
@@ -6552,7 +6425,7 @@ PyUnicode_DecodeASCII(const char *s,
const char *errors)
{
const char *starts = s;
- PyObject *unicode;
+ _PyUnicodeWriter writer;
int kind;
void *data;
Py_ssize_t startinpos;
@@ -6569,46 +6442,45 @@ PyUnicode_DecodeASCII(const char *s,
if (size == 1 && (unsigned char)s[0] < 128)
return get_latin1_char((unsigned char)s[0]);
- unicode = PyUnicode_New(size, 127);
- if (unicode == NULL)
+ _PyUnicodeWriter_Init(&writer, 0);
+ if (_PyUnicodeWriter_Prepare(&writer, size, 127) == -1)
goto onError;
e = s + size;
- data = PyUnicode_1BYTE_DATA(unicode);
+ data = writer.data;
outpos = ascii_decode(s, e, (Py_UCS1 *)data);
- if (outpos == size)
- return unicode;
+ writer.pos = outpos;
+ if (writer.pos == size)
+ return _PyUnicodeWriter_Finish(&writer);
- s += outpos;
- kind = PyUnicode_1BYTE_KIND;
+ s += writer.pos;
+ kind = writer.kind;
while (s < e) {
register unsigned char c = (unsigned char)*s;
if (c < 128) {
- PyUnicode_WRITE(kind, data, outpos++, c);
+ PyUnicode_WRITE(kind, data, writer.pos, c);
+ writer.pos++;
++s;
}
else {
startinpos = s-starts;
endinpos = startinpos + 1;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"ascii", "ordinal not in range(128)",
&starts, &e, &startinpos, &endinpos, &exc, &s,
- &unicode, &outpos))
+ &writer))
goto onError;
- kind = PyUnicode_KIND(unicode);
- data = PyUnicode_DATA(unicode);
+ kind = writer.kind;
+ data = writer.data;
}
}
- if (unicode_resize(&unicode, outpos) < 0)
- goto onError;
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- assert(_PyUnicode_CheckConsistency(unicode, 1));
- return unicode;
+ return _PyUnicodeWriter_Finish(&writer);
onError:
- Py_XDECREF(unicode);
+ _PyUnicodeWriter_Dealloc(&writer);
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return NULL;
@@ -6718,8 +6590,8 @@ decode_code_page_flags(UINT code_page)
* Decode a byte string from a Windows code page into unicode object in strict
* mode.
*
- * Returns consumed size if succeed, returns -2 on decode error, or raise a
- * WindowsError and returns -1 on other error.
+ * Returns consumed size if succeed, returns -2 on decode error, or raise an
+ * OSError and returns -1 on other error.
*/
static int
decode_code_page_strict(UINT code_page,
@@ -6770,7 +6642,7 @@ error:
* Decode a byte string from a code page into unicode object with an error
* handler.
*
- * Returns consumed size if succeed, or raise a WindowsError or
+ * Returns consumed size if succeed, or raise an OSError or
* UnicodeDecodeError exception and returns -1 on error.
*/
static int
@@ -6869,7 +6741,7 @@ decode_code_page_errors(UINT code_page,
startinpos = in - startin;
endinpos = startinpos + 1;
outpos = out - PyUnicode_AS_UNICODE(*v);
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_wchar(
errors, &errorHandler,
encoding, reason,
&startin, &endin, &startinpos, &endinpos, &exc, &in,
@@ -7025,7 +6897,7 @@ encode_code_page_flags(UINT code_page, const char *errors)
* mode.
*
* Returns consumed characters if succeed, returns -2 on encode error, or raise
- * a WindowsError and returns -1 on other error.
+ * an OSError and returns -1 on other error.
*/
static int
encode_code_page_strict(UINT code_page, PyObject **outbytes,
@@ -7121,7 +6993,7 @@ error:
* Encode a Unicode string to a Windows code page into a byte string using a
* error handler.
*
- * Returns consumed characters if succeed, or raise a WindowsError and returns
+ * Returns consumed characters if succeed, or raise an OSError and returns
* -1 on other error.
*/
static int
@@ -7207,9 +7079,8 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes,
charsize = 1;
}
else {
- ch -= 0x10000;
- chars[0] = 0xd800 + (ch >> 10);
- chars[1] = 0xdc00 + (ch & 0x3ff);
+ chars[0] = Py_UNICODE_HIGH_SURROGATE(ch);
+ chars[1] = Py_UNICODE_LOW_SURROGATE(ch);
charsize = 2;
}
@@ -7411,10 +7282,8 @@ PyUnicode_DecodeCharmap(const char *s,
const char *starts = s;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
- Py_ssize_t outpos;
const char *e;
- PyObject *v;
- Py_ssize_t extrachars = 0;
+ _PyUnicodeWriter writer;
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
@@ -7422,12 +7291,12 @@ PyUnicode_DecodeCharmap(const char *s,
if (mapping == NULL)
return PyUnicode_DecodeLatin1(s, size, errors);
- v = PyUnicode_New(size, 127);
- if (v == NULL)
- goto onError;
if (size == 0)
- return v;
- outpos = 0;
+ _Py_RETURN_UNICODE_EMPTY();
+ _PyUnicodeWriter_Init(&writer, 0);
+ if (_PyUnicodeWriter_Prepare(&writer, size, 127) == -1)
+ goto onError;
+
e = s + size;
if (PyUnicode_CheckExact(mapping)) {
Py_ssize_t maplen;
@@ -7444,28 +7313,29 @@ PyUnicode_DecodeCharmap(const char *s,
while (s < e) {
unsigned char ch;
if (mapkind == PyUnicode_2BYTE_KIND && maplen >= 256) {
- enum PyUnicode_Kind outkind = PyUnicode_KIND(v);
+ enum PyUnicode_Kind outkind = writer.kind;
+ void *outdata = writer.data;
if (outkind == PyUnicode_1BYTE_KIND) {
- void *outdata = PyUnicode_DATA(v);
- Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(v);
+ Py_UCS4 maxchar = writer.maxchar;
while (s < e) {
unsigned char ch = *s;
x = PyUnicode_READ(PyUnicode_2BYTE_KIND, mapdata, ch);
if (x > maxchar)
goto Error;
- PyUnicode_WRITE(PyUnicode_1BYTE_KIND, outdata, outpos++, x);
+ PyUnicode_WRITE(PyUnicode_1BYTE_KIND, outdata, writer.pos, x);
+ writer.pos++;
++s;
}
break;
}
else if (outkind == PyUnicode_2BYTE_KIND) {
- void *outdata = PyUnicode_DATA(v);
while (s < e) {
unsigned char ch = *s;
x = PyUnicode_READ(PyUnicode_2BYTE_KIND, mapdata, ch);
if (x == 0xFFFE)
goto Error;
- PyUnicode_WRITE(PyUnicode_2BYTE_KIND, outdata, outpos++, x);
+ PyUnicode_WRITE(PyUnicode_2BYTE_KIND, outdata, writer.pos, x);
+ writer.pos++;
++s;
}
break;
@@ -7483,18 +7353,20 @@ Error:
/* undefined mapping */
startinpos = s-starts;
endinpos = startinpos+1;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"charmap", "character maps to <undefined>",
&starts, &e, &startinpos, &endinpos, &exc, &s,
- &v, &outpos)) {
+ &writer)) {
goto onError;
}
continue;
}
- if (unicode_putchar(&v, &outpos, x) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, x) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, x);
+ writer.pos++;
++s;
}
}
@@ -7532,46 +7404,29 @@ Error:
Py_DECREF(x);
goto onError;
}
- if (unicode_putchar(&v, &outpos, value) < 0)
+
+ if (_PyUnicodeWriter_Prepare(&writer, 1, value) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, value);
+ writer.pos++;
}
else if (PyUnicode_Check(x)) {
- Py_ssize_t targetsize;
-
if (PyUnicode_READY(x) == -1)
goto onError;
- targetsize = PyUnicode_GET_LENGTH(x);
-
- if (targetsize == 1) {
- /* 1-1 mapping */
+ if (PyUnicode_GET_LENGTH(x) == 1) {
Py_UCS4 value = PyUnicode_READ_CHAR(x, 0);
if (value == 0xFFFE)
goto Undefined;
- if (unicode_putchar(&v, &outpos, value) < 0)
+ if (_PyUnicodeWriter_Prepare(&writer, 1, value) == -1)
goto onError;
+ PyUnicode_WRITE(writer.kind, writer.data, writer.pos, value);
+ writer.pos++;
}
- else if (targetsize > 1) {
- /* 1-n mapping */
- if (targetsize > extrachars) {
- /* resize first */
- Py_ssize_t needed = (targetsize - extrachars) + \
- (targetsize << 2);
- extrachars += needed;
- /* XXX overflow detection missing */
- if (unicode_resize(&v,
- PyUnicode_GET_LENGTH(v) + needed) < 0)
- {
- Py_DECREF(x);
- goto onError;
- }
- }
- if (unicode_widen(&v, outpos, PyUnicode_MAX_CHAR_VALUE(x)) < 0)
+ else {
+ writer.overallocate = 1;
+ if (_PyUnicodeWriter_WriteStr(&writer, x) == -1)
goto onError;
- PyUnicode_CopyCharacters(v, outpos, x, 0, targetsize);
- outpos += targetsize;
- extrachars -= targetsize;
}
- /* 1-0 mapping: skip the character */
}
else {
/* wrong return value */
@@ -7588,25 +7443,23 @@ Undefined:
Py_XDECREF(x);
startinpos = s-starts;
endinpos = startinpos+1;
- if (unicode_decode_call_errorhandler(
+ if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"charmap", "character maps to <undefined>",
&starts, &e, &startinpos, &endinpos, &exc, &s,
- &v, &outpos)) {
+ &writer)) {
goto onError;
}
}
}
- if (unicode_resize(&v, outpos) < 0)
- goto onError;
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- return unicode_result(v);
+ return _PyUnicodeWriter_Finish(&writer);
onError:
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
- Py_XDECREF(v);
+ _PyUnicodeWriter_Dealloc(&writer);
return NULL;
}
@@ -8224,19 +8077,6 @@ make_translate_exception(PyObject **exceptionObject,
}
}
-/* raises a UnicodeTranslateError */
-static void
-raise_translate_exception(PyObject **exceptionObject,
- PyObject *unicode,
- Py_ssize_t startpos, Py_ssize_t endpos,
- const char *reason)
-{
- make_translate_exception(exceptionObject,
- unicode, startpos, endpos, reason);
- if (*exceptionObject != NULL)
- PyCodec_StrictErrors(*exceptionObject);
-}
-
/* error handling callback helper:
build arguments, call the callback and check the arguments,
put the result into newpos and return the replacement string, which
@@ -8512,8 +8352,10 @@ _PyUnicode_TranslateCharmap(PyObject *input,
}
switch (known_errorHandler) {
case 1: /* strict */
- raise_translate_exception(&exc, input, collstart,
- collend, reason);
+ make_translate_exception(&exc,
+ input, collstart, collend, reason);
+ if (exc != NULL)
+ PyCodec_StrictErrors(exc);
goto onError;
case 2: /* replace */
/* No need to check for space, this is a 1:1 replacement */
@@ -9105,7 +8947,7 @@ tailmatch(PyObject *self,
if (PyUnicode_READY(self) == -1 ||
PyUnicode_READY(substring) == -1)
- return 0;
+ return -1;
if (PyUnicode_GET_LENGTH(substring) == 0)
return 1;
@@ -9143,7 +8985,6 @@ tailmatch(PyObject *self,
/* We do not need to compare 0 and len(substring)-1 because
the if statement above ensured already that they are equal
when we end up here. */
- /* TODO: honor direction and do a forward or backwards search */
for (i = 1; i < end_sub; ++i) {
if (PyUnicode_READ(kind_self, data_self, offset + i) !=
PyUnicode_READ(kind_sub, data_sub, i))
@@ -10416,7 +10257,12 @@ unicode_compare(PyObject *str1, PyObject *str2)
{
int kind1, kind2;
void *data1, *data2;
- Py_ssize_t len1, len2, i;
+ Py_ssize_t len1, len2;
+ Py_ssize_t i, len;
+
+ /* a string is equal to itself */
+ if (str1 == str2)
+ return 0;
kind1 = PyUnicode_KIND(str1);
kind2 = PyUnicode_KIND(str2);
@@ -10424,19 +10270,61 @@ unicode_compare(PyObject *str1, PyObject *str2)
data2 = PyUnicode_DATA(str2);
len1 = PyUnicode_GET_LENGTH(str1);
len2 = PyUnicode_GET_LENGTH(str2);
+ len = Py_MIN(len1, len2);
- for (i = 0; i < len1 && i < len2; ++i) {
- Py_UCS4 c1, c2;
- c1 = PyUnicode_READ(kind1, data1, i);
- c2 = PyUnicode_READ(kind2, data2, i);
+ if (kind1 == 1 && kind2 == 1) {
+ int cmp = memcmp(data1, data2, len);
+ /* normalize result of memcmp() into the range [-1; 1] */
+ if (cmp < 0)
+ return -1;
+ if (cmp > 0)
+ return 1;
+ }
+ else {
+ for (i = 0; i < len; ++i) {
+ Py_UCS4 c1, c2;
+ c1 = PyUnicode_READ(kind1, data1, i);
+ c2 = PyUnicode_READ(kind2, data2, i);
- if (c1 != c2)
- return (c1 < c2) ? -1 : 1;
+ if (c1 != c2)
+ return (c1 < c2) ? -1 : 1;
+ }
}
- return (len1 < len2) ? -1 : (len1 != len2);
+ if (len1 == len2)
+ return 0;
+ if (len1 < len2)
+ return -1;
+ else
+ return 1;
}
+static int
+unicode_compare_eq(PyObject *str1, PyObject *str2)
+{
+ int kind;
+ void *data1, *data2;
+ Py_ssize_t len;
+ int cmp;
+
+ /* a string is equal to itself */
+ if (str1 == str2)
+ return 1;
+
+ len = PyUnicode_GET_LENGTH(str1);
+ if (PyUnicode_GET_LENGTH(str2) != len)
+ return 0;
+ kind = PyUnicode_KIND(str1);
+ if (PyUnicode_KIND(str2) != kind)
+ return 0;
+ data1 = PyUnicode_DATA(str1);
+ data2 = PyUnicode_DATA(str2);
+
+ cmp = memcmp(data1, data2, len * kind);
+ return (cmp == 0);
+}
+
+
int
PyUnicode_Compare(PyObject *left, PyObject *right)
{
@@ -10487,36 +10375,27 @@ PyObject *
PyUnicode_RichCompare(PyObject *left, PyObject *right, int op)
{
int result;
+ PyObject *v;
- if (PyUnicode_Check(left) && PyUnicode_Check(right)) {
- PyObject *v;
- if (PyUnicode_READY(left) == -1 ||
- PyUnicode_READY(right) == -1)
- return NULL;
- if (PyUnicode_GET_LENGTH(left) != PyUnicode_GET_LENGTH(right) ||
- PyUnicode_KIND(left) != PyUnicode_KIND(right)) {
- if (op == Py_EQ) {
- Py_INCREF(Py_False);
- return Py_False;
- }
- if (op == Py_NE) {
- Py_INCREF(Py_True);
- return Py_True;
- }
- }
- if (left == right)
- result = 0;
+ if (!PyUnicode_Check(left) || !PyUnicode_Check(right))
+ Py_RETURN_NOTIMPLEMENTED;
+
+ if (PyUnicode_READY(left) == -1 ||
+ PyUnicode_READY(right) == -1)
+ return NULL;
+
+ if (op == Py_EQ || op == Py_NE) {
+ result = unicode_compare_eq(left, right);
+ if (op == Py_EQ)
+ v = TEST_COND(result);
else
- result = unicode_compare(left, right);
+ v = TEST_COND(!result);
+ }
+ else {
+ result = unicode_compare(left, right);
/* Convert the return value to a Boolean */
switch (op) {
- case Py_EQ:
- v = TEST_COND(result == 0);
- break;
- case Py_NE:
- v = TEST_COND(result != 0);
- break;
case Py_LE:
v = TEST_COND(result <= 0);
break;
@@ -10533,11 +10412,9 @@ PyUnicode_RichCompare(PyObject *left, PyObject *right, int op)
PyErr_BadArgument();
return NULL;
}
- Py_INCREF(v);
- return v;
}
-
- Py_RETURN_NOTIMPLEMENTED;
+ Py_INCREF(v);
+ return v;
}
int
@@ -12727,6 +12604,8 @@ unicode_startswith(PyObject *self,
return NULL;
result = tailmatch(self, substring, start, end, -1);
Py_DECREF(substring);
+ if (result == -1)
+ return NULL;
if (result) {
Py_RETURN_TRUE;
}
@@ -12743,6 +12622,8 @@ unicode_startswith(PyObject *self,
}
result = tailmatch(self, substring, start, end, -1);
Py_DECREF(substring);
+ if (result == -1)
+ return NULL;
return PyBool_FromLong(result);
}
@@ -12776,6 +12657,8 @@ unicode_endswith(PyObject *self,
return NULL;
result = tailmatch(self, substring, start, end, +1);
Py_DECREF(substring);
+ if (result == -1)
+ return NULL;
if (result) {
Py_RETURN_TRUE;
}
@@ -12790,6 +12673,8 @@ unicode_endswith(PyObject *self,
return NULL;
}
result = tailmatch(self, substring, start, end, +1);
+ if (result == -1)
+ return NULL;
Py_DECREF(substring);
return PyBool_FromLong(result);
}
@@ -12916,6 +12801,19 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str)
return 0;
}
+int
+_PyUnicodeWriter_WriteCstr(_PyUnicodeWriter *writer, const char *str, Py_ssize_t len)
+{
+ Py_UCS4 maxchar;
+
+ maxchar = ucs1lib_find_max_char((Py_UCS1*)str, (Py_UCS1*)str + len);
+ if (_PyUnicodeWriter_Prepare(writer, len, maxchar) == -1)
+ return -1;
+ unicode_write_cstr(writer->buffer, writer->pos, str, len);
+ writer->pos += len;
+ return 0;
+}
+
PyObject *
_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer)
{
@@ -13191,16 +13089,39 @@ static PyMappingMethods unicode_as_mapping = {
/* Helpers for PyUnicode_Format() */
+struct unicode_formatter_t {
+ PyObject *args;
+ int args_owned;
+ Py_ssize_t arglen, argidx;
+ PyObject *dict;
+
+ enum PyUnicode_Kind fmtkind;
+ Py_ssize_t fmtcnt, fmtpos;
+ void *fmtdata;
+ PyObject *fmtstr;
+
+ _PyUnicodeWriter writer;
+};
+
+struct unicode_format_arg_t {
+ Py_UCS4 ch;
+ int flags;
+ Py_ssize_t width;
+ int prec;
+ int sign;
+};
+
static PyObject *
-getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx)
+unicode_format_getnextarg(struct unicode_formatter_t *ctx)
{
- Py_ssize_t argidx = *p_argidx;
- if (argidx < arglen) {
- (*p_argidx)++;
- if (arglen < 0)
- return args;
+ Py_ssize_t argidx = ctx->argidx;
+
+ if (argidx < ctx->arglen) {
+ ctx->argidx++;
+ if (ctx->arglen < 0)
+ return ctx->args;
else
- return PyTuple_GetItem(args, argidx);
+ return PyTuple_GetItem(ctx->args, argidx);
}
PyErr_SetString(PyExc_TypeError,
"not enough arguments for format string");
@@ -13209,23 +13130,34 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx)
/* Returns a new reference to a PyUnicode object, or NULL on failure. */
+/* Format a float into the writer if the writer is not NULL, or into *p_output
+ otherwise.
+
+ Return 0 on success, raise an exception and return -1 on error. */
static int
-formatfloat(PyObject *v, int flags, int prec, int type,
- PyObject **p_output, _PyUnicodeWriter *writer)
+formatfloat(PyObject *v, struct unicode_format_arg_t *arg,
+ PyObject **p_output,
+ _PyUnicodeWriter *writer)
{
char *p;
double x;
Py_ssize_t len;
+ int prec;
+ int dtoa_flags;
x = PyFloat_AsDouble(v);
if (x == -1.0 && PyErr_Occurred())
return -1;
+ prec = arg->prec;
if (prec < 0)
prec = 6;
- p = PyOS_double_to_string(x, type, prec,
- (flags & F_ALT) ? Py_DTSF_ALT : 0, NULL);
+ if (arg->flags & F_ALT)
+ dtoa_flags = Py_DTSF_ALT;
+ else
+ dtoa_flags = 0;
+ p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL);
if (p == NULL)
return -1;
len = strlen(p);
@@ -13262,7 +13194,7 @@ formatfloat(PyObject *v, int flags, int prec, int type,
* produce a '-' sign, but can for Python's unbounded ints.
*/
static PyObject*
-formatlong(PyObject *val, int flags, int prec, int type)
+formatlong(PyObject *val, struct unicode_format_arg_t *arg)
{
PyObject *result = NULL;
char *buf;
@@ -13272,6 +13204,8 @@ formatlong(PyObject *val, int flags, int prec, int type)
Py_ssize_t llen;
int numdigits; /* len == numnondigits + numdigits */
int numnondigits = 0;
+ int prec = arg->prec;
+ int type = arg->ch;
/* Avoid exceeding SSIZE_T_MAX */
if (prec > INT_MAX-3) {
@@ -13283,7 +13217,10 @@ formatlong(PyObject *val, int flags, int prec, int type)
assert(PyLong_Check(val));
switch (type) {
+ default:
+ assert(!"'type' not in [diuoxX]");
case 'd':
+ case 'i':
case 'u':
/* Special-case boolean: we want 0/1 */
if (PyBool_Check(val))
@@ -13300,8 +13237,6 @@ formatlong(PyObject *val, int flags, int prec, int type)
numnondigits = 2;
result = PyNumber_ToBase(val, 16);
break;
- default:
- assert(!"'type' not in [duoxX]");
}
if (!result)
return NULL;
@@ -13329,7 +13264,7 @@ formatlong(PyObject *val, int flags, int prec, int type)
assert(numdigits > 0);
/* Get rid of base marker unless F_ALT */
- if (((flags & F_ALT) == 0 &&
+ if (((arg->flags & F_ALT) == 0 &&
(type == 'o' || type == 'x' || type == 'X'))) {
assert(buf[sign] == '0');
assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' ||
@@ -13374,15 +13309,100 @@ formatlong(PyObject *val, int flags, int prec, int type)
if (buf[i] >= 'a' && buf[i] <= 'x')
buf[i] -= 'a'-'A';
}
- if (!PyUnicode_Check(result) || len != PyUnicode_GET_LENGTH(result)) {
+ if (!PyUnicode_Check(result)
+ || buf != PyUnicode_DATA(result)) {
PyObject *unicode;
unicode = _PyUnicode_FromASCII(buf, len);
Py_DECREF(result);
result = unicode;
}
+ else if (len != PyUnicode_GET_LENGTH(result)) {
+ if (PyUnicode_Resize(&result, len) < 0)
+ Py_CLEAR(result);
+ }
return result;
}
+/* Format an integer.
+ * Return 1 if the number has been formatted into the writer,
+ * 0 if the number has been formatted into *p_output
+ * -1 and raise an exception on error */
+static int
+mainformatlong(PyObject *v,
+ struct unicode_format_arg_t *arg,
+ PyObject **p_output,
+ _PyUnicodeWriter *writer)
+{
+ PyObject *iobj, *res;
+ char type = (char)arg->ch;
+
+ if (!PyNumber_Check(v))
+ goto wrongtype;
+
+ if (!PyLong_Check(v)) {
+ iobj = PyNumber_Long(v);
+ if (iobj == NULL) {
+ if (PyErr_ExceptionMatches(PyExc_TypeError))
+ goto wrongtype;
+ return -1;
+ }
+ assert(PyLong_Check(iobj));
+ }
+ else {
+ iobj = v;
+ Py_INCREF(iobj);
+ }
+
+ if (PyLong_CheckExact(v)
+ && arg->width == -1 && arg->prec == -1
+ && !(arg->flags & (F_SIGN | F_BLANK))
+ && type != 'X')
+ {
+ /* Fast path */
+ int alternate = arg->flags & F_ALT;
+ int base;
+
+ switch(type)
+ {
+ default:
+ assert(0 && "'type' not in [diuoxX]");
+ case 'd':
+ case 'i':
+ case 'u':
+ base = 10;
+ break;
+ case 'o':
+ base = 8;
+ break;
+ case 'x':
+ case 'X':
+ base = 16;
+ break;
+ }
+
+ if (_PyLong_FormatWriter(writer, v, base, alternate) == -1) {
+ Py_DECREF(iobj);
+ return -1;
+ }
+ Py_DECREF(iobj);
+ return 1;
+ }
+
+ res = formatlong(iobj, arg);
+ Py_DECREF(iobj);
+ if (res == NULL)
+ return -1;
+ *p_output = res;
+ return 0;
+
+wrongtype:
+ PyErr_Format(PyExc_TypeError,
+ "%%%c format: a number is required, "
+ "not %.200s",
+ type, Py_TYPE(v)->tp_name);
+ return -1;
+}
+
static Py_UCS4
formatchar(PyObject *v)
{
@@ -13415,540 +13435,592 @@ formatchar(PyObject *v)
return (Py_UCS4) -1;
}
-PyObject *
-PyUnicode_Format(PyObject *format, PyObject *args)
-{
- Py_ssize_t fmtcnt, fmtpos, arglen, argidx;
- int args_owned = 0;
- PyObject *dict = NULL;
- PyObject *temp = NULL;
- PyObject *second = NULL;
- PyObject *uformat;
- void *fmt;
- enum PyUnicode_Kind kind, fmtkind;
- _PyUnicodeWriter writer;
- Py_ssize_t sublen;
- Py_UCS4 maxchar;
+/* Parse options of an argument: flags, width, precision.
+ Handle also "%(name)" syntax.
- if (format == NULL || args == NULL) {
- PyErr_BadInternalCall();
- return NULL;
- }
- uformat = PyUnicode_FromObject(format);
- if (uformat == NULL)
- return NULL;
- if (PyUnicode_READY(uformat) == -1) {
- Py_DECREF(uformat);
- return NULL;
- }
+ Return 0 if the argument has been formatted into arg->str.
+ Return 1 if the argument has been written into ctx->writer,
+ Raise an exception and return -1 on error. */
+static int
+unicode_format_arg_parse(struct unicode_formatter_t *ctx,
+ struct unicode_format_arg_t *arg)
+{
+#define FORMAT_READ(ctx) \
+ PyUnicode_READ((ctx)->fmtkind, (ctx)->fmtdata, (ctx)->fmtpos)
- fmt = PyUnicode_DATA(uformat);
- fmtkind = PyUnicode_KIND(uformat);
- fmtcnt = PyUnicode_GET_LENGTH(uformat);
- fmtpos = 0;
+ PyObject *v;
- _PyUnicodeWriter_Init(&writer, fmtcnt + 100);
+ if (arg->ch == '(') {
+ /* Get argument value from a dictionary. Example: "%(name)s". */
+ Py_ssize_t keystart;
+ Py_ssize_t keylen;
+ PyObject *key;
+ int pcount = 1;
- if (PyTuple_Check(args)) {
- arglen = PyTuple_Size(args);
- argidx = 0;
- }
- else {
- arglen = -1;
- argidx = -2;
+ if (ctx->dict == NULL) {
+ PyErr_SetString(PyExc_TypeError,
+ "format requires a mapping");
+ return -1;
+ }
+ ++ctx->fmtpos;
+ --ctx->fmtcnt;
+ keystart = ctx->fmtpos;
+ /* Skip over balanced parentheses */
+ while (pcount > 0 && --ctx->fmtcnt >= 0) {
+ arg->ch = FORMAT_READ(ctx);
+ if (arg->ch == ')')
+ --pcount;
+ else if (arg->ch == '(')
+ ++pcount;
+ ctx->fmtpos++;
+ }
+ keylen = ctx->fmtpos - keystart - 1;
+ if (ctx->fmtcnt < 0 || pcount > 0) {
+ PyErr_SetString(PyExc_ValueError,
+ "incomplete format key");
+ return -1;
+ }
+ key = PyUnicode_Substring(ctx->fmtstr,
+ keystart, keystart + keylen);
+ if (key == NULL)
+ return -1;
+ if (ctx->args_owned) {
+ Py_DECREF(ctx->args);
+ ctx->args_owned = 0;
+ }
+ ctx->args = PyObject_GetItem(ctx->dict, key);
+ Py_DECREF(key);
+ if (ctx->args == NULL)
+ return -1;
+ ctx->args_owned = 1;
+ ctx->arglen = -1;
+ ctx->argidx = -2;
+ }
+
+ /* Parse flags. Example: "%+i" => flags=F_SIGN. */
+ while (--ctx->fmtcnt >= 0) {
+ arg->ch = FORMAT_READ(ctx);
+ ctx->fmtpos++;
+ switch (arg->ch) {
+ case '-': arg->flags |= F_LJUST; continue;
+ case '+': arg->flags |= F_SIGN; continue;
+ case ' ': arg->flags |= F_BLANK; continue;
+ case '#': arg->flags |= F_ALT; continue;
+ case '0': arg->flags |= F_ZERO; continue;
+ }
+ break;
}
- if (PyMapping_Check(args) && !PyTuple_Check(args) && !PyUnicode_Check(args))
- dict = args;
-
- while (--fmtcnt >= 0) {
- if (PyUnicode_READ(fmtkind, fmt, fmtpos) != '%') {
- Py_ssize_t nonfmtpos;
- nonfmtpos = fmtpos++;
- while (fmtcnt >= 0 &&
- PyUnicode_READ(fmtkind, fmt, fmtpos) != '%') {
- fmtpos++;
- fmtcnt--;
- }
- if (fmtcnt < 0)
- fmtpos--;
- sublen = fmtpos - nonfmtpos;
- maxchar = _PyUnicode_FindMaxChar(uformat,
- nonfmtpos, nonfmtpos + sublen);
- if (_PyUnicodeWriter_Prepare(&writer, sublen, maxchar) == -1)
- goto onError;
- _PyUnicode_FastCopyCharacters(writer.buffer, writer.pos,
- uformat, nonfmtpos, sublen);
- writer.pos += sublen;
+ /* Parse width. Example: "%10s" => width=10 */
+ if (arg->ch == '*') {
+ v = unicode_format_getnextarg(ctx);
+ if (v == NULL)
+ return -1;
+ if (!PyLong_Check(v)) {
+ PyErr_SetString(PyExc_TypeError,
+ "* wants int");
+ return -1;
}
- else {
- /* Got a format specifier */
- int flags = 0;
- Py_ssize_t width = -1;
- int prec = -1;
- Py_UCS4 c = '\0';
- Py_UCS4 fill;
- int sign;
- Py_UCS4 signchar;
- int isnumok;
- PyObject *v = NULL;
- void *pbuf = NULL;
- Py_ssize_t pindex, len;
- Py_UCS4 bufmaxchar;
- Py_ssize_t buflen;
-
- fmtpos++;
- c = PyUnicode_READ(fmtkind, fmt, fmtpos);
- if (c == '(') {
- Py_ssize_t keystart;
- Py_ssize_t keylen;
- PyObject *key;
- int pcount = 1;
-
- if (dict == NULL) {
- PyErr_SetString(PyExc_TypeError,
- "format requires a mapping");
- goto onError;
- }
- ++fmtpos;
- --fmtcnt;
- keystart = fmtpos;
- /* Skip over balanced parentheses */
- while (pcount > 0 && --fmtcnt >= 0) {
- c = PyUnicode_READ(fmtkind, fmt, fmtpos);
- if (c == ')')
- --pcount;
- else if (c == '(')
- ++pcount;
- fmtpos++;
- }
- keylen = fmtpos - keystart - 1;
- if (fmtcnt < 0 || pcount > 0) {
- PyErr_SetString(PyExc_ValueError,
- "incomplete format key");
- goto onError;
- }
- key = PyUnicode_Substring(uformat,
- keystart, keystart + keylen);
- if (key == NULL)
- goto onError;
- if (args_owned) {
- Py_DECREF(args);
- args_owned = 0;
- }
- args = PyObject_GetItem(dict, key);
- Py_DECREF(key);
- if (args == NULL) {
- goto onError;
- }
- args_owned = 1;
- arglen = -1;
- argidx = -2;
- }
- while (--fmtcnt >= 0) {
- c = PyUnicode_READ(fmtkind, fmt, fmtpos++);
- switch (c) {
- case '-': flags |= F_LJUST; continue;
- case '+': flags |= F_SIGN; continue;
- case ' ': flags |= F_BLANK; continue;
- case '#': flags |= F_ALT; continue;
- case '0': flags |= F_ZERO; continue;
- }
+ arg->width = PyLong_AsSsize_t(v);
+ if (arg->width == -1 && PyErr_Occurred())
+ return -1;
+ if (arg->width < 0) {
+ arg->flags |= F_LJUST;
+ arg->width = -arg->width;
+ }
+ if (--ctx->fmtcnt >= 0) {
+ arg->ch = FORMAT_READ(ctx);
+ ctx->fmtpos++;
+ }
+ }
+ else if (arg->ch >= '0' && arg->ch <= '9') {
+ arg->width = arg->ch - '0';
+ while (--ctx->fmtcnt >= 0) {
+ arg->ch = FORMAT_READ(ctx);
+ ctx->fmtpos++;
+ if (arg->ch < '0' || arg->ch > '9')
break;
+ /* Since arg->ch is unsigned, the RHS would end up as unsigned,
+ mixing signed and unsigned comparison. Since arg->ch is between
+ '0' and '9', casting to int is safe. */
+ if (arg->width > (PY_SSIZE_T_MAX - ((int)arg->ch - '0')) / 10) {
+ PyErr_SetString(PyExc_ValueError,
+ "width too big");
+ return -1;
}
- if (c == '*') {
- v = getnextarg(args, arglen, &argidx);
- if (v == NULL)
- goto onError;
- if (!PyLong_Check(v)) {
- PyErr_SetString(PyExc_TypeError,
- "* wants int");
- goto onError;
- }
- width = PyLong_AsSsize_t(v);
- if (width == -1 && PyErr_Occurred())
- goto onError;
- if (width < 0) {
- flags |= F_LJUST;
- width = -width;
- }
- if (--fmtcnt >= 0)
- c = PyUnicode_READ(fmtkind, fmt, fmtpos++);
- }
- else if (c >= '0' && c <= '9') {
- width = c - '0';
- while (--fmtcnt >= 0) {
- c = PyUnicode_READ(fmtkind, fmt, fmtpos++);
- if (c < '0' || c > '9')
- break;
- /* Since c is unsigned, the RHS would end up as unsigned,
- mixing signed and unsigned comparison. Since c is between
- '0' and '9', casting to int is safe. */
- if (width > (PY_SSIZE_T_MAX - ((int)c - '0')) / 10) {
- PyErr_SetString(PyExc_ValueError,
- "width too big");
- goto onError;
- }
- width = width*10 + (c - '0');
- }
+ arg->width = arg->width*10 + (arg->ch - '0');
+ }
+ }
+
+ /* Parse precision. Example: "%.3f" => prec=3 */
+ if (arg->ch == '.') {
+ arg->prec = 0;
+ if (--ctx->fmtcnt >= 0) {
+ arg->ch = FORMAT_READ(ctx);
+ ctx->fmtpos++;
+ }
+ if (arg->ch == '*') {
+ v = unicode_format_getnextarg(ctx);
+ if (v == NULL)
+ return -1;
+ if (!PyLong_Check(v)) {
+ PyErr_SetString(PyExc_TypeError,
+ "* wants int");
+ return -1;
}
- if (c == '.') {
- prec = 0;
- if (--fmtcnt >= 0)
- c = PyUnicode_READ(fmtkind, fmt, fmtpos++);
- if (c == '*') {
- v = getnextarg(args, arglen, &argidx);
- if (v == NULL)
- goto onError;
- if (!PyLong_Check(v)) {
- PyErr_SetString(PyExc_TypeError,
- "* wants int");
- goto onError;
- }
- prec = _PyLong_AsInt(v);
- if (prec == -1 && PyErr_Occurred())
- goto onError;
- if (prec < 0)
- prec = 0;
- if (--fmtcnt >= 0)
- c = PyUnicode_READ(fmtkind, fmt, fmtpos++);
- }
- else if (c >= '0' && c <= '9') {
- prec = c - '0';
- while (--fmtcnt >= 0) {
- c = PyUnicode_READ(fmtkind, fmt, fmtpos++);
- if (c < '0' || c > '9')
- break;
- if (prec > (INT_MAX - ((int)c - '0')) / 10) {
- PyErr_SetString(PyExc_ValueError,
- "prec too big");
- goto onError;
- }
- prec = prec*10 + (c - '0');
- }
- }
- } /* prec */
- if (fmtcnt >= 0) {
- if (c == 'h' || c == 'l' || c == 'L') {
- if (--fmtcnt >= 0)
- c = PyUnicode_READ(fmtkind, fmt, fmtpos++);
+ arg->prec = _PyLong_AsInt(v);
+ if (arg->prec == -1 && PyErr_Occurred())
+ return -1;
+ if (arg->prec < 0)
+ arg->prec = 0;
+ if (--ctx->fmtcnt >= 0) {
+ arg->ch = FORMAT_READ(ctx);
+ ctx->fmtpos++;
+ }
+ }
+ else if (arg->ch >= '0' && arg->ch <= '9') {
+ arg->prec = arg->ch - '0';
+ while (--ctx->fmtcnt >= 0) {
+ arg->ch = FORMAT_READ(ctx);
+ ctx->fmtpos++;
+ if (arg->ch < '0' || arg->ch > '9')
+ break;
+ if (arg->prec > (INT_MAX - ((int)arg->ch - '0')) / 10) {
+ PyErr_SetString(PyExc_ValueError,
+ "precision too big");
+ return -1;
}
+ arg->prec = arg->prec*10 + (arg->ch - '0');
}
- if (fmtcnt < 0) {
- PyErr_SetString(PyExc_ValueError,
- "incomplete format");
- goto onError;
- }
- if (fmtcnt == 0)
- writer.overallocate = 0;
+ }
+ }
- if (c == '%') {
- if (_PyUnicodeWriter_Prepare(&writer, 1, '%') == -1)
- goto onError;
- PyUnicode_WRITE(writer.kind, writer.data, writer.pos, '%');
- writer.pos += 1;
- continue;
+ /* Ignore "h", "l" and "L" format prefix (ex: "%hi" or "%ls") */
+ if (ctx->fmtcnt >= 0) {
+ if (arg->ch == 'h' || arg->ch == 'l' || arg->ch == 'L') {
+ if (--ctx->fmtcnt >= 0) {
+ arg->ch = FORMAT_READ(ctx);
+ ctx->fmtpos++;
}
+ }
+ }
+ if (ctx->fmtcnt < 0) {
+ PyErr_SetString(PyExc_ValueError,
+ "incomplete format");
+ return -1;
+ }
+ return 0;
- v = getnextarg(args, arglen, &argidx);
- if (v == NULL)
- goto onError;
+#undef FORMAT_READ
+}
- sign = 0;
- signchar = '\0';
- fill = ' ';
- switch (c) {
-
- case 's':
- case 'r':
- case 'a':
- if (PyLong_CheckExact(v) && width == -1 && prec == -1) {
- /* Fast path */
- if (_PyLong_FormatWriter(&writer, v, 10, flags & F_ALT) == -1)
- goto onError;
- goto nextarg;
- }
+/* Format one argument. Supported conversion specifiers:
- if (PyUnicode_CheckExact(v) && c == 's') {
- temp = v;
- Py_INCREF(temp);
- }
- else {
- if (c == 's')
- temp = PyObject_Str(v);
- else if (c == 'r')
- temp = PyObject_Repr(v);
- else
- temp = PyObject_ASCII(v);
- }
- break;
+ - "s", "r", "a": any type
+ - "i", "d", "u", "o", "x", "X": int
+ - "e", "E", "f", "F", "g", "G": float
+ - "c": int or str (1 character)
- case 'i':
- case 'd':
- case 'u':
- case 'o':
- case 'x':
- case 'X':
- if (PyLong_CheckExact(v)
- && width == -1 && prec == -1
- && !(flags & (F_SIGN | F_BLANK)))
- {
- /* Fast path */
- switch(c)
- {
- case 'd':
- case 'i':
- case 'u':
- if (_PyLong_FormatWriter(&writer, v, 10, flags & F_ALT) == -1)
- goto onError;
- goto nextarg;
- case 'x':
- if (_PyLong_FormatWriter(&writer, v, 16, flags & F_ALT) == -1)
- goto onError;
- goto nextarg;
- case 'o':
- if (_PyLong_FormatWriter(&writer, v, 8, flags & F_ALT) == -1)
- goto onError;
- goto nextarg;
- default:
- break;
- }
- }
+ When possible, the output is written directly into the Unicode writer
+ (ctx->writer). A string is created when padding is required.
- isnumok = 0;
- if (PyNumber_Check(v)) {
- PyObject *iobj=NULL;
+ Return 0 if the argument has been formatted into *p_str,
+ 1 if the argument has been written into ctx->writer,
+ -1 on error. */
+static int
+unicode_format_arg_format(struct unicode_formatter_t *ctx,
+ struct unicode_format_arg_t *arg,
+ PyObject **p_str)
+{
+ PyObject *v;
+ _PyUnicodeWriter *writer = &ctx->writer;
- if (PyLong_Check(v)) {
- iobj = v;
- Py_INCREF(iobj);
- }
- else {
- iobj = PyNumber_Long(v);
- }
- if (iobj!=NULL) {
- if (PyLong_Check(iobj)) {
- isnumok = 1;
- sign = 1;
- temp = formatlong(iobj, flags, prec, (c == 'i'? 'd': c));
- Py_DECREF(iobj);
- }
- else {
- Py_DECREF(iobj);
- }
- }
- }
- if (!isnumok) {
- PyErr_Format(PyExc_TypeError,
- "%%%c format: a number is required, "
- "not %.200s", (char)c, Py_TYPE(v)->tp_name);
- goto onError;
- }
- if (flags & F_ZERO)
- fill = '0';
- break;
+ if (ctx->fmtcnt == 0)
+ ctx->writer.overallocate = 0;
- case 'e':
- case 'E':
- case 'f':
- case 'F':
- case 'g':
- case 'G':
- if (width == -1 && prec == -1
- && !(flags & (F_SIGN | F_BLANK)))
- {
- /* Fast path */
- if (formatfloat(v, flags, prec, c, NULL, &writer) == -1)
- goto onError;
- goto nextarg;
- }
+ if (arg->ch == '%') {
+ if (_PyUnicodeWriter_Prepare(writer, 1, '%') == -1)
+ return -1;
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '%');
+ writer->pos += 1;
+ return 1;
+ }
- sign = 1;
- if (flags & F_ZERO)
- fill = '0';
- if (formatfloat(v, flags, prec, c, &temp, NULL) == -1)
- temp = NULL;
- break;
+ v = unicode_format_getnextarg(ctx);
+ if (v == NULL)
+ return -1;
- case 'c':
- {
- Py_UCS4 ch = formatchar(v);
- if (ch == (Py_UCS4) -1)
- goto onError;
- if (width == -1 && prec == -1) {
- /* Fast path */
- if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1)
- goto onError;
- PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch);
- writer.pos += 1;
- goto nextarg;
- }
- temp = PyUnicode_FromOrdinal(ch);
- break;
- }
- default:
- PyErr_Format(PyExc_ValueError,
- "unsupported format character '%c' (0x%x) "
- "at index %zd",
- (31<=c && c<=126) ? (char)c : '?',
- (int)c,
- fmtpos - 1);
- goto onError;
- }
- if (temp == NULL)
- goto onError;
- assert (PyUnicode_Check(temp));
+ switch (arg->ch) {
+ case 's':
+ case 'r':
+ case 'a':
+ if (PyLong_CheckExact(v) && arg->width == -1 && arg->prec == -1) {
+ /* Fast path */
+ if (_PyLong_FormatWriter(writer, v, 10, arg->flags & F_ALT) == -1)
+ return -1;
+ return 1;
+ }
- if (width == -1 && prec == -1
- && !(flags & (F_SIGN | F_BLANK)))
- {
- /* Fast path */
- if (_PyUnicodeWriter_WriteStr(&writer, temp) == -1)
- goto onError;
- goto nextarg;
- }
+ if (PyUnicode_CheckExact(v) && arg->ch == 's') {
+ *p_str = v;
+ Py_INCREF(*p_str);
+ }
+ else {
+ if (arg->ch == 's')
+ *p_str = PyObject_Str(v);
+ else if (arg->ch == 'r')
+ *p_str = PyObject_Repr(v);
+ else
+ *p_str = PyObject_ASCII(v);
+ }
+ break;
- if (PyUnicode_READY(temp) == -1) {
- Py_CLEAR(temp);
- goto onError;
- }
- kind = PyUnicode_KIND(temp);
- pbuf = PyUnicode_DATA(temp);
- len = PyUnicode_GET_LENGTH(temp);
+ case 'i':
+ case 'd':
+ case 'u':
+ case 'o':
+ case 'x':
+ case 'X':
+ {
+ int ret = mainformatlong(v, arg, p_str, writer);
+ if (ret != 0)
+ return ret;
+ arg->sign = 1;
+ break;
+ }
- if (c == 's' || c == 'r' || c == 'a') {
- if (prec >= 0 && len > prec)
- len = prec;
- }
+ case 'e':
+ case 'E':
+ case 'f':
+ case 'F':
+ case 'g':
+ case 'G':
+ if (arg->width == -1 && arg->prec == -1
+ && !(arg->flags & (F_SIGN | F_BLANK)))
+ {
+ /* Fast path */
+ if (formatfloat(v, arg, NULL, writer) == -1)
+ return -1;
+ return 1;
+ }
- /* pbuf is initialized here. */
- pindex = 0;
- if (sign) {
- Py_UCS4 ch = PyUnicode_READ(kind, pbuf, pindex);
- if (ch == '-' || ch == '+') {
- signchar = ch;
- len--;
- pindex++;
- }
- else if (flags & F_SIGN)
- signchar = '+';
- else if (flags & F_BLANK)
- signchar = ' ';
- else
- sign = 0;
- }
- if (width < len)
- width = len;
-
- /* Compute the length and maximum character of the
- written characters */
- bufmaxchar = 127;
- if (!(flags & F_LJUST)) {
- if (sign) {
- if ((width-1) > len)
- bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill);
- }
- else {
- if (width > len)
- bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill);
- }
- }
- maxchar = _PyUnicode_FindMaxChar(temp, 0, pindex+len);
- bufmaxchar = MAX_MAXCHAR(bufmaxchar, maxchar);
+ arg->sign = 1;
+ if (formatfloat(v, arg, p_str, NULL) == -1)
+ return -1;
+ break;
- buflen = width;
- if (sign && len == width)
- buflen++;
+ case 'c':
+ {
+ Py_UCS4 ch = formatchar(v);
+ if (ch == (Py_UCS4) -1)
+ return -1;
+ if (arg->width == -1 && arg->prec == -1) {
+ /* Fast path */
+ if (_PyUnicodeWriter_Prepare(writer, 1, ch) == -1)
+ return -1;
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch);
+ writer->pos += 1;
+ return 1;
+ }
+ *p_str = PyUnicode_FromOrdinal(ch);
+ break;
+ }
- if (_PyUnicodeWriter_Prepare(&writer, buflen, bufmaxchar) == -1)
- goto onError;
+ default:
+ PyErr_Format(PyExc_ValueError,
+ "unsupported format character '%c' (0x%x) "
+ "at index %zd",
+ (31<=arg->ch && arg->ch<=126) ? (char)arg->ch : '?',
+ (int)arg->ch,
+ ctx->fmtpos - 1);
+ return -1;
+ }
+ if (*p_str == NULL)
+ return -1;
+ assert (PyUnicode_Check(*p_str));
+ return 0;
+}
- /* Write characters */
- if (sign) {
- if (fill != ' ') {
- PyUnicode_WRITE(writer.kind, writer.data, writer.pos, signchar);
- writer.pos += 1;
- }
- if (width > len)
- width--;
- }
- if ((flags & F_ALT) && (c == 'x' || c == 'X' || c == 'o')) {
- assert(PyUnicode_READ(kind, pbuf, pindex) == '0');
- assert(PyUnicode_READ(kind, pbuf, pindex + 1) == c);
- if (fill != ' ') {
- PyUnicode_WRITE(writer.kind, writer.data, writer.pos, '0');
- PyUnicode_WRITE(writer.kind, writer.data, writer.pos+1, c);
- writer.pos += 2;
- pindex += 2;
- }
- width -= 2;
- if (width < 0)
- width = 0;
- len -= 2;
- }
- if (width > len && !(flags & F_LJUST)) {
- sublen = width - len;
- FILL(writer.kind, writer.data, fill, writer.pos, sublen);
- writer.pos += sublen;
- width = len;
- }
- if (fill == ' ') {
- if (sign) {
- PyUnicode_WRITE(writer.kind, writer.data, writer.pos, signchar);
- writer.pos += 1;
- }
- if ((flags & F_ALT) && (c == 'x' || c == 'X' || c == 'o')) {
- assert(PyUnicode_READ(kind, pbuf, pindex) == '0');
- assert(PyUnicode_READ(kind, pbuf, pindex+1) == c);
- PyUnicode_WRITE(writer.kind, writer.data, writer.pos, '0');
- PyUnicode_WRITE(writer.kind, writer.data, writer.pos+1, c);
- writer.pos += 2;
- pindex += 2;
- }
- }
+static int
+unicode_format_arg_output(struct unicode_formatter_t *ctx,
+ struct unicode_format_arg_t *arg,
+ PyObject *str)
+{
+ Py_ssize_t len;
+ enum PyUnicode_Kind kind;
+ void *pbuf;
+ Py_ssize_t pindex;
+ Py_UCS4 signchar;
+ Py_ssize_t buflen;
+ Py_UCS4 maxchar, bufmaxchar;
+ Py_ssize_t sublen;
+ _PyUnicodeWriter *writer = &ctx->writer;
+ Py_UCS4 fill;
- if (len) {
- _PyUnicode_FastCopyCharacters(writer.buffer, writer.pos,
- temp, pindex, len);
- writer.pos += len;
- }
- if (width > len) {
- sublen = width - len;
- FILL(writer.kind, writer.data, ' ', writer.pos, sublen);
- writer.pos += sublen;
- }
+ fill = ' ';
+ if (arg->sign && arg->flags & F_ZERO)
+ fill = '0';
-nextarg:
- if (dict && (argidx < arglen) && c != '%') {
- PyErr_SetString(PyExc_TypeError,
- "not all arguments converted during string formatting");
+ if (PyUnicode_READY(str) == -1)
+ return -1;
+
+ len = PyUnicode_GET_LENGTH(str);
+ if ((arg->width == -1 || arg->width <= len)
+ && (arg->prec == -1 || arg->prec >= len)
+ && !(arg->flags & (F_SIGN | F_BLANK)))
+ {
+ /* Fast path */
+ if (_PyUnicodeWriter_WriteStr(writer, str) == -1)
+ return -1;
+ return 0;
+ }
+
+ /* Truncate the string for "s", "r" and "a" formats
+ if the precision is set */
+ if (arg->ch == 's' || arg->ch == 'r' || arg->ch == 'a') {
+ if (arg->prec >= 0 && len > arg->prec)
+ len = arg->prec;
+ }
+
+ /* Adjust sign and width */
+ kind = PyUnicode_KIND(str);
+ pbuf = PyUnicode_DATA(str);
+ pindex = 0;
+ signchar = '\0';
+ if (arg->sign) {
+ Py_UCS4 ch = PyUnicode_READ(kind, pbuf, pindex);
+ if (ch == '-' || ch == '+') {
+ signchar = ch;
+ len--;
+ pindex++;
+ }
+ else if (arg->flags & F_SIGN)
+ signchar = '+';
+ else if (arg->flags & F_BLANK)
+ signchar = ' ';
+ else
+ arg->sign = 0;
+ }
+ if (arg->width < len)
+ arg->width = len;
+
+ /* Prepare the writer */
+ bufmaxchar = 127;
+ if (!(arg->flags & F_LJUST)) {
+ if (arg->sign) {
+ if ((arg->width-1) > len)
+ bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill);
+ }
+ else {
+ if (arg->width > len)
+ bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill);
+ }
+ }
+ maxchar = _PyUnicode_FindMaxChar(str, 0, pindex+len);
+ bufmaxchar = MAX_MAXCHAR(bufmaxchar, maxchar);
+ buflen = arg->width;
+ if (arg->sign && len == arg->width)
+ buflen++;
+ if (_PyUnicodeWriter_Prepare(writer, buflen, bufmaxchar) == -1)
+ return -1;
+
+ /* Write the sign if needed */
+ if (arg->sign) {
+ if (fill != ' ') {
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar);
+ writer->pos += 1;
+ }
+ if (arg->width > len)
+ arg->width--;
+ }
+
+ /* Write the numeric prefix for "x", "X" and "o" formats
+ if the alternate form is used.
+ For example, write "0x" for the "%#x" format. */
+ if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) {
+ assert(PyUnicode_READ(kind, pbuf, pindex) == '0');
+ assert(PyUnicode_READ(kind, pbuf, pindex + 1) == arg->ch);
+ if (fill != ' ') {
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0');
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch);
+ writer->pos += 2;
+ pindex += 2;
+ }
+ arg->width -= 2;
+ if (arg->width < 0)
+ arg->width = 0;
+ len -= 2;
+ }
+
+ /* Pad left with the fill character if needed */
+ if (arg->width > len && !(arg->flags & F_LJUST)) {
+ sublen = arg->width - len;
+ FILL(writer->kind, writer->data, fill, writer->pos, sublen);
+ writer->pos += sublen;
+ arg->width = len;
+ }
+
+ /* If padding with spaces: write sign if needed and/or numeric prefix if
+ the alternate form is used */
+ if (fill == ' ') {
+ if (arg->sign) {
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar);
+ writer->pos += 1;
+ }
+ if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) {
+ assert(PyUnicode_READ(kind, pbuf, pindex) == '0');
+ assert(PyUnicode_READ(kind, pbuf, pindex+1) == arg->ch);
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0');
+ PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch);
+ writer->pos += 2;
+ pindex += 2;
+ }
+ }
+
+ /* Write characters */
+ if (len) {
+ _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos,
+ str, pindex, len);
+ writer->pos += len;
+ }
+
+ /* Pad right with the fill character if needed */
+ if (arg->width > len) {
+ sublen = arg->width - len;
+ FILL(writer->kind, writer->data, ' ', writer->pos, sublen);
+ writer->pos += sublen;
+ }
+ return 0;
+}
+
+/* Helper of PyUnicode_Format(): format one arg.
+ Return 0 on success, raise an exception and return -1 on error. */
+static int
+unicode_format_arg(struct unicode_formatter_t *ctx)
+{
+ struct unicode_format_arg_t arg;
+ PyObject *str;
+ int ret;
+
+ arg.ch = PyUnicode_READ(ctx->fmtkind, ctx->fmtdata, ctx->fmtpos);
+ arg.flags = 0;
+ arg.width = -1;
+ arg.prec = -1;
+ arg.sign = 0;
+ str = NULL;
+
+ ret = unicode_format_arg_parse(ctx, &arg);
+ if (ret == -1)
+ return -1;
+
+ ret = unicode_format_arg_format(ctx, &arg, &str);
+ if (ret == -1)
+ return -1;
+
+ if (ret != 1) {
+ ret = unicode_format_arg_output(ctx, &arg, str);
+ Py_DECREF(str);
+ if (ret == -1)
+ return -1;
+ }
+
+ if (ctx->dict && (ctx->argidx < ctx->arglen) && arg.ch != '%') {
+ PyErr_SetString(PyExc_TypeError,
+ "not all arguments converted during string formatting");
+ return -1;
+ }
+ return 0;
+}
+
+PyObject *
+PyUnicode_Format(PyObject *format, PyObject *args)
+{
+ struct unicode_formatter_t ctx;
+
+ if (format == NULL || args == NULL) {
+ PyErr_BadInternalCall();
+ return NULL;
+ }
+
+ ctx.fmtstr = PyUnicode_FromObject(format);
+ if (ctx.fmtstr == NULL)
+ return NULL;
+ if (PyUnicode_READY(ctx.fmtstr) == -1) {
+ Py_DECREF(ctx.fmtstr);
+ return NULL;
+ }
+ ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr);
+ ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr);
+ ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr);
+ ctx.fmtpos = 0;
+
+ _PyUnicodeWriter_Init(&ctx.writer, ctx.fmtcnt + 100);
+
+ if (PyTuple_Check(args)) {
+ ctx.arglen = PyTuple_Size(args);
+ ctx.argidx = 0;
+ }
+ else {
+ ctx.arglen = -1;
+ ctx.argidx = -2;
+ }
+ ctx.args_owned = 0;
+ if (PyMapping_Check(args) && !PyTuple_Check(args) && !PyUnicode_Check(args))
+ ctx.dict = args;
+ else
+ ctx.dict = NULL;
+ ctx.args = args;
+
+ while (--ctx.fmtcnt >= 0) {
+ if (PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') {
+ Py_ssize_t nonfmtpos, sublen;
+ Py_UCS4 maxchar;
+
+ nonfmtpos = ctx.fmtpos++;
+ while (ctx.fmtcnt >= 0 &&
+ PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') {
+ ctx.fmtpos++;
+ ctx.fmtcnt--;
+ }
+ if (ctx.fmtcnt < 0) {
+ ctx.fmtpos--;
+ ctx.writer.overallocate = 0;
+ }
+ sublen = ctx.fmtpos - nonfmtpos;
+ maxchar = _PyUnicode_FindMaxChar(ctx.fmtstr,
+ nonfmtpos, nonfmtpos + sublen);
+ if (_PyUnicodeWriter_Prepare(&ctx.writer, sublen, maxchar) == -1)
goto onError;
- }
- Py_CLEAR(temp);
- } /* '%' */
- } /* until end */
- if (argidx < arglen && !dict) {
+
+ _PyUnicode_FastCopyCharacters(ctx.writer.buffer, ctx.writer.pos,
+ ctx.fmtstr, nonfmtpos, sublen);
+ ctx.writer.pos += sublen;
+ }
+ else {
+ ctx.fmtpos++;
+ if (unicode_format_arg(&ctx) == -1)
+ goto onError;
+ }
+ }
+
+ if (ctx.argidx < ctx.arglen && !ctx.dict) {
PyErr_SetString(PyExc_TypeError,
"not all arguments converted during string formatting");
goto onError;
}
- if (args_owned) {
- Py_DECREF(args);
+ if (ctx.args_owned) {
+ Py_DECREF(ctx.args);
}
- Py_DECREF(uformat);
- Py_XDECREF(temp);
- Py_XDECREF(second);
- return _PyUnicodeWriter_Finish(&writer);
+ Py_DECREF(ctx.fmtstr);
+ return _PyUnicodeWriter_Finish(&ctx.writer);
onError:
- Py_DECREF(uformat);
- Py_XDECREF(temp);
- Py_XDECREF(second);
- _PyUnicodeWriter_Dealloc(&writer);
- if (args_owned) {
- Py_DECREF(args);
+ Py_DECREF(ctx.fmtstr);
+ _PyUnicodeWriter_Dealloc(&ctx.writer);
+ if (ctx.args_owned) {
+ Py_DECREF(ctx.args);
}
return NULL;
}
diff --git a/Objects/unicodetype_db.h b/Objects/unicodetype_db.h
index 46a92bb..1009bb3 100644
--- a/Objects/unicodetype_db.h
+++ b/Objects/unicodetype_db.h
@@ -1919,7 +1919,7 @@ static unsigned short index2[] = {
246, 247, 248, 249, 250, 251, 5, 5, 5, 5, 5, 95, 245, 26, 22, 23, 246,
247, 248, 249, 250, 251, 5, 5, 5, 5, 5, 0, 95, 95, 95, 95, 95, 95, 95,
95, 95, 95, 95, 95, 95, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
- 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 25, 6, 6, 6, 6, 25, 6, 6, 6, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 113, 5, 5,
@@ -2593,10 +2593,10 @@ static unsigned short index2[] = {
0, 0, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141,
141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141,
141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141,
- 141, 141, 141, 141, 141, 141, 141, 141, 141, 252, 252, 141, 141, 141,
141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141,
141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141,
- 141, 141, 141, 252, 252, 141, 141, 141, 141, 141, 141, 141, 141, 141,
+ 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141,
+ 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141,
141, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55,
55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55,
@@ -2925,6 +2925,9 @@ static unsigned short index2[] = {
double _PyUnicode_ToNumeric(Py_UCS4 ch)
{
switch (ch) {
+ case 0x12456:
+ case 0x12457:
+ return (double) -1.0;
case 0x0F33:
return (double) -1.0/2.0;
case 0x0030:
@@ -3427,6 +3430,8 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch)
return (double) 20000.0;
case 0x3251:
return (double) 21.0;
+ case 0x12432:
+ return (double) 216000.0;
case 0x3252:
return (double) 22.0;
case 0x3253:
@@ -3721,6 +3726,8 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch)
return (double) 42.0;
case 0x32B8:
return (double) 43.0;
+ case 0x12433:
+ return (double) 432000.0;
case 0x32B9:
return (double) 44.0;
case 0x32BA:
generated by cgit v0.12 at 2024-10-20 05:44:34 (GMT)