diff options
Diffstat (limited to 'Objects')
55 files changed, 20862 insertions, 11522 deletions
diff --git a/Objects/abstract.c b/Objects/abstract.c index 7705d05..a2737dd 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -74,7 +74,7 @@ PyObject_Length(PyObject *o) Py_ssize_t _PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue) { - static PyObject *hintstrobj = NULL; + _Py_IDENTIFIER(__length_hint__); PyObject *ro, *hintmeth; Py_ssize_t rv; @@ -89,7 +89,7 @@ _PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue) } /* try o.__length_hint__() */ - hintmeth = _PyObject_LookupSpecial(o, "__length_hint__", &hintstrobj); + hintmeth = _PyObject_LookupSpecial(o, &PyId___length_hint__); if (hintmeth == NULL) { if (PyErr_Occurred()) return -1; @@ -237,7 +237,8 @@ PyObject_AsCharBuffer(PyObject *obj, pb = obj->ob_type->tp_as_buffer; if (pb == NULL || pb->bf_getbuffer == NULL) { PyErr_SetString(PyExc_TypeError, - "expected an object with the buffer interface"); + "expected bytes, bytearray " + "or buffer compatible object"); return -1; } if ((*pb->bf_getbuffer)(obj, &view, PyBUF_SIMPLE)) return -1; @@ -331,7 +332,7 @@ PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { if (!PyObject_CheckBuffer(obj)) { PyErr_Format(PyExc_TypeError, - "'%100s' does not support the buffer interface", + "'%.100s' does not support the buffer interface", Py_TYPE(obj)->tp_name); return -1; } @@ -339,7 +340,7 @@ PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) } static int -_IsFortranContiguous(Py_buffer *view) +_IsFortranContiguous(const Py_buffer *view) { Py_ssize_t sd, dim; int i; @@ -360,7 +361,7 @@ _IsFortranContiguous(Py_buffer *view) } static int -_IsCContiguous(Py_buffer *view) +_IsCContiguous(const Py_buffer *view) { Py_ssize_t sd, dim; int i; @@ -381,16 +382,16 @@ _IsCContiguous(Py_buffer *view) } int -PyBuffer_IsContiguous(Py_buffer *view, char fort) +PyBuffer_IsContiguous(const Py_buffer *view, char order) { if (view->suboffsets != NULL) return 0; - if (fort == 'C') + if (order == 'C') return _IsCContiguous(view); - else if (fort == 'F') + else if (order == 'F') return _IsFortranContiguous(view); - else if (fort == 'A') + else if (order == 'A') return (_IsCContiguous(view) || _IsFortranContiguous(view)); return 0; } @@ -444,62 +445,6 @@ _Py_add_one_to_index_C(int nd, Py_ssize_t *index, const Py_ssize_t *shape) } } - /* view is not checked for consistency in either of these. It is - assumed that the size of the buffer is view->len in - view->len / view->itemsize elements. - */ - -int -PyBuffer_ToContiguous(void *buf, Py_buffer *view, Py_ssize_t len, char fort) -{ - int k; - void (*addone)(int, Py_ssize_t *, const Py_ssize_t *); - Py_ssize_t *indices, elements; - char *dest, *ptr; - - if (len > view->len) { - len = view->len; - } - - if (PyBuffer_IsContiguous(view, fort)) { - /* simplest copy is all that is needed */ - memcpy(buf, view->buf, len); - return 0; - } - - /* Otherwise a more elaborate scheme is needed */ - - /* XXX(nnorwitz): need to check for overflow! */ - indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*(view->ndim)); - if (indices == NULL) { - PyErr_NoMemory(); - return -1; - } - for (k=0; k<view->ndim;k++) { - indices[k] = 0; - } - - if (fort == 'F') { - addone = _Py_add_one_to_index_F; - } - else { - addone = _Py_add_one_to_index_C; - } - dest = buf; - /* XXX : This is not going to be the fastest code in the world - several optimizations are possible. - */ - elements = len / view->itemsize; - while (elements--) { - addone(view->ndim, indices, view->shape); - ptr = PyBuffer_GetPointer(view, indices); - memcpy(dest, ptr, view->itemsize); - dest += view->itemsize; - } - PyMem_Free(indices); - return 0; -} - int PyBuffer_FromContiguous(Py_buffer *view, void *buf, Py_ssize_t len, char fort) { @@ -648,9 +593,9 @@ PyBuffer_FillContiguousStrides(int nd, Py_ssize_t *shape, int PyBuffer_FillInfo(Py_buffer *view, PyObject *obj, void *buf, Py_ssize_t len, - int readonly, int flags) + int readonly, int flags) { - if (view == NULL) return 0; + if (view == NULL) return 0; /* XXX why not -1? */ if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && (readonly == 1)) { PyErr_SetString(PyExc_BufferError, @@ -696,16 +641,16 @@ PyObject_Format(PyObject *obj, PyObject *format_spec) PyObject *meth; PyObject *empty = NULL; PyObject *result = NULL; - static PyObject *format_cache = NULL; + _Py_IDENTIFIER(__format__); /* If no format_spec is provided, use an empty string */ if (format_spec == NULL) { - empty = PyUnicode_FromUnicode(NULL, 0); + empty = PyUnicode_New(0, 0); format_spec = empty; } /* Find the (unbound!) __format__ method (a borrowed reference) */ - meth = _PyObject_LookupSpecial(obj, "__format__", &format_cache); + meth = _PyObject_LookupSpecial(obj, &PyId___format__); if (meth == NULL) { if (!PyErr_Occurred()) PyErr_Format(PyExc_TypeError, @@ -792,8 +737,7 @@ binary_op1(PyObject *v, PyObject *w, const int op_slot) return x; Py_DECREF(x); /* can't do it */ } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static PyObject * @@ -1268,37 +1212,31 @@ PyNumber_AsSsize_t(PyObject *item, PyObject *err) } -PyObject * -_PyNumber_ConvertIntegralToInt(PyObject *integral, const char* error_format) -{ - static PyObject *int_name = NULL; - if (int_name == NULL) { - int_name = PyUnicode_InternFromString("__int__"); - if (int_name == NULL) - return NULL; - } - - if (integral && !PyLong_Check(integral)) { - /* Don't go through tp_as_number->nb_int to avoid - hitting the classic class fallback to __trunc__. */ - PyObject *int_func = PyObject_GetAttr(integral, int_name); - if (int_func == NULL) { - PyErr_Clear(); /* Raise a different error. */ - goto non_integral_error; - } - Py_DECREF(integral); - integral = PyEval_CallObject(int_func, NULL); - Py_DECREF(int_func); - if (integral && !PyLong_Check(integral)) { - goto non_integral_error; +/* + Returns the Integral instance converted to an int. The instance is expected + to be an int or have an __int__ method. Steals integral's + reference. error_format will be used to create the TypeError if integral + isn't actually an Integral instance. error_format should be a format string + that can accept a char* naming integral's type. +*/ +static PyObject * +convert_integral_to_int(PyObject *integral, const char *error_format) +{ + PyNumberMethods *nb; + if (PyLong_Check(integral)) + return integral; + nb = Py_TYPE(integral)->tp_as_number; + if (nb->nb_int) { + PyObject *as_int = nb->nb_int(integral); + if (!as_int || PyLong_Check(as_int)) { + Py_DECREF(integral); + return as_int; } + Py_DECREF(as_int); } - return integral; - -non_integral_error: PyErr_Format(PyExc_TypeError, error_format, Py_TYPE(integral)->tp_name); Py_DECREF(integral); - return NULL; + return NULL; } @@ -1325,16 +1263,10 @@ PyObject * PyNumber_Long(PyObject *o) { PyNumberMethods *m; - static PyObject *trunc_name = NULL; PyObject *trunc_func; const char *buffer; Py_ssize_t buffer_len; - - if (trunc_name == NULL) { - trunc_name = PyUnicode_InternFromString("__trunc__"); - if (trunc_name == NULL) - return NULL; - } + _Py_IDENTIFIER(__trunc__); if (o == NULL) return null_error(); @@ -1356,32 +1288,30 @@ PyNumber_Long(PyObject *o) } if (PyLong_Check(o)) /* An int subclass without nb_int */ return _PyLong_Copy((PyLongObject *)o); - trunc_func = PyObject_GetAttr(o, trunc_name); + trunc_func = _PyObject_LookupSpecial(o, &PyId___trunc__); if (trunc_func) { PyObject *truncated = PyEval_CallObject(trunc_func, NULL); PyObject *int_instance; Py_DECREF(trunc_func); /* __trunc__ is specified to return an Integral type, - but long() needs to return a long. */ - int_instance = _PyNumber_ConvertIntegralToInt( - truncated, + but int() needs to return a int. */ + int_instance = convert_integral_to_int(truncated, "__trunc__ returned non-Integral (type %.200s)"); return int_instance; } - PyErr_Clear(); /* It's not an error if o.__trunc__ doesn't exist. */ + if (PyErr_Occurred()) + return NULL; if (PyBytes_Check(o)) /* need to do extra error checking that PyLong_FromString() - * doesn't do. In particular long('9.5') must raise an + * doesn't do. In particular int('9.5') must raise an * exception, not truncate the float. */ return long_from_string(PyBytes_AS_STRING(o), PyBytes_GET_SIZE(o)); if (PyUnicode_Check(o)) /* The above check is done in PyLong_FromUnicode(). */ - return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o), - PyUnicode_GET_SIZE(o), - 10); + return PyLong_FromUnicodeObject(o, 10); if (!PyObject_AsCharBuffer(o, &buffer, &buffer_len)) return long_from_string(buffer, buffer_len); @@ -1986,7 +1916,7 @@ PySequence_Index(PyObject *s, PyObject *o) int PyMapping_Check(PyObject *o) { - return o && o->ob_type->tp_as_mapping && + return o && o->ob_type->tp_as_mapping && o->ob_type->tp_as_mapping->mp_subscript; } @@ -2084,10 +2014,11 @@ PyMapping_Keys(PyObject *o) { PyObject *keys; PyObject *fast; + _Py_IDENTIFIER(keys); if (PyDict_CheckExact(o)) return PyDict_Keys(o); - keys = PyObject_CallMethod(o, "keys", NULL); + keys = _PyObject_CallMethodId(o, &PyId_keys, NULL); if (keys == NULL) return NULL; fast = PySequence_Fast(keys, "o.keys() are not iterable"); @@ -2100,10 +2031,11 @@ PyMapping_Items(PyObject *o) { PyObject *items; PyObject *fast; + _Py_IDENTIFIER(items); if (PyDict_CheckExact(o)) return PyDict_Items(o); - items = PyObject_CallMethod(o, "items", NULL); + items = _PyObject_CallMethodId(o, &PyId_items, NULL); if (items == NULL) return NULL; fast = PySequence_Fast(items, "o.items() are not iterable"); @@ -2116,10 +2048,11 @@ PyMapping_Values(PyObject *o) { PyObject *values; PyObject *fast; + _Py_IDENTIFIER(values); if (PyDict_CheckExact(o)) return PyDict_Values(o); - values = PyObject_CallMethod(o, "values", NULL); + values = _PyObject_CallMethodId(o, &PyId_values, NULL); if (values == NULL) return NULL; fast = PySequence_Fast(values, "o.values() are not iterable"); @@ -2225,22 +2158,11 @@ _PyObject_CallFunction_SizeT(PyObject *callable, char *format, ...) return call_function_tail(callable, args); } -PyObject * -PyObject_CallMethod(PyObject *o, char *name, char *format, ...) +static PyObject* +callmethod(PyObject* func, char *format, va_list va, int is_size_t) { - va_list va; - PyObject *args; - PyObject *func = NULL; PyObject *retval = NULL; - - if (o == NULL || name == NULL) - return null_error(); - - func = PyObject_GetAttrString(o, name); - if (func == NULL) { - PyErr_SetString(PyExc_AttributeError, name); - return 0; - } + PyObject *args; if (!PyCallable_Check(func)) { type_error("attribute of type '%.200s' is not callable", func); @@ -2248,9 +2170,10 @@ PyObject_CallMethod(PyObject *o, char *name, char *format, ...) } if (format && *format) { - va_start(va, format); - args = Py_VaBuildValue(format, va); - va_end(va); + if (is_size_t) + args = _Py_VaBuildValue_SizeT(format, va); + else + args = Py_VaBuildValue(format, va); } else args = PyTuple_New(0); @@ -2265,10 +2188,9 @@ PyObject_CallMethod(PyObject *o, char *name, char *format, ...) } PyObject * -_PyObject_CallMethod_SizeT(PyObject *o, char *name, char *format, ...) +PyObject_CallMethod(PyObject *o, char *name, char *format, ...) { va_list va; - PyObject *args; PyObject *func = NULL; PyObject *retval = NULL; @@ -2277,32 +2199,75 @@ _PyObject_CallMethod_SizeT(PyObject *o, char *name, char *format, ...) func = PyObject_GetAttrString(o, name); if (func == NULL) { - PyErr_SetString(PyExc_AttributeError, name); return 0; } - if (!PyCallable_Check(func)) { - type_error("attribute of type '%.200s' is not callable", func); - goto exit; - } + va_start(va, format); + retval = callmethod(func, format, va, 0); + va_end(va); + return retval; +} - if (format && *format) { - va_start(va, format); - args = _Py_VaBuildValue_SizeT(format, va); - va_end(va); +PyObject * +_PyObject_CallMethodId(PyObject *o, _Py_Identifier *name, char *format, ...) +{ + va_list va; + PyObject *func = NULL; + PyObject *retval = NULL; + + if (o == NULL || name == NULL) + return null_error(); + + func = _PyObject_GetAttrId(o, name); + if (func == NULL) { + return 0; } - else - args = PyTuple_New(0); - retval = call_function_tail(func, args); + va_start(va, format); + retval = callmethod(func, format, va, 0); + va_end(va); + return retval; +} - exit: - /* args gets consumed in call_function_tail */ - Py_XDECREF(func); +PyObject * +_PyObject_CallMethod_SizeT(PyObject *o, char *name, char *format, ...) +{ + va_list va; + PyObject *func = NULL; + PyObject *retval; + if (o == NULL || name == NULL) + return null_error(); + + func = PyObject_GetAttrString(o, name); + if (func == NULL) { + return 0; + } + va_start(va, format); + retval = callmethod(func, format, va, 1); + va_end(va); return retval; } +PyObject * +_PyObject_CallMethodId_SizeT(PyObject *o, _Py_Identifier *name, char *format, ...) +{ + va_list va; + PyObject *func = NULL; + PyObject *retval; + + if (o == NULL || name == NULL) + return null_error(); + + func = _PyObject_GetAttrId(o, name); + if (func == NULL) { + return NULL; + } + va_start(va, format); + retval = callmethod(func, format, va, 1); + va_end(va); + return retval; +} static PyObject * objargs_mktuple(va_list va) @@ -2355,6 +2320,35 @@ PyObject_CallMethodObjArgs(PyObject *callable, PyObject *name, ...) } PyObject * +_PyObject_CallMethodObjIdArgs(PyObject *callable, + struct _Py_Identifier *name, ...) +{ + PyObject *args, *tmp; + va_list vargs; + + if (callable == NULL || name == NULL) + return null_error(); + + callable = _PyObject_GetAttrId(callable, name); + if (callable == NULL) + return NULL; + + /* count the args */ + va_start(vargs, name); + args = objargs_mktuple(vargs); + va_end(vargs); + if (args == NULL) { + Py_DECREF(callable); + return NULL; + } + tmp = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + Py_DECREF(callable); + + return tmp; +} + +PyObject * PyObject_CallFunctionObjArgs(PyObject *callable, ...) { PyObject *args, *tmp; @@ -2378,10 +2372,8 @@ PyObject_CallFunctionObjArgs(PyObject *callable, ...) /* isinstance(), issubclass() */ -/* abstract_get_bases() has logically 4 return states, with a sort of 0th - * state that will almost never happen. +/* abstract_get_bases() has logically 4 return states: * - * 0. creating the __bases__ static string could get a MemoryError * 1. getattr(cls, '__bases__') could raise an AttributeError * 2. getattr(cls, '__bases__') could raise some other exception * 3. getattr(cls, '__bases__') could return a tuple @@ -2407,16 +2399,11 @@ PyObject_CallFunctionObjArgs(PyObject *callable, ...) static PyObject * abstract_get_bases(PyObject *cls) { - static PyObject *__bases__ = NULL; + _Py_IDENTIFIER(__bases__); PyObject *bases; - if (__bases__ == NULL) { - __bases__ = PyUnicode_InternFromString("__bases__"); - if (__bases__ == NULL) - return NULL; - } Py_ALLOW_RECURSION - bases = PyObject_GetAttr(cls, __bases__); + bases = _PyObject_GetAttrId(cls, &PyId___bases__); Py_END_ALLOW_RECURSION if (bases == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) @@ -2486,19 +2473,13 @@ static int recursive_isinstance(PyObject *inst, PyObject *cls) { PyObject *icls; - static PyObject *__class__ = NULL; int retval = 0; - - if (__class__ == NULL) { - __class__ = PyUnicode_InternFromString("__class__"); - if (__class__ == NULL) - return -1; - } + _Py_IDENTIFIER(__class__); if (PyType_Check(cls)) { retval = PyObject_TypeCheck(inst, (PyTypeObject *)cls); if (retval == 0) { - PyObject *c = PyObject_GetAttr(inst, __class__); + PyObject *c = _PyObject_GetAttrId(inst, &PyId___class__); if (c == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); @@ -2519,7 +2500,7 @@ recursive_isinstance(PyObject *inst, PyObject *cls) if (!check_class(cls, "isinstance() arg 2 must be a type or tuple of types")) return -1; - icls = PyObject_GetAttr(inst, __class__); + icls = _PyObject_GetAttrId(inst, &PyId___class__); if (icls == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); @@ -2538,7 +2519,7 @@ recursive_isinstance(PyObject *inst, PyObject *cls) int PyObject_IsInstance(PyObject *inst, PyObject *cls) { - static PyObject *name = NULL; + _Py_IDENTIFIER(__instancecheck__); PyObject *checker; /* Quick test for an exact match */ @@ -2564,7 +2545,7 @@ PyObject_IsInstance(PyObject *inst, PyObject *cls) return r; } - checker = _PyObject_LookupSpecial(cls, "__instancecheck__", &name); + checker = _PyObject_LookupSpecial(cls, &PyId___instancecheck__); if (checker != NULL) { PyObject *res; int ok = -1; @@ -2607,7 +2588,7 @@ recursive_issubclass(PyObject *derived, PyObject *cls) int PyObject_IsSubclass(PyObject *derived, PyObject *cls) { - static PyObject *name = NULL; + _Py_IDENTIFIER(__subclasscheck__); PyObject *checker; if (PyTuple_Check(cls)) { @@ -2629,7 +2610,7 @@ PyObject_IsSubclass(PyObject *derived, PyObject *cls) return r; } - checker = _PyObject_LookupSpecial(cls, "__subclasscheck__", &name); + checker = _PyObject_LookupSpecial(cls, &PyId___subclasscheck__); if (checker != NULL) { PyObject *res; int ok = -1; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 55b4df6..2bb3a29 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -139,7 +139,7 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) } else { alloc = size + 1; - new->ob_bytes = PyMem_Malloc(alloc); + new->ob_bytes = PyObject_Malloc(alloc); if (new->ob_bytes == NULL) { Py_DECREF(new); return PyErr_NoMemory(); @@ -209,7 +209,7 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t size) alloc = size + 1; } - sval = PyMem_Realloc(((PyByteArrayObject *)self)->ob_bytes, alloc); + sval = PyObject_Realloc(((PyByteArrayObject *)self)->ob_bytes, alloc); if (sval == NULL) { PyErr_NoMemory(); return -1; @@ -789,7 +789,7 @@ bytearray_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds) size = view.len; if (PyByteArray_Resize((PyObject *)self, size) < 0) goto fail; if (PyBuffer_ToContiguous(self->ob_bytes, &view, size, 'C') < 0) - goto fail; + goto fail; PyBuffer_Release(&view); return 0; fail: @@ -850,87 +850,82 @@ bytearray_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * bytearray_repr(PyByteArrayObject *self) { - static const char *hexdigits = "0123456789abcdef"; const char *quote_prefix = "bytearray(b"; const char *quote_postfix = ")"; Py_ssize_t length = Py_SIZE(self); - /* 14 == strlen(quote_prefix) + 2 + strlen(quote_postfix) */ + /* 15 == strlen(quote_prefix) + 2 + strlen(quote_postfix) + 1 */ size_t newsize; PyObject *v; - if (length > (PY_SSIZE_T_MAX - 14) / 4) { + register Py_ssize_t i; + register char c; + register char *p; + int quote; + char *test, *start; + char *buffer; + + if (length > (PY_SSIZE_T_MAX - 15) / 4) { PyErr_SetString(PyExc_OverflowError, "bytearray object is too large to make repr"); return NULL; } - newsize = 14 + 4 * length; - v = PyUnicode_FromUnicode(NULL, newsize); - if (v == NULL) { + + newsize = 15 + length * 4; + buffer = PyObject_Malloc(newsize); + if (buffer == NULL) { + PyErr_NoMemory(); return NULL; } - else { - register Py_ssize_t i; - register Py_UNICODE c; - register Py_UNICODE *p; - int quote; - - /* Figure out which quote to use; single is preferred */ - quote = '\''; - { - char *test, *start; - start = PyByteArray_AS_STRING(self); - for (test = start; test < start+length; ++test) { - if (*test == '"') { - quote = '\''; /* back to single */ - goto decided; - } - else if (*test == '\'') - quote = '"'; - } - decided: - ; - } - p = PyUnicode_AS_UNICODE(v); - while (*quote_prefix) - *p++ = *quote_prefix++; - *p++ = quote; - - for (i = 0; i < length; i++) { - /* There's at least enough room for a hex escape - and a closing quote. */ - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5); - c = self->ob_bytes[i]; - if (c == '\'' || c == '\\') - *p++ = '\\', *p++ = c; - else if (c == '\t') - *p++ = '\\', *p++ = 't'; - else if (c == '\n') - *p++ = '\\', *p++ = 'n'; - else if (c == '\r') - *p++ = '\\', *p++ = 'r'; - else if (c == 0) - *p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0'; - else if (c < ' ' || c >= 0x7f) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(c & 0xf0) >> 4]; - *p++ = hexdigits[c & 0xf]; - } - else - *p++ = c; - } - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1); - *p++ = quote; - while (*quote_postfix) { - *p++ = *quote_postfix++; + /* Figure out which quote to use; single is preferred */ + quote = '\''; + start = PyByteArray_AS_STRING(self); + for (test = start; test < start+length; ++test) { + if (*test == '"') { + quote = '\''; /* back to single */ + break; } - *p = '\0'; - if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) { - Py_DECREF(v); - return NULL; + else if (*test == '\'') + quote = '"'; + } + + p = buffer; + while (*quote_prefix) + *p++ = *quote_prefix++; + *p++ = quote; + + for (i = 0; i < length; i++) { + /* There's at least enough room for a hex escape + and a closing quote. */ + assert(newsize - (p - buffer) >= 5); + c = self->ob_bytes[i]; + if (c == '\'' || c == '\\') + *p++ = '\\', *p++ = c; + else if (c == '\t') + *p++ = '\\', *p++ = 't'; + else if (c == '\n') + *p++ = '\\', *p++ = 'n'; + else if (c == '\r') + *p++ = '\\', *p++ = 'r'; + else if (c == 0) + *p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0'; + else if (c < ' ' || c >= 0x7f) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = Py_hexdigits[(c & 0xf0) >> 4]; + *p++ = Py_hexdigits[c & 0xf]; } - return v; + else + *p++ = c; + } + assert(newsize - (p - buffer) >= 1); + *p++ = quote; + while (*quote_postfix) { + *p++ = *quote_postfix++; } + + v = PyUnicode_DecodeASCII(buffer, p - buffer, NULL); + PyObject_Free(buffer); + return v; } static PyObject * @@ -964,23 +959,20 @@ bytearray_richcompare(PyObject *self, PyObject *other, int op) return NULL; } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } self_size = _getbuffer(self, &self_bytes); if (self_size < 0) { PyErr_Clear(); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } other_size = _getbuffer(other, &other_bytes); if (other_size < 0) { PyErr_Clear(); PyBuffer_Release(&self_bytes); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } if (self_size != other_size && (op == Py_EQ || op == Py_NE)) { @@ -1028,7 +1020,7 @@ bytearray_dealloc(PyByteArrayObject *self) PyErr_Print(); } if (self->ob_bytes != 0) { - PyMem_Free(self->ob_bytes); + PyObject_Free(self->ob_bytes); } Py_TYPE(self)->tp_free((PyObject *)self); } @@ -1037,6 +1029,8 @@ bytearray_dealloc(PyByteArrayObject *self) /* -------------------------------------------------------------------- */ /* Methods */ +#define FASTSEARCH fastsearch +#define STRINGLIB(F) stringlib_##F #define STRINGLIB_CHAR char #define STRINGLIB_LEN PyByteArray_GET_SIZE #define STRINGLIB_STR PyByteArray_AS_STRING @@ -1077,24 +1071,41 @@ Py_LOCAL_INLINE(Py_ssize_t) bytearray_find_internal(PyByteArrayObject *self, PyObject *args, int dir) { PyObject *subobj; + char byte; Py_buffer subbuf; + const char *sub; + Py_ssize_t sub_len; Py_ssize_t start=0, end=PY_SSIZE_T_MAX; Py_ssize_t res; - if (!stringlib_parse_args_finds("find/rfind/index/rindex", - args, &subobj, &start, &end)) - return -2; - if (_getbuffer(subobj, &subbuf) < 0) + if (!stringlib_parse_args_finds_byte("find/rfind/index/rindex", + args, &subobj, &byte, &start, &end)) return -2; + + if (subobj) { + if (_getbuffer(subobj, &subbuf) < 0) + return -2; + + sub = subbuf.buf; + sub_len = subbuf.len; + } + else { + sub = &byte; + sub_len = 1; + } + if (dir > 0) res = stringlib_find_slice( PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - subbuf.buf, subbuf.len, start, end); + sub, sub_len, start, end); else res = stringlib_rfind_slice( PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - subbuf.buf, subbuf.len, start, end); - PyBuffer_Release(&subbuf); + sub, sub_len, start, end); + + if (subobj) + PyBuffer_Release(&subbuf); + return res; } @@ -1127,26 +1138,66 @@ static PyObject * bytearray_count(PyByteArrayObject *self, PyObject *args) { PyObject *sub_obj; - const char *str = PyByteArray_AS_STRING(self); + const char *str = PyByteArray_AS_STRING(self), *sub; + Py_ssize_t sub_len; + char byte; Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; + Py_buffer vsub; PyObject *count_obj; - if (!stringlib_parse_args_finds("count", args, &sub_obj, &start, &end)) + if (!stringlib_parse_args_finds_byte("count", args, &sub_obj, &byte, + &start, &end)) return NULL; - if (_getbuffer(sub_obj, &vsub) < 0) - return NULL; + if (sub_obj) { + if (_getbuffer(sub_obj, &vsub) < 0) + return NULL; + + sub = vsub.buf; + sub_len = vsub.len; + } + else { + sub = &byte; + sub_len = 1; + } ADJUST_INDICES(start, end, PyByteArray_GET_SIZE(self)); count_obj = PyLong_FromSsize_t( - stringlib_count(str + start, end - start, vsub.buf, vsub.len, PY_SSIZE_T_MAX) + stringlib_count(str + start, end - start, sub, sub_len, PY_SSIZE_T_MAX) ); - PyBuffer_Release(&vsub); + + if (sub_obj) + PyBuffer_Release(&vsub); + return count_obj; } +PyDoc_STRVAR(clear__doc__, +"B.clear() -> None\n\ +\n\ +Remove all items from B."); + +static PyObject * +bytearray_clear(PyByteArrayObject *self) +{ + if (PyByteArray_Resize((PyObject *)self, 0) < 0) + return NULL; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(copy__doc__, +"B.copy() -> bytearray\n\ +\n\ +Return a copy of B."); + +static PyObject * +bytearray_copy(PyByteArrayObject *self) +{ + return PyByteArray_FromStringAndSize(PyByteArray_AS_STRING((PyObject *)self), + PyByteArray_GET_SIZE(self)); +} PyDoc_STRVAR(index__doc__, "B.index(sub[, start[, end]]) -> int\n\ @@ -1988,7 +2039,7 @@ bytearray_replace(PyByteArrayObject *self, PyObject *args) } PyDoc_STRVAR(split__doc__, -"B.split([sep[, maxsplit]]) -> list of bytearrays\n\ +"B.split(sep=None, maxsplit=-1) -> list of bytearrays\n\ \n\ Return a list of the sections in B, using sep as the delimiter.\n\ If sep is not given, B is split on ASCII whitespace characters\n\ @@ -1996,15 +2047,17 @@ If sep is not given, B is split on ASCII whitespace characters\n\ If maxsplit is given, at most maxsplit splits are done."); static PyObject * -bytearray_split(PyByteArrayObject *self, PyObject *args) +bytearray_split(PyByteArrayObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sep", "maxsplit", 0}; Py_ssize_t len = PyByteArray_GET_SIZE(self), n; Py_ssize_t maxsplit = -1; const char *s = PyByteArray_AS_STRING(self), *sub; PyObject *list, *subobj = Py_None; Py_buffer vsub; - if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|On:split", + kwlist, &subobj, &maxsplit)) return NULL; if (maxsplit < 0) maxsplit = PY_SSIZE_T_MAX; @@ -2080,7 +2133,7 @@ bytearray_rpartition(PyByteArrayObject *self, PyObject *sep_obj) } PyDoc_STRVAR(rsplit__doc__, -"B.rsplit(sep[, maxsplit]) -> list of bytearrays\n\ +"B.rsplit(sep=None, maxsplit=-1) -> list of bytearrays\n\ \n\ Return a list of the sections in B, using sep as the delimiter,\n\ starting at the end of B and working to the front.\n\ @@ -2089,15 +2142,17 @@ If sep is not given, B is split on ASCII whitespace characters\n\ If maxsplit is given, at most maxsplit splits are done."); static PyObject * -bytearray_rsplit(PyByteArrayObject *self, PyObject *args) +bytearray_rsplit(PyByteArrayObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sep", "maxsplit", 0}; Py_ssize_t len = PyByteArray_GET_SIZE(self), n; Py_ssize_t maxsplit = -1; const char *s = PyByteArray_AS_STRING(self), *sub; PyObject *list, *subobj = Py_None; Py_buffer vsub; - if (!PyArg_ParseTuple(args, "|On:rsplit", &subobj, &maxsplit)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|On:rsplit", + kwlist, &subobj, &maxsplit)) return NULL; if (maxsplit < 0) maxsplit = PY_SSIZE_T_MAX; @@ -2448,7 +2503,7 @@ If the argument is omitted, strip trailing ASCII whitespace."); static PyObject * bytearray_rstrip(PyByteArrayObject *self, PyObject *args) { - Py_ssize_t left, right, mysize, argsize; + Py_ssize_t right, mysize, argsize; void *myptr, *argptr; PyObject *arg = Py_None; Py_buffer varg; @@ -2466,11 +2521,10 @@ bytearray_rstrip(PyByteArrayObject *self, PyObject *args) } myptr = self->ob_bytes; mysize = Py_SIZE(self); - left = 0; right = rstrip_helper(myptr, mysize, argptr, argsize); if (arg != Py_None) PyBuffer_Release(&varg); - return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); + return PyByteArray_FromStringAndSize(self->ob_bytes, right); } PyDoc_STRVAR(decode_doc, @@ -2592,11 +2646,13 @@ Line breaks are not included in the resulting list unless keepends\n\ is given and true."); static PyObject* -bytearray_splitlines(PyObject *self, PyObject *args) +bytearray_splitlines(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"keepends", 0}; int keepends = 0; - if (!PyArg_ParseTuple(args, "|i:splitlines", &keepends)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i:splitlines", + kwlist, &keepends)) return NULL; return stringlib_splitlines( @@ -2613,7 +2669,7 @@ Spaces between two numbers are accepted.\n\ Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef')."); static int -hex_digit_to_int(Py_UNICODE c) +hex_digit_to_int(Py_UCS4 c) { if (c >= 128) return -1; @@ -2633,15 +2689,20 @@ bytearray_fromhex(PyObject *cls, PyObject *args) { PyObject *newbytes, *hexobj; char *buf; - Py_UNICODE *hex; Py_ssize_t hexlen, byteslen, i, j; int top, bot; + void *data; + unsigned int kind; if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj)) return NULL; assert(PyUnicode_Check(hexobj)); - hexlen = PyUnicode_GET_SIZE(hexobj); - hex = PyUnicode_AS_UNICODE(hexobj); + if (PyUnicode_READY(hexobj)) + return NULL; + kind = PyUnicode_KIND(hexobj); + data = PyUnicode_DATA(hexobj); + hexlen = PyUnicode_GET_LENGTH(hexobj); + byteslen = hexlen/2; /* This overestimates if there are spaces */ newbytes = PyByteArray_FromStringAndSize(NULL, byteslen); if (!newbytes) @@ -2649,12 +2710,12 @@ bytearray_fromhex(PyObject *cls, PyObject *args) buf = PyByteArray_AS_STRING(newbytes); for (i = j = 0; i < hexlen; i += 2) { /* skip over spaces in the input */ - while (hex[i] == ' ') + while (PyUnicode_READ(kind, data, i) == ' ') i++; if (i >= hexlen) break; - top = hex_digit_to_int(hex[i]); - bot = hex_digit_to_int(hex[i+1]); + top = hex_digit_to_int(PyUnicode_READ(kind, data, i)); + bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1)); if (top == -1 || bot == -1) { PyErr_Format(PyExc_ValueError, "non-hexadecimal number found in " @@ -2672,26 +2733,59 @@ bytearray_fromhex(PyObject *cls, PyObject *args) return NULL; } -PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyObject * -bytearray_reduce(PyByteArrayObject *self) +_common_reduce(PyByteArrayObject *self, int proto) { - PyObject *latin1, *dict; - if (self->ob_bytes) - latin1 = PyUnicode_DecodeLatin1(self->ob_bytes, - Py_SIZE(self), NULL); - else - latin1 = PyUnicode_FromString(""); + PyObject *dict; + _Py_IDENTIFIER(__dict__); - dict = PyObject_GetAttrString((PyObject *)self, "__dict__"); + dict = _PyObject_GetAttrId((PyObject *)self, &PyId___dict__); if (dict == NULL) { PyErr_Clear(); dict = Py_None; Py_INCREF(dict); } - return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", dict); + if (proto < 3) { + /* use str based reduction for backwards compatibility with Python 2.x */ + PyObject *latin1; + if (self->ob_bytes) + latin1 = PyUnicode_DecodeLatin1(self->ob_bytes, Py_SIZE(self), NULL); + else + latin1 = PyUnicode_FromString(""); + return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", dict); + } + else { + /* use more efficient byte based reduction */ + if (self->ob_bytes) { + return Py_BuildValue("(O(y#)N)", Py_TYPE(self), self->ob_bytes, Py_SIZE(self), dict); + } + else { + return Py_BuildValue("(O()N)", Py_TYPE(self), dict); + } + } +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); + +static PyObject * +bytearray_reduce(PyByteArrayObject *self) +{ + return _common_reduce(self, 2); +} + +PyDoc_STRVAR(reduce_ex_doc, "Return state information for pickling."); + +static PyObject * +bytearray_reduce_ex(PyByteArrayObject *self, PyObject *args) +{ + int proto = 0; + + if (!PyArg_ParseTuple(args, "|i:__reduce_ex__", &proto)) + return NULL; + + return _common_reduce(self, proto); } PyDoc_STRVAR(sizeof_doc, @@ -2735,11 +2829,14 @@ static PyMethodDef bytearray_methods[] = { {"__alloc__", (PyCFunction)bytearray_alloc, METH_NOARGS, alloc_doc}, {"__reduce__", (PyCFunction)bytearray_reduce, METH_NOARGS, reduce_doc}, + {"__reduce_ex__", (PyCFunction)bytearray_reduce_ex, METH_VARARGS, reduce_ex_doc}, {"__sizeof__", (PyCFunction)bytearray_sizeof, METH_NOARGS, sizeof_doc}, {"append", (PyCFunction)bytearray_append, METH_O, append__doc__}, {"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS, _Py_capitalize__doc__}, {"center", (PyCFunction)stringlib_center, METH_VARARGS, center__doc__}, + {"clear", (PyCFunction)bytearray_clear, METH_NOARGS, clear__doc__}, + {"copy", (PyCFunction)bytearray_copy, METH_NOARGS, copy__doc__}, {"count", (PyCFunction)bytearray_count, METH_VARARGS, count__doc__}, {"decode", (PyCFunction)bytearray_decode, METH_VARARGS | METH_KEYWORDS, decode_doc}, {"endswith", (PyCFunction)bytearray_endswith, METH_VARARGS, endswith__doc__}, @@ -2780,11 +2877,11 @@ bytearray_methods[] = { {"rindex", (PyCFunction)bytearray_rindex, METH_VARARGS, rindex__doc__}, {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, rjust__doc__}, {"rpartition", (PyCFunction)bytearray_rpartition, METH_O, rpartition__doc__}, - {"rsplit", (PyCFunction)bytearray_rsplit, METH_VARARGS, rsplit__doc__}, + {"rsplit", (PyCFunction)bytearray_rsplit, METH_VARARGS | METH_KEYWORDS, rsplit__doc__}, {"rstrip", (PyCFunction)bytearray_rstrip, METH_VARARGS, rstrip__doc__}, - {"split", (PyCFunction)bytearray_split, METH_VARARGS, split__doc__}, - {"splitlines", (PyCFunction)bytearray_splitlines, METH_VARARGS, - splitlines__doc__}, + {"split", (PyCFunction)bytearray_split, METH_VARARGS | METH_KEYWORDS, split__doc__}, + {"splitlines", (PyCFunction)bytearray_splitlines, + METH_VARARGS | METH_KEYWORDS, splitlines__doc__}, {"startswith", (PyCFunction)bytearray_startswith, METH_VARARGS , startswith__doc__}, {"strip", (PyCFunction)bytearray_strip, METH_VARARGS, strip__doc__}, @@ -2906,7 +3003,7 @@ bytearrayiter_next(bytesiterobject *it) } static PyObject * -bytesarrayiter_length_hint(bytesiterobject *it) +bytearrayiter_length_hint(bytesiterobject *it) { Py_ssize_t len = 0; if (it->it_seq) @@ -2917,9 +3014,41 @@ bytesarrayiter_length_hint(bytesiterobject *it) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +static PyObject * +bytearrayiter_reduce(bytesiterobject *it) +{ + if (it->it_seq != NULL) { + return Py_BuildValue("N(O)n", _PyObject_GetBuiltin("iter"), + it->it_seq, it->it_index); + } else { + PyObject *u = PyUnicode_FromUnicode(NULL, 0); + if (u == NULL) + return NULL; + return Py_BuildValue("N(N)", _PyObject_GetBuiltin("iter"), u); + } +} + +static PyObject * +bytearrayiter_setstate(bytesiterobject *it, PyObject *state) +{ + Py_ssize_t index = PyLong_AsSsize_t(state); + if (index == -1 && PyErr_Occurred()) + return NULL; + if (index < 0) + index = 0; + it->it_index = index; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); + static PyMethodDef bytearrayiter_methods[] = { - {"__length_hint__", (PyCFunction)bytesarrayiter_length_hint, METH_NOARGS, + {"__length_hint__", (PyCFunction)bytearrayiter_length_hint, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)bytearrayiter_reduce, METH_NOARGS, + reduce_doc}, + {"__setstate__", (PyCFunction)bytearrayiter_setstate, METH_O, + setstate_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index 7233cea..ef3c2f7 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -248,12 +248,8 @@ _Py_bytes_lower(char *result, const char *cptr, Py_ssize_t len) { Py_ssize_t i; - Py_MEMCPY(result, cptr, len); - for (i = 0; i < len; i++) { - int c = Py_CHARMASK(result[i]); - if (Py_ISUPPER(c)) - result[i] = Py_TOLOWER(c); + result[i] = Py_TOLOWER((unsigned char) cptr[i]); } } @@ -268,12 +264,8 @@ _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len) { Py_ssize_t i; - Py_MEMCPY(result, cptr, len); - for (i = 0; i < len; i++) { - int c = Py_CHARMASK(result[i]); - if (Py_ISLOWER(c)) - result[i] = Py_TOUPPER(c); + result[i] = Py_TOUPPER((unsigned char) cptr[i]); } } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 796e400..bf9259f 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -41,10 +41,6 @@ static PyBytesObject *nullstring; #define PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) /* - For both PyBytes_FromString() and PyBytes_FromStringAndSize(), the - parameter `size' denotes number of characters to allocate, not counting any - null terminating character. - For PyBytes_FromString(), the parameter `str' points to a null-terminated string containing exactly `size' bytes. @@ -61,8 +57,8 @@ static PyBytesObject *nullstring; The PyObject member `op->ob_size', which denotes the number of "extra items" in a variable-size object, will contain the number of bytes - allocated for string data, not counting the null terminating character. It - is therefore equal to the equal to the `size' parameter (for + allocated for string data, not counting the null terminating character. + It is therefore equal to the `size' parameter (for PyBytes_FromStringAndSize()) or the length of the string in the `str' parameter (for PyBytes_FromString()). */ @@ -568,76 +564,70 @@ PyBytes_AsStringAndSize(register PyObject *obj, PyObject * PyBytes_Repr(PyObject *obj, int smartquotes) { - static const char *hexdigits = "0123456789abcdef"; register PyBytesObject* op = (PyBytesObject*) obj; - Py_ssize_t length = Py_SIZE(op); - size_t newsize; + Py_ssize_t i, length = Py_SIZE(op); + size_t newsize, squotes, dquotes; PyObject *v; - if (length > (PY_SSIZE_T_MAX - 3) / 4) { + unsigned char quote, *s, *p; + + /* Compute size of output string */ + squotes = dquotes = 0; + newsize = 3; /* b'' */ + s = (unsigned char*)op->ob_sval; + for (i = 0; i < length; i++) { + switch(s[i]) { + case '\'': squotes++; newsize++; break; + case '"': dquotes++; newsize++; break; + case '\\': case '\t': case '\n': case '\r': + newsize += 2; break; /* \C */ + default: + if (s[i] < ' ' || s[i] >= 0x7f) + newsize += 4; /* \xHH */ + else + newsize++; + } + } + quote = '\''; + if (smartquotes && squotes && !dquotes) + quote = '"'; + if (squotes && quote == '\'') + newsize += squotes; + + if (newsize > (PY_SSIZE_T_MAX - sizeof(PyUnicodeObject) - 1)) { PyErr_SetString(PyExc_OverflowError, "bytes object is too large to make repr"); return NULL; } - newsize = 3 + 4 * length; - v = PyUnicode_FromUnicode(NULL, newsize); + + v = PyUnicode_New(newsize, 127); if (v == NULL) { return NULL; } - else { - register Py_ssize_t i; - register Py_UNICODE c; - register Py_UNICODE *p = PyUnicode_AS_UNICODE(v); - int quote; - - /* Figure out which quote to use; single is preferred */ - quote = '\''; - if (smartquotes) { - char *test, *start; - start = PyBytes_AS_STRING(op); - for (test = start; test < start+length; ++test) { - if (*test == '"') { - quote = '\''; /* back to single */ - goto decided; - } - else if (*test == '\'') - quote = '"'; - } - decided: - ; - } - - *p++ = 'b', *p++ = quote; - for (i = 0; i < length; i++) { - /* There's at least enough room for a hex escape - and a closing quote. */ - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5); - c = op->ob_sval[i]; - if (c == quote || c == '\\') - *p++ = '\\', *p++ = c; - else if (c == '\t') - *p++ = '\\', *p++ = 't'; - else if (c == '\n') - *p++ = '\\', *p++ = 'n'; - else if (c == '\r') - *p++ = '\\', *p++ = 'r'; - else if (c < ' ' || c >= 0x7f) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(c & 0xf0) >> 4]; - *p++ = hexdigits[c & 0xf]; - } - else - *p++ = c; - } - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1); - *p++ = quote; - *p = '\0'; - if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) { - Py_DECREF(v); - return NULL; + p = PyUnicode_1BYTE_DATA(v); + + *p++ = 'b', *p++ = quote; + for (i = 0; i < length; i++) { + unsigned char c = op->ob_sval[i]; + if (c == quote || c == '\\') + *p++ = '\\', *p++ = c; + else if (c == '\t') + *p++ = '\\', *p++ = 't'; + else if (c == '\n') + *p++ = '\\', *p++ = 'n'; + else if (c == '\r') + *p++ = '\\', *p++ = 'r'; + else if (c < ' ' || c >= 0x7f) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = Py_hexdigits[(c & 0xf0) >> 4]; + *p++ = Py_hexdigits[c & 0xf]; } - return v; + else + *p++ = c; } + *p++ = quote; + assert(_PyUnicode_CheckConsistency(v, 1)); + return v; } static PyObject * @@ -871,35 +861,11 @@ bytes_richcompare(PyBytesObject *a, PyBytesObject *b, int op) static Py_hash_t bytes_hash(PyBytesObject *a) { - register Py_ssize_t len; - register unsigned char *p; - register Py_hash_t x; - -#ifdef Py_DEBUG - assert(_Py_HashSecret_Initialized); -#endif - if (a->ob_shash != -1) - return a->ob_shash; - len = Py_SIZE(a); - /* - We make the hash of the empty string be 0, rather than using - (prefix ^ suffix), since this slightly obfuscates the hash secret - */ - if (len == 0) { - a->ob_shash = 0; - return 0; - } - p = (unsigned char *) a->ob_sval; - x = _Py_HashSecret.prefix; - x ^= *p << 7; - while (--len >= 0) - x = (_PyHASH_MULTIPLIER*x) ^ *p++; - x ^= Py_SIZE(a); - x ^= _Py_HashSecret.suffix; - if (x == -1) - x = -2; - a->ob_shash = x; - return x; + if (a->ob_shash == -1) { + /* Can't fail */ + a->ob_shash = _Py_HashBytes((unsigned char *) a->ob_sval, Py_SIZE(a)); + } + return a->ob_shash; } static PyObject* @@ -1007,7 +973,7 @@ static const char *stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; #define STRIPNAME(i) (stripformat[i]+3) PyDoc_STRVAR(split__doc__, -"B.split([sep[, maxsplit]]) -> list of bytes\n\ +"B.split(sep=None, maxsplit=-1) -> list of bytes\n\ \n\ Return a list of the sections in B, using sep as the delimiter.\n\ If sep is not specified or is None, B is split on ASCII whitespace\n\ @@ -1015,15 +981,17 @@ characters (space, tab, return, newline, formfeed, vertical tab).\n\ If maxsplit is given, at most maxsplit splits are done."); static PyObject * -bytes_split(PyBytesObject *self, PyObject *args) +bytes_split(PyBytesObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sep", "maxsplit", 0}; Py_ssize_t len = PyBytes_GET_SIZE(self), n; Py_ssize_t maxsplit = -1; const char *s = PyBytes_AS_STRING(self), *sub; Py_buffer vsub; PyObject *list, *subobj = Py_None; - if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|On:split", + kwlist, &subobj, &maxsplit)) return NULL; if (maxsplit < 0) maxsplit = PY_SSIZE_T_MAX; @@ -1095,7 +1063,7 @@ bytes_rpartition(PyBytesObject *self, PyObject *sep_obj) } PyDoc_STRVAR(rsplit__doc__, -"B.rsplit([sep[, maxsplit]]) -> list of bytes\n\ +"B.rsplit(sep=None, maxsplit=-1) -> list of bytes\n\ \n\ Return a list of the sections in B, using sep as the delimiter,\n\ starting at the end of B and working to the front.\n\ @@ -1105,15 +1073,17 @@ If maxsplit is given, at most maxsplit splits are done."); static PyObject * -bytes_rsplit(PyBytesObject *self, PyObject *args) +bytes_rsplit(PyBytesObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sep", "maxsplit", 0}; Py_ssize_t len = PyBytes_GET_SIZE(self), n; Py_ssize_t maxsplit = -1; const char *s = PyBytes_AS_STRING(self), *sub; Py_buffer vsub; PyObject *list, *subobj = Py_None; - if (!PyArg_ParseTuple(args, "|On:rsplit", &subobj, &maxsplit)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|On:rsplit", + kwlist, &subobj, &maxsplit)) return NULL; if (maxsplit < 0) maxsplit = PY_SSIZE_T_MAX; @@ -1254,31 +1224,42 @@ Py_LOCAL_INLINE(Py_ssize_t) bytes_find_internal(PyBytesObject *self, PyObject *args, int dir) { PyObject *subobj; + char byte; + Py_buffer subbuf; const char *sub; Py_ssize_t sub_len; Py_ssize_t start=0, end=PY_SSIZE_T_MAX; + Py_ssize_t res; - if (!stringlib_parse_args_finds("find/rfind/index/rindex", - args, &subobj, &start, &end)) + if (!stringlib_parse_args_finds_byte("find/rfind/index/rindex", + args, &subobj, &byte, &start, &end)) return -2; - if (PyBytes_Check(subobj)) { - sub = PyBytes_AS_STRING(subobj); - sub_len = PyBytes_GET_SIZE(subobj); + if (subobj) { + if (_getbuffer(subobj, &subbuf) < 0) + return -2; + + sub = subbuf.buf; + sub_len = subbuf.len; + } + else { + sub = &byte; + sub_len = 1; } - else if (PyObject_AsCharBuffer(subobj, &sub, &sub_len)) - /* XXX - the "expected a character buffer object" is pretty - confusing for a non-expert. remap to something else ? */ - return -2; if (dir > 0) - return stringlib_find_slice( + res = stringlib_find_slice( PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), sub, sub_len, start, end); else - return stringlib_rfind_slice( + res = stringlib_rfind_slice( PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), sub, sub_len, start, end); + + if (subobj) + PyBuffer_Release(&subbuf); + + return res; } @@ -1504,23 +1485,38 @@ bytes_count(PyBytesObject *self, PyObject *args) PyObject *sub_obj; const char *str = PyBytes_AS_STRING(self), *sub; Py_ssize_t sub_len; + char byte; Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; - if (!stringlib_parse_args_finds("count", args, &sub_obj, &start, &end)) + Py_buffer vsub; + PyObject *count_obj; + + if (!stringlib_parse_args_finds_byte("count", args, &sub_obj, &byte, + &start, &end)) return NULL; - if (PyBytes_Check(sub_obj)) { - sub = PyBytes_AS_STRING(sub_obj); - sub_len = PyBytes_GET_SIZE(sub_obj); + if (sub_obj) { + if (_getbuffer(sub_obj, &vsub) < 0) + return NULL; + + sub = vsub.buf; + sub_len = vsub.len; + } + else { + sub = &byte; + sub_len = 1; } - else if (PyObject_AsCharBuffer(sub_obj, &sub, &sub_len)) - return NULL; ADJUST_INDICES(start, end, PyBytes_GET_SIZE(self)); - return PyLong_FromSsize_t( + count_obj = PyLong_FromSsize_t( stringlib_count(str + start, end - start, sub, sub_len, PY_SSIZE_T_MAX) ); + + if (sub_obj) + PyBuffer_Release(&vsub); + + return count_obj; } @@ -2329,11 +2325,13 @@ Line breaks are not included in the resulting list unless keepends\n\ is given and true."); static PyObject* -bytes_splitlines(PyObject *self, PyObject *args) +bytes_splitlines(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"keepends", 0}; int keepends = 0; - if (!PyArg_ParseTuple(args, "|i:splitlines", &keepends)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i:splitlines", + kwlist, &keepends)) return NULL; return stringlib_splitlines( @@ -2351,7 +2349,7 @@ Spaces between two numbers are accepted.\n\ Example: bytes.fromhex('B9 01EF') -> b'\\xb9\\x01\\xef'."); static int -hex_digit_to_int(Py_UNICODE c) +hex_digit_to_int(Py_UCS4 c) { if (c >= 128) return -1; @@ -2371,15 +2369,20 @@ bytes_fromhex(PyObject *cls, PyObject *args) { PyObject *newstring, *hexobj; char *buf; - Py_UNICODE *hex; Py_ssize_t hexlen, byteslen, i, j; int top, bot; + void *data; + unsigned int kind; if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj)) return NULL; assert(PyUnicode_Check(hexobj)); - hexlen = PyUnicode_GET_SIZE(hexobj); - hex = PyUnicode_AS_UNICODE(hexobj); + if (PyUnicode_READY(hexobj)) + return NULL; + kind = PyUnicode_KIND(hexobj); + data = PyUnicode_DATA(hexobj); + hexlen = PyUnicode_GET_LENGTH(hexobj); + byteslen = hexlen/2; /* This overestimates if there are spaces */ newstring = PyBytes_FromStringAndSize(NULL, byteslen); if (!newstring) @@ -2387,12 +2390,12 @@ bytes_fromhex(PyObject *cls, PyObject *args) buf = PyBytes_AS_STRING(newstring); for (i = j = 0; i < hexlen; i += 2) { /* skip over spaces in the input */ - while (hex[i] == ' ') + while (PyUnicode_READ(kind, data, i) == ' ') i++; if (i >= hexlen) break; - top = hex_digit_to_int(hex[i]); - bot = hex_digit_to_int(hex[i+1]); + top = hex_digit_to_int(PyUnicode_READ(kind, data, i)); + bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1)); if (top == -1 || bot == -1) { PyErr_Format(PyExc_ValueError, "non-hexadecimal number found in " @@ -2472,10 +2475,10 @@ bytes_methods[] = { {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, rjust__doc__}, {"rpartition", (PyCFunction)bytes_rpartition, METH_O, rpartition__doc__}, - {"rsplit", (PyCFunction)bytes_rsplit, METH_VARARGS, rsplit__doc__}, + {"rsplit", (PyCFunction)bytes_rsplit, METH_VARARGS | METH_KEYWORDS, rsplit__doc__}, {"rstrip", (PyCFunction)bytes_rstrip, METH_VARARGS, rstrip__doc__}, - {"split", (PyCFunction)bytes_split, METH_VARARGS, split__doc__}, - {"splitlines", (PyCFunction)bytes_splitlines, METH_VARARGS, + {"split", (PyCFunction)bytes_split, METH_VARARGS | METH_KEYWORDS, split__doc__}, + {"splitlines", (PyCFunction)bytes_splitlines, METH_VARARGS | METH_KEYWORDS, splitlines__doc__}, {"startswith", (PyCFunction)bytes_startswith, METH_VARARGS, startswith__doc__}, @@ -2574,6 +2577,12 @@ PyBytes_FromObject(PyObject *x) PyErr_BadInternalCall(); return NULL; } + + if (PyBytes_CheckExact(x)) { + Py_INCREF(x); + return x; + } + /* Use the modern buffer interface */ if (PyObject_CheckBuffer(x)) { Py_buffer view; @@ -2582,7 +2591,6 @@ PyBytes_FromObject(PyObject *x) new = PyBytes_FromStringAndSize(NULL, view.len); if (!new) goto fail; - /* XXX(brett.cannon): Better way to get to internal buffer? */ if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval, &view, view.len, 'C') < 0) goto fail; @@ -2857,149 +2865,6 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) return 0; } -/* _PyBytes_FormatLong emulates the format codes d, u, o, x and X, and - * the F_ALT flag, for Python's long (unbounded) ints. It's not used for - * Python's regular ints. - * Return value: a new PyBytes*, or NULL if error. - * . *pbuf is set to point into it, - * *plen set to the # of chars following that. - * Caller must decref it when done using pbuf. - * The string starting at *pbuf is of the form - * "-"? ("0x" | "0X")? digit+ - * "0x"/"0X" are present only for x and X conversions, with F_ALT - * set in flags. The case of hex digits will be correct, - * There will be at least prec digits, zero-filled on the left if - * necessary to get that many. - * val object to be converted - * flags bitmask of format flags; only F_ALT is looked at - * prec minimum number of digits; 0-fill on left if needed - * type a character in [duoxX]; u acts the same as d - * - * CAUTION: o, x and X conversions on regular ints can never - * produce a '-' sign, but can for Python's unbounded ints. - */ -PyObject* -_PyBytes_FormatLong(PyObject *val, int flags, int prec, int type, - char **pbuf, int *plen) -{ - PyObject *result = NULL; - char *buf; - Py_ssize_t i; - int sign; /* 1 if '-', else 0 */ - int len; /* number of characters */ - Py_ssize_t llen; - int numdigits; /* len == numnondigits + numdigits */ - int numnondigits = 0; - - /* Avoid exceeding SSIZE_T_MAX */ - if (prec > INT_MAX-3) { - PyErr_SetString(PyExc_OverflowError, - "precision too large"); - return NULL; - } - - switch (type) { - case 'd': - case 'u': - /* Special-case boolean: we want 0/1 */ - if (PyBool_Check(val)) - result = PyNumber_ToBase(val, 10); - else - result = Py_TYPE(val)->tp_str(val); - break; - case 'o': - numnondigits = 2; - result = PyNumber_ToBase(val, 8); - break; - case 'x': - case 'X': - numnondigits = 2; - result = PyNumber_ToBase(val, 16); - break; - default: - assert(!"'type' not in [duoxX]"); - } - if (!result) - return NULL; - - buf = _PyUnicode_AsString(result); - if (!buf) { - Py_DECREF(result); - return NULL; - } - - /* To modify the string in-place, there can only be one reference. */ - if (Py_REFCNT(result) != 1) { - PyErr_BadInternalCall(); - return NULL; - } - llen = PyUnicode_GetSize(result); - if (llen > INT_MAX) { - PyErr_SetString(PyExc_ValueError, - "string too large in _PyBytes_FormatLong"); - return NULL; - } - len = (int)llen; - if (buf[len-1] == 'L') { - --len; - buf[len] = '\0'; - } - sign = buf[0] == '-'; - numnondigits += sign; - numdigits = len - numnondigits; - assert(numdigits > 0); - - /* Get rid of base marker unless F_ALT */ - if (((flags & F_ALT) == 0 && - (type == 'o' || type == 'x' || type == 'X'))) { - assert(buf[sign] == '0'); - assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' || - buf[sign+1] == 'o'); - numnondigits -= 2; - buf += 2; - len -= 2; - if (sign) - buf[0] = '-'; - assert(len == numnondigits + numdigits); - assert(numdigits > 0); - } - - /* Fill with leading zeroes to meet minimum width. */ - if (prec > numdigits) { - PyObject *r1 = PyBytes_FromStringAndSize(NULL, - numnondigits + prec); - char *b1; - if (!r1) { - Py_DECREF(result); - return NULL; - } - b1 = PyBytes_AS_STRING(r1); - for (i = 0; i < numnondigits; ++i) - *b1++ = *buf++; - for (i = 0; i < prec - numdigits; i++) - *b1++ = '0'; - for (i = 0; i < numdigits; i++) - *b1++ = *buf++; - *b1 = '\0'; - Py_DECREF(result); - result = r1; - buf = PyBytes_AS_STRING(result); - len = numnondigits + prec; - } - - /* Fix up case for hex conversions. */ - if (type == 'X') { - /* Need to convert all lower case letters to upper case. - and need to convert 0x to 0X (and -0x to -0X). */ - for (i = 0; i < len; i++) - if (buf[i] >= 'a' && buf[i] <= 'x') - buf[i] -= 'a'-'A'; - } - *pbuf = buf; - *plen = len; - return result; -} - void PyBytes_Fini(void) { @@ -3072,9 +2937,43 @@ striter_len(striterobject *it) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +static PyObject * +striter_reduce(striterobject *it) +{ + if (it->it_seq != NULL) { + return Py_BuildValue("N(O)n", _PyObject_GetBuiltin("iter"), + it->it_seq, it->it_index); + } else { + PyObject *u = PyUnicode_FromUnicode(NULL, 0); + if (u == NULL) + return NULL; + return Py_BuildValue("N(N)", _PyObject_GetBuiltin("iter"), u); + } +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); + +static PyObject * +striter_setstate(striterobject *it, PyObject *state) +{ + Py_ssize_t index = PyLong_AsSsize_t(state); + if (index == -1 && PyErr_Occurred()) + return NULL; + if (index < 0) + index = 0; + it->it_index = index; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); + static PyMethodDef striter_methods[] = { {"__length_hint__", (PyCFunction)striter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)striter_reduce, METH_NOARGS, + reduce_doc}, + {"__setstate__", (PyCFunction)striter_setstate, METH_O, + setstate_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/classobject.c b/Objects/classobject.c index f956852..cdc9b1c 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -14,6 +14,8 @@ static int numfree = 0; #define PyMethod_MAXFREELIST 256 #endif +_Py_IDENTIFIER(__name__); + PyObject * PyMethod_Function(PyObject *im) { @@ -190,8 +192,7 @@ method_richcompare(PyObject *self, PyObject *other, int op) !PyMethod_Check(self) || !PyMethod_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } a = (PyMethodObject *)self; b = (PyMethodObject *)other; @@ -228,7 +229,7 @@ method_repr(PyMethodObject *a) } klass = (PyObject*)Py_TYPE(self); - funcname = PyObject_GetAttrString(func, "__name__"); + funcname = _PyObject_GetAttrId(func, &PyId___name__); if (funcname == NULL) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return NULL; @@ -242,7 +243,7 @@ method_repr(PyMethodObject *a) if (klass == NULL) klassname = NULL; else { - klassname = PyObject_GetAttrString(klass, "__name__"); + klassname = _PyObject_GetAttrId(klass, &PyId___name__); if (klassname == NULL) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { Py_XDECREF(funcname); @@ -402,6 +403,15 @@ PyMethod_Fini(void) (void)PyMethod_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyMethod_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyMethodObject", + numfree, sizeof(PyMethodObject)); +} + /* ------------------------------------------------------------------------ * instance method */ @@ -519,8 +529,7 @@ instancemethod_richcompare(PyObject *self, PyObject *other, int op) !PyInstanceMethod_Check(self) || !PyInstanceMethod_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } a = (PyInstanceMethodObject *)self; b = (PyInstanceMethodObject *)other; @@ -547,7 +556,7 @@ instancemethod_repr(PyObject *self) return NULL; } - funcname = PyObject_GetAttrString(func, "__name__"); + funcname = _PyObject_GetAttrId(func, &PyId___name__); if (funcname == NULL) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return NULL; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index bb938ea..9713f61 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -8,19 +8,24 @@ /* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */ static int -all_name_chars(Py_UNICODE *s) +all_name_chars(PyObject *o) { static char ok_name_char[256]; static unsigned char *name_chars = (unsigned char *)NAME_CHARS; + PyUnicodeObject *u = (PyUnicodeObject *)o; + const unsigned char *s; + + if (!PyUnicode_Check(o) || PyUnicode_READY(u) == -1 || + PyUnicode_MAX_CHAR_VALUE(u) >= 128) + return 0; if (ok_name_char[*name_chars] == 0) { unsigned char *p; for (p = name_chars; *p; p++) ok_name_char[*p] = 1; } + s = PyUnicode_1BYTE_DATA(u); while (*s) { - if (*s >= 128) - return 0; if (ok_name_char[*s++] == 0) return 0; } @@ -51,7 +56,8 @@ PyCode_New(int argcount, int kwonlyargcount, PyObject *lnotab) { PyCodeObject *co; - Py_ssize_t i; + unsigned char *cell2arg = NULL; + Py_ssize_t i, n_cellvars; /* Check argument types */ if (argcount < 0 || kwonlyargcount < 0 || nlocals < 0 || @@ -68,48 +74,79 @@ PyCode_New(int argcount, int kwonlyargcount, PyErr_BadInternalCall(); return NULL; } + n_cellvars = PyTuple_GET_SIZE(cellvars); intern_strings(names); intern_strings(varnames); intern_strings(freevars); intern_strings(cellvars); /* Intern selected string constants */ - for (i = PyTuple_Size(consts); --i >= 0; ) { + for (i = PyTuple_GET_SIZE(consts); --i >= 0; ) { PyObject *v = PyTuple_GetItem(consts, i); - if (!PyUnicode_Check(v)) - continue; - if (!all_name_chars(PyUnicode_AS_UNICODE(v))) + if (!all_name_chars(v)) continue; PyUnicode_InternInPlace(&PyTuple_GET_ITEM(consts, i)); } + /* Create mapping between cells and arguments if needed. */ + if (n_cellvars) { + Py_ssize_t total_args = argcount + kwonlyargcount + + ((flags & CO_VARARGS) != 0) + ((flags & CO_VARKEYWORDS) != 0); + Py_ssize_t alloc_size = sizeof(unsigned char) * n_cellvars; + int used_cell2arg = 0; + cell2arg = PyMem_MALLOC(alloc_size); + if (cell2arg == NULL) + return NULL; + memset(cell2arg, CO_CELL_NOT_AN_ARG, alloc_size); + /* Find cells which are also arguments. */ + for (i = 0; i < n_cellvars; i++) { + Py_ssize_t j; + PyObject *cell = PyTuple_GET_ITEM(cellvars, i); + for (j = 0; j < total_args; j++) { + PyObject *arg = PyTuple_GET_ITEM(varnames, j); + if (!PyUnicode_Compare(cell, arg)) { + cell2arg[i] = j; + used_cell2arg = 1; + break; + } + } + } + if (!used_cell2arg) { + PyMem_FREE(cell2arg); + cell2arg = NULL; + } + } co = PyObject_NEW(PyCodeObject, &PyCode_Type); - if (co != NULL) { - co->co_argcount = argcount; - co->co_kwonlyargcount = kwonlyargcount; - co->co_nlocals = nlocals; - co->co_stacksize = stacksize; - co->co_flags = flags; - Py_INCREF(code); - co->co_code = code; - Py_INCREF(consts); - co->co_consts = consts; - Py_INCREF(names); - co->co_names = names; - Py_INCREF(varnames); - co->co_varnames = varnames; - Py_INCREF(freevars); - co->co_freevars = freevars; - Py_INCREF(cellvars); - co->co_cellvars = cellvars; - Py_INCREF(filename); - co->co_filename = filename; - Py_INCREF(name); - co->co_name = name; - co->co_firstlineno = firstlineno; - Py_INCREF(lnotab); - co->co_lnotab = lnotab; - co->co_zombieframe = NULL; - co->co_weakreflist = NULL; + if (co == NULL) { + if (cell2arg) + PyMem_FREE(cell2arg); + return NULL; } + co->co_argcount = argcount; + co->co_kwonlyargcount = kwonlyargcount; + co->co_nlocals = nlocals; + co->co_stacksize = stacksize; + co->co_flags = flags; + Py_INCREF(code); + co->co_code = code; + Py_INCREF(consts); + co->co_consts = consts; + Py_INCREF(names); + co->co_names = names; + Py_INCREF(varnames); + co->co_varnames = varnames; + Py_INCREF(freevars); + co->co_freevars = freevars; + Py_INCREF(cellvars); + co->co_cellvars = cellvars; + co->co_cell2arg = cell2arg; + Py_INCREF(filename); + co->co_filename = filename; + Py_INCREF(name); + co->co_name = name; + co->co_firstlineno = firstlineno; + Py_INCREF(lnotab); + co->co_lnotab = lnotab; + co->co_zombieframe = NULL; + co->co_weakreflist = NULL; return co; } @@ -212,9 +249,7 @@ validate_and_copy_tuple(PyObject *tup) return NULL; } else { - item = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(item), - PyUnicode_GET_SIZE(item)); + item = _PyUnicode_Copy(item); if (item == NULL) { Py_DECREF(newtuple); return NULL; @@ -330,6 +365,8 @@ code_dealloc(PyCodeObject *co) Py_XDECREF(co->co_filename); Py_XDECREF(co->co_name); Py_XDECREF(co->co_lnotab); + if (co->co_cell2arg != NULL) + PyMem_FREE(co->co_cell2arg); if (co->co_zombieframe != NULL) PyObject_GC_Del(co->co_zombieframe); if (co->co_weakreflist != NULL) @@ -338,6 +375,17 @@ code_dealloc(PyCodeObject *co) } static PyObject * +code_sizeof(PyCodeObject *co, void *unused) +{ + Py_ssize_t res; + + res = sizeof(PyCodeObject); + if (co->co_cell2arg != NULL && co->co_cellvars != NULL) + res += PyTuple_GET_SIZE(co->co_cellvars) * sizeof(unsigned char); + return PyLong_FromSsize_t(res); +} + +static PyObject * code_repr(PyCodeObject *co) { int lineno; @@ -366,8 +414,7 @@ code_richcompare(PyObject *self, PyObject *other, int op) if ((op != Py_EQ && op != Py_NE) || !PyCode_Check(self) || !PyCode_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } co = (PyCodeObject *)self; @@ -444,6 +491,11 @@ code_hash(PyCodeObject *co) /* XXX code objects need to participate in GC? */ +static struct PyMethodDef code_methods[] = { + {"__sizeof__", (PyCFunction)code_sizeof, METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; + PyTypeObject PyCode_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "code", @@ -472,7 +524,7 @@ PyTypeObject PyCode_Type = { offsetof(PyCodeObject, co_weakreflist), /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ - 0, /* tp_methods */ + code_methods, /* tp_methods */ code_memberlist, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ diff --git a/Objects/complexobject.c b/Objects/complexobject.c index e247ba9..403c60c 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -265,9 +265,9 @@ PyComplex_ImagAsDouble(PyObject *op) static PyObject * try_complex_special_method(PyObject *op) { PyObject *f; - static PyObject *complexstr; + _Py_IDENTIFIER(__complex__); - f = _PyObject_LookupSpecial(op, "__complex__", &complexstr); + f = _PyObject_LookupSpecial(op, &PyId___complex__); if (f) { PyObject *res = PyObject_CallFunctionObjArgs(f, NULL); Py_DECREF(f); @@ -330,12 +330,10 @@ complex_repr(PyComplexObject *v) int precision = 0; char format_code = 'r'; PyObject *result = NULL; - Py_ssize_t len; /* If these are non-NULL, they'll need to be freed. */ char *pre = NULL; char *im = NULL; - char *buf = NULL; /* These do not need to be freed. re is either an alias for pre or a pointer to a constant. lead and tail @@ -374,20 +372,10 @@ complex_repr(PyComplexObject *v) lead = "("; tail = ")"; } - /* Alloc the final buffer. Add one for the "j" in the format string, - and one for the trailing zero byte. */ - len = strlen(lead) + strlen(re) + strlen(im) + strlen(tail) + 2; - buf = PyMem_Malloc(len); - if (!buf) { - PyErr_NoMemory(); - goto done; - } - PyOS_snprintf(buf, len, "%s%s%sj%s", lead, re, im, tail); - result = PyUnicode_FromString(buf); + result = PyUnicode_FromFormat("%s%s%sj%s", lead, re, im, tail); done: PyMem_Free(im); PyMem_Free(pre); - PyMem_Free(buf); return result; } @@ -662,8 +650,7 @@ complex_richcompare(PyObject *v, PyObject *w, int op) return res; Unimplemented: - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static PyObject * @@ -712,12 +699,22 @@ static PyObject * complex__format__(PyObject* self, PyObject* args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) - return NULL; - return _PyComplex_FormatAdvanced(self, - PyUnicode_AS_UNICODE(format_spec), - PyUnicode_GET_SIZE(format_spec)); + return NULL; + + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyComplex_FormatAdvancedWriter( + &writer, + self, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } #if 0 @@ -768,20 +765,10 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) Py_ssize_t len; if (PyUnicode_Check(v)) { - Py_ssize_t i, buflen = PyUnicode_GET_SIZE(v); - Py_UNICODE *bufptr; - s_buffer = PyUnicode_TransformDecimalToASCII( - PyUnicode_AS_UNICODE(v), buflen); + s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v); if (s_buffer == NULL) return NULL; - /* Replace non-ASCII whitespace with ' ' */ - bufptr = PyUnicode_AS_UNICODE(s_buffer); - for (i = 0; i < buflen; i++) { - Py_UNICODE ch = bufptr[i]; - if (ch > 127 && Py_UNICODE_ISSPACE(ch)) - bufptr[i] = ' '; - } - s = _PyUnicode_AsStringAndSize(s_buffer, &len); + s = PyUnicode_AsUTF8AndSize(s_buffer, &len); if (s == NULL) goto error; } diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 9934f23..abcc002 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -9,6 +9,7 @@ descr_dealloc(PyDescrObject *descr) _PyObject_GC_UNTRACK(descr); Py_XDECREF(descr->d_type); Py_XDECREF(descr->d_name); + Py_XDECREF(descr->d_qualname); PyObject_GC_Del(descr); } @@ -359,6 +360,44 @@ method_get_doc(PyMethodDescrObject *descr, void *closure) return PyUnicode_FromString(descr->d_method->ml_doc); } +static PyObject * +calculate_qualname(PyDescrObject *descr) +{ + PyObject *type_qualname, *res; + _Py_IDENTIFIER(__qualname__); + + if (descr->d_name == NULL || !PyUnicode_Check(descr->d_name)) { + PyErr_SetString(PyExc_TypeError, + "<descriptor>.__name__ is not a unicode object"); + return NULL; + } + + type_qualname = _PyObject_GetAttrId((PyObject *)descr->d_type, + &PyId___qualname__); + if (type_qualname == NULL) + return NULL; + + if (!PyUnicode_Check(type_qualname)) { + PyErr_SetString(PyExc_TypeError, "<descriptor>.__objclass__." + "__qualname__ is not a unicode object"); + Py_XDECREF(type_qualname); + return NULL; + } + + res = PyUnicode_FromFormat("%S.%S", type_qualname, descr->d_name); + Py_DECREF(type_qualname); + return res; +} + +static PyObject * +descr_get_qualname(PyDescrObject *descr) +{ + if (descr->d_qualname == NULL) + descr->d_qualname = calculate_qualname(descr); + Py_XINCREF(descr->d_qualname); + return descr->d_qualname; +} + static PyMemberDef descr_members[] = { {"__objclass__", T_OBJECT, offsetof(PyDescrObject, d_type), READONLY}, {"__name__", T_OBJECT, offsetof(PyDescrObject, d_name), READONLY}, @@ -367,6 +406,7 @@ static PyMemberDef descr_members[] = { static PyGetSetDef method_getset[] = { {"__doc__", (getter)method_get_doc}, + {"__qualname__", (getter)descr_get_qualname}, {0} }; @@ -382,6 +422,7 @@ member_get_doc(PyMemberDescrObject *descr, void *closure) static PyGetSetDef member_getset[] = { {"__doc__", (getter)member_get_doc}, + {"__qualname__", (getter)descr_get_qualname}, {0} }; @@ -397,6 +438,7 @@ getset_get_doc(PyGetSetDescrObject *descr, void *closure) static PyGetSetDef getset_getset[] = { {"__doc__", (getter)getset_get_doc}, + {"__qualname__", (getter)descr_get_qualname}, {0} }; @@ -412,6 +454,7 @@ wrapperdescr_get_doc(PyWrapperDescrObject *descr, void *closure) static PyGetSetDef wrapperdescr_getset[] = { {"__doc__", (getter)wrapperdescr_get_doc}, + {"__qualname__", (getter)descr_get_qualname}, {0} }; @@ -623,6 +666,9 @@ descr_new(PyTypeObject *descrtype, PyTypeObject *type, const char *name) Py_DECREF(descr); descr = NULL; } + else { + descr->d_qualname = NULL; + } } return descr; } @@ -690,41 +736,44 @@ PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *base, void *wrapped) } -/* --- Readonly proxy for dictionaries (actually any mapping) --- */ +/* --- mappingproxy: read-only proxy for mappings --- */ /* This has no reason to be in this file except that adding new files is a bit of a pain */ typedef struct { PyObject_HEAD - PyObject *dict; -} proxyobject; + PyObject *mapping; +} mappingproxyobject; static Py_ssize_t -proxy_len(proxyobject *pp) +mappingproxy_len(mappingproxyobject *pp) { - return PyObject_Size(pp->dict); + return PyObject_Size(pp->mapping); } static PyObject * -proxy_getitem(proxyobject *pp, PyObject *key) +mappingproxy_getitem(mappingproxyobject *pp, PyObject *key) { - return PyObject_GetItem(pp->dict, key); + return PyObject_GetItem(pp->mapping, key); } -static PyMappingMethods proxy_as_mapping = { - (lenfunc)proxy_len, /* mp_length */ - (binaryfunc)proxy_getitem, /* mp_subscript */ +static PyMappingMethods mappingproxy_as_mapping = { + (lenfunc)mappingproxy_len, /* mp_length */ + (binaryfunc)mappingproxy_getitem, /* mp_subscript */ 0, /* mp_ass_subscript */ }; static int -proxy_contains(proxyobject *pp, PyObject *key) +mappingproxy_contains(mappingproxyobject *pp, PyObject *key) { - return PyDict_Contains(pp->dict, key); + if (PyDict_CheckExact(pp->mapping)) + return PyDict_Contains(pp->mapping, key); + else + return PySequence_Contains(pp->mapping, key); } -static PySequenceMethods proxy_as_sequence = { +static PySequenceMethods mappingproxy_as_sequence = { 0, /* sq_length */ 0, /* sq_concat */ 0, /* sq_repeat */ @@ -732,147 +781,199 @@ static PySequenceMethods proxy_as_sequence = { 0, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ - (objobjproc)proxy_contains, /* sq_contains */ + (objobjproc)mappingproxy_contains, /* sq_contains */ 0, /* sq_inplace_concat */ 0, /* sq_inplace_repeat */ }; static PyObject * -proxy_get(proxyobject *pp, PyObject *args) +mappingproxy_get(mappingproxyobject *pp, PyObject *args) { PyObject *key, *def = Py_None; + _Py_IDENTIFIER(get); if (!PyArg_UnpackTuple(args, "get", 1, 2, &key, &def)) return NULL; - return PyObject_CallMethod(pp->dict, "get", "(OO)", key, def); + return _PyObject_CallMethodId(pp->mapping, &PyId_get, "(OO)", key, def); } static PyObject * -proxy_keys(proxyobject *pp) +mappingproxy_keys(mappingproxyobject *pp) { - return PyObject_CallMethod(pp->dict, "keys", NULL); + _Py_IDENTIFIER(keys); + return _PyObject_CallMethodId(pp->mapping, &PyId_keys, NULL); } static PyObject * -proxy_values(proxyobject *pp) +mappingproxy_values(mappingproxyobject *pp) { - return PyObject_CallMethod(pp->dict, "values", NULL); + _Py_IDENTIFIER(values); + return _PyObject_CallMethodId(pp->mapping, &PyId_values, NULL); } static PyObject * -proxy_items(proxyobject *pp) +mappingproxy_items(mappingproxyobject *pp) { - return PyObject_CallMethod(pp->dict, "items", NULL); + _Py_IDENTIFIER(items); + return _PyObject_CallMethodId(pp->mapping, &PyId_items, NULL); } static PyObject * -proxy_copy(proxyobject *pp) +mappingproxy_copy(mappingproxyobject *pp) { - return PyObject_CallMethod(pp->dict, "copy", NULL); + _Py_IDENTIFIER(copy); + return _PyObject_CallMethodId(pp->mapping, &PyId_copy, NULL); } -static PyMethodDef proxy_methods[] = { - {"get", (PyCFunction)proxy_get, METH_VARARGS, +/* WARNING: mappingproxy methods must not give access + to the underlying mapping */ + +static PyMethodDef mappingproxy_methods[] = { + {"get", (PyCFunction)mappingproxy_get, METH_VARARGS, PyDoc_STR("D.get(k[,d]) -> D[k] if k in D, else d." - " d defaults to None.")}, - {"keys", (PyCFunction)proxy_keys, METH_NOARGS, + " d defaults to None.")}, + {"keys", (PyCFunction)mappingproxy_keys, METH_NOARGS, PyDoc_STR("D.keys() -> list of D's keys")}, - {"values", (PyCFunction)proxy_values, METH_NOARGS, + {"values", (PyCFunction)mappingproxy_values, METH_NOARGS, PyDoc_STR("D.values() -> list of D's values")}, - {"items", (PyCFunction)proxy_items, METH_NOARGS, + {"items", (PyCFunction)mappingproxy_items, METH_NOARGS, PyDoc_STR("D.items() -> list of D's (key, value) pairs, as 2-tuples")}, - {"copy", (PyCFunction)proxy_copy, METH_NOARGS, + {"copy", (PyCFunction)mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, {0} }; static void -proxy_dealloc(proxyobject *pp) +mappingproxy_dealloc(mappingproxyobject *pp) { _PyObject_GC_UNTRACK(pp); - Py_DECREF(pp->dict); + Py_DECREF(pp->mapping); PyObject_GC_Del(pp); } static PyObject * -proxy_getiter(proxyobject *pp) +mappingproxy_getiter(mappingproxyobject *pp) { - return PyObject_GetIter(pp->dict); + return PyObject_GetIter(pp->mapping); } static PyObject * -proxy_str(proxyobject *pp) +mappingproxy_str(mappingproxyobject *pp) { - return PyObject_Str(pp->dict); + return PyObject_Str(pp->mapping); } static PyObject * -proxy_repr(proxyobject *pp) +mappingproxy_repr(mappingproxyobject *pp) { - return PyUnicode_FromFormat("dict_proxy(%R)", pp->dict); + return PyUnicode_FromFormat("mappingproxy(%R)", pp->mapping); } static int -proxy_traverse(PyObject *self, visitproc visit, void *arg) +mappingproxy_traverse(PyObject *self, visitproc visit, void *arg) { - proxyobject *pp = (proxyobject *)self; - Py_VISIT(pp->dict); + mappingproxyobject *pp = (mappingproxyobject *)self; + Py_VISIT(pp->mapping); return 0; } static PyObject * -proxy_richcompare(proxyobject *v, PyObject *w, int op) +mappingproxy_richcompare(mappingproxyobject *v, PyObject *w, int op) +{ + return PyObject_RichCompare(v->mapping, w, op); +} + +static int +mappingproxy_check_mapping(PyObject *mapping) { - return PyObject_RichCompare(v->dict, w, op); + if (!PyMapping_Check(mapping) + || PyList_Check(mapping) + || PyTuple_Check(mapping)) { + PyErr_Format(PyExc_TypeError, + "mappingproxy() argument must be a mapping, not %s", + Py_TYPE(mapping)->tp_name); + return -1; + } + return 0; +} + +static PyObject* +mappingproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"mapping", NULL}; + PyObject *mapping; + mappingproxyobject *mappingproxy; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:mappingproxy", + kwlist, &mapping)) + return NULL; + + if (mappingproxy_check_mapping(mapping) == -1) + return NULL; + + mappingproxy = PyObject_GC_New(mappingproxyobject, &PyDictProxy_Type); + if (mappingproxy == NULL) + return NULL; + Py_INCREF(mapping); + mappingproxy->mapping = mapping; + _PyObject_GC_TRACK(mappingproxy); + return (PyObject *)mappingproxy; } PyTypeObject PyDictProxy_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "dict_proxy", /* tp_name */ - sizeof(proxyobject), /* tp_basicsize */ + "mappingproxy", /* tp_name */ + sizeof(mappingproxyobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)proxy_dealloc, /* tp_dealloc */ + (destructor)mappingproxy_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_reserved */ - (reprfunc)proxy_repr, /* tp_repr */ + (reprfunc)mappingproxy_repr, /* tp_repr */ 0, /* tp_as_number */ - &proxy_as_sequence, /* tp_as_sequence */ - &proxy_as_mapping, /* tp_as_mapping */ + &mappingproxy_as_sequence, /* tp_as_sequence */ + &mappingproxy_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ - (reprfunc)proxy_str, /* tp_str */ + (reprfunc)mappingproxy_str, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - proxy_traverse, /* tp_traverse */ + mappingproxy_traverse, /* tp_traverse */ 0, /* tp_clear */ - (richcmpfunc)proxy_richcompare, /* tp_richcompare */ + (richcmpfunc)mappingproxy_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ - (getiterfunc)proxy_getiter, /* tp_iter */ + (getiterfunc)mappingproxy_getiter, /* tp_iter */ 0, /* tp_iternext */ - proxy_methods, /* tp_methods */ + mappingproxy_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + mappingproxy_new, /* tp_new */ }; PyObject * -PyDictProxy_New(PyObject *dict) +PyDictProxy_New(PyObject *mapping) { - proxyobject *pp; + mappingproxyobject *pp; + + if (mappingproxy_check_mapping(mapping) == -1) + return NULL; - pp = PyObject_GC_New(proxyobject, &PyDictProxy_Type); + pp = PyObject_GC_New(mappingproxyobject, &PyDictProxy_Type); if (pp != NULL) { - Py_INCREF(dict); - pp->dict = dict; + Py_INCREF(mapping); + pp->mapping = mapping; _PyObject_GC_TRACK(pp); } return (PyObject *)pp; @@ -1020,9 +1121,16 @@ wrapper_doc(wrapperobject *wp) } } +static PyObject * +wrapper_qualname(wrapperobject *wp) +{ + return descr_get_qualname((PyDescrObject *)wp->descr); +} + static PyGetSetDef wrapper_getsets[] = { {"__objclass__", (getter)wrapper_objclass}, {"__name__", (getter)wrapper_name}, + {"__qualname__", (getter)wrapper_qualname}, {"__doc__", (getter)wrapper_doc}, {0} }; @@ -1332,7 +1440,8 @@ property_init(PyObject *self, PyObject *args, PyObject *kwds) /* if no docstring given and the getter has one, use that one */ if ((doc == NULL || doc == Py_None) && get != NULL) { - PyObject *get_doc = PyObject_GetAttrString(get, "__doc__"); + _Py_IDENTIFIER(__doc__); + PyObject *get_doc = _PyObject_GetAttrId(get, &PyId___doc__); if (get_doc) { if (Py_TYPE(self) == &PyProperty_Type) { Py_XDECREF(prop->prop_doc); @@ -1343,7 +1452,7 @@ property_init(PyObject *self, PyObject *args, PyObject *kwds) in dict of the subclass instance instead, otherwise it gets shadowed by __doc__ in the class's dict. */ - int err = PyObject_SetAttrString(self, "__doc__", get_doc); + int err = _PyObject_SetAttrId(self, &PyId___doc__, get_doc); Py_DECREF(get_doc); if (err < 0) return -1; @@ -1361,6 +1470,43 @@ property_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } +static PyObject * +property_get___isabstractmethod__(propertyobject *prop, void *closure) +{ + int res = _PyObject_IsAbstract(prop->prop_get); + if (res == -1) { + return NULL; + } + else if (res) { + Py_RETURN_TRUE; + } + + res = _PyObject_IsAbstract(prop->prop_set); + if (res == -1) { + return NULL; + } + else if (res) { + Py_RETURN_TRUE; + } + + res = _PyObject_IsAbstract(prop->prop_del); + if (res == -1) { + return NULL; + } + else if (res) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static PyGetSetDef property_getsetlist[] = { + {"__isabstractmethod__", + (getter)property_get___isabstractmethod__, NULL, + NULL, + NULL}, + {NULL} /* Sentinel */ +}; + PyDoc_STRVAR(property_doc, "property(fget=None, fset=None, fdel=None, doc=None) -> property attribute\n" "\n" @@ -1426,7 +1572,7 @@ PyTypeObject PyProperty_Type = { 0, /* tp_iternext */ property_methods, /* tp_methods */ property_members, /* tp_members */ - 0, /* tp_getset */ + property_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ property_descr_get, /* tp_descr_get */ diff --git a/Objects/dictnotes.txt b/Objects/dictnotes.txt index d3aa774..f89720c 100644 --- a/Objects/dictnotes.txt +++ b/Objects/dictnotes.txt @@ -1,7 +1,6 @@ -NOTES ON OPTIMIZING DICTIONARIES +NOTES ON DICTIONARIES ================================ - Principal Use Cases for Dictionaries ------------------------------------ @@ -21,7 +20,7 @@ Instance attribute lookup and Global variables Builtins Frequent reads. Almost never written. - Size 126 interned strings (as of Py2.3b1). + About 150 interned strings (as of Py3.3). A few keys are accessed much more frequently than others. Uniquification @@ -59,55 +58,20 @@ Dynamic Mappings Characterized by deletions interspersed with adds and replacements. Performance benefits greatly from the re-use of dummy entries. +Data Layout +----------- -Data Layout (assuming a 32-bit box with 64 bytes per cache line) ----------------------------------------------------------------- - -Smalldicts (8 entries) are attached to the dictobject structure -and the whole group nearly fills two consecutive cache lines. - -Larger dicts use the first half of the dictobject structure (one cache -line) and a separate, continuous block of entries (at 12 bytes each -for a total of 5.333 entries per cache line). +Dictionaries are composed of 3 components: +The dictobject struct itself +A dict-keys object (keys & hashes) +A values array Tunable Dictionary Parameters ----------------------------- -* PyDict_MINSIZE. Currently set to 8. - Must be a power of two. New dicts have to zero-out every cell. - Each additional 8 consumes 1.5 cache lines. Increasing improves - the sparseness of small dictionaries but costs time to read in - the additional cache lines if they are not already in cache. - That case is common when keyword arguments are passed. - -* Maximum dictionary load in PyDict_SetItem. Currently set to 2/3. - Increasing this ratio makes dictionaries more dense resulting - in more collisions. Decreasing it improves sparseness at the - expense of spreading entries over more cache lines and at the - cost of total memory consumed. - - The load test occurs in highly time sensitive code. Efforts - to make the test more complex (for example, varying the load - for different sizes) have degraded performance. - -* Growth rate upon hitting maximum load. Currently set to *2. - Raising this to *4 results in half the number of resizes, - less effort to resize, better sparseness for some (but not - all dict sizes), and potentially doubles memory consumption - depending on the size of the dictionary. Setting to *4 - eliminates every other resize step. - -* Maximum sparseness (minimum dictionary load). What percentage - of entries can be unused before the dictionary shrinks to - free up memory and speed up iteration? (The current CPython - code does not represent this parameter directly.) - -* Shrinkage rate upon exceeding maximum sparseness. The current - CPython code never even checks sparseness when deleting a - key. When a new key is added, it resizes based on the number - of active keys, so that the addition may trigger shrinkage - rather than growth. +See comments for PyDict_MINSIZE_SPLIT, PyDict_MINSIZE_COMBINED, +USABLE_FRACTION and GROWTH_RATE in dictobject.c Tune-ups should be measured across a broad range of applications and use cases. A change to any parameter will help in some situations and @@ -126,8 +90,8 @@ __iter__(), iterkeys(), iteritems(), itervalues(), and update(). Also, every dictionary iterates at least twice, once for the memset() when it is created and once by dealloc(). -Dictionary operations involving only a single key can be O(1) unless -resizing is possible. By checking for a resize only when the +Dictionary operations involving only a single key can be O(1) unless +resizing is possible. By checking for a resize only when the dictionary can grow (and may *require* resizing), other operations remain O(1), and the odds of resize thrashing or memory fragmentation are reduced. In particular, an algorithm that empties a dictionary @@ -135,136 +99,51 @@ by repeatedly invoking .pop will see no resizing, which might not be necessary at all because the dictionary is eventually discarded entirely. +The key differences between this implementation and earlier versions are: + 1. The table can be split into two parts, the keys and the values. + + 2. There is an additional key-value combination: (key, NULL). + Unlike (<dummy>, NULL) which represents a deleted value, (key, NULL) + represented a yet to be inserted value. This combination can only occur + when the table is split. + + 3. No small table embedded in the dict, + as this would make sharing of key-tables impossible. + + +These changes have the following consequences. + 1. General dictionaries are slightly larger. + + 2. All object dictionaries of a single class can share a single key-table, + saving about 60% memory for such cases. Results of Cache Locality Experiments -------------------------------------- - -When an entry is retrieved from memory, 4.333 adjacent entries are also -retrieved into a cache line. Since accessing items in cache is *much* -cheaper than a cache miss, an enticing idea is to probe the adjacent -entries as a first step in collision resolution. Unfortunately, the -introduction of any regularity into collision searches results in more -collisions than the current random chaining approach. - -Exploiting cache locality at the expense of additional collisions fails -to payoff when the entries are already loaded in cache (the expense -is paid with no compensating benefit). This occurs in small dictionaries -where the whole dictionary fits into a pair of cache lines. It also -occurs frequently in large dictionaries which have a common access pattern -where some keys are accessed much more frequently than others. The -more popular entries *and* their collision chains tend to remain in cache. - -To exploit cache locality, change the collision resolution section -in lookdict() and lookdict_string(). Set i^=1 at the top of the -loop and move the i = (i << 2) + i + perturb + 1 to an unrolled -version of the loop. - -This optimization strategy can be leveraged in several ways: - -* If the dictionary is kept sparse (through the tunable parameters), -then the occurrence of additional collisions is lessened. - -* If lookdict() and lookdict_string() are specialized for small dicts -and for largedicts, then the versions for large_dicts can be given -an alternate search strategy without increasing collisions in small dicts -which already have the maximum benefit of cache locality. - -* If the use case for a dictionary is known to have a random key -access pattern (as opposed to a more common pattern with a Zipf's law -distribution), then there will be more benefit for large dictionaries -because any given key is no more likely than another to already be -in cache. - -* In use cases with paired accesses to the same key, the second access -is always in cache and gets no benefit from efforts to further improve -cache locality. - -Optimizing the Search of Small Dictionaries -------------------------------------------- - -If lookdict() and lookdict_string() are specialized for smaller dictionaries, -then a custom search approach can be implemented that exploits the small -search space and cache locality. - -* The simplest example is a linear search of contiguous entries. This is - simple to implement, guaranteed to terminate rapidly, never searches - the same entry twice, and precludes the need to check for dummy entries. - -* A more advanced example is a self-organizing search so that the most - frequently accessed entries get probed first. The organization - adapts if the access pattern changes over time. Treaps are ideally - suited for self-organization with the most common entries at the - top of the heap and a rapid binary search pattern. Most probes and - results are all located at the top of the tree allowing them all to - be located in one or two cache lines. - -* Also, small dictionaries may be made more dense, perhaps filling all - eight cells to take the maximum advantage of two cache lines. - - -Strategy Pattern ----------------- - -Consider allowing the user to set the tunable parameters or to select a -particular search method. Since some dictionary use cases have known -sizes and access patterns, the user may be able to provide useful hints. - -1) For example, if membership testing or lookups dominate runtime and memory - is not at a premium, the user may benefit from setting the maximum load - ratio at 5% or 10% instead of the usual 66.7%. This will sharply - curtail the number of collisions but will increase iteration time. - The builtin namespace is a prime example of a dictionary that can - benefit from being highly sparse. - -2) Dictionary creation time can be shortened in cases where the ultimate - size of the dictionary is known in advance. The dictionary can be - pre-sized so that no resize operations are required during creation. - Not only does this save resizes, but the key insertion will go - more quickly because the first half of the keys will be inserted into - a more sparse environment than before. The preconditions for this - strategy arise whenever a dictionary is created from a key or item - sequence and the number of *unique* keys is known. - -3) If the key space is large and the access pattern is known to be random, - then search strategies exploiting cache locality can be fruitful. - The preconditions for this strategy arise in simulations and - numerical analysis. - -4) If the keys are fixed and the access pattern strongly favors some of - the keys, then the entries can be stored contiguously and accessed - with a linear search or treap. This exploits knowledge of the data, - cache locality, and a simplified search routine. It also eliminates - the need to test for dummy entries on each probe. The preconditions - for this strategy arise in symbol tables and in the builtin dictionary. - - -Readonly Dictionaries ---------------------- -Some dictionary use cases pass through a build stage and then move to a -more heavily exercised lookup stage with no further changes to the -dictionary. - -An idea that emerged on python-dev is to be able to convert a dictionary -to a read-only state. This can help prevent programming errors and also -provide knowledge that can be exploited for lookup optimization. - -The dictionary can be immediately rebuilt (eliminating dummy entries), -resized (to an appropriate level of sparseness), and the keys can be -jostled (to minimize collisions). The lookdict() routine can then -eliminate the test for dummy entries (saving about 1/4 of the time -spent in the collision resolution loop). - -An additional possibility is to insert links into the empty spaces -so that dictionary iteration can proceed in len(d) steps instead of -(mp->mask + 1) steps. Alternatively, a separate tuple of keys can be -kept just for iteration. - - -Caching Lookups ---------------- -The idea is to exploit key access patterns by anticipating future lookups -based on previous lookups. - -The simplest incarnation is to save the most recently accessed entry. -This gives optimal performance for use cases where every get is followed -by a set or del to the same key. +-------------------------------------- + +Experiments on an earlier design of dictionary, in which all tables were +combined, showed the following: + + When an entry is retrieved from memory, several adjacent entries are also + retrieved into a cache line. Since accessing items in cache is *much* + cheaper than a cache miss, an enticing idea is to probe the adjacent + entries as a first step in collision resolution. Unfortunately, the + introduction of any regularity into collision searches results in more + collisions than the current random chaining approach. + + Exploiting cache locality at the expense of additional collisions fails + to payoff when the entries are already loaded in cache (the expense + is paid with no compensating benefit). This occurs in small dictionaries + where the whole dictionary fits into a pair of cache lines. It also + occurs frequently in large dictionaries which have a common access pattern + where some keys are accessed much more frequently than others. The + more popular entries *and* their collision chains tend to remain in cache. + + To exploit cache locality, change the collision resolution section + in lookdict() and lookdict_string(). Set i^=1 at the top of the + loop and move the i = (i << 2) + i + perturb + 1 to an unrolled + version of the loop. + +For split tables, the above will apply to the keys, but the value will +always be in a different cache line from the key. + + diff --git a/Objects/dictobject.c b/Objects/dictobject.c index d3c5eac..02dcf7b 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -7,9 +7,93 @@ tuning dictionaries, and several ideas for possible optimizations. */ + +/* +There are four kinds of slots in the table: + +1. Unused. me_key == me_value == NULL + Does not hold an active (key, value) pair now and never did. Unused can + transition to Active upon key insertion. This is the only case in which + me_key is NULL, and is each slot's initial state. + +2. Active. me_key != NULL and me_key != dummy and me_value != NULL + Holds an active (key, value) pair. Active can transition to Dummy or + Pending upon key deletion (for combined and split tables respectively). + This is the only case in which me_value != NULL. + +3. Dummy. me_key == dummy and me_value == NULL + Previously held an active (key, value) pair, but that was deleted and an + active pair has not yet overwritten the slot. Dummy can transition to + Active upon key insertion. Dummy slots cannot be made Unused again + (cannot have me_key set to NULL), else the probe sequence in case of + collision would have no way to know they were once active. + +4. Pending. Not yet inserted or deleted from a split-table. + key != NULL, key != dummy and value == NULL + +The DictObject can be in one of two forms. +Either: + A combined table: + ma_values == NULL, dk_refcnt == 1. + Values are stored in the me_value field of the PyDictKeysObject. + Slot kind 4 is not allowed i.e. + key != NULL, key != dummy and value == NULL is illegal. +Or: + A split table: + ma_values != NULL, dk_refcnt >= 1 + Values are stored in the ma_values array. + Only string (unicode) keys are allowed, no <dummy> keys are present. + +Note: .popitem() abuses the me_hash field of an Unused or Dummy slot to +hold a search finger. The me_hash field of Unused or Dummy slots has no +meaning otherwise. As a consequence of this popitem always converts the dict +to the combined-table form. +*/ + +/* PyDict_MINSIZE_SPLIT is the minimum size of a split dictionary. + * It must be a power of 2, and at least 4. + * Resizing of split dictionaries is very rare, so the saving memory is more + * important than the cost of resizing. + */ +#define PyDict_MINSIZE_SPLIT 4 + +/* PyDict_MINSIZE_COMBINED is the starting size for any new, non-split dict. + * 8 allows dicts with no more than 5 active entries; experiments suggested + * this suffices for the majority of dicts (consisting mostly of usually-small + * dicts created to pass keyword arguments). + * Making this 8, rather than 4 reduces the number of resizes for most + * dictionaries, without any significant extra memory use. + */ +#define PyDict_MINSIZE_COMBINED 8 + #include "Python.h" #include "stringlib/eq.h" +typedef struct { + /* Cached hash code of me_key. */ + Py_hash_t me_hash; + PyObject *me_key; + PyObject *me_value; /* This field is only meaningful for combined tables */ +} PyDictKeyEntry; + +typedef PyDictKeyEntry *(*dict_lookup_func) +(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject ***value_addr); + +struct _dictkeysobject { + Py_ssize_t dk_refcnt; + Py_ssize_t dk_size; + dict_lookup_func dk_lookup; + Py_ssize_t dk_usable; + PyDictKeyEntry dk_entries[1]; +}; + + +/* +To ensure the lookup algorithm terminates, there must be at least one Unused +slot (NULL key) in the table. +To avoid slowing down lookups on a near-full table, we resize the table when +it's USABLE_FRACTION (currently two-thirds) full. +*/ /* Set a key error with the specified argument, wrapping it in a * tuple automatically so that tuple keys are not unpacked as the @@ -25,10 +109,6 @@ set_key_error(PyObject *arg) Py_DECREF(tup); } -/* Define this out if you don't want conversion statistics on exit. */ -#undef SHOW_CONVERSION_COUNTS - -/* See large comment block below. This must be >= 1. */ #define PERTURB_SHIFT 5 /* @@ -126,8 +206,13 @@ equally good collision statistics, needed less code & used less memory. */ -/* Object used as dummy key to fill deleted entries */ -static PyObject *dummy = NULL; /* Initialized by first call to newPyDictObject() */ +/* Object used as dummy key to fill deleted entries + * This could be any unique object, + * use a custom type in order to minimise coupling. +*/ +static PyObject _dummy_struct; + +#define dummy (&_dummy_struct) #ifdef Py_REF_DEBUG PyObject * @@ -138,152 +223,213 @@ _PyDict_Dummy(void) #endif /* forward declarations */ -static PyDictEntry * -lookdict_unicode(PyDictObject *mp, PyObject *key, Py_hash_t hash); +static PyDictKeyEntry *lookdict(PyDictObject *mp, PyObject *key, + Py_hash_t hash, PyObject ***value_addr); +static PyDictKeyEntry *lookdict_unicode(PyDictObject *mp, PyObject *key, + Py_hash_t hash, PyObject ***value_addr); +static PyDictKeyEntry * +lookdict_unicode_nodummy(PyDictObject *mp, PyObject *key, + Py_hash_t hash, PyObject ***value_addr); +static PyDictKeyEntry *lookdict_split(PyDictObject *mp, PyObject *key, + Py_hash_t hash, PyObject ***value_addr); + +static int dictresize(PyDictObject *mp, Py_ssize_t minused); -#ifdef SHOW_CONVERSION_COUNTS -static long created = 0L; -static long converted = 0L; +/* Dictionary reuse scheme to save calls to malloc, free, and memset */ +#ifndef PyDict_MAXFREELIST +#define PyDict_MAXFREELIST 80 +#endif +static PyDictObject *free_list[PyDict_MAXFREELIST]; +static int numfree = 0; -static void -show_counts(void) +int +PyDict_ClearFreeList(void) { - fprintf(stderr, "created %ld string dicts\n", created); - fprintf(stderr, "converted %ld to normal dicts\n", converted); - fprintf(stderr, "%.2f%% conversion rate\n", (100.0*converted)/created); + PyDictObject *op; + int ret = numfree; + while (numfree) { + op = free_list[--numfree]; + assert(PyDict_CheckExact(op)); + PyObject_GC_Del(op); + } + return ret; } -#endif -/* Debug statistic to compare allocations with reuse through the free list */ -#undef SHOW_ALLOC_COUNT -#ifdef SHOW_ALLOC_COUNT -static size_t count_alloc = 0; -static size_t count_reuse = 0; - -static void -show_alloc(void) +/* Print summary info about the state of the optimized allocator */ +void +_PyDict_DebugMallocStats(FILE *out) { - fprintf(stderr, "Dict allocations: %" PY_FORMAT_SIZE_T "d\n", - count_alloc); - fprintf(stderr, "Dict reuse through freelist: %" PY_FORMAT_SIZE_T - "d\n", count_reuse); - fprintf(stderr, "%.2f%% reuse rate\n\n", - (100.0*count_reuse/(count_alloc+count_reuse))); + _PyDebugAllocatorStats(out, + "free PyDictObject", numfree, sizeof(PyDictObject)); } -#endif -/* Debug statistic to count GC tracking of dicts */ -#ifdef SHOW_TRACK_COUNT -static Py_ssize_t count_untracked = 0; -static Py_ssize_t count_tracked = 0; -static void -show_track(void) +void +PyDict_Fini(void) { - fprintf(stderr, "Dicts created: %" PY_FORMAT_SIZE_T "d\n", - count_tracked + count_untracked); - fprintf(stderr, "Dicts tracked by the GC: %" PY_FORMAT_SIZE_T - "d\n", count_tracked); - fprintf(stderr, "%.2f%% dict tracking rate\n\n", - (100.0*count_tracked/(count_untracked+count_tracked))); + PyDict_ClearFreeList(); } -#endif +#define DK_DEBUG_INCREF _Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA +#define DK_DEBUG_DECREF _Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA + +#define DK_INCREF(dk) (DK_DEBUG_INCREF ++(dk)->dk_refcnt) +#define DK_DECREF(dk) if (DK_DEBUG_DECREF (--(dk)->dk_refcnt) == 0) free_keys_object(dk) +#define DK_SIZE(dk) ((dk)->dk_size) +#define DK_MASK(dk) (((dk)->dk_size)-1) +#define IS_POWER_OF_2(x) (((x) & (x-1)) == 0) + +/* USABLE_FRACTION is the maximum dictionary load. + * Currently set to (2n+1)/3. Increasing this ratio makes dictionaries more + * dense resulting in more collisions. Decreasing it improves sparseness + * at the expense of spreading entries over more cache lines and at the + * cost of total memory consumed. + * + * USABLE_FRACTION must obey the following: + * (0 < USABLE_FRACTION(n) < n) for all n >= 2 + * + * USABLE_FRACTION should be very quick to calculate. + * Fractions around 5/8 to 2/3 seem to work well in practice. + */ -/* Initialization macros. - There are two ways to create a dict: PyDict_New() is the main C API - function, and the tp_new slot maps to dict_new(). In the latter case we - can save a little time over what PyDict_New does because it's guaranteed - that the PyDictObject struct is already zeroed out. - Everyone except dict_new() should use EMPTY_TO_MINSIZE (unless they have - an excellent reason not to). +/* Use (2n+1)/3 rather than 2n+3 because: it makes no difference for + * combined tables (the two fractions round to the same number n < ), + * but 2*4/3 is 2 whereas (2*4+1)/3 is 3 which potentially saves quite + * a lot of space for small, split tables */ +#define USABLE_FRACTION(n) ((((n) << 1)+1)/3) + +/* Alternative fraction that is otherwise close enough to (2n+1)/3 to make + * little difference. 8 * 2/3 == 8 * 5/8 == 5. 16 * 2/3 == 16 * 5/8 == 10. + * 32 * 2/3 = 21, 32 * 5/8 = 20. + * Its advantage is that it is faster to compute on machines with slow division. + * #define USABLE_FRACTION(n) (((n) >> 1) + ((n) >> 2) - ((n) >> 3)) */ -#define INIT_NONZERO_DICT_SLOTS(mp) do { \ - (mp)->ma_table = (mp)->ma_smalltable; \ - (mp)->ma_mask = PyDict_MINSIZE - 1; \ - } while(0) +/* GROWTH_RATE. Growth rate upon hitting maximum load. Currently set to *2. + * Raising this to *4 doubles memory consumption depending on the size of + * the dictionary, but results in half the number of resizes, less effort to + * resize and better sparseness for some (but not all dict sizes). + * Setting to *4 eliminates every other resize step. + * GROWTH_RATE was set to *4 up to version 3.2. + */ +#define GROWTH_RATE(x) ((x) * 2) -#define EMPTY_TO_MINSIZE(mp) do { \ - memset((mp)->ma_smalltable, 0, sizeof((mp)->ma_smalltable)); \ - (mp)->ma_used = (mp)->ma_fill = 0; \ - INIT_NONZERO_DICT_SLOTS(mp); \ - } while(0) +#define ENSURE_ALLOWS_DELETIONS(d) \ + if ((d)->ma_keys->dk_lookup == lookdict_unicode_nodummy) { \ + (d)->ma_keys->dk_lookup = lookdict_unicode; \ + } -/* Dictionary reuse scheme to save calls to malloc, free, and memset */ -#ifndef PyDict_MAXFREELIST -#define PyDict_MAXFREELIST 80 -#endif -static PyDictObject *free_list[PyDict_MAXFREELIST]; -static int numfree = 0; +/* This immutable, empty PyDictKeysObject is used for PyDict_Clear() + * (which cannot fail and thus can do no allocation). + */ +static PyDictKeysObject empty_keys_struct = { + 2, /* dk_refcnt 1 for this struct, 1 for dummy_struct */ + 1, /* dk_size */ + lookdict_split, /* dk_lookup */ + 0, /* dk_usable (immutable) */ + { + { 0, 0, 0 } /* dk_entries (empty) */ + } +}; -void -PyDict_Fini(void) +static PyObject *empty_values[1] = { NULL }; + +#define Py_EMPTY_KEYS &empty_keys_struct + +static PyDictKeysObject *new_keys_object(Py_ssize_t size) { - PyDictObject *op; + PyDictKeysObject *dk; + Py_ssize_t i; + PyDictKeyEntry *ep0; - while (numfree) { - op = free_list[--numfree]; - assert(PyDict_CheckExact(op)); - PyObject_GC_Del(op); + assert(size >= PyDict_MINSIZE_SPLIT); + assert(IS_POWER_OF_2(size)); + dk = PyMem_MALLOC(sizeof(PyDictKeysObject) + + sizeof(PyDictKeyEntry) * (size-1)); + if (dk == NULL) { + PyErr_NoMemory(); + return NULL; } + DK_DEBUG_INCREF dk->dk_refcnt = 1; + dk->dk_size = size; + dk->dk_usable = USABLE_FRACTION(size); + ep0 = &dk->dk_entries[0]; + /* Hash value of slot 0 is used by popitem, so it must be initialized */ + ep0->me_hash = 0; + for (i = 0; i < size; i++) { + ep0[i].me_key = NULL; + ep0[i].me_value = NULL; + } + dk->dk_lookup = lookdict_unicode_nodummy; + return dk; } -PyObject * -PyDict_New(void) +static void +free_keys_object(PyDictKeysObject *keys) { - register PyDictObject *mp; - if (dummy == NULL) { /* Auto-initialize dummy */ - dummy = PyUnicode_FromString("<dummy key>"); - if (dummy == NULL) - return NULL; -#ifdef SHOW_CONVERSION_COUNTS - Py_AtExit(show_counts); -#endif -#ifdef SHOW_ALLOC_COUNT - Py_AtExit(show_alloc); -#endif -#ifdef SHOW_TRACK_COUNT - Py_AtExit(show_track); -#endif + PyDictKeyEntry *entries = &keys->dk_entries[0]; + Py_ssize_t i, n; + for (i = 0, n = DK_SIZE(keys); i < n; i++) { + Py_XDECREF(entries[i].me_key); + Py_XDECREF(entries[i].me_value); } + PyMem_FREE(keys); +} + +#define new_values(size) PyMem_NEW(PyObject *, size) + +#define free_values(values) PyMem_FREE(values) + +/* Consumes a reference to the keys object */ +static PyObject * +new_dict(PyDictKeysObject *keys, PyObject **values) +{ + PyDictObject *mp; if (numfree) { mp = free_list[--numfree]; assert (mp != NULL); assert (Py_TYPE(mp) == &PyDict_Type); _Py_NewReference((PyObject *)mp); - if (mp->ma_fill) { - EMPTY_TO_MINSIZE(mp); - } else { - /* At least set ma_table and ma_mask; these are wrong - if an empty but presized dict is added to freelist */ - INIT_NONZERO_DICT_SLOTS(mp); - } - assert (mp->ma_used == 0); - assert (mp->ma_table == mp->ma_smalltable); - assert (mp->ma_mask == PyDict_MINSIZE - 1); -#ifdef SHOW_ALLOC_COUNT - count_reuse++; -#endif - } else { + } + else { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); - if (mp == NULL) + if (mp == NULL) { + DK_DECREF(keys); + free_values(values); return NULL; - EMPTY_TO_MINSIZE(mp); -#ifdef SHOW_ALLOC_COUNT - count_alloc++; -#endif + } } - mp->ma_lookup = lookdict_unicode; -#ifdef SHOW_TRACK_COUNT - count_untracked++; -#endif -#ifdef SHOW_CONVERSION_COUNTS - ++created; -#endif + mp->ma_keys = keys; + mp->ma_values = values; + mp->ma_used = 0; return (PyObject *)mp; } +/* Consumes a reference to the keys object */ +static PyObject * +new_dict_with_shared_keys(PyDictKeysObject *keys) +{ + PyObject **values; + Py_ssize_t i, size; + + size = DK_SIZE(keys); + values = new_values(size); + if (values == NULL) { + DK_DECREF(keys); + return PyErr_NoMemory(); + } + for (i = 0; i < size; i++) { + values[i] = NULL; + } + return new_dict(keys, values); +} + +PyObject * +PyDict_New(void) +{ + return new_dict(new_keys_object(PyDict_MINSIZE_COMBINED), NULL); +} + /* The basic lookup function used by all operations. This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4. @@ -302,29 +448,35 @@ Christian Tismer. lookdict() is general-purpose, and may return NULL if (and only if) a comparison raises an exception (this was new in Python 2.5). lookdict_unicode() below is specialized to string keys, comparison of which can -never raise an exception; that function can never return NULL. For both, when -the key isn't found a PyDictEntry* is returned for which the me_value field is -NULL; this is the slot in the dict at which the key would have been found, and -the caller can (if it wishes) add the <key, value> pair to the returned -PyDictEntry*. +never raise an exception; that function can never return NULL. +lookdict_unicode_nodummy is further specialized for string keys that cannot be +the <dummy> value. +For both, when the key isn't found a PyDictEntry* is returned +where the key would have been found, *value_addr points to the matching value +slot. */ -static PyDictEntry * -lookdict(PyDictObject *mp, PyObject *key, register Py_hash_t hash) +static PyDictKeyEntry * +lookdict(PyDictObject *mp, PyObject *key, + Py_hash_t hash, PyObject ***value_addr) { register size_t i; register size_t perturb; - register PyDictEntry *freeslot; - register size_t mask = (size_t)mp->ma_mask; - PyDictEntry *ep0 = mp->ma_table; - register PyDictEntry *ep; + register PyDictKeyEntry *freeslot; + register size_t mask; + PyDictKeyEntry *ep0; + register PyDictKeyEntry *ep; register int cmp; PyObject *startkey; +top: + mask = DK_MASK(mp->ma_keys); + ep0 = &mp->ma_keys->dk_entries[0]; i = (size_t)hash & mask; ep = &ep0[i]; - if (ep->me_key == NULL || ep->me_key == key) + if (ep->me_key == NULL || ep->me_key == key) { + *value_addr = &ep->me_value; return ep; - + } if (ep->me_key == dummy) freeslot = ep; else { @@ -335,17 +487,15 @@ lookdict(PyDictObject *mp, PyObject *key, register Py_hash_t hash) Py_DECREF(startkey); if (cmp < 0) return NULL; - if (ep0 == mp->ma_table && ep->me_key == startkey) { - if (cmp > 0) + if (ep0 == mp->ma_keys->dk_entries && ep->me_key == startkey) { + if (cmp > 0) { + *value_addr = &ep->me_value; return ep; + } } else { - /* The compare did major nasty stuff to the - * dict: start over. - * XXX A clever adversary could prevent this - * XXX from terminating. - */ - return lookdict(mp, key, hash); + /* The dict was mutated, restart */ + goto top; } } freeslot = NULL; @@ -356,28 +506,37 @@ lookdict(PyDictObject *mp, PyObject *key, register Py_hash_t hash) for (perturb = hash; ; perturb >>= PERTURB_SHIFT) { i = (i << 2) + i + perturb + 1; ep = &ep0[i & mask]; - if (ep->me_key == NULL) - return freeslot == NULL ? ep : freeslot; - if (ep->me_key == key) + if (ep->me_key == NULL) { + if (freeslot == NULL) { + *value_addr = &ep->me_value; + return ep; + } else { + *value_addr = &freeslot->me_value; + return freeslot; + } + } + if (ep->me_key == key) { + *value_addr = &ep->me_value; return ep; + } if (ep->me_hash == hash && ep->me_key != dummy) { startkey = ep->me_key; Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); - if (cmp < 0) + if (cmp < 0) { + *value_addr = NULL; return NULL; - if (ep0 == mp->ma_table && ep->me_key == startkey) { - if (cmp > 0) + } + if (ep0 == mp->ma_keys->dk_entries && ep->me_key == startkey) { + if (cmp > 0) { + *value_addr = &ep->me_value; return ep; + } } else { - /* The compare did major nasty stuff to the - * dict: start over. - * XXX A clever adversary could prevent this - * XXX from terminating. - */ - return lookdict(mp, key, hash); + /* The dict was mutated, restart */ + goto top; } } else if (ep->me_key == dummy && freeslot == NULL) @@ -387,46 +546,39 @@ lookdict(PyDictObject *mp, PyObject *key, register Py_hash_t hash) return 0; } -/* - * Hacked up version of lookdict which can assume keys are always - * unicodes; this assumption allows testing for errors during - * PyObject_RichCompareBool() to be dropped; unicode-unicode - * comparisons never raise exceptions. This also means we don't need - * to go through PyObject_RichCompareBool(); we can always use - * unicode_eq() directly. - * - * This is valuable because dicts with only unicode keys are very common. - */ -static PyDictEntry * -lookdict_unicode(PyDictObject *mp, PyObject *key, register Py_hash_t hash) +/* Specialized version for string-only keys */ +static PyDictKeyEntry * +lookdict_unicode(PyDictObject *mp, PyObject *key, + Py_hash_t hash, PyObject ***value_addr) { register size_t i; register size_t perturb; - register PyDictEntry *freeslot; - register size_t mask = (size_t)mp->ma_mask; - PyDictEntry *ep0 = mp->ma_table; - register PyDictEntry *ep; + register PyDictKeyEntry *freeslot; + register size_t mask = DK_MASK(mp->ma_keys); + PyDictKeyEntry *ep0 = &mp->ma_keys->dk_entries[0]; + register PyDictKeyEntry *ep; /* Make sure this function doesn't have to handle non-unicode keys, including subclasses of str; e.g., one reason to subclass unicodes is to override __eq__, and for speed we don't cater to that here. */ if (!PyUnicode_CheckExact(key)) { -#ifdef SHOW_CONVERSION_COUNTS - ++converted; -#endif - mp->ma_lookup = lookdict; - return lookdict(mp, key, hash); + mp->ma_keys->dk_lookup = lookdict; + return lookdict(mp, key, hash, value_addr); } - i = hash & mask; + i = (size_t)hash & mask; ep = &ep0[i]; - if (ep->me_key == NULL || ep->me_key == key) + if (ep->me_key == NULL || ep->me_key == key) { + *value_addr = &ep->me_value; return ep; + } if (ep->me_key == dummy) freeslot = ep; else { - if (ep->me_hash == hash && unicode_eq(ep->me_key, key)) + if (ep->me_hash == hash && unicode_eq(ep->me_key, key)) { + *value_addr = &ep->me_value; return ep; + } freeslot = NULL; } @@ -435,13 +587,22 @@ lookdict_unicode(PyDictObject *mp, PyObject *key, register Py_hash_t hash) for (perturb = hash; ; perturb >>= PERTURB_SHIFT) { i = (i << 2) + i + perturb + 1; ep = &ep0[i & mask]; - if (ep->me_key == NULL) - return freeslot == NULL ? ep : freeslot; + if (ep->me_key == NULL) { + if (freeslot == NULL) { + *value_addr = &ep->me_value; + return ep; + } else { + *value_addr = &freeslot->me_value; + return freeslot; + } + } if (ep->me_key == key || (ep->me_hash == hash && ep->me_key != dummy - && unicode_eq(ep->me_key, key))) + && unicode_eq(ep->me_key, key))) { + *value_addr = &ep->me_value; return ep; + } if (ep->me_key == dummy && freeslot == NULL) freeslot = ep; } @@ -449,6 +610,92 @@ lookdict_unicode(PyDictObject *mp, PyObject *key, register Py_hash_t hash) return 0; } +/* Faster version of lookdict_unicode when it is known that no <dummy> keys + * will be present. */ +static PyDictKeyEntry * +lookdict_unicode_nodummy(PyDictObject *mp, PyObject *key, + Py_hash_t hash, PyObject ***value_addr) +{ + register size_t i; + register size_t perturb; + register size_t mask = DK_MASK(mp->ma_keys); + PyDictKeyEntry *ep0 = &mp->ma_keys->dk_entries[0]; + register PyDictKeyEntry *ep; + + /* Make sure this function doesn't have to handle non-unicode keys, + including subclasses of str; e.g., one reason to subclass + unicodes is to override __eq__, and for speed we don't cater to + that here. */ + if (!PyUnicode_CheckExact(key)) { + mp->ma_keys->dk_lookup = lookdict; + return lookdict(mp, key, hash, value_addr); + } + i = (size_t)hash & mask; + ep = &ep0[i]; + assert(ep->me_key == NULL || PyUnicode_CheckExact(ep->me_key)); + if (ep->me_key == NULL || ep->me_key == key || + (ep->me_hash == hash && unicode_eq(ep->me_key, key))) { + *value_addr = &ep->me_value; + return ep; + } + for (perturb = hash; ; perturb >>= PERTURB_SHIFT) { + i = (i << 2) + i + perturb + 1; + ep = &ep0[i & mask]; + assert(ep->me_key == NULL || PyUnicode_CheckExact(ep->me_key)); + if (ep->me_key == NULL || ep->me_key == key || + (ep->me_hash == hash && unicode_eq(ep->me_key, key))) { + *value_addr = &ep->me_value; + return ep; + } + } + assert(0); /* NOT REACHED */ + return 0; +} + +/* Version of lookdict for split tables. + * All split tables and only split tables use this lookup function. + * Split tables only contain unicode keys and no dummy keys, + * so algorithm is the same as lookdict_unicode_nodummy. + */ +static PyDictKeyEntry * +lookdict_split(PyDictObject *mp, PyObject *key, + Py_hash_t hash, PyObject ***value_addr) +{ + register size_t i; + register size_t perturb; + register size_t mask = DK_MASK(mp->ma_keys); + PyDictKeyEntry *ep0 = &mp->ma_keys->dk_entries[0]; + register PyDictKeyEntry *ep; + + if (!PyUnicode_CheckExact(key)) { + ep = lookdict(mp, key, hash, value_addr); + /* lookdict expects a combined-table, so fix value_addr */ + i = ep - ep0; + *value_addr = &mp->ma_values[i]; + return ep; + } + i = (size_t)hash & mask; + ep = &ep0[i]; + assert(ep->me_key == NULL || PyUnicode_CheckExact(ep->me_key)); + if (ep->me_key == NULL || ep->me_key == key || + (ep->me_hash == hash && unicode_eq(ep->me_key, key))) { + *value_addr = &mp->ma_values[i]; + return ep; + } + for (perturb = hash; ; perturb >>= PERTURB_SHIFT) { + i = (i << 2) + i + perturb + 1; + ep = &ep0[i & mask]; + assert(ep->me_key == NULL || PyUnicode_CheckExact(ep->me_key)); + if (ep->me_key == NULL || ep->me_key == key || + (ep->me_hash == hash && unicode_eq(ep->me_key, key))) { + *value_addr = &mp->ma_values[i & mask]; + return ep; + } + } + assert(0); /* NOT REACHED */ + return 0; +} + int _PyDict_HasOnlyStringKeys(PyObject *dict) { @@ -456,7 +703,7 @@ _PyDict_HasOnlyStringKeys(PyObject *dict) PyObject *key, *value; assert(PyDict_Check(dict)); /* Shortcut */ - if (((PyDictObject *)dict)->ma_lookup == lookdict_unicode) + if (((PyDictObject *)dict)->ma_keys->dk_lookup != lookdict) return 1; while (PyDict_Next(dict, &pos, &key, &value)) if (!PyUnicode_Check(key)) @@ -464,23 +711,12 @@ _PyDict_HasOnlyStringKeys(PyObject *dict) return 1; } -#ifdef SHOW_TRACK_COUNT -#define INCREASE_TRACK_COUNT \ - (count_tracked++, count_untracked--); -#define DECREASE_TRACK_COUNT \ - (count_tracked--, count_untracked++); -#else -#define INCREASE_TRACK_COUNT -#define DECREASE_TRACK_COUNT -#endif - #define MAINTAIN_TRACKING(mp, key, value) \ do { \ if (!_PyObject_GC_IS_TRACKED(mp)) { \ if (_PyObject_GC_MAY_BE_TRACKED(key) || \ _PyObject_GC_MAY_BE_TRACKED(value)) { \ _PyObject_GC_TRACK(mp); \ - INCREASE_TRACK_COUNT \ } \ } \ } while(0) @@ -490,78 +726,136 @@ _PyDict_MaybeUntrack(PyObject *op) { PyDictObject *mp; PyObject *value; - Py_ssize_t mask, i; - PyDictEntry *ep; + Py_ssize_t i, size; if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) return; mp = (PyDictObject *) op; - ep = mp->ma_table; - mask = mp->ma_mask; - for (i = 0; i <= mask; i++) { - if ((value = ep[i].me_value) == NULL) - continue; - if (_PyObject_GC_MAY_BE_TRACKED(value) || - _PyObject_GC_MAY_BE_TRACKED(ep[i].me_key)) - return; - } - DECREASE_TRACK_COUNT - _PyObject_GC_UNTRACK(op); -} - -/* -Internal routine to insert a new item into the table when you have entry object. -Used by insertdict. -*/ -static int -insertdict_by_entry(register PyDictObject *mp, PyObject *key, Py_hash_t hash, - PyDictEntry *ep, PyObject *value) -{ - PyObject *old_value; - - MAINTAIN_TRACKING(mp, key, value); - if (ep->me_value != NULL) { - old_value = ep->me_value; - ep->me_value = value; - Py_DECREF(old_value); /* which **CAN** re-enter */ - Py_DECREF(key); + size = DK_SIZE(mp->ma_keys); + if (_PyDict_HasSplitTable(mp)) { + for (i = 0; i < size; i++) { + if ((value = mp->ma_values[i]) == NULL) + continue; + if (_PyObject_GC_MAY_BE_TRACKED(value)) { + assert(!_PyObject_GC_MAY_BE_TRACKED( + mp->ma_keys->dk_entries[i].me_key)); + return; + } + } } else { - if (ep->me_key == NULL) - mp->ma_fill++; - else { - assert(ep->me_key == dummy); - Py_DECREF(dummy); + PyDictKeyEntry *ep0 = &mp->ma_keys->dk_entries[0]; + for (i = 0; i < size; i++) { + if ((value = ep0[i].me_value) == NULL) + continue; + if (_PyObject_GC_MAY_BE_TRACKED(value) || + _PyObject_GC_MAY_BE_TRACKED(ep0[i].me_key)) + return; } - ep->me_key = key; - ep->me_hash = hash; - ep->me_value = value; - mp->ma_used++; } - return 0; + _PyObject_GC_UNTRACK(op); +} + +/* Internal function to find slot for an item from its hash + * when it is known that the key is not present in the dict. + */ +static PyDictKeyEntry * +find_empty_slot(PyDictObject *mp, PyObject *key, Py_hash_t hash, + PyObject ***value_addr) +{ + size_t i; + size_t perturb; + size_t mask = DK_MASK(mp->ma_keys); + PyDictKeyEntry *ep0 = &mp->ma_keys->dk_entries[0]; + PyDictKeyEntry *ep; + + assert(key != NULL); + if (!PyUnicode_CheckExact(key)) + mp->ma_keys->dk_lookup = lookdict; + i = hash & mask; + ep = &ep0[i]; + for (perturb = hash; ep->me_key != NULL; perturb >>= PERTURB_SHIFT) { + i = (i << 2) + i + perturb + 1; + ep = &ep0[i & mask]; + } + assert(ep->me_value == NULL); + if (mp->ma_values) + *value_addr = &mp->ma_values[i & mask]; + else + *value_addr = &ep->me_value; + return ep; } +static int +insertion_resize(PyDictObject *mp) +{ + return dictresize(mp, GROWTH_RATE(mp->ma_used)); +} /* Internal routine to insert a new item into the table. Used both by the internal resize routine and by the public insert routine. -Eats a reference to key and one to value. Returns -1 if an error occurred, or 0 on success. */ static int -insertdict(register PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) +insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { - register PyDictEntry *ep; + PyObject *old_value; + PyObject **value_addr; + PyDictKeyEntry *ep; + assert(key != dummy); - assert(mp->ma_lookup != NULL); - ep = mp->ma_lookup(mp, key, hash); + if (mp->ma_values != NULL && !PyUnicode_CheckExact(key)) { + if (insertion_resize(mp) < 0) + return -1; + } + + ep = mp->ma_keys->dk_lookup(mp, key, hash, &value_addr); if (ep == NULL) { - Py_DECREF(key); - Py_DECREF(value); return -1; } - return insertdict_by_entry(mp, key, hash, ep, value); + Py_INCREF(value); + MAINTAIN_TRACKING(mp, key, value); + old_value = *value_addr; + if (old_value != NULL) { + assert(ep->me_key != NULL && ep->me_key != dummy); + *value_addr = value; + Py_DECREF(old_value); /* which **CAN** re-enter */ + } + else { + if (ep->me_key == NULL) { + Py_INCREF(key); + if (mp->ma_keys->dk_usable <= 0) { + /* Need to resize. */ + if (insertion_resize(mp) < 0) { + Py_DECREF(key); + Py_DECREF(value); + return -1; + } + ep = find_empty_slot(mp, key, hash, &value_addr); + } + mp->ma_keys->dk_usable--; + assert(mp->ma_keys->dk_usable >= 0); + ep->me_key = key; + ep->me_hash = hash; + } + else { + if (ep->me_key == dummy) { + Py_INCREF(key); + ep->me_key = key; + ep->me_hash = hash; + Py_DECREF(dummy); + } else { + assert(_PyDict_HasSplitTable(mp)); + } + } + mp->ma_used++; + *value_addr = value; + } + assert(ep->me_key != NULL && ep->me_key != dummy); + assert(PyUnicode_CheckExact(key) || mp->ma_keys->dk_lookup == lookdict); + return 0; } /* @@ -571,18 +865,25 @@ the dict contains no deleted entries. Besides the performance benefit, using insertdict() in dictresize() is dangerous (SF bug #1456209). Note that no refcounts are changed by this routine; if needed, the caller is responsible for incref'ing `key` and `value`. +Neither mp->ma_used nor k->dk_usable are modified by this routine; the caller +must set them correctly */ static void -insertdict_clean(register PyDictObject *mp, PyObject *key, Py_hash_t hash, +insertdict_clean(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { - register size_t i; - register size_t perturb; - register size_t mask = (size_t)mp->ma_mask; - PyDictEntry *ep0 = mp->ma_table; - register PyDictEntry *ep; - - MAINTAIN_TRACKING(mp, key, value); + size_t i; + size_t perturb; + PyDictKeysObject *k = mp->ma_keys; + size_t mask = (size_t)DK_SIZE(k)-1; + PyDictKeyEntry *ep0 = &k->dk_entries[0]; + PyDictKeyEntry *ep; + + assert(k->dk_lookup != NULL); + assert(value != NULL); + assert(key != NULL); + assert(key != dummy); + assert(PyUnicode_CheckExact(key) || k->dk_lookup == lookdict); i = hash & mask; ep = &ep0[i]; for (perturb = hash; ep->me_key != NULL; perturb >>= PERTURB_SHIFT) { @@ -590,31 +891,31 @@ insertdict_clean(register PyDictObject *mp, PyObject *key, Py_hash_t hash, ep = &ep0[i & mask]; } assert(ep->me_value == NULL); - mp->ma_fill++; ep->me_key = key; ep->me_hash = hash; ep->me_value = value; - mp->ma_used++; } /* Restructure the table by allocating a new table and reinserting all items again. When entries have been deleted, the new table may actually be smaller than the old one. +If a table is split (its keys and hashes are shared, its values are not), +then the values are temporarily copied into the table, it is resized as +a combined table, then the me_value slots in the old table are NULLed out. +After resizing a table is always combined, +but can be resplit by make_keys_shared(). */ static int dictresize(PyDictObject *mp, Py_ssize_t minused) { Py_ssize_t newsize; - PyDictEntry *oldtable, *newtable, *ep; - Py_ssize_t i; - int is_oldtable_malloced; - PyDictEntry small_copy[PyDict_MINSIZE]; + PyDictKeysObject *oldkeys; + PyObject **oldvalues; + Py_ssize_t i, oldsize; - assert(minused >= 0); - - /* Find the smallest table size > minused. */ - for (newsize = PyDict_MINSIZE; +/* Find the smallest table size > minused. */ + for (newsize = PyDict_MINSIZE_COMBINED; newsize <= minused && newsize > 0; newsize <<= 1) ; @@ -622,83 +923,125 @@ dictresize(PyDictObject *mp, Py_ssize_t minused) PyErr_NoMemory(); return -1; } - - /* Get space for a new table. */ - oldtable = mp->ma_table; - assert(oldtable != NULL); - is_oldtable_malloced = oldtable != mp->ma_smalltable; - - if (newsize == PyDict_MINSIZE) { - /* A large table is shrinking, or we can't get any smaller. */ - newtable = mp->ma_smalltable; - if (newtable == oldtable) { - if (mp->ma_fill == mp->ma_used) { - /* No dummies, so no point doing anything. */ - return 0; + oldkeys = mp->ma_keys; + oldvalues = mp->ma_values; + /* Allocate a new table. */ + mp->ma_keys = new_keys_object(newsize); + if (mp->ma_keys == NULL) { + mp->ma_keys = oldkeys; + return -1; + } + if (oldkeys->dk_lookup == lookdict) + mp->ma_keys->dk_lookup = lookdict; + oldsize = DK_SIZE(oldkeys); + mp->ma_values = NULL; + /* If empty then nothing to copy so just return */ + if (oldsize == 1) { + assert(oldkeys == Py_EMPTY_KEYS); + DK_DECREF(oldkeys); + return 0; + } + /* Main loop below assumes we can transfer refcount to new keys + * and that value is stored in me_value. + * Increment ref-counts and copy values here to compensate + * This (resizing a split table) should be relatively rare */ + if (oldvalues != NULL) { + for (i = 0; i < oldsize; i++) { + if (oldvalues[i] != NULL) { + Py_INCREF(oldkeys->dk_entries[i].me_key); + oldkeys->dk_entries[i].me_value = oldvalues[i]; } - /* We're not going to resize it, but rebuild the - table anyway to purge old dummy entries. - Subtle: This is *necessary* if fill==size, - as lookdict needs at least one virgin slot to - terminate failing searches. If fill < size, it's - merely desirable, as dummies slow searches. */ - assert(mp->ma_fill > mp->ma_used); - memcpy(small_copy, oldtable, sizeof(small_copy)); - oldtable = small_copy; } } + /* Main loop */ + for (i = 0; i < oldsize; i++) { + PyDictKeyEntry *ep = &oldkeys->dk_entries[i]; + if (ep->me_value != NULL) { + assert(ep->me_key != dummy); + insertdict_clean(mp, ep->me_key, ep->me_hash, ep->me_value); + } + } + mp->ma_keys->dk_usable -= mp->ma_used; + if (oldvalues != NULL) { + /* NULL out me_value slot in oldkeys, in case it was shared */ + for (i = 0; i < oldsize; i++) + oldkeys->dk_entries[i].me_value = NULL; + assert(oldvalues != empty_values); + free_values(oldvalues); + DK_DECREF(oldkeys); + } else { - newtable = PyMem_NEW(PyDictEntry, newsize); - if (newtable == NULL) { - PyErr_NoMemory(); - return -1; + assert(oldkeys->dk_lookup != lookdict_split); + if (oldkeys->dk_lookup != lookdict_unicode_nodummy) { + PyDictKeyEntry *ep0 = &oldkeys->dk_entries[0]; + for (i = 0; i < oldsize; i++) { + if (ep0[i].me_key == dummy) + Py_DECREF(dummy); + } } + assert(oldkeys->dk_refcnt == 1); + DK_DEBUG_DECREF PyMem_FREE(oldkeys); } + return 0; +} - /* Make the dict empty, using the new table. */ - assert(newtable != oldtable); - mp->ma_table = newtable; - mp->ma_mask = newsize - 1; - memset(newtable, 0, sizeof(PyDictEntry) * newsize); - mp->ma_used = 0; - i = mp->ma_fill; - mp->ma_fill = 0; - - /* Copy the data over; this is refcount-neutral for active entries; - dummy entries aren't copied over, of course */ - for (ep = oldtable; i > 0; ep++) { - if (ep->me_value != NULL) { /* active entry */ - --i; - insertdict_clean(mp, ep->me_key, ep->me_hash, ep->me_value); +/* Returns NULL if unable to split table. + * A NULL return does not necessarily indicate an error */ +static PyDictKeysObject * +make_keys_shared(PyObject *op) +{ + Py_ssize_t i; + Py_ssize_t size; + PyDictObject *mp = (PyDictObject *)op; + + if (!PyDict_CheckExact(op)) + return NULL; + if (!_PyDict_HasSplitTable(mp)) { + PyDictKeyEntry *ep0; + PyObject **values; + assert(mp->ma_keys->dk_refcnt == 1); + if (mp->ma_keys->dk_lookup == lookdict) { + return NULL; } - else if (ep->me_key != NULL) { /* dummy entry */ - --i; - assert(ep->me_key == dummy); - Py_DECREF(ep->me_key); + else if (mp->ma_keys->dk_lookup == lookdict_unicode) { + /* Remove dummy keys */ + if (dictresize(mp, DK_SIZE(mp->ma_keys))) + return NULL; } - /* else key == value == NULL: nothing to do */ + assert(mp->ma_keys->dk_lookup == lookdict_unicode_nodummy); + /* Copy values into a new array */ + ep0 = &mp->ma_keys->dk_entries[0]; + size = DK_SIZE(mp->ma_keys); + values = new_values(size); + if (values == NULL) { + PyErr_SetString(PyExc_MemoryError, + "Not enough memory to allocate new values array"); + return NULL; + } + for (i = 0; i < size; i++) { + values[i] = ep0[i].me_value; + ep0[i].me_value = NULL; + } + mp->ma_keys->dk_lookup = lookdict_split; + mp->ma_values = values; } - - if (is_oldtable_malloced) - PyMem_DEL(oldtable); - return 0; + DK_INCREF(mp->ma_keys); + return mp->ma_keys; } -/* Create a new dictionary pre-sized to hold an estimated number of elements. - Underestimates are okay because the dictionary will resize as necessary. - Overestimates just mean the dictionary will be more sparse than usual. -*/ - PyObject * _PyDict_NewPresized(Py_ssize_t minused) { - PyObject *op = PyDict_New(); - - if (minused>5 && op != NULL && dictresize((PyDictObject *)op, minused) == -1) { - Py_DECREF(op); + Py_ssize_t newsize; + PyDictKeysObject *new_keys; + for (newsize = PyDict_MINSIZE_COMBINED; + newsize <= minused && newsize > 0; + newsize <<= 1) + ; + new_keys = new_keys_object(newsize); + if (new_keys == NULL) return NULL; - } - return op; + return new_dict(new_keys, NULL); } /* Note that, for historical reasons, PyDict_GetItem() suppresses all errors @@ -716,12 +1059,14 @@ PyDict_GetItem(PyObject *op, PyObject *key) { Py_hash_t hash; PyDictObject *mp = (PyDictObject *)op; - PyDictEntry *ep; + PyDictKeyEntry *ep; PyThreadState *tstate; + PyObject **value_addr; + if (!PyDict_Check(op)) return NULL; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) { @@ -741,20 +1086,20 @@ PyDict_GetItem(PyObject *op, PyObject *key) /* preserve the existing exception */ PyObject *err_type, *err_value, *err_tb; PyErr_Fetch(&err_type, &err_value, &err_tb); - ep = (mp->ma_lookup)(mp, key, hash); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); /* ignore errors */ PyErr_Restore(err_type, err_value, err_tb); if (ep == NULL) return NULL; } else { - ep = (mp->ma_lookup)(mp, key, hash); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); if (ep == NULL) { PyErr_Clear(); return NULL; } } - return ep->me_value; + return *value_addr; } /* Variant of PyDict_GetItem() that doesn't suppress exceptions. @@ -766,14 +1111,15 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) { Py_hash_t hash; PyDictObject*mp = (PyDictObject *)op; - PyDictEntry *ep; + PyDictKeyEntry *ep; + PyObject **value_addr; if (!PyDict_Check(op)) { PyErr_BadInternalCall(); return NULL; } if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) { @@ -781,49 +1127,55 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) } } - ep = (mp->ma_lookup)(mp, key, hash); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); if (ep == NULL) return NULL; - return ep->me_value; + return *value_addr; } -static int -dict_set_item_by_hash_or_entry(register PyObject *op, PyObject *key, - Py_hash_t hash, PyDictEntry *ep, PyObject *value) +PyObject * +_PyDict_GetItemIdWithError(PyObject *dp, struct _Py_Identifier *key) { - register PyDictObject *mp; - register Py_ssize_t n_used; + PyObject *kv; + kv = _PyUnicode_FromId(key); /* borrowed */ + if (kv == NULL) + return NULL; + return PyDict_GetItemWithError(dp, kv); +} - mp = (PyDictObject *)op; - assert(mp->ma_fill <= mp->ma_mask); /* at least one empty slot */ - n_used = mp->ma_used; - Py_INCREF(value); - Py_INCREF(key); - if (ep == NULL) { - if (insertdict(mp, key, hash, value) != 0) - return -1; - } - else { - if (insertdict_by_entry(mp, key, hash, ep, value) != 0) - return -1; +/* Fast version of global value lookup. + * Lookup in globals, then builtins. + */ +PyObject * +_PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) +{ + PyObject *x; + if (PyUnicode_CheckExact(key)) { + PyObject **value_addr; + Py_hash_t hash = ((PyASCIIObject *)key)->hash; + if (hash != -1) { + PyDictKeyEntry *e; + e = globals->ma_keys->dk_lookup(globals, key, hash, &value_addr); + if (e == NULL) { + return NULL; + } + x = *value_addr; + if (x != NULL) + return x; + e = builtins->ma_keys->dk_lookup(builtins, key, hash, &value_addr); + if (e == NULL) { + return NULL; + } + x = *value_addr; + return x; + } } - /* If we added a key, we can safely resize. Otherwise just return! - * If fill >= 2/3 size, adjust size. Normally, this doubles or - * quaduples the size, but it's also possible for the dict to shrink - * (if ma_fill is much larger than ma_used, meaning a lot of dict - * keys have been * deleted). - * - * Quadrupling the size improves average dictionary sparseness - * (reducing collisions) at the cost of some memory and iteration - * speed (which loops over every possible entry). It also halves - * the number of expensive resize operations in a growing dictionary. - * - * Very large dictionaries (over 50K items) use doubling instead. - * This may help applications with severe memory constraints. - */ - if (!(mp->ma_used > n_used && mp->ma_fill*3 >= (mp->ma_mask+1)*2)) - return 0; - return dictresize(mp, (mp->ma_used > 50000 ? 2 : 4) * mp->ma_used); + x = PyDict_GetItemWithError((PyObject *)globals, key); + if (x != NULL) + return x; + if (PyErr_Occurred()) + return NULL; + return PyDict_GetItemWithError((PyObject *)builtins, key); } /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the @@ -833,36 +1185,37 @@ dict_set_item_by_hash_or_entry(register PyObject *op, PyObject *key, * remove them. */ int -PyDict_SetItem(register PyObject *op, PyObject *key, PyObject *value) +PyDict_SetItem(PyObject *op, PyObject *key, PyObject *value) { - register Py_hash_t hash; - + PyDictObject *mp; + Py_hash_t hash; if (!PyDict_Check(op)) { PyErr_BadInternalCall(); return -1; } assert(key); assert(value); - if (PyUnicode_CheckExact(key)) { - hash = ((PyUnicodeObject *) key)->hash; - if (hash == -1) - hash = PyObject_Hash(key); - } - else { + mp = (PyDictObject *)op; + if (!PyUnicode_CheckExact(key) || + (hash = ((PyASCIIObject *) key)->hash) == -1) + { hash = PyObject_Hash(key); if (hash == -1) return -1; } - return dict_set_item_by_hash_or_entry(op, key, hash, NULL, value); + + /* insertdict() handles any resizing that might be necessary */ + return insertdict(mp, key, hash, value); } int PyDict_DelItem(PyObject *op, PyObject *key) { - register PyDictObject *mp; - register Py_hash_t hash; - register PyDictEntry *ep; - PyObject *old_value, *old_key; + PyDictObject *mp; + Py_hash_t hash; + PyDictKeyEntry *ep; + PyObject *old_key, *old_value; + PyObject **value_addr; if (!PyDict_Check(op)) { PyErr_BadInternalCall(); @@ -870,27 +1223,30 @@ PyDict_DelItem(PyObject *op, PyObject *key) } assert(key); if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; } mp = (PyDictObject *)op; - ep = (mp->ma_lookup)(mp, key, hash); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); if (ep == NULL) return -1; - if (ep->me_value == NULL) { + if (*value_addr == NULL) { set_key_error(key); return -1; } - old_key = ep->me_key; - Py_INCREF(dummy); - ep->me_key = dummy; - old_value = ep->me_value; - ep->me_value = NULL; + old_value = *value_addr; + *value_addr = NULL; mp->ma_used--; + if (!_PyDict_HasSplitTable(mp)) { + ENSURE_ALLOWS_DELETIONS(mp); + old_key = ep->me_key; + Py_INCREF(dummy); + ep->me_key = dummy; + Py_DECREF(old_key); + } Py_DECREF(old_value); - Py_DECREF(old_key); return 0; } @@ -898,69 +1254,70 @@ void PyDict_Clear(PyObject *op) { PyDictObject *mp; - PyDictEntry *ep, *table; - int table_is_malloced; - Py_ssize_t fill; - PyDictEntry small_copy[PyDict_MINSIZE]; -#ifdef Py_DEBUG + PyDictKeysObject *oldkeys; + PyObject **oldvalues; Py_ssize_t i, n; -#endif if (!PyDict_Check(op)) return; - mp = (PyDictObject *)op; -#ifdef Py_DEBUG - n = mp->ma_mask + 1; - i = 0; -#endif + mp = ((PyDictObject *)op); + oldkeys = mp->ma_keys; + oldvalues = mp->ma_values; + if (oldvalues == empty_values) + return; + /* Empty the dict... */ + DK_INCREF(Py_EMPTY_KEYS); + mp->ma_keys = Py_EMPTY_KEYS; + mp->ma_values = empty_values; + mp->ma_used = 0; + /* ...then clear the keys and values */ + if (oldvalues != NULL) { + n = DK_SIZE(oldkeys); + for (i = 0; i < n; i++) + Py_CLEAR(oldvalues[i]); + free_values(oldvalues); + DK_DECREF(oldkeys); + } + else { + assert(oldkeys->dk_refcnt == 1); + DK_DECREF(oldkeys); + } +} - table = mp->ma_table; - assert(table != NULL); - table_is_malloced = table != mp->ma_smalltable; +/* Returns -1 if no more items (or op is not a dict), + * index of item otherwise. Stores value in pvalue + */ +Py_LOCAL_INLINE(Py_ssize_t) +dict_next(PyObject *op, Py_ssize_t i, PyObject **pvalue) +{ + Py_ssize_t mask, offset; + PyDictObject *mp; + PyObject **value_ptr; - /* This is delicate. During the process of clearing the dict, - * decrefs can cause the dict to mutate. To avoid fatal confusion - * (voice of experience), we have to make the dict empty before - * clearing the slots, and never refer to anything via mp->xxx while - * clearing. - */ - fill = mp->ma_fill; - if (table_is_malloced) - EMPTY_TO_MINSIZE(mp); - - else if (fill > 0) { - /* It's a small table with something that needs to be cleared. - * Afraid the only safe way is to copy the dict entries into - * another small table first. - */ - memcpy(small_copy, table, sizeof(small_copy)); - table = small_copy; - EMPTY_TO_MINSIZE(mp); - } - /* else it's a small table that's already empty */ - /* Now we can finally clear things. If C had refcounts, we could - * assert that the refcount on table is 1 now, i.e. that this function - * has unique access to it, so decref side-effects can't alter it. - */ - for (ep = table; fill > 0; ++ep) { -#ifdef Py_DEBUG - assert(i < n); - ++i; -#endif - if (ep->me_key) { - --fill; - Py_DECREF(ep->me_key); - Py_XDECREF(ep->me_value); - } -#ifdef Py_DEBUG - else - assert(ep->me_value == NULL); -#endif + if (!PyDict_Check(op)) + return -1; + mp = (PyDictObject *)op; + if (i < 0) + return -1; + if (mp->ma_values) { + value_ptr = &mp->ma_values[i]; + offset = sizeof(PyObject *); } - - if (table_is_malloced) - PyMem_DEL(table); + else { + value_ptr = &mp->ma_keys->dk_entries[i].me_value; + offset = sizeof(PyDictKeyEntry); + } + mask = DK_MASK(mp->ma_keys); + while (i <= mask && *value_ptr == NULL) { + value_ptr = (PyObject **)(((char *)value_ptr) + offset); + i++; + } + if (i > mask) + return -1; + if (pvalue) + *pvalue = *value_ptr; + return i; } /* @@ -981,75 +1338,59 @@ PyDict_Clear(PyObject *op) int PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) { - register Py_ssize_t i; - register Py_ssize_t mask; - register PyDictEntry *ep; - - if (!PyDict_Check(op)) - return 0; - i = *ppos; + PyDictObject *mp; + Py_ssize_t i = dict_next(op, *ppos, pvalue); if (i < 0) return 0; - ep = ((PyDictObject *)op)->ma_table; - mask = ((PyDictObject *)op)->ma_mask; - while (i <= mask && ep[i].me_value == NULL) - i++; + mp = (PyDictObject *)op; *ppos = i+1; - if (i > mask) - return 0; if (pkey) - *pkey = ep[i].me_key; - if (pvalue) - *pvalue = ep[i].me_value; + *pkey = mp->ma_keys->dk_entries[i].me_key; return 1; } -/* Internal version of PyDict_Next that returns a hash value in addition to the key and value.*/ +/* Internal version of PyDict_Next that returns a hash value in addition + * to the key and value. + */ int -_PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue, Py_hash_t *phash) +_PyDict_Next(PyObject *op, Py_ssize_t *ppos, PyObject **pkey, + PyObject **pvalue, Py_hash_t *phash) { - register Py_ssize_t i; - register Py_ssize_t mask; - register PyDictEntry *ep; - - if (!PyDict_Check(op)) - return 0; - i = *ppos; + PyDictObject *mp; + Py_ssize_t i = dict_next(op, *ppos, pvalue); if (i < 0) return 0; - ep = ((PyDictObject *)op)->ma_table; - mask = ((PyDictObject *)op)->ma_mask; - while (i <= mask && ep[i].me_value == NULL) - i++; + mp = (PyDictObject *)op; *ppos = i+1; - if (i > mask) - return 0; - *phash = ep[i].me_hash; + *phash = mp->ma_keys->dk_entries[i].me_hash; if (pkey) - *pkey = ep[i].me_key; - if (pvalue) - *pvalue = ep[i].me_value; + *pkey = mp->ma_keys->dk_entries[i].me_key; return 1; } /* Methods */ static void -dict_dealloc(register PyDictObject *mp) +dict_dealloc(PyDictObject *mp) { - register PyDictEntry *ep; - Py_ssize_t fill = mp->ma_fill; + PyObject **values = mp->ma_values; + PyDictKeysObject *keys = mp->ma_keys; + Py_ssize_t i, n; PyObject_GC_UnTrack(mp); Py_TRASHCAN_SAFE_BEGIN(mp) - for (ep = mp->ma_table; fill > 0; ep++) { - if (ep->me_key) { - --fill; - Py_DECREF(ep->me_key); - Py_XDECREF(ep->me_value); + if (values != NULL) { + if (values != empty_values) { + for (i = 0, n = DK_SIZE(mp->ma_keys); i < n; i++) { + Py_XDECREF(values[i]); + } + free_values(values); } + DK_DECREF(keys); + } + else { + assert(keys->dk_refcnt == 1); + DK_DECREF(keys); } - if (mp->ma_table != mp->ma_smalltable) - PyMem_DEL(mp->ma_table); if (numfree < PyDict_MAXFREELIST && Py_TYPE(mp) == &PyDict_Type) free_list[numfree++] = mp; else @@ -1057,6 +1398,7 @@ dict_dealloc(register PyDictObject *mp) Py_TRASHCAN_SAFE_END(mp) } + static PyObject * dict_repr(PyDictObject *mp) { @@ -1088,11 +1430,13 @@ dict_repr(PyDictObject *mp) i = 0; while (PyDict_Next((PyObject *)mp, &i, &key, &value)) { int status; - /* Prevent repr from deleting value during key format. */ + /* Prevent repr from deleting key or value during key format. */ + Py_INCREF(key); Py_INCREF(value); s = PyObject_Repr(key); PyUnicode_Append(&s, colon); PyUnicode_AppendAndDel(&s, PyObject_Repr(value)); + Py_DECREF(key); Py_DECREF(value); if (s == NULL) goto Done; @@ -1147,26 +1491,25 @@ dict_subscript(PyDictObject *mp, register PyObject *key) { PyObject *v; Py_hash_t hash; - PyDictEntry *ep; - assert(mp->ma_table != NULL); + PyDictKeyEntry *ep; + PyObject **value_addr; + if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; } - ep = (mp->ma_lookup)(mp, key, hash); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); if (ep == NULL) return NULL; - v = ep->me_value; + v = *value_addr; if (v == NULL) { if (!PyDict_CheckExact(mp)) { /* Look up __missing__ method if we're a subclass. */ PyObject *missing, *res; - static PyObject *missing_str = NULL; - missing = _PyObject_LookupSpecial((PyObject *)mp, - "__missing__", - &missing_str); + _Py_IDENTIFIER(__missing__); + missing = _PyObject_LookupSpecial((PyObject *)mp, &PyId___missing__); if (missing != NULL) { res = PyObject_CallFunctionObjArgs(missing, key, NULL); @@ -1204,8 +1547,9 @@ dict_keys(register PyDictObject *mp) { register PyObject *v; register Py_ssize_t i, j; - PyDictEntry *ep; - Py_ssize_t mask, n; + PyDictKeyEntry *ep; + Py_ssize_t size, n, offset; + PyObject **value_ptr; again: n = mp->ma_used; @@ -1219,15 +1563,24 @@ dict_keys(register PyDictObject *mp) Py_DECREF(v); goto again; } - ep = mp->ma_table; - mask = mp->ma_mask; - for (i = 0, j = 0; i <= mask; i++) { - if (ep[i].me_value != NULL) { + ep = &mp->ma_keys->dk_entries[0]; + size = DK_SIZE(mp->ma_keys); + if (mp->ma_values) { + value_ptr = mp->ma_values; + offset = sizeof(PyObject *); + } + else { + value_ptr = &ep[0].me_value; + offset = sizeof(PyDictKeyEntry); + } + for (i = 0, j = 0; i < size; i++) { + if (*value_ptr != NULL) { PyObject *key = ep[i].me_key; Py_INCREF(key); PyList_SET_ITEM(v, j, key); j++; } + value_ptr = (PyObject **)(((char *)value_ptr) + offset); } assert(j == n); return v; @@ -1238,8 +1591,8 @@ dict_values(register PyDictObject *mp) { register PyObject *v; register Py_ssize_t i, j; - PyDictEntry *ep; - Py_ssize_t mask, n; + Py_ssize_t size, n, offset; + PyObject **value_ptr; again: n = mp->ma_used; @@ -1253,11 +1606,19 @@ dict_values(register PyDictObject *mp) Py_DECREF(v); goto again; } - ep = mp->ma_table; - mask = mp->ma_mask; - for (i = 0, j = 0; i <= mask; i++) { - if (ep[i].me_value != NULL) { - PyObject *value = ep[i].me_value; + size = DK_SIZE(mp->ma_keys); + if (mp->ma_values) { + value_ptr = mp->ma_values; + offset = sizeof(PyObject *); + } + else { + value_ptr = &mp->ma_keys->dk_entries[0].me_value; + offset = sizeof(PyDictKeyEntry); + } + for (i = 0, j = 0; i < size; i++) { + PyObject *value = *value_ptr; + value_ptr = (PyObject **)(((char *)value_ptr) + offset); + if (value != NULL) { Py_INCREF(value); PyList_SET_ITEM(v, j, value); j++; @@ -1272,9 +1633,10 @@ dict_items(register PyDictObject *mp) { register PyObject *v; register Py_ssize_t i, j, n; - Py_ssize_t mask; - PyObject *item, *key, *value; - PyDictEntry *ep; + Py_ssize_t size, offset; + PyObject *item, *key; + PyDictKeyEntry *ep; + PyObject **value_ptr; /* Preallocate the list of tuples, to avoid allocations during * the loop over the items, which could trigger GC, which @@ -1301,10 +1663,20 @@ dict_items(register PyDictObject *mp) goto again; } /* Nothing we do below makes any function calls. */ - ep = mp->ma_table; - mask = mp->ma_mask; - for (i = 0, j = 0; i <= mask; i++) { - if ((value=ep[i].me_value) != NULL) { + ep = mp->ma_keys->dk_entries; + size = DK_SIZE(mp->ma_keys); + if (mp->ma_values) { + value_ptr = mp->ma_values; + offset = sizeof(PyObject *); + } + else { + value_ptr = &ep[0].me_value; + offset = sizeof(PyDictKeyEntry); + } + for (i = 0, j = 0; i < size; i++) { + PyObject *value = *value_ptr; + value_ptr = (PyObject **)(((char *)value_ptr) + offset); + if (value != NULL) { key = ep[i].me_key; item = PyList_GET_ITEM(v, j); Py_INCREF(key); @@ -1349,8 +1721,6 @@ dict_fromkeys(PyObject *cls, PyObject *args) } while (_PyDict_Next(seq, &pos, &key, &oldvalue, &hash)) { - Py_INCREF(key); - Py_INCREF(value); if (insertdict(mp, key, hash, value)) { Py_DECREF(d); return NULL; @@ -1370,8 +1740,6 @@ dict_fromkeys(PyObject *cls, PyObject *args) } while (_PySet_NextEntry(seq, &pos, &key, &hash)) { - Py_INCREF(key); - Py_INCREF(value); if (insertdict(mp, key, hash, value)) { Py_DECREF(d); return NULL; @@ -1424,7 +1792,8 @@ dict_update_common(PyObject *self, PyObject *args, PyObject *kwds, char *methnam result = -1; else if (arg != NULL) { - if (PyObject_HasAttrString(arg, "keys")) + _Py_IDENTIFIER(keys); + if (_PyObject_HasAttrId(arg, &PyId_keys)) result = PyDict_Merge(self, arg, 1); else result = PyDict_MergeFromSeq2(self, arg, 1); @@ -1536,8 +1905,8 @@ int PyDict_Merge(PyObject *a, PyObject *b, int override) { register PyDictObject *mp, *other; - register Py_ssize_t i; - PyDictEntry *entry; + register Py_ssize_t i, n; + PyDictKeyEntry *entry; /* We accept for the argument either a concrete dictionary object, * or an abstract "mapping" object. For the former, we can do @@ -1564,20 +1933,23 @@ PyDict_Merge(PyObject *a, PyObject *b, int override) * incrementally resizing as we insert new items. Expect * that there will be no (or few) overlapping keys. */ - if ((mp->ma_fill + other->ma_used)*3 >= (mp->ma_mask+1)*2) { - if (dictresize(mp, (mp->ma_used + other->ma_used)*2) != 0) + if (mp->ma_keys->dk_usable * 3 < other->ma_used * 2) + if (dictresize(mp, (mp->ma_used + other->ma_used)*2) != 0) return -1; - } - for (i = 0; i <= other->ma_mask; i++) { - entry = &other->ma_table[i]; - if (entry->me_value != NULL && + for (i = 0, n = DK_SIZE(other->ma_keys); i < n; i++) { + PyObject *value; + entry = &other->ma_keys->dk_entries[i]; + if (other->ma_values) + value = other->ma_values[i]; + else + value = entry->me_value; + + if (value != NULL && (override || PyDict_GetItem(a, entry->me_key) == NULL)) { - Py_INCREF(entry->me_key); - Py_INCREF(entry->me_value); if (insertdict(mp, entry->me_key, entry->me_hash, - entry->me_value) != 0) + value) != 0) return -1; } } @@ -1639,11 +2011,37 @@ PyObject * PyDict_Copy(PyObject *o) { PyObject *copy; + PyDictObject *mp; + Py_ssize_t i, n; if (o == NULL || !PyDict_Check(o)) { PyErr_BadInternalCall(); return NULL; } + mp = (PyDictObject *)o; + if (_PyDict_HasSplitTable(mp)) { + PyDictObject *split_copy; + PyObject **newvalues = new_values(DK_SIZE(mp->ma_keys)); + if (newvalues == NULL) + return PyErr_NoMemory(); + split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); + if (split_copy == NULL) { + free_values(newvalues); + return NULL; + } + split_copy->ma_values = newvalues; + split_copy->ma_keys = mp->ma_keys; + split_copy->ma_used = mp->ma_used; + DK_INCREF(mp->ma_keys); + for (i = 0, n = DK_SIZE(mp->ma_keys); i < n; i++) { + PyObject *value = mp->ma_values[i]; + Py_XINCREF(value); + split_copy->ma_values[i] = value; + } + if (_PyObject_GC_IS_TRACKED(mp)) + _PyObject_GC_TRACK(split_copy); + return (PyObject *)split_copy; + } copy = PyDict_New(); if (copy == NULL) return NULL; @@ -1705,14 +2103,18 @@ dict_equal(PyDictObject *a, PyDictObject *b) if (a->ma_used != b->ma_used) /* can't be equal if # of entries differ */ return 0; - /* Same # of entries -- check all of 'em. Exit early on any diff. */ - for (i = 0; i <= a->ma_mask; i++) { - PyObject *aval = a->ma_table[i].me_value; + for (i = 0; i < DK_SIZE(a->ma_keys); i++) { + PyDictKeyEntry *ep = &a->ma_keys->dk_entries[i]; + PyObject *aval; + if (a->ma_values) + aval = a->ma_values[i]; + else + aval = ep->me_value; if (aval != NULL) { int cmp; PyObject *bval; - PyObject *key = a->ma_table[i].me_key; + PyObject *key = ep->me_key; /* temporarily bump aval's refcount to ensure it stays alive until we're done with it */ Py_INCREF(aval); @@ -1733,7 +2135,7 @@ dict_equal(PyDictObject *a, PyDictObject *b) } } return 1; - } +} static PyObject * dict_richcompare(PyObject *v, PyObject *w, int op) @@ -1754,24 +2156,25 @@ dict_richcompare(PyObject *v, PyObject *w, int op) res = Py_NotImplemented; Py_INCREF(res); return res; - } +} static PyObject * dict_contains(register PyDictObject *mp, PyObject *key) { Py_hash_t hash; - PyDictEntry *ep; + PyDictKeyEntry *ep; + PyObject **value_addr; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; } - ep = (mp->ma_lookup)(mp, key, hash); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); if (ep == NULL) return NULL; - return PyBool_FromLong(ep->me_value != NULL); + return PyBool_FromLong(*value_addr != NULL); } static PyObject * @@ -1781,28 +2184,28 @@ dict_get(register PyDictObject *mp, PyObject *args) PyObject *failobj = Py_None; PyObject *val = NULL; Py_hash_t hash; - PyDictEntry *ep; + PyDictKeyEntry *ep; + PyObject **value_addr; if (!PyArg_UnpackTuple(args, "get", 1, 2, &key, &failobj)) return NULL; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; } - ep = (mp->ma_lookup)(mp, key, hash); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); if (ep == NULL) return NULL; - val = ep->me_value; + val = *value_addr; if (val == NULL) val = failobj; Py_INCREF(val); return val; } - static PyObject * dict_setdefault(register PyDictObject *mp, PyObject *args) { @@ -1810,27 +2213,40 @@ dict_setdefault(register PyDictObject *mp, PyObject *args) PyObject *failobj = Py_None; PyObject *val = NULL; Py_hash_t hash; - PyDictEntry *ep; + PyDictKeyEntry *ep; + PyObject **value_addr; if (!PyArg_UnpackTuple(args, "setdefault", 1, 2, &key, &failobj)) return NULL; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; } - ep = (mp->ma_lookup)(mp, key, hash); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); if (ep == NULL) return NULL; - val = ep->me_value; + val = *value_addr; if (val == NULL) { - if (dict_set_item_by_hash_or_entry((PyObject*)mp, key, hash, ep, - failobj) == 0) - val = failobj; + Py_INCREF(failobj); + Py_INCREF(key); + if (mp->ma_keys->dk_usable <= 0) { + /* Need to resize. */ + if (insertion_resize(mp) < 0) + return NULL; + ep = find_empty_slot(mp, key, hash, &value_addr); + } + MAINTAIN_TRACKING(mp, key, failobj); + ep->me_key = key; + ep->me_hash = hash; + *value_addr = failobj; + val = failobj; + mp->ma_keys->dk_usable--; + mp->ma_used++; } - Py_XINCREF(val); + Py_INCREF(val); return val; } @@ -1846,9 +2262,10 @@ static PyObject * dict_pop(PyDictObject *mp, PyObject *args) { Py_hash_t hash; - PyDictEntry *ep; PyObject *old_value, *old_key; PyObject *key, *deflt = NULL; + PyDictKeyEntry *ep; + PyObject **value_addr; if(!PyArg_UnpackTuple(args, "pop", 1, 2, &key, &deflt)) return NULL; @@ -1861,15 +2278,16 @@ dict_pop(PyDictObject *mp, PyObject *args) return NULL; } if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return NULL; } - ep = (mp->ma_lookup)(mp, key, hash); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); if (ep == NULL) return NULL; - if (ep->me_value == NULL) { + old_value = *value_addr; + if (old_value == NULL) { if (deflt) { Py_INCREF(deflt); return deflt; @@ -1877,13 +2295,15 @@ dict_pop(PyDictObject *mp, PyObject *args) set_key_error(key); return NULL; } - old_key = ep->me_key; - Py_INCREF(dummy); - ep->me_key = dummy; - old_value = ep->me_value; - ep->me_value = NULL; + *value_addr = NULL; mp->ma_used--; - Py_DECREF(old_key); + if (!_PyDict_HasSplitTable(mp)) { + ENSURE_ALLOWS_DELETIONS(mp); + old_key = ep->me_key; + Py_INCREF(dummy); + ep->me_key = dummy; + Py_DECREF(old_key); + } return old_value; } @@ -1891,9 +2311,10 @@ static PyObject * dict_popitem(PyDictObject *mp) { Py_hash_t i = 0; - PyDictEntry *ep; + PyDictKeyEntry *ep; PyObject *res; + /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which * could empty the dict, so if we checked the size first and that @@ -1912,25 +2333,34 @@ dict_popitem(PyDictObject *mp) "popitem(): dictionary is empty"); return NULL; } + /* Convert split table to combined table */ + if (mp->ma_keys->dk_lookup == lookdict_split) { + if (dictresize(mp, DK_SIZE(mp->ma_keys))) { + Py_DECREF(res); + return NULL; + } + } + ENSURE_ALLOWS_DELETIONS(mp); /* Set ep to "the first" dict entry with a value. We abuse the hash * field of slot 0 to hold a search finger: * If slot 0 has a value, use slot 0. * Else slot 0 is being used to hold a search finger, * and we use its hash value as the first index to look. */ - ep = &mp->ma_table[0]; + ep = &mp->ma_keys->dk_entries[0]; if (ep->me_value == NULL) { + Py_ssize_t mask = DK_MASK(mp->ma_keys); i = ep->me_hash; /* The hash field may be a real hash value, or it may be a * legit search finger, or it may be a once-legit search * finger that's out of bounds now because it wrapped around * or the table shrunk -- simply make sure it's in bounds now. */ - if (i > mp->ma_mask || i < 1) + if (i > mask || i < 1) i = 1; /* skip slot 0 */ - while ((ep = &mp->ma_table[i])->me_value == NULL) { + while ((ep = &mp->ma_keys->dk_entries[i])->me_value == NULL) { i++; - if (i > mp->ma_mask) + if (i > mask) i = 1; } } @@ -1940,21 +2370,34 @@ dict_popitem(PyDictObject *mp) ep->me_key = dummy; ep->me_value = NULL; mp->ma_used--; - assert(mp->ma_table[0].me_value == NULL); - mp->ma_table[0].me_hash = i + 1; /* next place to start */ + assert(mp->ma_keys->dk_entries[0].me_value == NULL); + mp->ma_keys->dk_entries[0].me_hash = i + 1; /* next place to start */ return res; } static int dict_traverse(PyObject *op, visitproc visit, void *arg) { - Py_ssize_t i = 0; - PyObject *pk; - PyObject *pv; - - while (PyDict_Next(op, &i, &pk, &pv)) { - Py_VISIT(pk); - Py_VISIT(pv); + Py_ssize_t i, n; + PyDictObject *mp = (PyDictObject *)op; + if (mp->ma_keys->dk_lookup == lookdict) { + for (i = 0; i < DK_SIZE(mp->ma_keys); i++) { + if (mp->ma_keys->dk_entries[i].me_value != NULL) { + Py_VISIT(mp->ma_keys->dk_entries[i].me_value); + Py_VISIT(mp->ma_keys->dk_entries[i].me_key); + } + } + } else { + if (mp->ma_values != NULL) { + for (i = 0, n = DK_SIZE(mp->ma_keys); i < n; i++) { + Py_VISIT(mp->ma_values[i]); + } + } + else { + for (i = 0, n = DK_SIZE(mp->ma_keys); i < n; i++) { + Py_VISIT(mp->ma_keys->dk_entries[i].me_value); + } + } } return 0; } @@ -1971,14 +2414,25 @@ static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); static PyObject * dict_sizeof(PyDictObject *mp) { - Py_ssize_t res; + Py_ssize_t size, res; + size = DK_SIZE(mp->ma_keys); res = sizeof(PyDictObject); - if (mp->ma_table != mp->ma_smalltable) - res = res + (mp->ma_mask + 1) * sizeof(PyDictEntry); + if (mp->ma_values) + res += size * sizeof(PyObject*); + /* If the dictionary is split, the keys portion is accounted-for + in the type object. */ + if (mp->ma_keys->dk_refcnt == 1) + res += sizeof(PyDictKeysObject) + (size-1) * sizeof(PyDictKeyEntry); return PyLong_FromSsize_t(res); } +Py_ssize_t +_PyDict_KeysSize(PyDictKeysObject *keys) +{ + return sizeof(PyDictKeysObject) + (DK_SIZE(keys)-1) * sizeof(PyDictKeyEntry); +} + PyDoc_STRVAR(contains__doc__, "D.__contains__(k) -> True if D has a key k, else False"); @@ -2067,16 +2521,17 @@ PyDict_Contains(PyObject *op, PyObject *key) { Py_hash_t hash; PyDictObject *mp = (PyDictObject *)op; - PyDictEntry *ep; + PyDictKeyEntry *ep; + PyObject **value_addr; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; } - ep = (mp->ma_lookup)(mp, key, hash); - return ep == NULL ? -1 : (ep->me_value != NULL); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); + return (ep == NULL) ? -1 : (*value_addr != NULL); } /* Internal version of PyDict_Contains used when the hash value is already known */ @@ -2084,10 +2539,11 @@ int _PyDict_Contains(PyObject *op, PyObject *key, Py_hash_t hash) { PyDictObject *mp = (PyDictObject *)op; - PyDictEntry *ep; + PyDictKeyEntry *ep; + PyObject **value_addr; - ep = (mp->ma_lookup)(mp, key, hash); - return ep == NULL ? -1 : (ep->me_value != NULL); + ep = (mp->ma_keys->dk_lookup)(mp, key, hash, &value_addr); + return (ep == NULL) ? -1 : (*value_addr != NULL); } /* Hack to implement "key in dict" */ @@ -2113,22 +2569,17 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self = type->tp_alloc(type, 0); if (self != NULL) { PyDictObject *d = (PyDictObject *)self; - /* It's guaranteed that tp->alloc zeroed out the struct. */ - assert(d->ma_table == NULL && d->ma_fill == 0 && d->ma_used == 0); - INIT_NONZERO_DICT_SLOTS(d); - d->ma_lookup = lookdict_unicode; + d->ma_keys = new_keys_object(PyDict_MINSIZE_COMBINED); + /* XXX - Should we raise a no-memory error? */ + if (d->ma_keys == NULL) { + DK_INCREF(Py_EMPTY_KEYS); + d->ma_keys = Py_EMPTY_KEYS; + d->ma_values = empty_values; + } + d->ma_used = 0; /* The object has been implicitly tracked by tp_alloc */ if (type == &PyDict_Type) _PyObject_GC_UNTRACK(d); -#ifdef SHOW_CONVERSION_COUNTS - ++created; -#endif -#ifdef SHOW_TRACK_COUNT - if (_PyObject_GC_IS_TRACKED(d)) - count_tracked++; - else - count_untracked++; -#endif } return self; } @@ -2199,6 +2650,16 @@ PyTypeObject PyDict_Type = { PyObject_GC_Del, /* tp_free */ }; +PyObject * +_PyDict_GetItemId(PyObject *dp, struct _Py_Identifier *key) +{ + PyObject *kv; + kv = _PyUnicode_FromId(key); /* borrowed */ + if (kv == NULL) + return NULL; + return PyDict_GetItem(dp, kv); +} + /* For backward compatibility with old dictionary interface */ PyObject * @@ -2214,6 +2675,16 @@ PyDict_GetItemString(PyObject *v, const char *key) } int +_PyDict_SetItemId(PyObject *v, struct _Py_Identifier *key, PyObject *item) +{ + PyObject *kv; + kv = _PyUnicode_FromId(key); /* borrowed */ + if (kv == NULL) + return -1; + return PyDict_SetItem(v, kv, item); +} + +int PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) { PyObject *kv; @@ -2304,18 +2775,26 @@ dictiter_len(dictiterobject *di) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +static PyObject * +dictiter_reduce(dictiterobject *di); + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); + static PyMethodDef dictiter_methods[] = { {"__length_hint__", (PyCFunction)dictiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)dictiter_reduce, METH_NOARGS, + reduce_doc}, {NULL, NULL} /* sentinel */ }; static PyObject *dictiter_iternextkey(dictiterobject *di) { PyObject *key; - register Py_ssize_t i, mask; - register PyDictEntry *ep; + register Py_ssize_t i, mask, offset; + register PyDictKeysObject *k; PyDictObject *d = di->di_dict; + PyObject **value_ptr; if (d == NULL) return NULL; @@ -2331,15 +2810,25 @@ static PyObject *dictiter_iternextkey(dictiterobject *di) i = di->di_pos; if (i < 0) goto fail; - ep = d->ma_table; - mask = d->ma_mask; - while (i <= mask && ep[i].me_value == NULL) + k = d->ma_keys; + if (d->ma_values) { + value_ptr = &d->ma_values[i]; + offset = sizeof(PyObject *); + } + else { + value_ptr = &k->dk_entries[i].me_value; + offset = sizeof(PyDictKeyEntry); + } + mask = DK_SIZE(k)-1; + while (i <= mask && *value_ptr == NULL) { + value_ptr = (PyObject **)(((char *)value_ptr) + offset); i++; + } di->di_pos = i+1; if (i > mask) goto fail; di->len--; - key = ep[i].me_key; + key = k->dk_entries[i].me_key; Py_INCREF(key); return key; @@ -2385,9 +2874,9 @@ PyTypeObject PyDictIterKey_Type = { static PyObject *dictiter_iternextvalue(dictiterobject *di) { PyObject *value; - register Py_ssize_t i, mask; - register PyDictEntry *ep; + register Py_ssize_t i, mask, offset; PyDictObject *d = di->di_dict; + PyObject **value_ptr; if (d == NULL) return NULL; @@ -2401,17 +2890,26 @@ static PyObject *dictiter_iternextvalue(dictiterobject *di) } i = di->di_pos; - mask = d->ma_mask; + mask = DK_SIZE(d->ma_keys)-1; if (i < 0 || i > mask) goto fail; - ep = d->ma_table; - while ((value=ep[i].me_value) == NULL) { + if (d->ma_values) { + value_ptr = &d->ma_values[i]; + offset = sizeof(PyObject *); + } + else { + value_ptr = &d->ma_keys->dk_entries[i].me_value; + offset = sizeof(PyDictKeyEntry); + } + while (i <= mask && *value_ptr == NULL) { + value_ptr = (PyObject **)(((char *)value_ptr) + offset); i++; if (i > mask) goto fail; } di->di_pos = i+1; di->len--; + value = *value_ptr; Py_INCREF(value); return value; @@ -2457,9 +2955,9 @@ PyTypeObject PyDictIterValue_Type = { static PyObject *dictiter_iternextitem(dictiterobject *di) { PyObject *key, *value, *result = di->di_result; - register Py_ssize_t i, mask; - register PyDictEntry *ep; + register Py_ssize_t i, mask, offset; PyDictObject *d = di->di_dict; + PyObject **value_ptr; if (d == NULL) return NULL; @@ -2475,10 +2973,19 @@ static PyObject *dictiter_iternextitem(dictiterobject *di) i = di->di_pos; if (i < 0) goto fail; - ep = d->ma_table; - mask = d->ma_mask; - while (i <= mask && ep[i].me_value == NULL) + mask = DK_SIZE(d->ma_keys)-1; + if (d->ma_values) { + value_ptr = &d->ma_values[i]; + offset = sizeof(PyObject *); + } + else { + value_ptr = &d->ma_keys->dk_entries[i].me_value; + offset = sizeof(PyDictKeyEntry); + } + while (i <= mask && *value_ptr == NULL) { + value_ptr = (PyObject **)(((char *)value_ptr) + offset); i++; + } di->di_pos = i+1; if (i > mask) goto fail; @@ -2493,8 +3000,8 @@ static PyObject *dictiter_iternextitem(dictiterobject *di) return NULL; } di->len--; - key = ep[i].me_key; - value = ep[i].me_value; + key = d->ma_keys->dk_entries[i].me_key; + value = *value_ptr; Py_INCREF(key); Py_INCREF(value); PyTuple_SET_ITEM(result, 0, key); @@ -2541,6 +3048,52 @@ PyTypeObject PyDictIterItem_Type = { }; +static PyObject * +dictiter_reduce(dictiterobject *di) +{ + PyObject *list; + dictiterobject tmp; + + list = PyList_New(0); + if (!list) + return NULL; + + /* copy the itertor state */ + tmp = *di; + Py_XINCREF(tmp.di_dict); + + /* iterate the temporary into a list */ + for(;;) { + PyObject *element = 0; + if (Py_TYPE(di) == &PyDictIterItem_Type) + element = dictiter_iternextitem(&tmp); + else if (Py_TYPE(di) == &PyDictIterKey_Type) + element = dictiter_iternextkey(&tmp); + else if (Py_TYPE(di) == &PyDictIterValue_Type) + element = dictiter_iternextvalue(&tmp); + else + assert(0); + if (element) { + if (PyList_Append(list, element)) { + Py_DECREF(element); + Py_DECREF(list); + Py_XDECREF(tmp.di_dict); + return NULL; + } + Py_DECREF(element); + } else + break; + } + Py_XDECREF(tmp.di_dict); + /* check for error */ + if (tmp.di_dict != NULL) { + /* we have an error */ + Py_DECREF(list); + return NULL; + } + return Py_BuildValue("N(N)", _PyObject_GetBuiltin("iter"), list); +} + /***********************************************/ /* View objects for keys(), items(), values(). */ /***********************************************/ @@ -2645,10 +3198,8 @@ dictview_richcompare(PyObject *self, PyObject *other, int op) assert(PyDictViewSet_Check(self)); assert(other != NULL); - if (!PyAnySet_Check(other) && !PyDictViewSet_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyAnySet_Check(other) && !PyDictViewSet_Check(other)) + Py_RETURN_NOTIMPLEMENTED; len_self = PyObject_Size(self); if (len_self < 0) @@ -2746,10 +3297,12 @@ dictviews_sub(PyObject* self, PyObject *other) { PyObject *result = PySet_New(self); PyObject *tmp; + _Py_IDENTIFIER(difference_update); + if (result == NULL) return NULL; - tmp = PyObject_CallMethod(result, "difference_update", "O", other); + tmp = _PyObject_CallMethodId(result, &PyId_difference_update, "O", other); if (tmp == NULL) { Py_DECREF(result); return NULL; @@ -2764,10 +3317,12 @@ dictviews_and(PyObject* self, PyObject *other) { PyObject *result = PySet_New(self); PyObject *tmp; + _Py_IDENTIFIER(intersection_update); + if (result == NULL) return NULL; - tmp = PyObject_CallMethod(result, "intersection_update", "O", other); + tmp = _PyObject_CallMethodId(result, &PyId_intersection_update, "O", other); if (tmp == NULL) { Py_DECREF(result); return NULL; @@ -2782,10 +3337,12 @@ dictviews_or(PyObject* self, PyObject *other) { PyObject *result = PySet_New(self); PyObject *tmp; + _Py_IDENTIFIER(update); + if (result == NULL) return NULL; - tmp = PyObject_CallMethod(result, "update", "O", other); + tmp = _PyObject_CallMethodId(result, &PyId_update, "O", other); if (tmp == NULL) { Py_DECREF(result); return NULL; @@ -2800,10 +3357,12 @@ dictviews_xor(PyObject* self, PyObject *other) { PyObject *result = PySet_New(self); PyObject *tmp; + _Py_IDENTIFIER(symmetric_difference_update); + if (result == NULL) return NULL; - tmp = PyObject_CallMethod(result, "symmetric_difference_update", "O", + tmp = _PyObject_CallMethodId(result, &PyId_symmetric_difference_update, "O", other); if (tmp == NULL) { Py_DECREF(result); @@ -3082,3 +3641,151 @@ dictvalues_new(PyObject *dict) { return dictview_new(dict, &PyDictValues_Type); } + +/* Returns NULL if cannot allocate a new PyDictKeysObject, + but does not set an error */ +PyDictKeysObject * +_PyDict_NewKeysForClass(void) +{ + PyDictKeysObject *keys = new_keys_object(PyDict_MINSIZE_SPLIT); + if (keys == NULL) + PyErr_Clear(); + else + keys->dk_lookup = lookdict_split; + return keys; +} + +#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys) + +PyObject * +PyObject_GenericGetDict(PyObject *obj, void *context) +{ + PyObject *dict, **dictptr = _PyObject_GetDictPtr(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return NULL; + } + dict = *dictptr; + if (dict == NULL) { + PyTypeObject *tp = Py_TYPE(obj); + if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { + DK_INCREF(CACHED_KEYS(tp)); + *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + } + else { + *dictptr = dict = PyDict_New(); + } + } + Py_XINCREF(dict); + return dict; +} + +int +_PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, + PyObject *key, PyObject *value) +{ + PyObject *dict; + int res; + PyDictKeysObject *cached; + + assert(dictptr != NULL); + if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { + assert(dictptr != NULL); + dict = *dictptr; + if (dict == NULL) { + DK_INCREF(cached); + dict = new_dict_with_shared_keys(cached); + if (dict == NULL) + return -1; + *dictptr = dict; + } + if (value == NULL) { + res = PyDict_DelItem(dict, key); + if (cached != ((PyDictObject *)dict)->ma_keys) { + CACHED_KEYS(tp) = NULL; + DK_DECREF(cached); + } + } else { + res = PyDict_SetItem(dict, key, value); + if (cached != ((PyDictObject *)dict)->ma_keys) { + /* Either update tp->ht_cached_keys or delete it */ + if (cached->dk_refcnt == 1) { + CACHED_KEYS(tp) = make_keys_shared(dict); + } else { + CACHED_KEYS(tp) = NULL; + } + DK_DECREF(cached); + if (CACHED_KEYS(tp) == NULL && PyErr_Occurred()) + return -1; + } + } + } else { + dict = *dictptr; + if (dict == NULL) { + dict = PyDict_New(); + if (dict == NULL) + return -1; + *dictptr = dict; + } + if (value == NULL) { + res = PyDict_DelItem(dict, key); + } else { + res = PyDict_SetItem(dict, key, value); + } + } + return res; +} + +void +_PyDictKeys_DecRef(PyDictKeysObject *keys) +{ + DK_DECREF(keys); +} + + +/* ARGSUSED */ +static PyObject * +dummy_repr(PyObject *op) +{ + return PyUnicode_FromString("<dummy key>"); +} + +/* ARGUSED */ +static void +dummy_dealloc(PyObject* ignore) +{ + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref dummy-key out of existence. + */ + Py_FatalError("deallocating <dummy key>"); +} + +static PyTypeObject PyDictDummy_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "<dummy key> type", + 0, + 0, + dummy_dealloc, /*tp_dealloc*/ /*never called*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + dummy_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call */ + 0, /*tp_str */ + 0, /*tp_getattro */ + 0, /*tp_setattro */ + 0, /*tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /*tp_flags */ +}; + +static PyObject _dummy_struct = { + _PyObject_EXTRA_INIT + 2, &PyDictDummy_Type +}; + diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 00a3346..c458cfe 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -158,6 +158,22 @@ enum_next(enumobject *en) return result; } +static PyObject * +enum_reduce(enumobject *en) +{ + if (en->en_longindex != NULL) + return Py_BuildValue("O(OO)", Py_TYPE(en), en->en_sit, en->en_longindex); + else + return Py_BuildValue("O(On)", Py_TYPE(en), en->en_sit, en->en_index); +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); + +static PyMethodDef enum_methods[] = { + {"__reduce__", (PyCFunction)enum_reduce, METH_NOARGS, reduce_doc}, + {NULL, NULL} /* sentinel */ +}; + PyDoc_STRVAR(enum_doc, "enumerate(iterable[, start]) -> iterator for index, value of iterable\n" "\n" @@ -197,7 +213,7 @@ PyTypeObject PyEnum_Type = { 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)enum_next, /* tp_iternext */ - 0, /* tp_methods */ + enum_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ @@ -224,8 +240,8 @@ reversed_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { Py_ssize_t n; PyObject *seq, *reversed_meth; - static PyObject *reversed_cache = NULL; reversedobject *ro; + _Py_IDENTIFIER(__reversed__); if (type == &PyReversed_Type && !_PyArg_NoKeywords("reversed()", kwds)) return NULL; @@ -233,7 +249,7 @@ reversed_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!PyArg_UnpackTuple(args, "reversed", 1, 1, &seq) ) return NULL; - reversed_meth = _PyObject_LookupSpecial(seq, "__reversed__", &reversed_cache); + reversed_meth = _PyObject_LookupSpecial(seq, &PyId___reversed__); if (reversed_meth != NULL) { PyObject *res = PyObject_CallFunctionObjArgs(reversed_meth, NULL); Py_DECREF(reversed_meth); @@ -319,8 +335,40 @@ reversed_len(reversedobject *ro) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +static PyObject * +reversed_reduce(reversedobject *ro) +{ + if (ro->seq) + return Py_BuildValue("O(O)n", Py_TYPE(ro), ro->seq, ro->index); + else + return Py_BuildValue("O(())", Py_TYPE(ro)); +} + +static PyObject * +reversed_setstate(reversedobject *ro, PyObject *state) +{ + Py_ssize_t index = PyLong_AsSsize_t(state); + if (index == -1 && PyErr_Occurred()) + return NULL; + if (ro->seq != 0) { + Py_ssize_t n = PySequence_Size(ro->seq); + if (n < 0) + return NULL; + if (index < -1) + index = -1; + else if (index > n-1) + index = n-1; + ro->index = index; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); + static PyMethodDef reversediter_methods[] = { {"__length_hint__", (PyCFunction)reversed_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)reversed_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", (PyCFunction)reversed_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 9daa12a..6b04700 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -10,6 +10,20 @@ #include "osdefs.h" +/* Compatibility aliases */ +PyObject *PyExc_EnvironmentError = NULL; +PyObject *PyExc_IOError = NULL; +#ifdef MS_WINDOWS +PyObject *PyExc_WindowsError = NULL; +#endif +#ifdef __VMS +PyObject *PyExc_VMSError = NULL; +#endif + +/* The dict map from errno codes to OSError subclasses */ +static PyObject *errnomap = NULL; + + /* NOTE: If the exception class hierarchy changes, don't forget to update * Lib/test/exception_hierarchy.txt */ @@ -28,6 +42,13 @@ BaseException_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* the dict is created on the fly in PyObject_GenericSetAttr */ self->dict = NULL; self->traceback = self->cause = self->context = NULL; + self->suppress_context = 0; + + if (args) { + self->args = args; + Py_INCREF(args); + return (PyObject *)self; + } self->args = PyTuple_New(0); if (!self->args) { @@ -41,12 +62,15 @@ BaseException_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int BaseException_init(PyBaseExceptionObject *self, PyObject *args, PyObject *kwds) { + PyObject *tmp; + if (!_PyArg_NoKeywords(Py_TYPE(self)->tp_name, kwds)) return -1; - Py_DECREF(self->args); + tmp = self->args; self->args = args; Py_INCREF(self->args); + Py_XDECREF(tmp); return 0; } @@ -163,36 +187,6 @@ static PyMethodDef BaseException_methods[] = { {NULL, NULL, 0, NULL}, }; - -static PyObject * -BaseException_get_dict(PyBaseExceptionObject *self) -{ - if (self->dict == NULL) { - self->dict = PyDict_New(); - if (!self->dict) - return NULL; - } - Py_INCREF(self->dict); - return self->dict; -} - -static int -BaseException_set_dict(PyBaseExceptionObject *self, PyObject *val) -{ - if (val == NULL) { - PyErr_SetString(PyExc_TypeError, "__dict__ may not be deleted"); - return -1; - } - if (!PyDict_Check(val)) { - PyErr_SetString(PyExc_TypeError, "__dict__ must be a dictionary"); - return -1; - } - Py_CLEAR(self->dict); - Py_INCREF(val); - self->dict = val; - return 0; -} - static PyObject * BaseException_get_args(PyBaseExceptionObject *self) { @@ -306,7 +300,7 @@ BaseException_set_cause(PyObject *self, PyObject *arg) { static PyGetSetDef BaseException_getset[] = { - {"__dict__", (getter)BaseException_get_dict, (setter)BaseException_set_dict}, + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, {"args", (getter)BaseException_get_args, (setter)BaseException_set_args}, {"__traceback__", (getter)BaseException_get_tb, (setter)BaseException_set_tb}, {"__context__", (getter)BaseException_get_context, @@ -342,6 +336,7 @@ void PyException_SetCause(PyObject *self, PyObject *cause) { PyObject *old_cause = ((PyBaseExceptionObject *)self)->cause; ((PyBaseExceptionObject *)self)->cause = cause; + ((PyBaseExceptionObject *)self)->suppress_context = 1; Py_XDECREF(old_cause); } @@ -361,6 +356,13 @@ PyException_SetContext(PyObject *self, PyObject *context) { } +static struct PyMemberDef BaseException_members[] = { + {"__suppress_context__", T_BOOL, + offsetof(PyBaseExceptionObject, suppress_context)}, + {NULL} +}; + + static PyTypeObject _PyExc_BaseException = { PyVarObject_HEAD_INIT(NULL, 0) "BaseException", /*tp_name*/ @@ -391,7 +393,7 @@ static PyTypeObject _PyExc_BaseException = { 0, /* tp_iter */ 0, /* tp_iternext */ BaseException_methods, /* tp_methods */ - 0, /* tp_members */ + BaseException_members, /* tp_members */ BaseException_getset, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ @@ -436,11 +438,13 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \ PyDoc_STR(EXCDOC), (traverseproc)EXCSTORE ## _traverse, \ (inquiry)EXCSTORE ## _clear, 0, 0, 0, 0, 0, 0, 0, &_ ## EXCBASE, \ 0, 0, 0, offsetof(Py ## EXCSTORE ## Object, dict), \ - (initproc)EXCSTORE ## _init, 0, BaseException_new,\ + (initproc)EXCSTORE ## _init, 0, 0, \ }; \ PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME -#define ComplexExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCDEALLOC, EXCMETHODS, EXCMEMBERS, EXCSTR, EXCDOC) \ +#define ComplexExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCNEW, \ + EXCMETHODS, EXCMEMBERS, EXCGETSET, \ + EXCSTR, EXCDOC) \ static PyTypeObject _PyExc_ ## EXCNAME = { \ PyVarObject_HEAD_INIT(NULL, 0) \ # EXCNAME, \ @@ -450,9 +454,9 @@ static PyTypeObject _PyExc_ ## EXCNAME = { \ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, \ PyDoc_STR(EXCDOC), (traverseproc)EXCSTORE ## _traverse, \ (inquiry)EXCSTORE ## _clear, 0, 0, 0, 0, EXCMETHODS, \ - EXCMEMBERS, 0, &_ ## EXCBASE, \ + EXCMEMBERS, EXCGETSET, &_ ## EXCBASE, \ 0, 0, 0, offsetof(Py ## EXCSTORE ## Object, dict), \ - (initproc)EXCSTORE ## _init, 0, BaseException_new,\ + (initproc)EXCSTORE ## _init, 0, EXCNEW,\ }; \ PyObject *PyExc_ ## EXCNAME = (PyObject *)&_PyExc_ ## EXCNAME @@ -474,8 +478,64 @@ SimpleExtendsException(PyExc_Exception, TypeError, /* * StopIteration extends Exception */ -SimpleExtendsException(PyExc_Exception, StopIteration, - "Signal the end from iterator.__next__()."); + +static PyMemberDef StopIteration_members[] = { + {"value", T_OBJECT, offsetof(PyStopIterationObject, value), 0, + PyDoc_STR("generator return value")}, + {NULL} /* Sentinel */ +}; + +static int +StopIteration_init(PyStopIterationObject *self, PyObject *args, PyObject *kwds) +{ + Py_ssize_t size = PyTuple_GET_SIZE(args); + PyObject *value; + + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + return -1; + Py_CLEAR(self->value); + if (size > 0) + value = PyTuple_GET_ITEM(args, 0); + else + value = Py_None; + Py_INCREF(value); + self->value = value; + return 0; +} + +static int +StopIteration_clear(PyStopIterationObject *self) +{ + Py_CLEAR(self->value); + return BaseException_clear((PyBaseExceptionObject *)self); +} + +static void +StopIteration_dealloc(PyStopIterationObject *self) +{ + _PyObject_GC_UNTRACK(self); + StopIteration_clear(self); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int +StopIteration_traverse(PyStopIterationObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->value); + return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); +} + +ComplexExtendsException( + PyExc_Exception, /* base */ + StopIteration, /* name */ + StopIteration, /* prefix for *_init, etc */ + 0, /* new */ + 0, /* methods */ + StopIteration_members, /* members */ + 0, /* getset */ + 0, /* str */ + "Signal the end from iterator.__next__()." +); /* @@ -502,7 +562,7 @@ SystemExit_init(PySystemExitObject *self, PyObject *args, PyObject *kwds) Py_CLEAR(self->code); if (size == 1) self->code = PyTuple_GET_ITEM(args, 0); - else if (size > 1) + else /* size > 1 */ self->code = args; Py_INCREF(self->code); return 0; @@ -537,7 +597,7 @@ static PyMemberDef SystemExit_members[] = { }; ComplexExtendsException(PyExc_BaseException, SystemExit, SystemExit, - SystemExit_dealloc, 0, SystemExit_members, 0, + 0, 0, SystemExit_members, 0, 0, "Request to exit from the interpreter."); /* @@ -550,129 +610,441 @@ SimpleExtendsException(PyExc_BaseException, KeyboardInterrupt, /* * ImportError extends Exception */ -SimpleExtendsException(PyExc_Exception, ImportError, - "Import can't find module, or can't find name in module."); +static int +ImportError_init(PyImportErrorObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *msg = NULL; + PyObject *name = NULL; + PyObject *path = NULL; + +/* Macro replacement doesn't allow ## to start the first line of a macro, + so we move the assignment and NULL check into the if-statement. */ +#define GET_KWD(kwd) { \ + kwd = PyDict_GetItemString(kwds, #kwd); \ + if (kwd) { \ + Py_CLEAR(self->kwd); \ + self->kwd = kwd; \ + Py_INCREF(self->kwd);\ + if (PyDict_DelItemString(kwds, #kwd)) \ + return -1; \ + } \ + } + + if (kwds) { + GET_KWD(name); + GET_KWD(path); + } + + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + return -1; + if (PyTuple_GET_SIZE(args) != 1) + return 0; + if (!PyArg_UnpackTuple(args, "ImportError", 1, 1, &msg)) + return -1; + + Py_CLEAR(self->msg); /* replacing */ + self->msg = msg; + Py_INCREF(self->msg); + + return 0; +} + +static int +ImportError_clear(PyImportErrorObject *self) +{ + Py_CLEAR(self->msg); + Py_CLEAR(self->name); + Py_CLEAR(self->path); + return BaseException_clear((PyBaseExceptionObject *)self); +} + +static void +ImportError_dealloc(PyImportErrorObject *self) +{ + _PyObject_GC_UNTRACK(self); + ImportError_clear(self); + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static int +ImportError_traverse(PyImportErrorObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->msg); + Py_VISIT(self->name); + Py_VISIT(self->path); + return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); +} + +static PyObject * +ImportError_str(PyImportErrorObject *self) +{ + if (self->msg && PyUnicode_CheckExact(self->msg)) { + Py_INCREF(self->msg); + return self->msg; + } + else { + return BaseException_str((PyBaseExceptionObject *)self); + } +} + +static PyMemberDef ImportError_members[] = { + {"msg", T_OBJECT, offsetof(PyImportErrorObject, msg), 0, + PyDoc_STR("exception message")}, + {"name", T_OBJECT, offsetof(PyImportErrorObject, name), 0, + PyDoc_STR("module name")}, + {"path", T_OBJECT, offsetof(PyImportErrorObject, path), 0, + PyDoc_STR("module path")}, + {NULL} /* Sentinel */ +}; + +static PyMethodDef ImportError_methods[] = { + {NULL} +}; + +ComplexExtendsException(PyExc_Exception, ImportError, + ImportError, 0 /* new */, + ImportError_methods, ImportError_members, + 0 /* getset */, ImportError_str, + "Import can't find module, or can't find name in " + "module."); /* - * EnvironmentError extends Exception + * OSError extends Exception */ +#ifdef MS_WINDOWS +#include "errmap.h" +#endif + /* Where a function has a single filename, such as open() or some * of the os module functions, PyErr_SetFromErrnoWithFilename() is * called, giving a third argument which is the filename. But, so * that old code using in-place unpacking doesn't break, e.g.: * - * except IOError, (errno, strerror): + * except OSError, (errno, strerror): * * we hack args so that it only contains two items. This also * means we need our own __str__() which prints out the filename * when it was supplied. */ + +/* This function doesn't cleanup on error, the caller should */ static int -EnvironmentError_init(PyEnvironmentErrorObject *self, PyObject *args, - PyObject *kwds) +oserror_parse_args(PyObject **p_args, + PyObject **myerrno, PyObject **strerror, + PyObject **filename +#ifdef MS_WINDOWS + , PyObject **winerror +#endif + ) { - PyObject *myerrno = NULL, *strerror = NULL, *filename = NULL; - PyObject *subslice = NULL; + Py_ssize_t nargs; + PyObject *args = *p_args; - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) - return -1; + nargs = PyTuple_GET_SIZE(args); - if (PyTuple_GET_SIZE(args) <= 1 || PyTuple_GET_SIZE(args) > 3) { - return 0; +#ifdef MS_WINDOWS + if (nargs >= 2 && nargs <= 4) { + if (!PyArg_UnpackTuple(args, "OSError", 2, 4, + myerrno, strerror, filename, winerror)) + return -1; + if (*winerror && PyLong_Check(*winerror)) { + long errcode, winerrcode; + PyObject *newargs; + Py_ssize_t i; + + winerrcode = PyLong_AsLong(*winerror); + if (winerrcode == -1 && PyErr_Occurred()) + return -1; + /* Set errno to the corresponding POSIX errno (overriding + first argument). Windows Socket error codes (>= 10000) + have the same value as their POSIX counterparts. + */ + if (winerrcode < 10000) + errcode = winerror_to_errno(winerrcode); + else + errcode = winerrcode; + *myerrno = PyLong_FromLong(errcode); + if (!*myerrno) + return -1; + newargs = PyTuple_New(nargs); + if (!newargs) + return -1; + PyTuple_SET_ITEM(newargs, 0, *myerrno); + for (i = 1; i < nargs; i++) { + PyObject *val = PyTuple_GET_ITEM(args, i); + Py_INCREF(val); + PyTuple_SET_ITEM(newargs, i, val); + } + Py_DECREF(args); + args = *p_args = newargs; + } } +#else + if (nargs >= 2 && nargs <= 3) { + if (!PyArg_UnpackTuple(args, "OSError", 2, 3, + myerrno, strerror, filename)) + return -1; + } +#endif - if (!PyArg_UnpackTuple(args, "EnvironmentError", 2, 3, - &myerrno, &strerror, &filename)) { - return -1; + return 0; +} + +static int +oserror_init(PyOSErrorObject *self, PyObject **p_args, + PyObject *myerrno, PyObject *strerror, + PyObject *filename +#ifdef MS_WINDOWS + , PyObject *winerror +#endif + ) +{ + PyObject *args = *p_args; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + + /* self->filename will remain Py_None otherwise */ + if (filename && filename != Py_None) { + if (Py_TYPE(self) == (PyTypeObject *) PyExc_BlockingIOError && + PyNumber_Check(filename)) { + /* BlockingIOError's 3rd argument can be the number of + * characters written. + */ + self->written = PyNumber_AsSsize_t(filename, PyExc_ValueError); + if (self->written == -1 && PyErr_Occurred()) + return -1; + } + else { + Py_INCREF(filename); + self->filename = filename; + + if (nargs >= 2 && nargs <= 3) { + /* filename is removed from the args tuple (for compatibility + purposes, see test_exceptions.py) */ + PyObject *subslice = PyTuple_GetSlice(args, 0, 2); + if (!subslice) + return -1; + + Py_DECREF(args); /* replacing args */ + *p_args = args = subslice; + } + } } - Py_CLEAR(self->myerrno); /* replacing */ + Py_XINCREF(myerrno); self->myerrno = myerrno; - Py_INCREF(self->myerrno); - Py_CLEAR(self->strerror); /* replacing */ + Py_XINCREF(strerror); self->strerror = strerror; - Py_INCREF(self->strerror); - /* self->filename will remain Py_None otherwise */ - if (filename != NULL) { - Py_CLEAR(self->filename); /* replacing */ - self->filename = filename; - Py_INCREF(self->filename); +#ifdef MS_WINDOWS + Py_XINCREF(winerror); + self->winerror = winerror; +#endif - subslice = PyTuple_GetSlice(args, 0, 2); - if (!subslice) - return -1; + /* Steals the reference to args */ + Py_CLEAR(self->args); + self->args = args; + args = NULL; + + return 0; +} - Py_DECREF(self->args); /* replacing args */ - self->args = subslice; +static PyObject * +OSError_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +static int +OSError_init(PyOSErrorObject *self, PyObject *args, PyObject *kwds); + +static int +oserror_use_init(PyTypeObject *type) +{ + /* When __init__ is defined in a OSError subclass, we want any + extraneous argument to __new__ to be ignored. The only reasonable + solution, given __new__ takes a variable number of arguments, + is to defer arg parsing and initialization to __init__. + + But when __new__ is overriden as well, it should call our __new__ + with the right arguments. + + (see http://bugs.python.org/issue12555#msg148829 ) + */ + if (type->tp_init != (initproc) OSError_init && + type->tp_new == (newfunc) OSError_new) { + assert((PyObject *) type != PyExc_OSError); + return 1; } return 0; } +static PyObject * +OSError_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyOSErrorObject *self = NULL; + PyObject *myerrno = NULL, *strerror = NULL, *filename = NULL; +#ifdef MS_WINDOWS + PyObject *winerror = NULL; +#endif + + if (!oserror_use_init(type)) { + if (!_PyArg_NoKeywords(type->tp_name, kwds)) + return NULL; + + Py_INCREF(args); + if (oserror_parse_args(&args, &myerrno, &strerror, &filename +#ifdef MS_WINDOWS + , &winerror +#endif + )) + goto error; + + if (myerrno && PyLong_Check(myerrno) && + errnomap && (PyObject *) type == PyExc_OSError) { + PyObject *newtype; + newtype = PyDict_GetItem(errnomap, myerrno); + if (newtype) { + assert(PyType_Check(newtype)); + type = (PyTypeObject *) newtype; + } + else if (PyErr_Occurred()) + goto error; + } + } + + self = (PyOSErrorObject *) type->tp_alloc(type, 0); + if (!self) + goto error; + + self->dict = NULL; + self->traceback = self->cause = self->context = NULL; + self->written = -1; + + if (!oserror_use_init(type)) { + if (oserror_init(self, &args, myerrno, strerror, filename +#ifdef MS_WINDOWS + , winerror +#endif + )) + goto error; + } + else { + self->args = PyTuple_New(0); + if (self->args == NULL) + goto error; + } + + return (PyObject *) self; + +error: + Py_XDECREF(args); + Py_XDECREF(self); + return NULL; +} + static int -EnvironmentError_clear(PyEnvironmentErrorObject *self) +OSError_init(PyOSErrorObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *myerrno = NULL, *strerror = NULL, *filename = NULL; +#ifdef MS_WINDOWS + PyObject *winerror = NULL; +#endif + + if (!oserror_use_init(Py_TYPE(self))) + /* Everything already done in OSError_new */ + return 0; + + if (!_PyArg_NoKeywords(Py_TYPE(self)->tp_name, kwds)) + return -1; + + Py_INCREF(args); + if (oserror_parse_args(&args, &myerrno, &strerror, &filename +#ifdef MS_WINDOWS + , &winerror +#endif + )) + goto error; + + if (oserror_init(self, &args, myerrno, strerror, filename +#ifdef MS_WINDOWS + , winerror +#endif + )) + goto error; + + return 0; + +error: + Py_XDECREF(args); + return -1; +} + +static int +OSError_clear(PyOSErrorObject *self) { Py_CLEAR(self->myerrno); Py_CLEAR(self->strerror); Py_CLEAR(self->filename); +#ifdef MS_WINDOWS + Py_CLEAR(self->winerror); +#endif return BaseException_clear((PyBaseExceptionObject *)self); } static void -EnvironmentError_dealloc(PyEnvironmentErrorObject *self) +OSError_dealloc(PyOSErrorObject *self) { _PyObject_GC_UNTRACK(self); - EnvironmentError_clear(self); + OSError_clear(self); Py_TYPE(self)->tp_free((PyObject *)self); } static int -EnvironmentError_traverse(PyEnvironmentErrorObject *self, visitproc visit, +OSError_traverse(PyOSErrorObject *self, visitproc visit, void *arg) { Py_VISIT(self->myerrno); Py_VISIT(self->strerror); Py_VISIT(self->filename); +#ifdef MS_WINDOWS + Py_VISIT(self->winerror); +#endif return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); } static PyObject * -EnvironmentError_str(PyEnvironmentErrorObject *self) +OSError_str(PyOSErrorObject *self) { +#ifdef MS_WINDOWS + /* If available, winerror has the priority over myerrno */ + if (self->winerror && self->filename) + return PyUnicode_FromFormat("[WinError %S] %S: %R", + self->winerror ? self->winerror: Py_None, + self->strerror ? self->strerror: Py_None, + self->filename); + if (self->winerror && self->strerror) + return PyUnicode_FromFormat("[WinError %S] %S", + self->winerror ? self->winerror: Py_None, + self->strerror ? self->strerror: Py_None); +#endif if (self->filename) return PyUnicode_FromFormat("[Errno %S] %S: %R", self->myerrno ? self->myerrno: Py_None, self->strerror ? self->strerror: Py_None, self->filename); - else if (self->myerrno && self->strerror) + if (self->myerrno && self->strerror) return PyUnicode_FromFormat("[Errno %S] %S", self->myerrno ? self->myerrno: Py_None, self->strerror ? self->strerror: Py_None); - else - return BaseException_str((PyBaseExceptionObject *)self); + return BaseException_str((PyBaseExceptionObject *)self); } -static PyMemberDef EnvironmentError_members[] = { - {"errno", T_OBJECT, offsetof(PyEnvironmentErrorObject, myerrno), 0, - PyDoc_STR("exception errno")}, - {"strerror", T_OBJECT, offsetof(PyEnvironmentErrorObject, strerror), 0, - PyDoc_STR("exception strerror")}, - {"filename", T_OBJECT, offsetof(PyEnvironmentErrorObject, filename), 0, - PyDoc_STR("exception filename")}, - {NULL} /* Sentinel */ -}; - - static PyObject * -EnvironmentError_reduce(PyEnvironmentErrorObject *self) +OSError_reduce(PyOSErrorObject *self) { PyObject *args = self->args; PyObject *res = NULL, *tmp; /* self->args is only the first two real arguments if there was a - * file name given to EnvironmentError. */ + * file name given to OSError. */ if (PyTuple_GET_SIZE(args) == 2 && self->filename) { args = PyTuple_New(3); if (!args) @@ -699,144 +1071,93 @@ EnvironmentError_reduce(PyEnvironmentErrorObject *self) return res; } - -static PyMethodDef EnvironmentError_methods[] = { - {"__reduce__", (PyCFunction)EnvironmentError_reduce, METH_NOARGS}, - {NULL} -}; - -ComplexExtendsException(PyExc_Exception, EnvironmentError, - EnvironmentError, EnvironmentError_dealloc, - EnvironmentError_methods, EnvironmentError_members, - EnvironmentError_str, - "Base class for I/O related errors."); - - -/* - * IOError extends EnvironmentError - */ -MiddlingExtendsException(PyExc_EnvironmentError, IOError, - EnvironmentError, "I/O operation failed."); - - -/* - * OSError extends EnvironmentError - */ -MiddlingExtendsException(PyExc_EnvironmentError, OSError, - EnvironmentError, "OS system call failed."); - - -/* - * WindowsError extends OSError - */ -#ifdef MS_WINDOWS -#include "errmap.h" - -static int -WindowsError_clear(PyWindowsErrorObject *self) -{ - Py_CLEAR(self->myerrno); - Py_CLEAR(self->strerror); - Py_CLEAR(self->filename); - Py_CLEAR(self->winerror); - return BaseException_clear((PyBaseExceptionObject *)self); -} - -static void -WindowsError_dealloc(PyWindowsErrorObject *self) -{ - _PyObject_GC_UNTRACK(self); - WindowsError_clear(self); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static int -WindowsError_traverse(PyWindowsErrorObject *self, visitproc visit, void *arg) +static PyObject * +OSError_written_get(PyOSErrorObject *self, void *context) { - Py_VISIT(self->myerrno); - Py_VISIT(self->strerror); - Py_VISIT(self->filename); - Py_VISIT(self->winerror); - return BaseException_traverse((PyBaseExceptionObject *)self, visit, arg); + if (self->written == -1) { + PyErr_SetString(PyExc_AttributeError, "characters_written"); + return NULL; + } + return PyLong_FromSsize_t(self->written); } static int -WindowsError_init(PyWindowsErrorObject *self, PyObject *args, PyObject *kwds) +OSError_written_set(PyOSErrorObject *self, PyObject *arg, void *context) { - PyObject *o_errcode = NULL; - long errcode; - long posix_errno; - - if (EnvironmentError_init((PyEnvironmentErrorObject *)self, args, kwds) - == -1) + Py_ssize_t n; + n = PyNumber_AsSsize_t(arg, PyExc_ValueError); + if (n == -1 && PyErr_Occurred()) return -1; - - if (self->myerrno == NULL) - return 0; - - /* Set errno to the POSIX errno, and winerror to the Win32 - error code. */ - errcode = PyLong_AsLong(self->myerrno); - if (errcode == -1 && PyErr_Occurred()) - return -1; - posix_errno = winerror_to_errno(errcode); - - Py_CLEAR(self->winerror); - self->winerror = self->myerrno; - - o_errcode = PyLong_FromLong(posix_errno); - if (!o_errcode) - return -1; - - self->myerrno = o_errcode; - + self->written = n; return 0; } - -static PyObject * -WindowsError_str(PyWindowsErrorObject *self) -{ - if (self->filename) - return PyUnicode_FromFormat("[Error %S] %S: %R", - self->winerror ? self->winerror: Py_None, - self->strerror ? self->strerror: Py_None, - self->filename); - else if (self->winerror && self->strerror) - return PyUnicode_FromFormat("[Error %S] %S", - self->winerror ? self->winerror: Py_None, - self->strerror ? self->strerror: Py_None); - else - return EnvironmentError_str((PyEnvironmentErrorObject *)self); -} - -static PyMemberDef WindowsError_members[] = { - {"errno", T_OBJECT, offsetof(PyWindowsErrorObject, myerrno), 0, +static PyMemberDef OSError_members[] = { + {"errno", T_OBJECT, offsetof(PyOSErrorObject, myerrno), 0, PyDoc_STR("POSIX exception code")}, - {"strerror", T_OBJECT, offsetof(PyWindowsErrorObject, strerror), 0, + {"strerror", T_OBJECT, offsetof(PyOSErrorObject, strerror), 0, PyDoc_STR("exception strerror")}, - {"filename", T_OBJECT, offsetof(PyWindowsErrorObject, filename), 0, + {"filename", T_OBJECT, offsetof(PyOSErrorObject, filename), 0, PyDoc_STR("exception filename")}, - {"winerror", T_OBJECT, offsetof(PyWindowsErrorObject, winerror), 0, +#ifdef MS_WINDOWS + {"winerror", T_OBJECT, offsetof(PyOSErrorObject, winerror), 0, PyDoc_STR("Win32 exception code")}, +#endif {NULL} /* Sentinel */ }; -ComplexExtendsException(PyExc_OSError, WindowsError, WindowsError, - WindowsError_dealloc, 0, WindowsError_members, - WindowsError_str, "MS-Windows OS system call failed."); +static PyMethodDef OSError_methods[] = { + {"__reduce__", (PyCFunction)OSError_reduce, METH_NOARGS}, + {NULL} +}; -#endif /* MS_WINDOWS */ +static PyGetSetDef OSError_getset[] = { + {"characters_written", (getter) OSError_written_get, + (setter) OSError_written_set, NULL}, + {NULL} +}; + + +ComplexExtendsException(PyExc_Exception, OSError, + OSError, OSError_new, + OSError_methods, OSError_members, OSError_getset, + OSError_str, + "Base class for I/O related errors."); /* - * VMSError extends OSError (I think) + * Various OSError subclasses */ -#ifdef __VMS -MiddlingExtendsException(PyExc_OSError, VMSError, EnvironmentError, - "OpenVMS OS system call failed."); -#endif - +MiddlingExtendsException(PyExc_OSError, BlockingIOError, OSError, + "I/O operation would block."); +MiddlingExtendsException(PyExc_OSError, ConnectionError, OSError, + "Connection error."); +MiddlingExtendsException(PyExc_OSError, ChildProcessError, OSError, + "Child process error."); +MiddlingExtendsException(PyExc_ConnectionError, BrokenPipeError, OSError, + "Broken pipe."); +MiddlingExtendsException(PyExc_ConnectionError, ConnectionAbortedError, OSError, + "Connection aborted."); +MiddlingExtendsException(PyExc_ConnectionError, ConnectionRefusedError, OSError, + "Connection refused."); +MiddlingExtendsException(PyExc_ConnectionError, ConnectionResetError, OSError, + "Connection reset."); +MiddlingExtendsException(PyExc_OSError, FileExistsError, OSError, + "File already exists."); +MiddlingExtendsException(PyExc_OSError, FileNotFoundError, OSError, + "File not found."); +MiddlingExtendsException(PyExc_OSError, IsADirectoryError, OSError, + "Operation doesn't work on directories."); +MiddlingExtendsException(PyExc_OSError, NotADirectoryError, OSError, + "Operation only works on directories."); +MiddlingExtendsException(PyExc_OSError, InterruptedError, OSError, + "Interrupted by signal."); +MiddlingExtendsException(PyExc_OSError, PermissionError, OSError, + "Not enough permissions."); +MiddlingExtendsException(PyExc_OSError, ProcessLookupError, OSError, + "Process not found."); +MiddlingExtendsException(PyExc_OSError, TimeoutError, OSError, + "Timeout expired."); /* * EOFError extends Exception @@ -967,21 +1288,23 @@ SyntaxError_traverse(PySyntaxErrorObject *self, visitproc visit, void *arg) static PyObject* my_basename(PyObject *name) { - Py_UNICODE *unicode; Py_ssize_t i, size, offset; + int kind; + void *data; - unicode = PyUnicode_AS_UNICODE(name); - size = PyUnicode_GET_SIZE(name); + if (PyUnicode_READY(name)) + return NULL; + kind = PyUnicode_KIND(name); + data = PyUnicode_DATA(name); + size = PyUnicode_GET_LENGTH(name); offset = 0; for(i=0; i < size; i++) { - if (unicode[i] == SEP) + if (PyUnicode_READ(kind, data, i) == SEP) offset = i + 1; } - if (offset != 0) { - return PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(name) + offset, - size - offset); - } else { + if (offset != 0) + return PyUnicode_Substring(name, offset, size); + else { Py_INCREF(name); return name; } @@ -1049,7 +1372,7 @@ static PyMemberDef SyntaxError_members[] = { }; ComplexExtendsException(PyExc_Exception, SyntaxError, SyntaxError, - SyntaxError_dealloc, 0, SyntaxError_members, + 0, 0, SyntaxError_members, 0, SyntaxError_str, "Invalid syntax."); @@ -1103,7 +1426,7 @@ KeyError_str(PyBaseExceptionObject *self) } ComplexExtendsException(PyExc_LookupError, KeyError, BaseException, - 0, 0, 0, KeyError_str, "Mapping key not found."); + 0, 0, 0, 0, KeyError_str, "Mapping key not found."); /* @@ -1202,7 +1525,7 @@ PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) if (!obj) return -1; *start = ((PyUnicodeErrorObject *)exc)->start; - size = PyUnicode_GET_SIZE(obj); + size = PyUnicode_GET_LENGTH(obj); if (*start<0) *start = 0; /*XXX check for values <0*/ if (*start>=size) @@ -1270,7 +1593,7 @@ PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) if (!obj) return -1; *end = ((PyUnicodeErrorObject *)exc)->end; - size = PyUnicode_GET_SIZE(obj); + size = PyUnicode_GET_LENGTH(obj); if (*end<1) *end = 1; if (*end>size) @@ -1442,6 +1765,11 @@ UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds) return -1; } + if (PyUnicode_READY(err->object) < -1) { + err->encoding = NULL; + return -1; + } + Py_INCREF(err->encoding); Py_INCREF(err->object); Py_INCREF(err->reason); @@ -1466,8 +1794,8 @@ UnicodeEncodeError_str(PyObject *self) if (encoding_str == NULL) goto done; - if (uself->start < PyUnicode_GET_SIZE(uself->object) && uself->end == uself->start+1) { - int badchar = (int)PyUnicode_AS_UNICODE(uself->object)[uself->start]; + if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) { + Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start); const char *fmt; if (badchar <= 0xff) fmt = "'%U' codec can't encode character '\\x%02x' in position %zd: %U"; @@ -1478,7 +1806,7 @@ UnicodeEncodeError_str(PyObject *self) result = PyUnicode_FromFormat( fmt, encoding_str, - badchar, + (int)badchar, uself->start, reason_str); } @@ -1675,8 +2003,8 @@ UnicodeTranslateError_str(PyObject *self) if (reason_str == NULL) goto done; - if (uself->start < PyUnicode_GET_SIZE(uself->object) && uself->end == uself->start+1) { - int badchar = (int)PyUnicode_AS_UNICODE(uself->object)[uself->start]; + if (uself->start < PyUnicode_GET_LENGTH(uself->object) && uself->end == uself->start+1) { + Py_UCS4 badchar = PyUnicode_ReadChar(uself->object, uself->start); const char *fmt; if (badchar <= 0xff) fmt = "can't translate character '\\x%02x' in position %zd: %U"; @@ -1686,7 +2014,7 @@ UnicodeTranslateError_str(PyObject *self) fmt = "can't translate character '\\U%08x' in position %zd: %U"; result = PyUnicode_FromFormat( fmt, - badchar, + (int)badchar, uself->start, reason_str ); @@ -1717,6 +2045,7 @@ static PyTypeObject _PyExc_UnicodeTranslateError = { }; PyObject *PyExc_UnicodeTranslateError = (PyObject *)&_PyExc_UnicodeTranslateError; +/* Deprecated. */ PyObject * PyUnicodeTranslateError_Create( const Py_UNICODE *object, Py_ssize_t length, @@ -1726,6 +2055,14 @@ PyUnicodeTranslateError_Create( object, length, start, end, reason); } +PyObject * +_PyUnicodeTranslateError_Create( + PyObject *object, + Py_ssize_t start, Py_ssize_t end, const char *reason) +{ + return PyObject_CallFunction(PyExc_UnicodeTranslateError, "Ons", + object, start, end, reason); +} /* * AssertionError extends Exception @@ -1975,11 +2312,80 @@ PyObject *PyExc_RecursionErrorInst = NULL; if (PyDict_SetItemString(bdict, # TYPE, PyExc_ ## TYPE)) \ Py_FatalError("Module dictionary insertion problem."); +#define INIT_ALIAS(NAME, TYPE) Py_INCREF(PyExc_ ## TYPE); \ + Py_XDECREF(PyExc_ ## NAME); \ + PyExc_ ## NAME = PyExc_ ## TYPE; \ + if (PyDict_SetItemString(bdict, # NAME, PyExc_ ## NAME)) \ + Py_FatalError("Module dictionary insertion problem."); + +#define ADD_ERRNO(TYPE, CODE) { \ + PyObject *_code = PyLong_FromLong(CODE); \ + assert(_PyObject_RealIsSubclass(PyExc_ ## TYPE, PyExc_OSError)); \ + if (!_code || PyDict_SetItem(errnomap, _code, PyExc_ ## TYPE)) \ + Py_FatalError("errmap insertion problem."); \ + Py_DECREF(_code); \ + } + +#ifdef MS_WINDOWS +#include <Winsock2.h> +/* The following constants were added to errno.h in VS2010 but have + preferred WSA equivalents. */ +#undef EADDRINUSE +#undef EADDRNOTAVAIL +#undef EAFNOSUPPORT +#undef EALREADY +#undef ECONNABORTED +#undef ECONNREFUSED +#undef ECONNRESET +#undef EDESTADDRREQ +#undef EHOSTUNREACH +#undef EINPROGRESS +#undef EISCONN +#undef ELOOP +#undef EMSGSIZE +#undef ENETDOWN +#undef ENETRESET +#undef ENETUNREACH +#undef ENOBUFS +#undef ENOPROTOOPT +#undef ENOTCONN +#undef ENOTSOCK +#undef EOPNOTSUPP +#undef EPROTONOSUPPORT +#undef EPROTOTYPE +#undef ETIMEDOUT +#undef EWOULDBLOCK + +#if defined(WSAEALREADY) && !defined(EALREADY) +#define EALREADY WSAEALREADY +#endif +#if defined(WSAECONNABORTED) && !defined(ECONNABORTED) +#define ECONNABORTED WSAECONNABORTED +#endif +#if defined(WSAECONNREFUSED) && !defined(ECONNREFUSED) +#define ECONNREFUSED WSAECONNREFUSED +#endif +#if defined(WSAECONNRESET) && !defined(ECONNRESET) +#define ECONNRESET WSAECONNRESET +#endif +#if defined(WSAEINPROGRESS) && !defined(EINPROGRESS) +#define EINPROGRESS WSAEINPROGRESS +#endif +#if defined(WSAESHUTDOWN) && !defined(ESHUTDOWN) +#define ESHUTDOWN WSAESHUTDOWN +#endif +#if defined(WSAETIMEDOUT) && !defined(ETIMEDOUT) +#define ETIMEDOUT WSAETIMEDOUT +#endif +#if defined(WSAEWOULDBLOCK) && !defined(EWOULDBLOCK) +#define EWOULDBLOCK WSAEWOULDBLOCK +#endif +#endif /* MS_WINDOWS */ void -_PyExc_Init(void) +_PyExc_Init(PyObject *bltinmod) { - PyObject *bltinmod, *bdict; + PyObject *bdict; PRE_INIT(BaseException) PRE_INIT(Exception) @@ -1989,15 +2395,7 @@ _PyExc_Init(void) PRE_INIT(SystemExit) PRE_INIT(KeyboardInterrupt) PRE_INIT(ImportError) - PRE_INIT(EnvironmentError) - PRE_INIT(IOError) PRE_INIT(OSError) -#ifdef MS_WINDOWS - PRE_INIT(WindowsError) -#endif -#ifdef __VMS - PRE_INIT(VMSError) -#endif PRE_INIT(EOFError) PRE_INIT(RuntimeError) PRE_INIT(NotImplementedError) @@ -2037,9 +2435,24 @@ _PyExc_Init(void) PRE_INIT(BytesWarning) PRE_INIT(ResourceWarning) - bltinmod = PyImport_ImportModule("builtins"); - if (bltinmod == NULL) - Py_FatalError("exceptions bootstrapping error."); + /* OSError subclasses */ + PRE_INIT(ConnectionError); + + PRE_INIT(BlockingIOError); + PRE_INIT(BrokenPipeError); + PRE_INIT(ChildProcessError); + PRE_INIT(ConnectionAbortedError); + PRE_INIT(ConnectionRefusedError); + PRE_INIT(ConnectionResetError); + PRE_INIT(FileExistsError); + PRE_INIT(FileNotFoundError); + PRE_INIT(IsADirectoryError); + PRE_INIT(NotADirectoryError); + PRE_INIT(InterruptedError); + PRE_INIT(PermissionError); + PRE_INIT(ProcessLookupError); + PRE_INIT(TimeoutError); + bdict = PyModule_GetDict(bltinmod); if (bdict == NULL) Py_FatalError("exceptions bootstrapping error."); @@ -2052,14 +2465,14 @@ _PyExc_Init(void) POST_INIT(SystemExit) POST_INIT(KeyboardInterrupt) POST_INIT(ImportError) - POST_INIT(EnvironmentError) - POST_INIT(IOError) POST_INIT(OSError) + INIT_ALIAS(EnvironmentError, OSError) + INIT_ALIAS(IOError, OSError) #ifdef MS_WINDOWS - POST_INIT(WindowsError) + INIT_ALIAS(WindowsError, OSError) #endif #ifdef __VMS - POST_INIT(VMSError) + INIT_ALIAS(VMSError, OSError) #endif POST_INIT(EOFError) POST_INIT(RuntimeError) @@ -2100,6 +2513,49 @@ _PyExc_Init(void) POST_INIT(BytesWarning) POST_INIT(ResourceWarning) + if (!errnomap) { + errnomap = PyDict_New(); + if (!errnomap) + Py_FatalError("Cannot allocate map from errnos to OSError subclasses"); + } + + /* OSError subclasses */ + POST_INIT(ConnectionError); + + POST_INIT(BlockingIOError); + ADD_ERRNO(BlockingIOError, EAGAIN); + ADD_ERRNO(BlockingIOError, EALREADY); + ADD_ERRNO(BlockingIOError, EINPROGRESS); + ADD_ERRNO(BlockingIOError, EWOULDBLOCK); + POST_INIT(BrokenPipeError); + ADD_ERRNO(BrokenPipeError, EPIPE); + ADD_ERRNO(BrokenPipeError, ESHUTDOWN); + POST_INIT(ChildProcessError); + ADD_ERRNO(ChildProcessError, ECHILD); + POST_INIT(ConnectionAbortedError); + ADD_ERRNO(ConnectionAbortedError, ECONNABORTED); + POST_INIT(ConnectionRefusedError); + ADD_ERRNO(ConnectionRefusedError, ECONNREFUSED); + POST_INIT(ConnectionResetError); + ADD_ERRNO(ConnectionResetError, ECONNRESET); + POST_INIT(FileExistsError); + ADD_ERRNO(FileExistsError, EEXIST); + POST_INIT(FileNotFoundError); + ADD_ERRNO(FileNotFoundError, ENOENT); + POST_INIT(IsADirectoryError); + ADD_ERRNO(IsADirectoryError, EISDIR); + POST_INIT(NotADirectoryError); + ADD_ERRNO(NotADirectoryError, ENOTDIR); + POST_INIT(InterruptedError); + ADD_ERRNO(InterruptedError, EINTR); + POST_INIT(PermissionError); + ADD_ERRNO(PermissionError, EACCES); + ADD_ERRNO(PermissionError, EPERM); + POST_INIT(ProcessLookupError); + ADD_ERRNO(ProcessLookupError, ESRCH); + POST_INIT(TimeoutError); + ADD_ERRNO(TimeoutError, ETIMEDOUT); + preallocate_memerrors(); if (!PyExc_RecursionErrorInst) { @@ -2126,7 +2582,6 @@ _PyExc_Init(void) Py_DECREF(args_tuple); } } - Py_DECREF(bltinmod); } void @@ -2134,4 +2589,5 @@ _PyExc_Fini(void) { Py_CLEAR(PyExc_RecursionErrorInst); free_preallocated_memerrors(); + Py_CLEAR(errnomap); } diff --git a/Objects/fileobject.c b/Objects/fileobject.c index d20f196..e1c47ce 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -30,11 +30,12 @@ PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding, char *errors, char *newline, int closefd) { PyObject *io, *stream; + _Py_IDENTIFIER(open); io = PyImport_ImportModule("io"); if (io == NULL) return NULL; - stream = PyObject_CallMethod(io, "open", "isisssi", fd, mode, + stream = _PyObject_CallMethodId(io, &PyId_open, "isisssi", fd, mode, buffering, encoding, errors, newline, closefd); Py_DECREF(io); @@ -58,8 +59,9 @@ PyFile_GetLine(PyObject *f, int n) { PyObject *reader; PyObject *args; + _Py_IDENTIFIER(readline); - reader = PyObject_GetAttrString(f, "readline"); + reader = _PyObject_GetAttrId(f, &PyId_readline); if (reader == NULL) return NULL; if (n <= 0) @@ -103,23 +105,18 @@ PyFile_GetLine(PyObject *f, int n) } } if (n < 0 && result != NULL && PyUnicode_Check(result)) { - Py_UNICODE *s = PyUnicode_AS_UNICODE(result); - Py_ssize_t len = PyUnicode_GET_SIZE(result); + Py_ssize_t len = PyUnicode_GET_LENGTH(result); if (len == 0) { Py_DECREF(result); result = NULL; PyErr_SetString(PyExc_EOFError, "EOF when reading a line"); } - else if (s[len-1] == '\n') { - if (result->ob_refcnt == 1) - PyUnicode_Resize(&result, len-1); - else { - PyObject *v; - v = PyUnicode_FromUnicode(s, len-1); - Py_DECREF(result); - result = v; - } + else if (PyUnicode_READ_CHAR(result, len-1) == '\n') { + PyObject *v; + v = PyUnicode_Substring(result, 0, len-1); + Py_DECREF(result); + result = v; } } return result; @@ -131,11 +128,13 @@ int PyFile_WriteObject(PyObject *v, PyObject *f, int flags) { PyObject *writer, *value, *args, *result; + _Py_IDENTIFIER(write); + if (f == NULL) { PyErr_SetString(PyExc_TypeError, "writeobject with NULL file"); return -1; } - writer = PyObject_GetAttrString(f, "write"); + writer = _PyObject_GetAttrId(f, &PyId_write); if (writer == NULL) return -1; if (flags & Py_PRINT_RAW) { @@ -198,11 +197,12 @@ PyObject_AsFileDescriptor(PyObject *o) { int fd; PyObject *meth; + _Py_IDENTIFIER(fileno); if (PyLong_Check(o)) { fd = PyLong_AsLong(o); } - else if ((meth = PyObject_GetAttrString(o, "fileno")) != NULL) + else if ((meth = _PyObject_GetAttrId(o, &PyId_fileno)) != NULL) { PyObject *fno = PyEval_CallObject(meth, NULL); Py_DECREF(meth); @@ -297,8 +297,8 @@ Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj) *p++ = c; if (c == '\n') break; } - if ( c == EOF && skipnextlf ) - newlinetypes |= NEWLINE_CR; + /* if ( c == EOF && skipnextlf ) + newlinetypes |= NEWLINE_CR; */ FUNLOCKFILE(stream); *p = '\0'; if ( skipnextlf ) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index e146a4f..a42be71 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -15,59 +15,17 @@ #define MIN(x, y) ((x) < (y) ? (x) : (y)) -#ifdef _OSF_SOURCE -/* OSF1 5.1 doesn't make this available with XOPEN_SOURCE_EXTENDED defined */ -extern int finite(double); -#endif - /* Special free list - - Since some Python programs can spend much of their time allocating - and deallocating floats, these operations should be very fast. - Therefore we use a dedicated allocation scheme with a much lower - overhead (in space and time) than straight malloc(): a simple - dedicated free list, filled when necessary with memory from malloc(). - - block_list is a singly-linked list of all PyFloatBlocks ever allocated, - linked via their next members. PyFloatBlocks are never returned to the - system before shutdown (PyFloat_Fini). - free_list is a singly-linked list of available PyFloatObjects, linked via abuse of their ob_type members. */ -#define BLOCK_SIZE 1000 /* 1K less typical malloc overhead */ -#define BHEAD_SIZE 8 /* Enough for a 64-bit pointer */ -#define N_FLOATOBJECTS ((BLOCK_SIZE - BHEAD_SIZE) / sizeof(PyFloatObject)) - -struct _floatblock { - struct _floatblock *next; - PyFloatObject objects[N_FLOATOBJECTS]; -}; - -typedef struct _floatblock PyFloatBlock; - -static PyFloatBlock *block_list = NULL; +#ifndef PyFloat_MAXFREELIST +#define PyFloat_MAXFREELIST 100 +#endif +static int numfree = 0; static PyFloatObject *free_list = NULL; -static PyFloatObject * -fill_free_list(void) -{ - PyFloatObject *p, *q; - /* XXX Float blocks escape the object heap. Use PyObject_MALLOC ??? */ - p = (PyFloatObject *) PyMem_MALLOC(sizeof(PyFloatBlock)); - if (p == NULL) - return (PyFloatObject *) PyErr_NoMemory(); - ((PyFloatBlock *)p)->next = block_list; - block_list = (PyFloatBlock *)p; - p = &((PyFloatBlock *)p)->objects[0]; - q = p + N_FLOATOBJECTS; - while (--q > p) - Py_TYPE(q) = (struct _typeobject *)(q-1); - Py_TYPE(q) = NULL; - return p + N_FLOATOBJECTS - 1; -} - double PyFloat_GetMax(void) { @@ -156,14 +114,16 @@ PyFloat_GetInfo(void) PyObject * PyFloat_FromDouble(double fval) { - register PyFloatObject *op; - if (free_list == NULL) { - if ((free_list = fill_free_list()) == NULL) - return NULL; + register PyFloatObject *op = free_list; + if (op != NULL) { + free_list = (PyFloatObject *) Py_TYPE(op); + numfree--; + } else { + op = (PyFloatObject*) PyObject_MALLOC(sizeof(PyFloatObject)); + if (!op) + return PyErr_NoMemory(); } /* Inline PyObject_New */ - op = free_list; - free_list = (PyFloatObject *)Py_TYPE(op); PyObject_INIT(op, &PyFloat_Type); op->ob_fval = fval; return (PyObject *) op; @@ -179,25 +139,14 @@ PyFloat_FromString(PyObject *v) PyObject *result = NULL; if (PyUnicode_Check(v)) { - Py_ssize_t i, buflen = PyUnicode_GET_SIZE(v); - Py_UNICODE *bufptr; - s_buffer = PyUnicode_TransformDecimalToASCII( - PyUnicode_AS_UNICODE(v), buflen); + s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v); if (s_buffer == NULL) return NULL; - /* Replace non-ASCII whitespace with ' ' */ - bufptr = PyUnicode_AS_UNICODE(s_buffer); - for (i = 0; i < buflen; i++) { - Py_UNICODE ch = bufptr[i]; - if (ch > 127 && Py_UNICODE_ISSPACE(ch)) - bufptr[i] = ' '; - } - s = _PyUnicode_AsStringAndSize(s_buffer, &len); + s = PyUnicode_AsUTF8AndSize(s_buffer, &len); if (s == NULL) { Py_DECREF(s_buffer); return NULL; } - last = s + len; } else if (PyObject_AsCharBuffer(v, &s, &len)) { PyErr_SetString(PyExc_TypeError, @@ -233,6 +182,11 @@ static void float_dealloc(PyFloatObject *op) { if (PyFloat_CheckExact(op)) { + if (numfree >= PyFloat_MAXFREELIST) { + PyObject_FREE(op); + return; + } + numfree++; Py_TYPE(op) = (struct _typeobject *)free_list; free_list = op; } @@ -313,13 +267,15 @@ static PyObject * float_repr(PyFloatObject *v) { PyObject *result; - char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v), - 'r', 0, - Py_DTSF_ADD_DOT_0, - NULL); + char *buf; + + buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v), + 'r', 0, + Py_DTSF_ADD_DOT_0, + NULL); if (!buf) return PyErr_NoMemory(); - result = PyUnicode_FromString(buf); + result = _PyUnicode_FromASCII(buf, strlen(buf)); PyMem_Free(buf); return result; } @@ -523,8 +479,7 @@ float_richcompare(PyObject *v, PyObject *w, int op) return PyBool_FromLong(r); Unimplemented: - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static Py_hash_t @@ -1080,7 +1035,7 @@ static char char_from_hex(int x) { assert(0 <= x && x < 16); - return "0123456789abcdef"[x]; + return Py_hexdigits[x]; } static int @@ -1750,12 +1705,22 @@ static PyObject * float__format__(PyObject *self, PyObject *args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyFloat_FormatAdvanced(self, - PyUnicode_AS_UNICODE(format_spec), - PyUnicode_GET_SIZE(format_spec)); + + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyFloat_FormatAdvancedWriter( + &writer, + self, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } PyDoc_STRVAR(float__format__doc, @@ -1950,98 +1915,34 @@ _PyFloat_Init(void) int PyFloat_ClearFreeList(void) { - PyFloatObject *p; - PyFloatBlock *list, *next; - int i; - int u; /* remaining unfreed floats per block */ - int freelist_size = 0; - - list = block_list; - block_list = NULL; - free_list = NULL; - while (list != NULL) { - u = 0; - for (i = 0, p = &list->objects[0]; - i < N_FLOATOBJECTS; - i++, p++) { - if (PyFloat_CheckExact(p) && Py_REFCNT(p) != 0) - u++; - } - next = list->next; - if (u) { - list->next = block_list; - block_list = list; - for (i = 0, p = &list->objects[0]; - i < N_FLOATOBJECTS; - i++, p++) { - if (!PyFloat_CheckExact(p) || - Py_REFCNT(p) == 0) { - Py_TYPE(p) = (struct _typeobject *) - free_list; - free_list = p; - } - } - } - else { - PyMem_FREE(list); - } - freelist_size += u; - list = next; + PyFloatObject *f = free_list, *next; + int i = numfree; + while (f) { + next = (PyFloatObject*) Py_TYPE(f); + PyObject_FREE(f); + f = next; } - return freelist_size; + free_list = NULL; + numfree = 0; + return i; } void PyFloat_Fini(void) { - PyFloatObject *p; - PyFloatBlock *list; - int i; - int u; /* total unfreed floats per block */ - - u = PyFloat_ClearFreeList(); + (void)PyFloat_ClearFreeList(); +} - if (!Py_VerboseFlag) - return; - fprintf(stderr, "# cleanup floats"); - if (!u) { - fprintf(stderr, "\n"); - } - else { - fprintf(stderr, - ": %d unfreed float%s\n", - u, u == 1 ? "" : "s"); - } - if (Py_VerboseFlag > 1) { - list = block_list; - while (list != NULL) { - for (i = 0, p = &list->objects[0]; - i < N_FLOATOBJECTS; - i++, p++) { - if (PyFloat_CheckExact(p) && - Py_REFCNT(p) != 0) { - char *buf = PyOS_double_to_string( - PyFloat_AS_DOUBLE(p), 'r', - 0, 0, NULL); - if (buf) { - /* XXX(twouters) cast - refcount to long - until %zd is - universally - available - */ - fprintf(stderr, - "# <float at %p, refcnt=%ld, val=%s>\n", - p, (long)Py_REFCNT(p), buf); - PyMem_Free(buf); - } - } - } - list = list->next; - } - } +/* Print summary info about the state of the optimized allocator */ +void +_PyFloat_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyFloatObject", + numfree, sizeof(PyFloatObject)); } + /*---------------------------------------------------------------------------- * _PyFloat_{Pack,Unpack}{4,8}. See floatobject.h. */ @@ -2251,7 +2152,7 @@ _PyFloat_Pack8(double x, unsigned char *p, int le) /* Eighth byte */ *p = flo & 0xFF; - p += incr; + /* p += incr; */ /* Done */ return 0; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index adce42b..808e595 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -15,11 +15,11 @@ #define OFF(x) offsetof(PyFrameObject, x) static PyMemberDef frame_memberlist[] = { - {"f_back", T_OBJECT, OFF(f_back), READONLY}, - {"f_code", T_OBJECT, OFF(f_code), READONLY}, - {"f_builtins", T_OBJECT, OFF(f_builtins),READONLY}, - {"f_globals", T_OBJECT, OFF(f_globals), READONLY}, - {"f_lasti", T_INT, OFF(f_lasti), READONLY}, + {"f_back", T_OBJECT, OFF(f_back), READONLY}, + {"f_code", T_OBJECT, OFF(f_code), READONLY}, + {"f_builtins", T_OBJECT, OFF(f_builtins), READONLY}, + {"f_globals", T_OBJECT, OFF(f_globals), READONLY}, + {"f_lasti", T_INT, OFF(f_lasti), READONLY}, {NULL} /* Sentinel */ }; @@ -614,10 +614,8 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, if (builtins) { if (PyModule_Check(builtins)) { builtins = PyModule_GetDict(builtins); - assert(!builtins || PyDict_Check(builtins)); + assert(builtins != NULL); } - else if (!PyDict_Check(builtins)) - builtins = NULL; } if (builtins == NULL) { /* No builtins! Make up a minimal one @@ -636,7 +634,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, /* If we share the globals, we share the builtins. Save a lookup and a call. */ builtins = back->f_builtins; - assert(builtins != NULL && PyDict_Check(builtins)); + assert(builtins != NULL); Py_INCREF(builtins); } if (code->co_zombieframe != NULL) { @@ -665,11 +663,13 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, f = free_list; free_list = free_list->f_back; if (Py_SIZE(f) < extras) { - f = PyObject_GC_Resize(PyFrameObject, f, extras); - if (f == NULL) { + PyFrameObject *new_f = PyObject_GC_Resize(PyFrameObject, f, extras); + if (new_f == NULL) { + PyObject_GC_Del(f); Py_DECREF(builtins); return NULL; } + f = new_f; } _Py_NewReference((PyObject *)f); } @@ -955,3 +955,13 @@ PyFrame_Fini(void) Py_XDECREF(builtin_object); builtin_object = NULL; } + +/* Print summary info about the state of the optimized allocator */ +void +_PyFrame_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyFrameObject", + numfree, sizeof(PyFrameObject)); +} + diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 2292a9e..49415b9 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -6,7 +6,7 @@ #include "structmember.h" PyObject * -PyFunction_New(PyObject *code, PyObject *globals) +PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) { PyFunctionObject *op = PyObject_GC_New(PyFunctionObject, &PyFunction_Type); @@ -54,6 +54,11 @@ PyFunction_New(PyObject *code, PyObject *globals) Py_INCREF(module); op->func_module = module; } + if (qualname) + op->func_qualname = qualname; + else + op->func_qualname = op->func_name; + Py_INCREF(op->func_qualname); } else return NULL; @@ -62,6 +67,12 @@ PyFunction_New(PyObject *code, PyObject *globals) } PyObject * +PyFunction_New(PyObject *code, PyObject *globals) +{ + return PyFunction_NewWithQualName(code, globals, NULL); +} + +PyObject * PyFunction_GetCode(PyObject *op) { if (!PyFunction_Check(op)) { @@ -234,42 +245,6 @@ static PyMemberDef func_memberlist[] = { }; static PyObject * -func_get_dict(PyFunctionObject *op) -{ - if (op->func_dict == NULL) { - op->func_dict = PyDict_New(); - if (op->func_dict == NULL) - return NULL; - } - Py_INCREF(op->func_dict); - return op->func_dict; -} - -static int -func_set_dict(PyFunctionObject *op, PyObject *value) -{ - PyObject *tmp; - - /* It is illegal to del f.func_dict */ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "function's dictionary may not be deleted"); - return -1; - } - /* Can only set func_dict to a dictionary */ - if (!PyDict_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "setting function's dictionary to a non-dict"); - return -1; - } - tmp = op->func_dict; - Py_INCREF(value); - op->func_dict = value; - Py_XDECREF(tmp); - return 0; -} - -static PyObject * func_get_code(PyFunctionObject *op) { Py_INCREF(op->func_code); @@ -334,6 +309,32 @@ func_set_name(PyFunctionObject *op, PyObject *value) } static PyObject * +func_get_qualname(PyFunctionObject *op) +{ + Py_INCREF(op->func_qualname); + return op->func_qualname; +} + +static int +func_set_qualname(PyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; + + /* Not legal to del f.__qualname__ or to set it to anything + * other than a string object. */ + if (value == NULL || !PyUnicode_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + tmp = op->func_qualname; + Py_INCREF(value); + op->func_qualname = value; + Py_DECREF(tmp); + return 0; +} + +static PyObject * func_get_defaults(PyFunctionObject *op) { if (op->func_defaults == NULL) { @@ -439,8 +440,9 @@ static PyGetSetDef func_getsetlist[] = { (setter)func_set_kwdefaults}, {"__annotations__", (getter)func_get_annotations, (setter)func_set_annotations}, - {"__dict__", (getter)func_get_dict, (setter)func_set_dict}, + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, {"__name__", (getter)func_get_name, (setter)func_set_name}, + {"__qualname__", (getter)func_get_qualname, (setter)func_set_qualname}, {NULL} /* Sentinel */ }; @@ -561,6 +563,7 @@ func_dealloc(PyFunctionObject *op) Py_XDECREF(op->func_dict); Py_XDECREF(op->func_closure); Py_XDECREF(op->func_annotations); + Py_XDECREF(op->func_qualname); PyObject_GC_Del(op); } @@ -568,7 +571,7 @@ static PyObject* func_repr(PyFunctionObject *op) { return PyUnicode_FromFormat("<function %U at %p>", - op->func_name, op); + op->func_qualname, op); } static int @@ -584,6 +587,7 @@ func_traverse(PyFunctionObject *f, visitproc visit, void *arg) Py_VISIT(f->func_dict); Py_VISIT(f->func_closure); Py_VISIT(f->func_annotations); + Py_VISIT(f->func_qualname); return 0; } @@ -667,8 +671,8 @@ PyTypeObject PyFunction_Type = { 0, /* tp_hash */ function_call, /* tp_call */ 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - PyObject_GenericSetAttr, /* tp_setattro */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ func_doc, /* tp_doc */ @@ -714,6 +718,7 @@ PyTypeObject PyFunction_Type = { typedef struct { PyObject_HEAD PyObject *cm_callable; + PyObject *cm_dict; } classmethod; static void @@ -721,6 +726,7 @@ cm_dealloc(classmethod *cm) { _PyObject_GC_UNTRACK((PyObject *)cm); Py_XDECREF(cm->cm_callable); + Py_XDECREF(cm->cm_dict); Py_TYPE(cm)->tp_free((PyObject *)cm); } @@ -728,6 +734,7 @@ static int cm_traverse(classmethod *cm, visitproc visit, void *arg) { Py_VISIT(cm->cm_callable); + Py_VISIT(cm->cm_dict); return 0; } @@ -735,6 +742,7 @@ static int cm_clear(classmethod *cm) { Py_CLEAR(cm->cm_callable); + Py_CLEAR(cm->cm_dict); return 0; } @@ -774,6 +782,28 @@ static PyMemberDef cm_memberlist[] = { {NULL} /* Sentinel */ }; +static PyObject * +cm_get___isabstractmethod__(classmethod *cm, void *closure) +{ + int res = _PyObject_IsAbstract(cm->cm_callable); + if (res == -1) { + return NULL; + } + else if (res) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static PyGetSetDef cm_getsetlist[] = { + {"__isabstractmethod__", + (getter)cm_get___isabstractmethod__, NULL, + NULL, + NULL}, + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {NULL} /* Sentinel */ +}; + PyDoc_STRVAR(classmethod_doc, "classmethod(function) -> method\n\ \n\ @@ -812,7 +842,7 @@ PyTypeObject PyClassMethod_Type = { 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, @@ -825,12 +855,12 @@ PyTypeObject PyClassMethod_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ cm_memberlist, /* tp_members */ - 0, /* tp_getset */ + cm_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ cm_descr_get, /* tp_descr_get */ 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ + offsetof(classmethod, cm_dict), /* tp_dictoffset */ cm_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ PyType_GenericNew, /* tp_new */ @@ -869,6 +899,7 @@ PyClassMethod_New(PyObject *callable) typedef struct { PyObject_HEAD PyObject *sm_callable; + PyObject *sm_dict; } staticmethod; static void @@ -876,6 +907,7 @@ sm_dealloc(staticmethod *sm) { _PyObject_GC_UNTRACK((PyObject *)sm); Py_XDECREF(sm->sm_callable); + Py_XDECREF(sm->sm_dict); Py_TYPE(sm)->tp_free((PyObject *)sm); } @@ -883,6 +915,7 @@ static int sm_traverse(staticmethod *sm, visitproc visit, void *arg) { Py_VISIT(sm->sm_callable); + Py_VISIT(sm->sm_dict); return 0; } @@ -890,6 +923,7 @@ static int sm_clear(staticmethod *sm) { Py_CLEAR(sm->sm_callable); + Py_CLEAR(sm->sm_dict); return 0; } @@ -927,6 +961,28 @@ static PyMemberDef sm_memberlist[] = { {NULL} /* Sentinel */ }; +static PyObject * +sm_get___isabstractmethod__(staticmethod *sm, void *closure) +{ + int res = _PyObject_IsAbstract(sm->sm_callable); + if (res == -1) { + return NULL; + } + else if (res) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static PyGetSetDef sm_getsetlist[] = { + {"__isabstractmethod__", + (getter)sm_get___isabstractmethod__, NULL, + NULL, + NULL}, + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {NULL} /* Sentinel */ +}; + PyDoc_STRVAR(staticmethod_doc, "staticmethod(function) -> method\n\ \n\ @@ -962,7 +1018,7 @@ PyTypeObject PyStaticMethod_Type = { 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, @@ -975,12 +1031,12 @@ PyTypeObject PyStaticMethod_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ sm_memberlist, /* tp_members */ - 0, /* tp_getset */ + sm_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ sm_descr_get, /* tp_descr_get */ 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ + offsetof(staticmethod, sm_dict), /* tp_dictoffset */ sm_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ PyType_GenericNew, /* tp_new */ diff --git a/Objects/genobject.c b/Objects/genobject.c index 6e25f13..597aed3 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -5,6 +5,8 @@ #include "structmember.h" #include "opcode.h" +static PyObject *gen_close(PyGenObject *gen, PyObject *args); + static int gen_traverse(PyGenObject *gen, visitproc visit, void *arg) { @@ -51,7 +53,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc) "generator already executing"); return NULL; } - if (f==NULL || f->f_stacktop == NULL) { + if (f == NULL || f->f_stacktop == NULL) { /* Only set exception if called from send() */ if (arg && !exc) PyErr_SetNone(PyExc_StopIteration); @@ -90,12 +92,19 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc) /* If the generator just returned (as opposed to yielding), signal * that the generator is exhausted. */ - if (result == Py_None && f->f_stacktop == NULL) { - Py_DECREF(result); - result = NULL; - /* Set exception if not called by gen_iternext() */ - if (arg) + if (result && f->f_stacktop == NULL) { + if (result == Py_None) { + /* Delay exception instantiation if we can */ PyErr_SetNone(PyExc_StopIteration); + } else { + PyObject *e = PyObject_CallFunctionObjArgs( + PyExc_StopIteration, result, NULL); + if (e != NULL) { + PyErr_SetObject(PyExc_StopIteration, e); + Py_DECREF(e); + } + } + Py_CLEAR(result); } if (!result || f->f_stacktop == NULL) { @@ -111,8 +120,8 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc) Py_XDECREF(t); Py_XDECREF(v); Py_XDECREF(tb); - Py_DECREF(f); gen->gi_frame = NULL; + Py_DECREF(f); } return result; @@ -122,8 +131,8 @@ PyDoc_STRVAR(send_doc, "send(arg) -> send 'arg' into generator,\n\ return next yielded value or raise StopIteration."); -static PyObject * -gen_send(PyGenObject *gen, PyObject *arg) +PyObject * +_PyGen_Send(PyGenObject *gen, PyObject *arg) { return gen_send_ex(gen, arg, 0); } @@ -131,11 +140,72 @@ gen_send(PyGenObject *gen, PyObject *arg) PyDoc_STRVAR(close_doc, "close() -> raise GeneratorExit inside generator."); +/* + * This helper function is used by gen_close and gen_throw to + * close a subiterator being delegated to by yield-from. + */ + +static int +gen_close_iter(PyObject *yf) +{ + PyObject *retval = NULL; + _Py_IDENTIFIER(close); + + if (PyGen_CheckExact(yf)) { + retval = gen_close((PyGenObject *)yf, NULL); + if (retval == NULL) + return -1; + } else { + PyObject *meth = _PyObject_GetAttrId(yf, &PyId_close); + if (meth == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + PyErr_WriteUnraisable(yf); + PyErr_Clear(); + } else { + retval = PyObject_CallFunction(meth, ""); + Py_DECREF(meth); + if (retval == NULL) + return -1; + } + } + Py_XDECREF(retval); + return 0; +} + +static PyObject * +gen_yf(PyGenObject *gen) +{ + PyObject *yf = NULL; + PyFrameObject *f = gen->gi_frame; + + if (f) { + PyObject *bytecode = f->f_code->co_code; + unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode); + + if (code[f->f_lasti + 1] != YIELD_FROM) + return NULL; + yf = f->f_stacktop[-1]; + Py_INCREF(yf); + } + + return yf; +} + static PyObject * gen_close(PyGenObject *gen, PyObject *args) { PyObject *retval; - PyErr_SetNone(PyExc_GeneratorExit); + PyObject *yf = gen_yf(gen); + int err = 0; + + if (yf) { + gen->gi_running = 1; + err = gen_close_iter(yf); + gen->gi_running = 0; + Py_DECREF(yf); + } + if (err == 0) + PyErr_SetNone(PyExc_GeneratorExit); retval = gen_send_ex(gen, Py_None, 1); if (retval) { Py_DECREF(retval); @@ -144,8 +214,7 @@ gen_close(PyGenObject *gen, PyObject *args) return NULL; } if (PyErr_ExceptionMatches(PyExc_StopIteration) - || PyErr_ExceptionMatches(PyExc_GeneratorExit)) - { + || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { PyErr_Clear(); /* ignore these errors */ Py_INCREF(Py_None); return Py_None; @@ -196,7 +265,7 @@ gen_del(PyObject *self) _Py_NewReference(self); self->ob_refcnt = refcnt; } - assert(PyType_IS_GC(self->ob_type) && + assert(PyType_IS_GC(Py_TYPE(self)) && _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so @@ -209,8 +278,8 @@ gen_del(PyObject *self) * undone. */ #ifdef COUNT_ALLOCS - --self->ob_type->tp_frees; - --self->ob_type->tp_allocs; + --(Py_TYPE(self)->tp_frees); + --(Py_TYPE(self)->tp_allocs); #endif } @@ -226,10 +295,64 @@ gen_throw(PyGenObject *gen, PyObject *args) PyObject *typ; PyObject *tb = NULL; PyObject *val = NULL; + PyObject *yf = gen_yf(gen); + _Py_IDENTIFIER(throw); if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb)) return NULL; + if (yf) { + PyObject *ret; + int err; + if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { + gen->gi_running = 1; + err = gen_close_iter(yf); + gen->gi_running = 0; + Py_DECREF(yf); + if (err < 0) + return gen_send_ex(gen, Py_None, 1); + goto throw_here; + } + if (PyGen_CheckExact(yf)) { + gen->gi_running = 1; + ret = gen_throw((PyGenObject *)yf, args); + gen->gi_running = 0; + } else { + PyObject *meth = _PyObject_GetAttrId(yf, &PyId_throw); + if (meth == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(yf); + return NULL; + } + PyErr_Clear(); + Py_DECREF(yf); + goto throw_here; + } + gen->gi_running = 1; + ret = PyObject_CallObject(meth, args); + gen->gi_running = 0; + Py_DECREF(meth); + } + Py_DECREF(yf); + if (!ret) { + PyObject *val; + /* Pop subiterator from stack */ + ret = *(--gen->gi_frame->f_stacktop); + assert(ret == yf); + Py_DECREF(ret); + /* Termination repetition of YIELD_FROM */ + gen->gi_frame->f_lasti++; + if (_PyGen_FetchStopIterationValue(&val) == 0) { + ret = gen_send_ex(gen, val, 0); + Py_DECREF(val); + } else { + ret = gen_send_ex(gen, Py_None, 1); + } + } + return ret; + } + +throw_here: /* First, check the traceback argument, replacing None with NULL. */ if (tb == Py_None) { @@ -272,7 +395,7 @@ gen_throw(PyGenObject *gen, PyObject *args) PyErr_Format(PyExc_TypeError, "exceptions must be classes or instances " "deriving from BaseException, not %s", - typ->ob_type->tp_name); + Py_TYPE(typ)->tp_name); goto failed_throw; } @@ -291,9 +414,46 @@ failed_throw: static PyObject * gen_iternext(PyGenObject *gen) { - return gen_send_ex(gen, NULL, 0); + PyObject *val = NULL; + PyObject *ret; + ret = gen_send_ex(gen, val, 0); + Py_XDECREF(val); + return ret; } +/* + * If StopIteration exception is set, fetches its 'value' + * attribute if any, otherwise sets pvalue to None. + * + * Returns 0 if no exception or StopIteration is set. + * If any other exception is set, returns -1 and leaves + * pvalue unchanged. + */ + +int +_PyGen_FetchStopIterationValue(PyObject **pvalue) { + PyObject *et, *ev, *tb; + PyObject *value = NULL; + + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Fetch(&et, &ev, &tb); + Py_XDECREF(et); + Py_XDECREF(tb); + if (ev) { + value = ((PyStopIterationObject *)ev)->value; + Py_INCREF(value); + Py_DECREF(ev); + } + } else if (PyErr_Occurred()) { + return -1; + } + if (value == NULL) { + value = Py_None; + Py_INCREF(value); + } + *pvalue = value; + return 0; +} static PyObject * gen_repr(PyGenObject *gen) @@ -324,13 +484,13 @@ static PyGetSetDef gen_getsetlist[] = { static PyMemberDef gen_memberlist[] = { {"gi_frame", T_OBJECT, offsetof(PyGenObject, gi_frame), READONLY}, - {"gi_running", T_INT, offsetof(PyGenObject, gi_running), READONLY}, + {"gi_running", T_BOOL, offsetof(PyGenObject, gi_running), READONLY}, {"gi_code", T_OBJECT, offsetof(PyGenObject, gi_code), READONLY}, {NULL} /* Sentinel */ }; static PyMethodDef gen_methods[] = { - {"send",(PyCFunction)gen_send, METH_O, send_doc}, + {"send",(PyCFunction)_PyGen_Send, METH_O, send_doc}, {"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, {NULL, NULL} /* Sentinel */ @@ -410,15 +570,13 @@ PyGen_NeedsFinalizing(PyGenObject *gen) int i; PyFrameObject *f = gen->gi_frame; - if (f == NULL || f->f_stacktop == NULL || f->f_iblock <= 0) + if (f == NULL || f->f_stacktop == NULL) return 0; /* no frame or empty blockstack == no finalization */ /* Any block type besides a loop requires cleanup. */ - i = f->f_iblock; - while (--i >= 0) { + for (i = 0; i < f->f_iblock; i++) if (f->f_blockstack[i].b_type != SETUP_LOOP) return 1; - } /* No blocks except loops, it's safe to skip finalization. */ return 0; diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 91a93f5..3cfbeaf 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -88,8 +88,38 @@ iter_len(seqiterobject *it) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +static PyObject * +iter_reduce(seqiterobject *it) +{ + if (it->it_seq != NULL) + return Py_BuildValue("N(O)n", _PyObject_GetBuiltin("iter"), + it->it_seq, it->it_index); + else + return Py_BuildValue("N(())", _PyObject_GetBuiltin("iter")); +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); + +static PyObject * +iter_setstate(seqiterobject *it, PyObject *state) +{ + Py_ssize_t index = PyLong_AsSsize_t(state); + if (index == -1 && PyErr_Occurred()) + return NULL; + if (it->it_seq != NULL) { + if (index < 0) + index = 0; + it->it_index = index; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); + static PyMethodDef seqiter_methods[] = { {"__length_hint__", (PyCFunction)iter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)iter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", (PyCFunction)iter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -195,6 +225,21 @@ calliter_iternext(calliterobject *it) return NULL; } +static PyObject * +calliter_reduce(calliterobject *it) +{ + if (it->it_callable != NULL && it->it_sentinel != NULL) + return Py_BuildValue("N(OO)", _PyObject_GetBuiltin("iter"), + it->it_callable, it->it_sentinel); + else + return Py_BuildValue("N(())", _PyObject_GetBuiltin("iter")); +} + +static PyMethodDef calliter_methods[] = { + {"__reduce__", (PyCFunction)calliter_reduce, METH_NOARGS, reduce_doc}, + {NULL, NULL} /* sentinel */ +}; + PyTypeObject PyCallIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "callable_iterator", /* tp_name */ @@ -224,7 +269,7 @@ PyTypeObject PyCallIter_Type = { 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)calliter_iternext, /* tp_iternext */ - 0, /* tp_methods */ + calliter_methods, /* tp_methods */ }; diff --git a/Objects/listobject.c b/Objects/listobject.c index b9ef0d0..6e0d094 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -98,16 +98,32 @@ show_alloc(void) static PyListObject *free_list[PyList_MAXFREELIST]; static int numfree = 0; -void -PyList_Fini(void) +int +PyList_ClearFreeList(void) { PyListObject *op; - + int ret = numfree; while (numfree) { op = free_list[--numfree]; assert(PyList_CheckExact(op)); PyObject_GC_Del(op); } + return ret; +} + +void +PyList_Fini(void) +{ + PyList_ClearFreeList(); +} + +/* Print summary info about the state of the optimized allocator */ +void +_PyList_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyListObject", + numfree, sizeof(PyListObject)); } PyObject * @@ -737,6 +753,19 @@ listinsert(PyListObject *self, PyObject *args) } static PyObject * +listclear(PyListObject *self) +{ + list_clear(self); + Py_RETURN_NONE; +} + +static PyObject * +listcopy(PyListObject *self) +{ + return list_slice(self, 0, Py_SIZE(self)); +} + +static PyObject * listappend(PyListObject *self, PyObject *v) { if (app1(self, v) == 0) @@ -2109,8 +2138,7 @@ static PyObject * listindex(PyListObject *self, PyObject *args) { Py_ssize_t i, start=0, stop=Py_SIZE(self); - PyObject *v, *format_tuple, *err_string; - static PyObject *err_format = NULL; + PyObject *v; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, _PyEval_SliceIndex, &start, @@ -2133,20 +2161,7 @@ listindex(PyListObject *self, PyObject *args) else if (cmp < 0) return NULL; } - if (err_format == NULL) { - err_format = PyUnicode_FromString("%r is not in list"); - if (err_format == NULL) - return NULL; - } - format_tuple = PyTuple_Pack(1, v); - if (format_tuple == NULL) - return NULL; - err_string = PyUnicode_Format(err_format, format_tuple); - Py_DECREF(format_tuple); - if (err_string == NULL) - return NULL; - PyErr_SetObject(PyExc_ValueError, err_string); - Py_DECREF(err_string); + PyErr_Format(PyExc_ValueError, "%R is not in list", v); return NULL; } @@ -2202,10 +2217,8 @@ list_richcompare(PyObject *v, PyObject *w, int op) PyListObject *vl, *wl; Py_ssize_t i; - if (!PyList_Check(v) || !PyList_Check(w)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyList_Check(v) || !PyList_Check(w)) + Py_RETURN_NOTIMPLEMENTED; vl = (PyListObject *)v; wl = (PyListObject *)w; @@ -2314,17 +2327,21 @@ PyDoc_STRVAR(reversed_doc, "L.__reversed__() -- return a reverse iterator over the list"); PyDoc_STRVAR(sizeof_doc, "L.__sizeof__() -- size of L in memory, in bytes"); +PyDoc_STRVAR(clear_doc, +"L.clear() -> None -- remove all items from L"); +PyDoc_STRVAR(copy_doc, +"L.copy() -> list -- a shallow copy of L"); PyDoc_STRVAR(append_doc, -"L.append(object) -- append object to end"); +"L.append(object) -> None -- append object to end"); PyDoc_STRVAR(extend_doc, -"L.extend(iterable) -- extend list by appending elements from the iterable"); +"L.extend(iterable) -> None -- extend list by appending elements from the iterable"); PyDoc_STRVAR(insert_doc, "L.insert(index, object) -- insert object before index"); PyDoc_STRVAR(pop_doc, "L.pop([index]) -> item -- remove and return item at index (default last).\n" "Raises IndexError if list is empty or index is out of range."); PyDoc_STRVAR(remove_doc, -"L.remove(value) -- remove first occurrence of value.\n" +"L.remove(value) -> None -- remove first occurrence of value.\n" "Raises ValueError if the value is not present."); PyDoc_STRVAR(index_doc, "L.index(value, [start, [stop]]) -> integer -- return first index of value.\n" @@ -2334,7 +2351,7 @@ PyDoc_STRVAR(count_doc, PyDoc_STRVAR(reverse_doc, "L.reverse() -- reverse *IN PLACE*"); PyDoc_STRVAR(sort_doc, -"L.sort(key=None, reverse=False) -- stable sort *IN PLACE*"); +"L.sort(key=None, reverse=False) -> None -- stable sort *IN PLACE*"); static PyObject *list_subscript(PyListObject*, PyObject*); @@ -2342,9 +2359,11 @@ static PyMethodDef list_methods[] = { {"__getitem__", (PyCFunction)list_subscript, METH_O|METH_COEXIST, getitem_doc}, {"__reversed__",(PyCFunction)list_reversed, METH_NOARGS, reversed_doc}, {"__sizeof__", (PyCFunction)list_sizeof, METH_NOARGS, sizeof_doc}, + {"clear", (PyCFunction)listclear, METH_NOARGS, clear_doc}, + {"copy", (PyCFunction)listcopy, METH_NOARGS, copy_doc}, {"append", (PyCFunction)listappend, METH_O, append_doc}, {"insert", (PyCFunction)listinsert, METH_VARARGS, insert_doc}, - {"extend", (PyCFunction)listextend, METH_O, extend_doc}, + {"extend", (PyCFunction)listextend, METH_O, extend_doc}, {"pop", (PyCFunction)listpop, METH_VARARGS, pop_doc}, {"remove", (PyCFunction)listremove, METH_O, remove_doc}, {"index", (PyCFunction)listindex, METH_VARARGS, index_doc}, @@ -2407,7 +2426,7 @@ list_subscript(PyListObject* self, PyObject* item) src = self->ob_item; dest = ((PyListObject *)result)->ob_item; for (cur = start, i = 0; i < slicelength; - cur += step, i++) { + cur += (size_t)step, i++) { it = src[cur]; Py_INCREF(it); dest[i] = it; @@ -2498,7 +2517,7 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) self->ob_item + cur + 1, lim * sizeof(PyObject *)); } - cur = start + slicelength*step; + cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { memmove(self->ob_item + cur - slicelength, self->ob_item + cur, @@ -2562,7 +2581,7 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) selfitems = self->ob_item; seqitems = PySequence_Fast_ITEMS(seq); for (cur = start, i = 0; i < slicelength; - cur += step, i++) { + cur += (size_t)step, i++) { garbage[i] = selfitems[cur]; ins = seqitems[i]; Py_INCREF(ins); @@ -2650,11 +2669,18 @@ static void listiter_dealloc(listiterobject *); static int listiter_traverse(listiterobject *, visitproc, void *); static PyObject *listiter_next(listiterobject *); static PyObject *listiter_len(listiterobject *); +static PyObject *listiter_reduce_general(void *_it, int forward); +static PyObject *listiter_reduce(listiterobject *); +static PyObject *listiter_setstate(listiterobject *, PyObject *state); PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); +PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); static PyMethodDef listiter_methods[] = { {"__length_hint__", (PyCFunction)listiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)listiter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", (PyCFunction)listiter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -2761,6 +2787,27 @@ listiter_len(listiterobject *it) } return PyLong_FromLong(0); } + +static PyObject * +listiter_reduce(listiterobject *it) +{ + return listiter_reduce_general(it, 1); +} + +static PyObject * +listiter_setstate(listiterobject *it, PyObject *state) +{ + long index = PyLong_AsLong(state); + if (index == -1 && PyErr_Occurred()) + return NULL; + if (it->it_seq != NULL) { + if (index < 0) + index = 0; + it->it_index = index; + } + Py_RETURN_NONE; +} + /*********************** List Reverse Iterator **************************/ typedef struct { @@ -2774,9 +2821,13 @@ static void listreviter_dealloc(listreviterobject *); static int listreviter_traverse(listreviterobject *, visitproc, void *); static PyObject *listreviter_next(listreviterobject *); static PyObject *listreviter_len(listreviterobject *); +static PyObject *listreviter_reduce(listreviterobject *); +static PyObject *listreviter_setstate(listreviterobject *, PyObject *); static PyMethodDef listreviter_methods[] = { {"__length_hint__", (PyCFunction)listreviter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)listreviter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", (PyCFunction)listreviter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -2873,3 +2924,51 @@ listreviter_len(listreviterobject *it) len = 0; return PyLong_FromSsize_t(len); } + +static PyObject * +listreviter_reduce(listreviterobject *it) +{ + return listiter_reduce_general(it, 0); +} + +static PyObject * +listreviter_setstate(listreviterobject *it, PyObject *state) +{ + Py_ssize_t index = PyLong_AsSsize_t(state); + if (index == -1 && PyErr_Occurred()) + return NULL; + if (it->it_seq != NULL) { + if (index < -1) + index = -1; + else if (index > PyList_GET_SIZE(it->it_seq) - 1) + index = PyList_GET_SIZE(it->it_seq) - 1; + it->it_index = index; + } + Py_RETURN_NONE; +} + +/* common pickling support */ + +static PyObject * +listiter_reduce_general(void *_it, int forward) +{ + PyObject *list; + + /* the objects are not the same, index is of different types! */ + if (forward) { + listiterobject *it = (listiterobject *)_it; + if (it->it_seq) + return Py_BuildValue("N(O)l", _PyObject_GetBuiltin("iter"), + it->it_seq, it->it_index); + } else { + listreviterobject *it = (listreviterobject *)_it; + if (it->it_seq) + return Py_BuildValue("N(O)n", _PyObject_GetBuiltin("reversed"), + it->it_seq, it->it_index); + } + /* empty iterator, create an empty list */ + list = PyList_New(0); + if (list == NULL) + return NULL; + return Py_BuildValue("N(N)", _PyObject_GetBuiltin("iter"), list); +} diff --git a/Objects/longobject.c b/Objects/longobject.c index 3a675c4..87150a9 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -320,8 +320,15 @@ PyLong_FromDouble(double dval) #define PY_ABS_LONG_MIN (0-(unsigned long)LONG_MIN) #define PY_ABS_SSIZE_T_MIN (0-(size_t)PY_SSIZE_T_MIN) -/* Get a C long int from a long int object. - Returns -1 and sets an error condition if overflow occurs. */ +/* Get a C long int from a long int object or any object that has an __int__ + method. + + On overflow, return -1 and set *overflow to 1 or -1 depending on the sign of + the result. Otherwise *overflow is 0. + + For other errors (e.g., TypeError), return -1 and set an error condition. + In this case *overflow will be 0. +*/ long PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) @@ -410,6 +417,9 @@ PyLong_AsLongAndOverflow(PyObject *vv, int *overflow) return res; } +/* Get a C long int from a long int object or any object that has an __int__ + method. Return -1 and set an error if overflow occurs. */ + long PyLong_AsLong(PyObject *obj) { @@ -658,10 +668,9 @@ _PyLong_NumBits(PyObject *vv) assert(ndigits == 0 || v->ob_digit[ndigits - 1] != 0); if (ndigits > 0) { digit msd = v->ob_digit[ndigits - 1]; - - result = (ndigits - 1) * PyLong_SHIFT; - if (result / PyLong_SHIFT != (size_t)(ndigits - 1)) + if ((size_t)(ndigits - 1) > PY_SIZE_MAX / (size_t)PyLong_SHIFT) goto Overflow; + result = (size_t)(ndigits - 1) * (size_t)PyLong_SHIFT; do { ++result; if (result == 0) @@ -921,7 +930,7 @@ _PyLong_AsByteArray(PyLongObject* v, } -/* Create a new long (or int) object from a C pointer */ +/* Create a new long int object from a C pointer */ PyObject * PyLong_FromVoidPtr(void *p) @@ -947,15 +956,11 @@ PyLong_FromVoidPtr(void *p) } -/* Get a C pointer from a long object (or an int object in some cases) */ +/* Get a C pointer from a long int object. */ void * PyLong_AsVoidPtr(PyObject *vv) { - /* This function will allow int or long objects. If vv is neither, - then the PyLong_AsLong*() functions will raise the exception: - PyExc_SystemError, "bad argument to internal function" - */ #if SIZEOF_VOID_P <= SIZEOF_LONG long x; @@ -1136,8 +1141,8 @@ PyLong_FromSize_t(size_t ival) return (PyObject *)v; } -/* Get a C PY_LONG_LONG int from a long int object. - Return -1 and set an error if overflow occurs. */ +/* Get a C long long int from a long int object or any object that has an + __int__ method. Return -1 and set an error if overflow occurs. */ PY_LONG_LONG PyLong_AsLongLong(PyObject *vv) @@ -1199,10 +1204,14 @@ PyLong_AsUnsignedLongLong(PyObject *vv) int one = 1; int res; - if (vv == NULL || !PyLong_Check(vv)) { + if (vv == NULL) { PyErr_BadInternalCall(); return (unsigned PY_LONG_LONG)-1; } + if (!PyLong_Check(vv)) { + PyErr_SetString(PyExc_TypeError, "an integer is required"); + return (unsigned PY_LONG_LONG)-1; + } v = (PyLongObject*)vv; switch(Py_SIZE(v)) { @@ -1289,12 +1298,14 @@ PyLong_AsUnsignedLongLongMask(register PyObject *op) } #undef IS_LITTLE_ENDIAN -/* Get a C long long int from a Python long or Python int object. - On overflow, returns -1 and sets *overflow to 1 or -1 depending - on the sign of the result. Otherwise *overflow is 0. +/* Get a C long long int from a long int object or any object that has an + __int__ method. - For other errors (e.g., type error), returns -1 and sets an error - condition. + On overflow, return -1 and set *overflow to 1 or -1 depending on the sign of + the result. Otherwise *overflow is 0. + + For other errors (e.g., TypeError), return -1 and set an error condition. + In this case *overflow will be 0. */ PY_LONG_LONG @@ -1388,10 +1399,8 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow) #define CHECK_BINOP(v,w) \ do { \ - if (!PyLong_Check(v) || !PyLong_Check(w)) { \ - Py_INCREF(Py_NotImplemented); \ - return Py_NotImplemented; \ - } \ + if (!PyLong_Check(v) || !PyLong_Check(w)) \ + Py_RETURN_NOTIMPLEMENTED; \ } while(0) /* bits_in_digit(d) returns the unique integer k such that 2**(k-1) <= d < @@ -1548,20 +1557,22 @@ divrem1(PyLongObject *a, digit n, digit *prem) string. (Return value is non-shared so that callers can modify the returned value if necessary.) */ -static PyObject * -long_to_decimal_string(PyObject *aa) +static int +long_to_decimal_string_internal(PyObject *aa, + PyObject **p_output, + _PyUnicodeWriter *writer) { PyLongObject *scratch, *a; PyObject *str; Py_ssize_t size, strlen, size_a, i, j; digit *pout, *pin, rem, tenpow; - Py_UNICODE *p; int negative; + enum PyUnicode_Kind kind; a = (PyLongObject *)aa; if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); - return NULL; + return -1; } size_a = ABS(Py_SIZE(a)); negative = Py_SIZE(a) < 0; @@ -1578,13 +1589,13 @@ long_to_decimal_string(PyObject *aa) if (size_a > PY_SSIZE_T_MAX / PyLong_SHIFT) { PyErr_SetString(PyExc_OverflowError, "long is too large to format"); - return NULL; + return -1; } /* the expression size_a * PyLong_SHIFT is now safe from overflow */ size = 1 + size_a * PyLong_SHIFT / (3 * _PyLong_DECIMAL_SHIFT); scratch = _PyLong_New(size); if (scratch == NULL) - return NULL; + return -1; /* convert array of base _PyLong_BASE digits in pin to an array of base _PyLong_DECIMAL_BASE digits in pout, following Knuth (TAOCP, @@ -1607,7 +1618,7 @@ long_to_decimal_string(PyObject *aa) /* check for keyboard interrupt */ SIGCHECK({ Py_DECREF(scratch); - return NULL; + return -1; }); } /* pout should have at least one digit, so that the case when a = 0 @@ -1623,64 +1634,118 @@ long_to_decimal_string(PyObject *aa) tenpow *= 10; strlen++; } - str = PyUnicode_FromUnicode(NULL, strlen); - if (str == NULL) { - Py_DECREF(scratch); - return NULL; + if (writer) { + if (_PyUnicodeWriter_Prepare(writer, strlen, '9') == -1) { + Py_DECREF(scratch); + return -1; + } + kind = writer->kind; + str = NULL; } + else { + str = PyUnicode_New(strlen, '9'); + if (str == NULL) { + Py_DECREF(scratch); + return -1; + } + kind = PyUnicode_KIND(str); + } + +#define WRITE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + strlen; \ + else \ + p = (TYPE*)PyUnicode_DATA(str) + strlen; \ + \ + *p = '\0'; \ + /* pout[0] through pout[size-2] contribute exactly \ + _PyLong_DECIMAL_SHIFT digits each */ \ + for (i=0; i < size - 1; i++) { \ + rem = pout[i]; \ + for (j = 0; j < _PyLong_DECIMAL_SHIFT; j++) { \ + *--p = '0' + rem % 10; \ + rem /= 10; \ + } \ + } \ + /* pout[size-1]: always produce at least one decimal digit */ \ + rem = pout[i]; \ + do { \ + *--p = '0' + rem % 10; \ + rem /= 10; \ + } while (rem != 0); \ + \ + /* and sign */ \ + if (negative) \ + *--p = '-'; \ + \ + /* check we've counted correctly */ \ + if (writer) \ + assert(p == ((TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos)); \ + else \ + assert(p == (TYPE*)PyUnicode_DATA(str)); \ + } while (0) /* fill the string right-to-left */ - p = PyUnicode_AS_UNICODE(str) + strlen; - *p = '\0'; - /* pout[0] through pout[size-2] contribute exactly - _PyLong_DECIMAL_SHIFT digits each */ - for (i=0; i < size - 1; i++) { - rem = pout[i]; - for (j = 0; j < _PyLong_DECIMAL_SHIFT; j++) { - *--p = '0' + rem % 10; - rem /= 10; - } + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_DIGITS(Py_UCS1); } - /* pout[size-1]: always produce at least one decimal digit */ - rem = pout[i]; - do { - *--p = '0' + rem % 10; - rem /= 10; - } while (rem != 0); - - /* and sign */ - if (negative) - *--p = '-'; + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_DIGITS(Py_UCS2); + } + else { + Py_UCS4 *p; + assert (kind == PyUnicode_4BYTE_KIND); + WRITE_DIGITS(Py_UCS4); + } +#undef WRITE_DIGITS - /* check we've counted correctly */ - assert(p == PyUnicode_AS_UNICODE(str)); Py_DECREF(scratch); - return (PyObject *)str; + if (writer) { + writer->pos += strlen; + } + else { + assert(_PyUnicode_CheckConsistency(str, 1)); + *p_output = (PyObject *)str; + } + return 0; +} + +static PyObject * +long_to_decimal_string(PyObject *aa) +{ + PyObject *v; + if (long_to_decimal_string_internal(aa, &v, NULL) == -1) + return NULL; + return v; } /* Convert a long int object to a string, using a given conversion base, - which should be one of 2, 8, 10 or 16. Return a string object. - If base is 2, 8 or 16, add the proper prefix '0b', '0o' or '0x'. */ + which should be one of 2, 8 or 16. Return a string object. + If base is 2, 8 or 16, add the proper prefix '0b', '0o' or '0x' + if alternate is nonzero. */ -PyObject * -_PyLong_Format(PyObject *aa, int base) +static int +long_format_binary(PyObject *aa, int base, int alternate, + PyObject **p_output, _PyUnicodeWriter *writer) { register PyLongObject *a = (PyLongObject *)aa; - PyObject *str; - Py_ssize_t i, sz; + PyObject *v; + Py_ssize_t sz; Py_ssize_t size_a; - Py_UNICODE *p, sign = '\0'; + enum PyUnicode_Kind kind; + int negative; int bits; - assert(base == 2 || base == 8 || base == 10 || base == 16); - if (base == 10) - return long_to_decimal_string((PyObject *)a); - + assert(base == 2 || base == 8 || base == 16); if (a == NULL || !PyLong_Check(a)) { PyErr_BadInternalCall(); - return NULL; + return -1; } size_a = ABS(Py_SIZE(a)); + negative = Py_SIZE(a) < 0; /* Compute a rough upper bound for the length of the string */ switch (base) { @@ -1697,70 +1762,137 @@ _PyLong_Format(PyObject *aa, int base) assert(0); /* shouldn't ever get here */ bits = 0; /* to silence gcc warning */ } - /* compute length of output string: allow 2 characters for prefix and - 1 for possible '-' sign. */ - if (size_a > (PY_SSIZE_T_MAX - 3) / PyLong_SHIFT) { - PyErr_SetString(PyExc_OverflowError, - "int is too large to format"); - return NULL; - } - /* now size_a * PyLong_SHIFT + 3 <= PY_SSIZE_T_MAX, so the RHS below - is safe from overflow */ - sz = 3 + (size_a * PyLong_SHIFT + (bits - 1)) / bits; - assert(sz >= 0); - str = PyUnicode_FromUnicode(NULL, sz); - if (str == NULL) - return NULL; - p = PyUnicode_AS_UNICODE(str) + sz; - *p = '\0'; - if (Py_SIZE(a) < 0) - sign = '-'; - if (Py_SIZE(a) == 0) { - *--p = '0'; + /* Compute exact length 'sz' of output string. */ + if (size_a == 0) { + sz = 1; } else { - /* JRH: special case for power-of-2 bases */ - twodigits accum = 0; - int accumbits = 0; /* # of bits in accum */ - for (i = 0; i < size_a; ++i) { - accum |= (twodigits)a->ob_digit[i] << accumbits; - accumbits += PyLong_SHIFT; - assert(accumbits >= bits); - do { - Py_UNICODE cdigit; - cdigit = (Py_UNICODE)(accum & (base - 1)); - cdigit += (cdigit < 10) ? '0' : 'a'-10; - assert(p > PyUnicode_AS_UNICODE(str)); - *--p = cdigit; - accumbits -= bits; - accum >>= bits; - } while (i < size_a-1 ? accumbits >= bits : accum > 0); + Py_ssize_t size_a_in_bits; + /* Ensure overflow doesn't occur during computation of sz. */ + if (size_a > (PY_SSIZE_T_MAX - 3) / PyLong_SHIFT) { + PyErr_SetString(PyExc_OverflowError, + "int is too large to format"); + return -1; } + size_a_in_bits = (size_a - 1) * PyLong_SHIFT + + bits_in_digit(a->ob_digit[size_a - 1]); + /* Allow 1 character for a '-' sign. */ + sz = negative + (size_a_in_bits + (bits - 1)) / bits; + } + if (alternate) { + /* 2 characters for prefix */ + sz += 2; } - if (base == 16) - *--p = 'x'; - else if (base == 8) - *--p = 'o'; - else /* (base == 2) */ - *--p = 'b'; - *--p = '0'; - if (sign) - *--p = sign; - if (p != PyUnicode_AS_UNICODE(str)) { - Py_UNICODE *q = PyUnicode_AS_UNICODE(str); - assert(p > q); - do { - } while ((*q++ = *p++) != '\0'); - q--; - if (PyUnicode_Resize(&str,(Py_ssize_t) (q - - PyUnicode_AS_UNICODE(str)))) { - Py_DECREF(str); - return NULL; - } + if (writer) { + if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) + return -1; + kind = writer->kind; + v = NULL; + } + else { + v = PyUnicode_New(sz, 'x'); + if (v == NULL) + return -1; + kind = PyUnicode_KIND(v); + } + +#define WRITE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + sz; \ + else \ + p = (TYPE*)PyUnicode_DATA(v) + sz; \ + \ + if (size_a == 0) { \ + *--p = '0'; \ + } \ + else { \ + /* JRH: special case for power-of-2 bases */ \ + twodigits accum = 0; \ + int accumbits = 0; /* # of bits in accum */ \ + Py_ssize_t i; \ + for (i = 0; i < size_a; ++i) { \ + accum |= (twodigits)a->ob_digit[i] << accumbits; \ + accumbits += PyLong_SHIFT; \ + assert(accumbits >= bits); \ + do { \ + char cdigit; \ + cdigit = (char)(accum & (base - 1)); \ + cdigit += (cdigit < 10) ? '0' : 'a'-10; \ + *--p = cdigit; \ + accumbits -= bits; \ + accum >>= bits; \ + } while (i < size_a-1 ? accumbits >= bits : accum > 0); \ + } \ + } \ + \ + if (alternate) { \ + if (base == 16) \ + *--p = 'x'; \ + else if (base == 8) \ + *--p = 'o'; \ + else /* (base == 2) */ \ + *--p = 'b'; \ + *--p = '0'; \ + } \ + if (negative) \ + *--p = '-'; \ + if (writer) \ + assert(p == ((TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos)); \ + else \ + assert(p == (TYPE*)PyUnicode_DATA(v)); \ + } while (0) + + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *p; + WRITE_DIGITS(Py_UCS1); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *p; + WRITE_DIGITS(Py_UCS2); } - return (PyObject *)str; + else { + Py_UCS4 *p; + assert (kind == PyUnicode_4BYTE_KIND); + WRITE_DIGITS(Py_UCS4); + } +#undef WRITE_DIGITS + + if (writer) { + writer->pos += sz; + } + else { + assert(_PyUnicode_CheckConsistency(v, 1)); + *p_output = v; + } + return 0; +} + +PyObject * +_PyLong_Format(PyObject *obj, int base) +{ + PyObject *str; + int err; + if (base == 10) + err = long_to_decimal_string_internal(obj, &str, NULL); + else + err = long_format_binary(obj, base, 1, &str, NULL); + if (err == -1) + return NULL; + return str; +} + +int +_PyLong_FormatWriter(_PyUnicodeWriter *writer, + PyObject *obj, + int base, int alternate) +{ + if (base == 10) + return long_to_decimal_string_internal(obj, NULL, writer); + else + return long_format_binary(obj, base, alternate, NULL, writer); } /* Table of digit values for 8-bit string -> integer conversion. @@ -2138,23 +2270,26 @@ digit beyond the first. PyObject * PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) { + PyObject *v, *unicode = PyUnicode_FromUnicode(u, length); + if (unicode == NULL) + return NULL; + v = PyLong_FromUnicodeObject(unicode, base); + Py_DECREF(unicode); + return v; +} + +PyObject * +PyLong_FromUnicodeObject(PyObject *u, int base) +{ PyObject *result; PyObject *asciidig; char *buffer, *end; - Py_ssize_t i, buflen; - Py_UNICODE *ptr; + Py_ssize_t buflen; - asciidig = PyUnicode_TransformDecimalToASCII(u, length); + asciidig = _PyUnicode_TransformDecimalAndSpaceToASCII(u); if (asciidig == NULL) return NULL; - /* Replace non-ASCII whitespace with ' ' */ - ptr = PyUnicode_AS_UNICODE(asciidig); - for (i = 0; i < length; i++) { - Py_UNICODE ch = ptr[i]; - if (ch > 127 && Py_UNICODE_ISSPACE(ch)) - ptr[i] = ' '; - } - buffer = _PyUnicode_AsStringAndSize(asciidig, &buflen); + buffer = PyUnicode_AsUTF8AndSize(asciidig, &buflen); if (buffer == NULL) { Py_DECREF(asciidig); return NULL; @@ -2451,8 +2586,7 @@ _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e) break; } } - assert(1 <= x_size && - x_size <= (Py_ssize_t)(sizeof(x_digits)/sizeof(digit))); + assert(1 <= x_size && x_size <= (Py_ssize_t)Py_ARRAY_LENGTH(x_digits)); /* Round, and convert to double. */ x_digits[0] += half_even_correction[x_digits[0] & 7]; @@ -2489,10 +2623,14 @@ PyLong_AsDouble(PyObject *v) Py_ssize_t exponent; double x; - if (v == NULL || !PyLong_Check(v)) { + if (v == NULL) { PyErr_BadInternalCall(); return -1.0; } + if (!PyLong_Check(v)) { + PyErr_SetString(PyExc_TypeError, "an integer is required"); + return -1.0; + } x = _PyLong_Frexp((PyLongObject *)v, &exponent); if ((x == -1.0 && PyErr_Occurred()) || exponent > DBL_MAX_EXP) { PyErr_SetString(PyExc_OverflowError, @@ -3617,8 +3755,7 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) else { Py_DECREF(a); Py_DECREF(b); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } if (Py_SIZE(b) < 0) { /* if exponent is negative */ @@ -4145,9 +4282,7 @@ long_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } if (PyUnicode_Check(x)) - return PyLong_FromUnicode(PyUnicode_AS_UNICODE(x), - PyUnicode_GET_SIZE(x), - (int)base); + return PyLong_FromUnicodeObject(x, (int)base); else if (PyByteArray_Check(x) || PyBytes_Check(x)) { /* Since PyLong_FromString doesn't have a length parameter, * check here for possible NULs in the string. */ @@ -4226,12 +4361,22 @@ static PyObject * long__format__(PyObject *self, PyObject *args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyLong_FormatAdvanced(self, - PyUnicode_AS_UNICODE(format_spec), - PyUnicode_GET_SIZE(format_spec)); + + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyLong_FormatAdvancedWriter( + &writer, + self, + format_spec, 0, PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } /* Return a pair (q, r) such that a = b * q + r, and diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 403aa68..d56faf8 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1,126 +1,954 @@ - /* Memoryview object implementation */ #include "Python.h" +#include <stddef.h> + + +/****************************************************************************/ +/* ManagedBuffer Object */ +/****************************************************************************/ + +/* + ManagedBuffer Object: + --------------------- + + The purpose of this object is to facilitate the handling of chained + memoryviews that have the same underlying exporting object. PEP-3118 + allows the underlying object to change while a view is exported. This + could lead to unexpected results when constructing a new memoryview + from an existing memoryview. + + Rather than repeatedly redirecting buffer requests to the original base + object, all chained memoryviews use a single buffer snapshot. This + snapshot is generated by the constructor _PyManagedBuffer_FromObject(). + + Ownership rules: + ---------------- + + The master buffer inside a managed buffer is filled in by the original + base object. shape, strides, suboffsets and format are read-only for + all consumers. -#define IS_RELEASED(memobj) \ - (((PyMemoryViewObject *) memobj)->view.buf == NULL) + A memoryview's buffer is a private copy of the exporter's buffer. shape, + strides and suboffsets belong to the memoryview and are thus writable. -#define CHECK_RELEASED(memobj) \ - if (IS_RELEASED(memobj)) { \ - PyErr_SetString(PyExc_ValueError, \ - "operation forbidden on released memoryview object"); \ - return NULL; \ + If a memoryview itself exports several buffers via memory_getbuf(), all + buffer copies share shape, strides and suboffsets. In this case, the + arrays are NOT writable. + + Reference count assumptions: + ---------------------------- + + The 'obj' member of a Py_buffer must either be NULL or refer to the + exporting base object. In the Python codebase, all getbufferprocs + return a new reference to view.obj (example: bytes_buffer_getbuffer()). + + PyBuffer_Release() decrements view.obj (if non-NULL), so the + releasebufferprocs must NOT decrement view.obj. +*/ + + +#define XSTRINGIZE(v) #v +#define STRINGIZE(v) XSTRINGIZE(v) + +#define CHECK_MBUF_RELEASED(mbuf) \ + if (((_PyManagedBufferObject *)mbuf)->flags&_Py_MANAGED_BUFFER_RELEASED) { \ + PyErr_SetString(PyExc_ValueError, \ + "operation forbidden on released memoryview object"); \ + return NULL; \ } -#define CHECK_RELEASED_INT(memobj) \ - if (IS_RELEASED(memobj)) { \ - PyErr_SetString(PyExc_ValueError, \ - "operation forbidden on released memoryview object"); \ - return -1; \ + +Py_LOCAL_INLINE(_PyManagedBufferObject *) +mbuf_alloc(void) +{ + _PyManagedBufferObject *mbuf; + + mbuf = (_PyManagedBufferObject *) + PyObject_GC_New(_PyManagedBufferObject, &_PyManagedBuffer_Type); + if (mbuf == NULL) + return NULL; + mbuf->flags = 0; + mbuf->exports = 0; + mbuf->master.obj = NULL; + _PyObject_GC_TRACK(mbuf); + + return mbuf; +} + +static PyObject * +_PyManagedBuffer_FromObject(PyObject *base) +{ + _PyManagedBufferObject *mbuf; + + mbuf = mbuf_alloc(); + if (mbuf == NULL) + return NULL; + + if (PyObject_GetBuffer(base, &mbuf->master, PyBUF_FULL_RO) < 0) { + mbuf->master.obj = NULL; + Py_DECREF(mbuf); + return NULL; } -static Py_ssize_t -get_shape0(Py_buffer *buf) -{ - if (buf->shape != NULL) - return buf->shape[0]; - if (buf->ndim == 0) - return 1; - PyErr_SetString(PyExc_TypeError, - "exported buffer does not have any shape information associated " - "to it"); - return -1; + return (PyObject *)mbuf; } static void -dup_buffer(Py_buffer *dest, Py_buffer *src) +mbuf_release(_PyManagedBufferObject *self) { - *dest = *src; - if (src->ndim == 1 && src->shape != NULL) { - dest->shape = &(dest->smalltable[0]); - dest->shape[0] = get_shape0(src); - } - if (src->ndim == 1 && src->strides != NULL) { - dest->strides = &(dest->smalltable[1]); - dest->strides[0] = src->strides[0]; - } + if (self->flags&_Py_MANAGED_BUFFER_RELEASED) + return; + + /* NOTE: at this point self->exports can still be > 0 if this function + is called from mbuf_clear() to break up a reference cycle. */ + self->flags |= _Py_MANAGED_BUFFER_RELEASED; + + /* PyBuffer_Release() decrements master->obj and sets it to NULL. */ + _PyObject_GC_UNTRACK(self); + PyBuffer_Release(&self->master); +} + +static void +mbuf_dealloc(_PyManagedBufferObject *self) +{ + assert(self->exports == 0); + mbuf_release(self); + if (self->flags&_Py_MANAGED_BUFFER_FREE_FORMAT) + PyMem_Free(self->master.format); + PyObject_GC_Del(self); } static int -memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) +mbuf_traverse(_PyManagedBufferObject *self, visitproc visit, void *arg) { - int res = 0; - CHECK_RELEASED_INT(self); - if (self->view.obj != NULL) - res = PyObject_GetBuffer(self->view.obj, view, flags); - if (view) - dup_buffer(view, &self->view); - return res; + Py_VISIT(self->master.obj); + return 0; } -static void -memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) +static int +mbuf_clear(_PyManagedBufferObject *self) { - PyBuffer_Release(view); + assert(self->exports >= 0); + mbuf_release(self); + return 0; } +PyTypeObject _PyManagedBuffer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "managedbuffer", + sizeof(_PyManagedBufferObject), + 0, + (destructor)mbuf_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)mbuf_traverse, /* tp_traverse */ + (inquiry)mbuf_clear /* tp_clear */ +}; + + +/****************************************************************************/ +/* MemoryView Object */ +/****************************************************************************/ + +/* In the process of breaking reference cycles mbuf_release() can be + called before memory_release(). */ +#define BASE_INACCESSIBLE(mv) \ + (((PyMemoryViewObject *)mv)->flags&_Py_MEMORYVIEW_RELEASED || \ + ((PyMemoryViewObject *)mv)->mbuf->flags&_Py_MANAGED_BUFFER_RELEASED) + +#define CHECK_RELEASED(mv) \ + if (BASE_INACCESSIBLE(mv)) { \ + PyErr_SetString(PyExc_ValueError, \ + "operation forbidden on released memoryview object"); \ + return NULL; \ + } + +#define CHECK_RELEASED_INT(mv) \ + if (BASE_INACCESSIBLE(mv)) { \ + PyErr_SetString(PyExc_ValueError, \ + "operation forbidden on released memoryview object"); \ + return -1; \ + } + +#define CHECK_LIST_OR_TUPLE(v) \ + if (!PyList_Check(v) && !PyTuple_Check(v)) { \ + PyErr_SetString(PyExc_TypeError, \ + #v " must be a list or a tuple"); \ + return NULL; \ + } + +#define VIEW_ADDR(mv) (&((PyMemoryViewObject *)mv)->view) + +/* Check for the presence of suboffsets in the first dimension. */ +#define HAVE_PTR(suboffsets) (suboffsets && suboffsets[0] >= 0) +/* Adjust ptr if suboffsets are present. */ +#define ADJUST_PTR(ptr, suboffsets) \ + (HAVE_PTR(suboffsets) ? *((char**)ptr) + suboffsets[0] : ptr) + +/* Memoryview buffer properties */ +#define MV_C_CONTIGUOUS(flags) (flags&(_Py_MEMORYVIEW_SCALAR|_Py_MEMORYVIEW_C)) +#define MV_F_CONTIGUOUS(flags) \ + (flags&(_Py_MEMORYVIEW_SCALAR|_Py_MEMORYVIEW_FORTRAN)) +#define MV_ANY_CONTIGUOUS(flags) \ + (flags&(_Py_MEMORYVIEW_SCALAR|_Py_MEMORYVIEW_C|_Py_MEMORYVIEW_FORTRAN)) + +/* Fast contiguity test. Caller must ensure suboffsets==NULL and ndim==1. */ +#define MV_CONTIGUOUS_NDIM1(view) \ + ((view)->shape[0] == 1 || (view)->strides[0] == (view)->itemsize) + +/* getbuffer() requests */ +#define REQ_INDIRECT(flags) ((flags&PyBUF_INDIRECT) == PyBUF_INDIRECT) +#define REQ_C_CONTIGUOUS(flags) ((flags&PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) +#define REQ_F_CONTIGUOUS(flags) ((flags&PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) +#define REQ_ANY_CONTIGUOUS(flags) ((flags&PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS) +#define REQ_STRIDES(flags) ((flags&PyBUF_STRIDES) == PyBUF_STRIDES) +#define REQ_SHAPE(flags) ((flags&PyBUF_ND) == PyBUF_ND) +#define REQ_WRITABLE(flags) (flags&PyBUF_WRITABLE) +#define REQ_FORMAT(flags) (flags&PyBUF_FORMAT) + + PyDoc_STRVAR(memory_doc, "memoryview(object)\n\ \n\ Create a new memoryview object which references the given object."); + +/**************************************************************************/ +/* Copy memoryview buffers */ +/**************************************************************************/ + +/* The functions in this section take a source and a destination buffer + with the same logical structure: format, itemsize, ndim and shape + are identical, with ndim > 0. + + NOTE: All buffers are assumed to have PyBUF_FULL information, which + is the case for memoryviews! */ + + +/* Assumptions: ndim >= 1. The macro tests for a corner case that should + perhaps be explicitly forbidden in the PEP. */ +#define HAVE_SUBOFFSETS_IN_LAST_DIM(view) \ + (view->suboffsets && view->suboffsets[dest->ndim-1] >= 0) + +Py_LOCAL_INLINE(int) +last_dim_is_contiguous(const Py_buffer *dest, const Py_buffer *src) +{ + assert(dest->ndim > 0 && src->ndim > 0); + return (!HAVE_SUBOFFSETS_IN_LAST_DIM(dest) && + !HAVE_SUBOFFSETS_IN_LAST_DIM(src) && + dest->strides[dest->ndim-1] == dest->itemsize && + src->strides[src->ndim-1] == src->itemsize); +} + +/* This is not a general function for determining format equivalence. + It is used in copy_single() and copy_buffer() to weed out non-matching + formats. Skipping the '@' character is specifically used in slice + assignments, where the lvalue is already known to have a single character + format. This is a performance hack that could be rewritten (if properly + benchmarked). */ +Py_LOCAL_INLINE(int) +equiv_format(const Py_buffer *dest, const Py_buffer *src) +{ + const char *dfmt, *sfmt; + + assert(dest->format && src->format); + dfmt = dest->format[0] == '@' ? dest->format+1 : dest->format; + sfmt = src->format[0] == '@' ? src->format+1 : src->format; + + if (strcmp(dfmt, sfmt) != 0 || + dest->itemsize != src->itemsize) { + return 0; + } + + return 1; +} + +/* Two shapes are equivalent if they are either equal or identical up + to a zero element at the same position. For example, in NumPy arrays + the shapes [1, 0, 5] and [1, 0, 7] are equivalent. */ +Py_LOCAL_INLINE(int) +equiv_shape(const Py_buffer *dest, const Py_buffer *src) +{ + int i; + + if (dest->ndim != src->ndim) + return 0; + + for (i = 0; i < dest->ndim; i++) { + if (dest->shape[i] != src->shape[i]) + return 0; + if (dest->shape[i] == 0) + break; + } + + return 1; +} + +/* Check that the logical structure of the destination and source buffers + is identical. */ +static int +equiv_structure(const Py_buffer *dest, const Py_buffer *src) +{ + if (!equiv_format(dest, src) || + !equiv_shape(dest, src)) { + PyErr_SetString(PyExc_ValueError, + "ndarray assignment: lvalue and rvalue have different structures"); + return 0; + } + + return 1; +} + +/* Base case for recursive multi-dimensional copying. Contiguous arrays are + copied with very little overhead. Assumptions: ndim == 1, mem == NULL or + sizeof(mem) == shape[0] * itemsize. */ +static void +copy_base(const Py_ssize_t *shape, Py_ssize_t itemsize, + char *dptr, const Py_ssize_t *dstrides, const Py_ssize_t *dsuboffsets, + char *sptr, const Py_ssize_t *sstrides, const Py_ssize_t *ssuboffsets, + char *mem) +{ + if (mem == NULL) { /* contiguous */ + Py_ssize_t size = shape[0] * itemsize; + if (dptr + size < sptr || sptr + size < dptr) + memcpy(dptr, sptr, size); /* no overlapping */ + else + memmove(dptr, sptr, size); + } + else { + char *p; + Py_ssize_t i; + for (i=0, p=mem; i < shape[0]; p+=itemsize, sptr+=sstrides[0], i++) { + char *xsptr = ADJUST_PTR(sptr, ssuboffsets); + memcpy(p, xsptr, itemsize); + } + for (i=0, p=mem; i < shape[0]; p+=itemsize, dptr+=dstrides[0], i++) { + char *xdptr = ADJUST_PTR(dptr, dsuboffsets); + memcpy(xdptr, p, itemsize); + } + } + +} + +/* Recursively copy a source buffer to a destination buffer. The two buffers + have the same ndim, shape and itemsize. */ +static void +copy_rec(const Py_ssize_t *shape, Py_ssize_t ndim, Py_ssize_t itemsize, + char *dptr, const Py_ssize_t *dstrides, const Py_ssize_t *dsuboffsets, + char *sptr, const Py_ssize_t *sstrides, const Py_ssize_t *ssuboffsets, + char *mem) +{ + Py_ssize_t i; + + assert(ndim >= 1); + + if (ndim == 1) { + copy_base(shape, itemsize, + dptr, dstrides, dsuboffsets, + sptr, sstrides, ssuboffsets, + mem); + return; + } + + for (i = 0; i < shape[0]; dptr+=dstrides[0], sptr+=sstrides[0], i++) { + char *xdptr = ADJUST_PTR(dptr, dsuboffsets); + char *xsptr = ADJUST_PTR(sptr, ssuboffsets); + + copy_rec(shape+1, ndim-1, itemsize, + xdptr, dstrides+1, dsuboffsets ? dsuboffsets+1 : NULL, + xsptr, sstrides+1, ssuboffsets ? ssuboffsets+1 : NULL, + mem); + } +} + +/* Faster copying of one-dimensional arrays. */ +static int +copy_single(Py_buffer *dest, Py_buffer *src) +{ + char *mem = NULL; + + assert(dest->ndim == 1); + + if (!equiv_structure(dest, src)) + return -1; + + if (!last_dim_is_contiguous(dest, src)) { + mem = PyMem_Malloc(dest->shape[0] * dest->itemsize); + if (mem == NULL) { + PyErr_NoMemory(); + return -1; + } + } + + copy_base(dest->shape, dest->itemsize, + dest->buf, dest->strides, dest->suboffsets, + src->buf, src->strides, src->suboffsets, + mem); + + if (mem) + PyMem_Free(mem); + + return 0; +} + +/* Recursively copy src to dest. Both buffers must have the same basic + structure. Copying is atomic, the function never fails with a partial + copy. */ +static int +copy_buffer(Py_buffer *dest, Py_buffer *src) +{ + char *mem = NULL; + + assert(dest->ndim > 0); + + if (!equiv_structure(dest, src)) + return -1; + + if (!last_dim_is_contiguous(dest, src)) { + mem = PyMem_Malloc(dest->shape[dest->ndim-1] * dest->itemsize); + if (mem == NULL) { + PyErr_NoMemory(); + return -1; + } + } + + copy_rec(dest->shape, dest->ndim, dest->itemsize, + dest->buf, dest->strides, dest->suboffsets, + src->buf, src->strides, src->suboffsets, + mem); + + if (mem) + PyMem_Free(mem); + + return 0; +} + +/* Initialize strides for a C-contiguous array. */ +Py_LOCAL_INLINE(void) +init_strides_from_shape(Py_buffer *view) +{ + Py_ssize_t i; + + assert(view->ndim > 0); + + view->strides[view->ndim-1] = view->itemsize; + for (i = view->ndim-2; i >= 0; i--) + view->strides[i] = view->strides[i+1] * view->shape[i+1]; +} + +/* Initialize strides for a Fortran-contiguous array. */ +Py_LOCAL_INLINE(void) +init_fortran_strides_from_shape(Py_buffer *view) +{ + Py_ssize_t i; + + assert(view->ndim > 0); + + view->strides[0] = view->itemsize; + for (i = 1; i < view->ndim; i++) + view->strides[i] = view->strides[i-1] * view->shape[i-1]; +} + +/* Copy src to a contiguous representation. order is one of 'C', 'F' (Fortran) + or 'A' (Any). Assumptions: src has PyBUF_FULL information, src->ndim >= 1, + len(mem) == src->len. */ +static int +buffer_to_contiguous(char *mem, Py_buffer *src, char order) +{ + Py_buffer dest; + Py_ssize_t *strides; + int ret; + + assert(src->ndim >= 1); + assert(src->shape != NULL); + assert(src->strides != NULL); + + strides = PyMem_Malloc(src->ndim * (sizeof *src->strides)); + if (strides == NULL) { + PyErr_NoMemory(); + return -1; + } + + /* initialize dest */ + dest = *src; + dest.buf = mem; + /* shape is constant and shared: the logical representation of the + array is unaltered. */ + + /* The physical representation determined by strides (and possibly + suboffsets) may change. */ + dest.strides = strides; + if (order == 'C' || order == 'A') { + init_strides_from_shape(&dest); + } + else { + init_fortran_strides_from_shape(&dest); + } + + dest.suboffsets = NULL; + + ret = copy_buffer(&dest, src); + + PyMem_Free(strides); + return ret; +} + + +/****************************************************************************/ +/* Constructors */ +/****************************************************************************/ + +/* Initialize values that are shared with the managed buffer. */ +Py_LOCAL_INLINE(void) +init_shared_values(Py_buffer *dest, const Py_buffer *src) +{ + dest->obj = src->obj; + dest->buf = src->buf; + dest->len = src->len; + dest->itemsize = src->itemsize; + dest->readonly = src->readonly; + dest->format = src->format ? src->format : "B"; + dest->internal = src->internal; +} + +/* Copy shape and strides. Reconstruct missing values. */ +static void +init_shape_strides(Py_buffer *dest, const Py_buffer *src) +{ + Py_ssize_t i; + + if (src->ndim == 0) { + dest->shape = NULL; + dest->strides = NULL; + return; + } + if (src->ndim == 1) { + dest->shape[0] = src->shape ? src->shape[0] : src->len / src->itemsize; + dest->strides[0] = src->strides ? src->strides[0] : src->itemsize; + return; + } + + for (i = 0; i < src->ndim; i++) + dest->shape[i] = src->shape[i]; + if (src->strides) { + for (i = 0; i < src->ndim; i++) + dest->strides[i] = src->strides[i]; + } + else { + init_strides_from_shape(dest); + } +} + +Py_LOCAL_INLINE(void) +init_suboffsets(Py_buffer *dest, const Py_buffer *src) +{ + Py_ssize_t i; + + if (src->suboffsets == NULL) { + dest->suboffsets = NULL; + return; + } + for (i = 0; i < src->ndim; i++) + dest->suboffsets[i] = src->suboffsets[i]; +} + +/* len = product(shape) * itemsize */ +Py_LOCAL_INLINE(void) +init_len(Py_buffer *view) +{ + Py_ssize_t i, len; + + len = 1; + for (i = 0; i < view->ndim; i++) + len *= view->shape[i]; + len *= view->itemsize; + + view->len = len; +} + +/* Initialize memoryview buffer properties. */ +static void +init_flags(PyMemoryViewObject *mv) +{ + const Py_buffer *view = &mv->view; + int flags = 0; + + switch (view->ndim) { + case 0: + flags |= (_Py_MEMORYVIEW_SCALAR|_Py_MEMORYVIEW_C| + _Py_MEMORYVIEW_FORTRAN); + break; + case 1: + if (MV_CONTIGUOUS_NDIM1(view)) + flags |= (_Py_MEMORYVIEW_C|_Py_MEMORYVIEW_FORTRAN); + break; + default: + if (PyBuffer_IsContiguous(view, 'C')) + flags |= _Py_MEMORYVIEW_C; + if (PyBuffer_IsContiguous(view, 'F')) + flags |= _Py_MEMORYVIEW_FORTRAN; + break; + } + + if (view->suboffsets) { + flags |= _Py_MEMORYVIEW_PIL; + flags &= ~(_Py_MEMORYVIEW_C|_Py_MEMORYVIEW_FORTRAN); + } + + mv->flags = flags; +} + +/* Allocate a new memoryview and perform basic initialization. New memoryviews + are exclusively created through the mbuf_add functions. */ +Py_LOCAL_INLINE(PyMemoryViewObject *) +memory_alloc(int ndim) +{ + PyMemoryViewObject *mv; + + mv = (PyMemoryViewObject *) + PyObject_GC_NewVar(PyMemoryViewObject, &PyMemoryView_Type, 3*ndim); + if (mv == NULL) + return NULL; + + mv->mbuf = NULL; + mv->hash = -1; + mv->flags = 0; + mv->exports = 0; + mv->view.ndim = ndim; + mv->view.shape = mv->ob_array; + mv->view.strides = mv->ob_array + ndim; + mv->view.suboffsets = mv->ob_array + 2 * ndim; + mv->weakreflist = NULL; + + _PyObject_GC_TRACK(mv); + return mv; +} + +/* + Return a new memoryview that is registered with mbuf. If src is NULL, + use mbuf->master as the underlying buffer. Otherwise, use src. + + The new memoryview has full buffer information: shape and strides + are always present, suboffsets as needed. Arrays are copied to + the memoryview's ob_array field. + */ +static PyObject * +mbuf_add_view(_PyManagedBufferObject *mbuf, const Py_buffer *src) +{ + PyMemoryViewObject *mv; + Py_buffer *dest; + + if (src == NULL) + src = &mbuf->master; + + if (src->ndim > PyBUF_MAX_NDIM) { + PyErr_SetString(PyExc_ValueError, + "memoryview: number of dimensions must not exceed " + STRINGIZE(PyBUF_MAX_NDIM)); + return NULL; + } + + mv = memory_alloc(src->ndim); + if (mv == NULL) + return NULL; + + dest = &mv->view; + init_shared_values(dest, src); + init_shape_strides(dest, src); + init_suboffsets(dest, src); + init_flags(mv); + + mv->mbuf = mbuf; + Py_INCREF(mbuf); + mbuf->exports++; + + return (PyObject *)mv; +} + +/* Register an incomplete view: shape, strides, suboffsets and flags still + need to be initialized. Use 'ndim' instead of src->ndim to determine the + size of the memoryview's ob_array. + + Assumption: ndim <= PyBUF_MAX_NDIM. */ +static PyObject * +mbuf_add_incomplete_view(_PyManagedBufferObject *mbuf, const Py_buffer *src, + int ndim) +{ + PyMemoryViewObject *mv; + Py_buffer *dest; + + if (src == NULL) + src = &mbuf->master; + + assert(ndim <= PyBUF_MAX_NDIM); + + mv = memory_alloc(ndim); + if (mv == NULL) + return NULL; + + dest = &mv->view; + init_shared_values(dest, src); + + mv->mbuf = mbuf; + Py_INCREF(mbuf); + mbuf->exports++; + + return (PyObject *)mv; +} + +/* Expose a raw memory area as a view of contiguous bytes. flags can be + PyBUF_READ or PyBUF_WRITE. view->format is set to "B" (unsigned bytes). + The memoryview has complete buffer information. */ +PyObject * +PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags) +{ + _PyManagedBufferObject *mbuf; + PyObject *mv; + int readonly; + + assert(mem != NULL); + assert(flags == PyBUF_READ || flags == PyBUF_WRITE); + + mbuf = mbuf_alloc(); + if (mbuf == NULL) + return NULL; + + readonly = (flags == PyBUF_WRITE) ? 0 : 1; + (void)PyBuffer_FillInfo(&mbuf->master, NULL, mem, size, readonly, + PyBUF_FULL_RO); + + mv = mbuf_add_view(mbuf, NULL); + Py_DECREF(mbuf); + + return mv; +} + +/* Create a memoryview from a given Py_buffer. For simple byte views, + PyMemoryView_FromMemory() should be used instead. + This function is the only entry point that can create a master buffer + without full information. Because of this fact init_shape_strides() + must be able to reconstruct missing values. */ PyObject * PyMemoryView_FromBuffer(Py_buffer *info) { - PyMemoryViewObject *mview; + _PyManagedBufferObject *mbuf; + PyObject *mv; if (info->buf == NULL) { PyErr_SetString(PyExc_ValueError, - "cannot make memory view from a buffer with a NULL data pointer"); + "PyMemoryView_FromBuffer(): info->buf must not be NULL"); return NULL; } - mview = (PyMemoryViewObject *) - PyObject_GC_New(PyMemoryViewObject, &PyMemoryView_Type); - if (mview == NULL) + + mbuf = mbuf_alloc(); + if (mbuf == NULL) return NULL; - dup_buffer(&mview->view, info); - /* NOTE: mview->view.obj should already have been incref'ed as - part of PyBuffer_FillInfo(). */ - _PyObject_GC_TRACK(mview); - return (PyObject *)mview; + + /* info->obj is either NULL or a borrowed reference. This reference + should not be decremented in PyBuffer_Release(). */ + mbuf->master = *info; + mbuf->master.obj = NULL; + + mv = mbuf_add_view(mbuf, NULL); + Py_DECREF(mbuf); + + return mv; } +/* Create a memoryview from an object that implements the buffer protocol. + If the object is a memoryview, the new memoryview must be registered + with the same managed buffer. Otherwise, a new managed buffer is created. */ PyObject * -PyMemoryView_FromObject(PyObject *base) +PyMemoryView_FromObject(PyObject *v) { - PyMemoryViewObject *mview; - Py_buffer view; + _PyManagedBufferObject *mbuf; - if (!PyObject_CheckBuffer(base)) { - PyErr_SetString(PyExc_TypeError, - "cannot make memory view because object does " - "not have the buffer interface"); + if (PyMemoryView_Check(v)) { + PyMemoryViewObject *mv = (PyMemoryViewObject *)v; + CHECK_RELEASED(mv); + return mbuf_add_view(mv->mbuf, &mv->view); + } + else if (PyObject_CheckBuffer(v)) { + PyObject *ret; + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(v); + if (mbuf == NULL) + return NULL; + ret = mbuf_add_view(mbuf, NULL); + Py_DECREF(mbuf); + return ret; + } + + PyErr_Format(PyExc_TypeError, + "memoryview: %.200s object does not have the buffer interface", + Py_TYPE(v)->tp_name); + return NULL; +} + +/* Copy the format string from a base object that might vanish. */ +static int +mbuf_copy_format(_PyManagedBufferObject *mbuf, const char *fmt) +{ + if (fmt != NULL) { + char *cp = PyMem_Malloc(strlen(fmt)+1); + if (cp == NULL) { + PyErr_NoMemory(); + return -1; + } + mbuf->master.format = strcpy(cp, fmt); + mbuf->flags |= _Py_MANAGED_BUFFER_FREE_FORMAT; + } + + return 0; +} + +/* + Return a memoryview that is based on a contiguous copy of src. + Assumptions: src has PyBUF_FULL_RO information, src->ndim > 0. + + Ownership rules: + 1) As usual, the returned memoryview has a private copy + of src->shape, src->strides and src->suboffsets. + 2) src->format is copied to the master buffer and released + in mbuf_dealloc(). The releasebufferproc of the bytes + object is NULL, so it does not matter that mbuf_release() + passes the altered format pointer to PyBuffer_Release(). +*/ +static PyObject * +memory_from_contiguous_copy(Py_buffer *src, char order) +{ + _PyManagedBufferObject *mbuf; + PyMemoryViewObject *mv; + PyObject *bytes; + Py_buffer *dest; + int i; + + assert(src->ndim > 0); + assert(src->shape != NULL); + + bytes = PyBytes_FromStringAndSize(NULL, src->len); + if (bytes == NULL) + return NULL; + + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(bytes); + Py_DECREF(bytes); + if (mbuf == NULL) + return NULL; + + if (mbuf_copy_format(mbuf, src->format) < 0) { + Py_DECREF(mbuf); return NULL; } - if (PyObject_GetBuffer(base, &view, PyBUF_FULL_RO) < 0) + mv = (PyMemoryViewObject *)mbuf_add_incomplete_view(mbuf, NULL, src->ndim); + Py_DECREF(mbuf); + if (mv == NULL) return NULL; - mview = (PyMemoryViewObject *)PyMemoryView_FromBuffer(&view); - if (mview == NULL) { - PyBuffer_Release(&view); + dest = &mv->view; + + /* shared values are initialized correctly except for itemsize */ + dest->itemsize = src->itemsize; + + /* shape and strides */ + for (i = 0; i < src->ndim; i++) { + dest->shape[i] = src->shape[i]; + } + if (order == 'C' || order == 'A') { + init_strides_from_shape(dest); + } + else { + init_fortran_strides_from_shape(dest); + } + /* suboffsets */ + dest->suboffsets = NULL; + + /* flags */ + init_flags(mv); + + if (copy_buffer(dest, src) < 0) { + Py_DECREF(mv); return NULL; } - return (PyObject *)mview; + return (PyObject *)mv; } +/* + Return a new memoryview object based on a contiguous exporter with + buffertype={PyBUF_READ, PyBUF_WRITE} and order={'C', 'F'ortran, or 'A'ny}. + The logical structure of the input and output buffers is the same + (i.e. tolist(input) == tolist(output)), but the physical layout in + memory can be explicitly chosen. + + As usual, if buffertype=PyBUF_WRITE, the exporter's buffer must be writable, + otherwise it may be writable or read-only. + + If the exporter is already contiguous with the desired target order, + the memoryview will be directly based on the exporter. + + Otherwise, if the buffertype is PyBUF_READ, the memoryview will be + based on a new bytes object. If order={'C', 'A'ny}, use 'C' order, + 'F'ortran order otherwise. +*/ +PyObject * +PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) +{ + PyMemoryViewObject *mv; + PyObject *ret; + Py_buffer *view; + + assert(buffertype == PyBUF_READ || buffertype == PyBUF_WRITE); + assert(order == 'C' || order == 'F' || order == 'A'); + + mv = (PyMemoryViewObject *)PyMemoryView_FromObject(obj); + if (mv == NULL) + return NULL; + + view = &mv->view; + if (buffertype == PyBUF_WRITE && view->readonly) { + PyErr_SetString(PyExc_BufferError, + "underlying buffer is not writable"); + Py_DECREF(mv); + return NULL; + } + + if (PyBuffer_IsContiguous(view, order)) + return (PyObject *)mv; + + if (buffertype == PyBUF_WRITE) { + PyErr_SetString(PyExc_BufferError, + "writable contiguous buffer requested " + "for a non-contiguous object."); + Py_DECREF(mv); + return NULL; + } + + ret = memory_from_contiguous_copy(view, order); + Py_DECREF(mv); + return ret; +} + + static PyObject * memory_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) { PyObject *obj; - static char *kwlist[] = {"object", 0}; + static char *kwlist[] = {"object", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:memoryview", kwlist, &obj)) { @@ -131,479 +959,1284 @@ memory_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) } -static void -_strided_copy_nd(char *dest, char *src, int nd, Py_ssize_t *shape, - Py_ssize_t *strides, Py_ssize_t itemsize, char fort) +/****************************************************************************/ +/* Previously in abstract.c */ +/****************************************************************************/ + +typedef struct { + Py_buffer view; + Py_ssize_t array[1]; +} Py_buffer_full; + +int +PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) { - int k; - Py_ssize_t outstride; + Py_buffer_full *fb = NULL; + int ret; - if (nd==0) { - memcpy(dest, src, itemsize); + assert(order == 'C' || order == 'F' || order == 'A'); + + if (len != src->len) { + PyErr_SetString(PyExc_ValueError, + "PyBuffer_ToContiguous: len != view->len"); + return -1; } - else if (nd == 1) { - for (k = 0; k<shape[0]; k++) { - memcpy(dest, src, itemsize); - dest += itemsize; - src += strides[0]; - } + + if (PyBuffer_IsContiguous(src, order)) { + memcpy((char *)buf, src->buf, len); + return 0; } - else { - if (fort == 'F') { - /* Copy first dimension first, - second dimension second, etc... - Set up the recursive loop backwards so that final - dimension is actually copied last. - */ - outstride = itemsize; - for (k=1; k<nd-1;k++) { - outstride *= shape[k]; - } - for (k=0; k<shape[nd-1]; k++) { - _strided_copy_nd(dest, src, nd-1, shape, - strides, itemsize, fort); - dest += outstride; - src += strides[nd-1]; - } - } - else { - /* Copy last dimension first, - second-to-last dimension second, etc. - Set up the recursion so that the - first dimension is copied last - */ - outstride = itemsize; - for (k=1; k < nd; k++) { - outstride *= shape[k]; - } - for (k=0; k<shape[0]; k++) { - _strided_copy_nd(dest, src, nd-1, shape+1, - strides+1, itemsize, - fort); - dest += outstride; - src += strides[0]; - } - } + /* buffer_to_contiguous() assumes PyBUF_FULL */ + fb = PyMem_Malloc(sizeof *fb + 3 * src->ndim * (sizeof *fb->array)); + if (fb == NULL) { + PyErr_NoMemory(); + return -1; } - return; + fb->view.ndim = src->ndim; + fb->view.shape = fb->array; + fb->view.strides = fb->array + src->ndim; + fb->view.suboffsets = fb->array + 2 * src->ndim; + + init_shared_values(&fb->view, src); + init_shape_strides(&fb->view, src); + init_suboffsets(&fb->view, src); + + src = &fb->view; + + ret = buffer_to_contiguous(buf, src, order); + PyMem_Free(fb); + return ret; } + +/****************************************************************************/ +/* Release/GC management */ +/****************************************************************************/ + +/* Inform the managed buffer that this particular memoryview will not access + the underlying buffer again. If no other memoryviews are registered with + the managed buffer, the underlying buffer is released instantly and + marked as inaccessible for both the memoryview and the managed buffer. + + This function fails if the memoryview itself has exported buffers. */ static int -_indirect_copy_nd(char *dest, Py_buffer *view, char fort) +_memory_release(PyMemoryViewObject *self) { - Py_ssize_t *indices; - int k; - Py_ssize_t elements; - char *ptr; - void (*func)(int, Py_ssize_t *, const Py_ssize_t *); - - if (view->ndim > PY_SSIZE_T_MAX / sizeof(Py_ssize_t)) { - PyErr_NoMemory(); + if (self->flags & _Py_MEMORYVIEW_RELEASED) + return 0; + + if (self->exports == 0) { + self->flags |= _Py_MEMORYVIEW_RELEASED; + assert(self->mbuf->exports > 0); + if (--self->mbuf->exports == 0) + mbuf_release(self->mbuf); + return 0; + } + if (self->exports > 0) { + PyErr_Format(PyExc_BufferError, + "memoryview has %zd exported buffer%s", self->exports, + self->exports==1 ? "" : "s"); return -1; } - indices = (Py_ssize_t *)PyMem_Malloc(sizeof(Py_ssize_t)*view->ndim); - if (indices == NULL) { - PyErr_NoMemory(); - return -1; + Py_FatalError("_memory_release(): negative export count"); + return -1; +} + +static PyObject * +memory_release(PyMemoryViewObject *self, PyObject *noargs) +{ + if (_memory_release(self) < 0) + return NULL; + Py_RETURN_NONE; +} + +static void +memory_dealloc(PyMemoryViewObject *self) +{ + assert(self->exports == 0); + _PyObject_GC_UNTRACK(self); + (void)_memory_release(self); + Py_CLEAR(self->mbuf); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + PyObject_GC_Del(self); +} + +static int +memory_traverse(PyMemoryViewObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->mbuf); + return 0; +} + +static int +memory_clear(PyMemoryViewObject *self) +{ + (void)_memory_release(self); + Py_CLEAR(self->mbuf); + return 0; +} + +static PyObject * +memory_enter(PyObject *self, PyObject *args) +{ + CHECK_RELEASED(self); + Py_INCREF(self); + return self; +} + +static PyObject * +memory_exit(PyObject *self, PyObject *args) +{ + return memory_release((PyMemoryViewObject *)self, NULL); +} + + +/****************************************************************************/ +/* Casting format and shape */ +/****************************************************************************/ + +#define IS_BYTE_FORMAT(f) (f == 'b' || f == 'B' || f == 'c') + +Py_LOCAL_INLINE(Py_ssize_t) +get_native_fmtchar(char *result, const char *fmt) +{ + Py_ssize_t size = -1; + + if (fmt[0] == '@') fmt++; + + switch (fmt[0]) { + case 'c': case 'b': case 'B': size = sizeof(char); break; + case 'h': case 'H': size = sizeof(short); break; + case 'i': case 'I': size = sizeof(int); break; + case 'l': case 'L': size = sizeof(long); break; + #ifdef HAVE_LONG_LONG + case 'q': case 'Q': size = sizeof(PY_LONG_LONG); break; + #endif + case 'n': case 'N': size = sizeof(Py_ssize_t); break; + case 'f': size = sizeof(float); break; + case 'd': size = sizeof(double); break; + #ifdef HAVE_C99_BOOL + case '?': size = sizeof(_Bool); break; + #else + case '?': size = sizeof(char); break; + #endif + case 'P': size = sizeof(void *); break; + } + + if (size > 0 && fmt[1] == '\0') { + *result = fmt[0]; + return size; + } + + return -1; +} + +/* Cast a memoryview's data type to 'format'. The input array must be + C-contiguous. At least one of input-format, output-format must have + byte size. The output array is 1-D, with the same byte length as the + input array. Thus, view->len must be a multiple of the new itemsize. */ +static int +cast_to_1D(PyMemoryViewObject *mv, PyObject *format) +{ + Py_buffer *view = &mv->view; + PyObject *asciifmt; + char srcchar, destchar; + Py_ssize_t itemsize; + int ret = -1; + + assert(view->ndim >= 1); + assert(Py_SIZE(mv) == 3*view->ndim); + assert(view->shape == mv->ob_array); + assert(view->strides == mv->ob_array + view->ndim); + assert(view->suboffsets == mv->ob_array + 2*view->ndim); + + if (get_native_fmtchar(&srcchar, view->format) < 0) { + PyErr_SetString(PyExc_ValueError, + "memoryview: source format must be a native single character " + "format prefixed with an optional '@'"); + return ret; } - for (k=0; k<view->ndim;k++) { - indices[k] = 0; + + asciifmt = PyUnicode_AsASCIIString(format); + if (asciifmt == NULL) + return ret; + + itemsize = get_native_fmtchar(&destchar, PyBytes_AS_STRING(asciifmt)); + if (itemsize < 0) { + PyErr_SetString(PyExc_ValueError, + "memoryview: destination format must be a native single " + "character format prefixed with an optional '@'"); + goto out; } - elements = 1; - for (k=0; k<view->ndim; k++) { - elements *= view->shape[k]; + if (!IS_BYTE_FORMAT(srcchar) && !IS_BYTE_FORMAT(destchar)) { + PyErr_SetString(PyExc_TypeError, + "memoryview: cannot cast between two non-byte formats"); + goto out; } - if (fort == 'F') { - func = _Py_add_one_to_index_F; + if (view->len % itemsize) { + PyErr_SetString(PyExc_TypeError, + "memoryview: length is not a multiple of itemsize"); + goto out; + } + + strncpy(mv->format, PyBytes_AS_STRING(asciifmt), + _Py_MEMORYVIEW_MAX_FORMAT); + mv->format[_Py_MEMORYVIEW_MAX_FORMAT-1] = '\0'; + view->format = mv->format; + view->itemsize = itemsize; + + view->ndim = 1; + view->shape[0] = view->len / view->itemsize; + view->strides[0] = view->itemsize; + view->suboffsets = NULL; + + init_flags(mv); + + ret = 0; + +out: + Py_DECREF(asciifmt); + return ret; +} + +/* The memoryview must have space for 3*len(seq) elements. */ +static Py_ssize_t +copy_shape(Py_ssize_t *shape, const PyObject *seq, Py_ssize_t ndim, + Py_ssize_t itemsize) +{ + Py_ssize_t x, i; + Py_ssize_t len = itemsize; + + for (i = 0; i < ndim; i++) { + PyObject *tmp = PySequence_Fast_GET_ITEM(seq, i); + if (!PyLong_Check(tmp)) { + PyErr_SetString(PyExc_TypeError, + "memoryview.cast(): elements of shape must be integers"); + return -1; + } + x = PyLong_AsSsize_t(tmp); + if (x == -1 && PyErr_Occurred()) { + return -1; + } + if (x <= 0) { + /* In general elements of shape may be 0, but not for casting. */ + PyErr_Format(PyExc_ValueError, + "memoryview.cast(): elements of shape must be integers > 0"); + return -1; + } + if (x > PY_SSIZE_T_MAX / len) { + PyErr_Format(PyExc_ValueError, + "memoryview.cast(): product(shape) > SSIZE_MAX"); + return -1; + } + len *= x; + shape[i] = x; + } + + return len; +} + +/* Cast a 1-D array to a new shape. The result array will be C-contiguous. + If the result array does not have exactly the same byte length as the + input array, raise ValueError. */ +static int +cast_to_ND(PyMemoryViewObject *mv, const PyObject *shape, int ndim) +{ + Py_buffer *view = &mv->view; + Py_ssize_t len; + + assert(view->ndim == 1); /* ndim from cast_to_1D() */ + assert(Py_SIZE(mv) == 3*(ndim==0?1:ndim)); /* ndim of result array */ + assert(view->shape == mv->ob_array); + assert(view->strides == mv->ob_array + (ndim==0?1:ndim)); + assert(view->suboffsets == NULL); + + view->ndim = ndim; + if (view->ndim == 0) { + view->shape = NULL; + view->strides = NULL; + len = view->itemsize; } else { - func = _Py_add_one_to_index_C; + len = copy_shape(view->shape, shape, ndim, view->itemsize); + if (len < 0) + return -1; + init_strides_from_shape(view); } - while (elements--) { - func(view->ndim, indices, view->shape); - ptr = PyBuffer_GetPointer(view, indices); - memcpy(dest, ptr, view->itemsize); - dest += view->itemsize; + + if (view->len != len) { + PyErr_SetString(PyExc_TypeError, + "memoryview: product(shape) * itemsize != buffer size"); + return -1; } - PyMem_Free(indices); + init_flags(mv); + + return 0; +} + +static int +zero_in_shape(PyMemoryViewObject *mv) +{ + Py_buffer *view = &mv->view; + Py_ssize_t i; + + for (i = 0; i < view->ndim; i++) + if (view->shape[i] == 0) + return 1; + return 0; } /* - Get a the data from an object as a contiguous chunk of memory (in - either 'C' or 'F'ortran order) even if it means copying it into a - separate memory area. - - Returns a new reference to a Memory view object. If no copy is needed, - the memory view object points to the original memory and holds a - lock on the original. If a copy is needed, then the memory view object - points to a brand-new Bytes object (and holds a memory lock on it). - - buffertype - - PyBUF_READ buffer only needs to be read-only - PyBUF_WRITE buffer needs to be writable (give error if not contiguous) - PyBUF_SHADOW buffer needs to be writable so shadow it with - a contiguous buffer if it is not. The view will point to - the shadow buffer which can be written to and then - will be copied back into the other buffer when the memory - view is de-allocated. While the shadow buffer is - being used, it will have an exclusive write lock on - the original buffer. - */ + Cast a copy of 'self' to a different view. The input view must + be C-contiguous. The function always casts the input view to a + 1-D output according to 'format'. At least one of input-format, + output-format must have byte size. -PyObject * -PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char fort) + If 'shape' is given, the 1-D view from the previous step will + be cast to a C-contiguous view with new shape and strides. + + All casts must result in views that will have the exact byte + size of the original input. Otherwise, an error is raised. +*/ +static PyObject * +memory_cast(PyMemoryViewObject *self, PyObject *args, PyObject *kwds) { - PyMemoryViewObject *mem; - PyObject *bytes; - Py_buffer *view; - int flags; - char *dest; + static char *kwlist[] = {"format", "shape", NULL}; + PyMemoryViewObject *mv = NULL; + PyObject *shape = NULL; + PyObject *format; + Py_ssize_t ndim = 1; - if (!PyObject_CheckBuffer(obj)) { + CHECK_RELEASED(self); + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O", kwlist, + &format, &shape)) { + return NULL; + } + if (!PyUnicode_Check(format)) { PyErr_SetString(PyExc_TypeError, - "object does not support the buffer interface"); + "memoryview: format argument must be a string"); return NULL; } - - mem = PyObject_GC_New(PyMemoryViewObject, &PyMemoryView_Type); - if (mem == NULL) + if (!MV_C_CONTIGUOUS(self->flags)) { + PyErr_SetString(PyExc_TypeError, + "memoryview: casts are restricted to C-contiguous views"); + return NULL; + } + if (zero_in_shape(self)) { + PyErr_SetString(PyExc_TypeError, + "memoryview: cannot cast view with zeros in shape or strides"); return NULL; - - view = &mem->view; - flags = PyBUF_FULL_RO; - switch(buffertype) { - case PyBUF_WRITE: - flags = PyBUF_FULL; - break; + } + if (shape) { + CHECK_LIST_OR_TUPLE(shape) + ndim = PySequence_Fast_GET_SIZE(shape); + if (ndim > PyBUF_MAX_NDIM) { + PyErr_SetString(PyExc_ValueError, + "memoryview: number of dimensions must not exceed " + STRINGIZE(PyBUF_MAX_NDIM)); + return NULL; + } + if (self->view.ndim != 1 && ndim != 1) { + PyErr_SetString(PyExc_TypeError, + "memoryview: cast must be 1D -> ND or ND -> 1D"); + return NULL; + } } - if (PyObject_GetBuffer(obj, view, flags) != 0) { - Py_DECREF(mem); + mv = (PyMemoryViewObject *) + mbuf_add_incomplete_view(self->mbuf, &self->view, ndim==0 ? 1 : (int)ndim); + if (mv == NULL) return NULL; + + if (cast_to_1D(mv, format) < 0) + goto error; + if (shape && cast_to_ND(mv, shape, (int)ndim) < 0) + goto error; + + return (PyObject *)mv; + +error: + Py_DECREF(mv); + return NULL; +} + + +/**************************************************************************/ +/* getbuffer */ +/**************************************************************************/ + +static int +memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) +{ + Py_buffer *base = &self->view; + int baseflags = self->flags; + + CHECK_RELEASED_INT(self); + + /* start with complete information */ + *view = *base; + view->obj = NULL; + + if (REQ_WRITABLE(flags) && base->readonly) { + PyErr_SetString(PyExc_BufferError, + "memoryview: underlying buffer is not writable"); + return -1; + } + if (!REQ_FORMAT(flags)) { + /* NULL indicates that the buffer's data type has been cast to 'B'. + view->itemsize is the _previous_ itemsize. If shape is present, + the equality product(shape) * itemsize = len still holds at this + point. The equality calcsize(format) = itemsize does _not_ hold + from here on! */ + view->format = NULL; } - if (PyBuffer_IsContiguous(view, fort)) { - /* no copy needed */ - _PyObject_GC_TRACK(mem); - return (PyObject *)mem; + if (REQ_C_CONTIGUOUS(flags) && !MV_C_CONTIGUOUS(baseflags)) { + PyErr_SetString(PyExc_BufferError, + "memoryview: underlying buffer is not C-contiguous"); + return -1; } - /* otherwise a copy is needed */ - if (buffertype == PyBUF_WRITE) { - Py_DECREF(mem); + if (REQ_F_CONTIGUOUS(flags) && !MV_F_CONTIGUOUS(baseflags)) { PyErr_SetString(PyExc_BufferError, - "writable contiguous buffer requested " - "for a non-contiguousobject."); - return NULL; + "memoryview: underlying buffer is not Fortran contiguous"); + return -1; } - bytes = PyBytes_FromStringAndSize(NULL, view->len); - if (bytes == NULL) { - Py_DECREF(mem); - return NULL; + if (REQ_ANY_CONTIGUOUS(flags) && !MV_ANY_CONTIGUOUS(baseflags)) { + PyErr_SetString(PyExc_BufferError, + "memoryview: underlying buffer is not contiguous"); + return -1; } - dest = PyBytes_AS_STRING(bytes); - /* different copying strategy depending on whether - or not any pointer de-referencing is needed - */ - /* strided or in-direct copy */ - if (view->suboffsets==NULL) { - _strided_copy_nd(dest, view->buf, view->ndim, view->shape, - view->strides, view->itemsize, fort); + if (!REQ_INDIRECT(flags) && (baseflags & _Py_MEMORYVIEW_PIL)) { + PyErr_SetString(PyExc_BufferError, + "memoryview: underlying buffer requires suboffsets"); + return -1; } - else { - if (_indirect_copy_nd(dest, view, fort) < 0) { - Py_DECREF(bytes); - Py_DECREF(mem); - return NULL; + if (!REQ_STRIDES(flags)) { + if (!MV_C_CONTIGUOUS(baseflags)) { + PyErr_SetString(PyExc_BufferError, + "memoryview: underlying buffer is not C-contiguous"); + return -1; } - PyBuffer_Release(view); /* XXX ? */ + view->strides = NULL; + } + if (!REQ_SHAPE(flags)) { + /* PyBUF_SIMPLE or PyBUF_WRITABLE: at this point buf is C-contiguous, + so base->buf = ndbuf->data. */ + if (view->format != NULL) { + /* PyBUF_SIMPLE|PyBUF_FORMAT and PyBUF_WRITABLE|PyBUF_FORMAT do + not make sense. */ + PyErr_Format(PyExc_BufferError, + "ndarray: cannot cast to unsigned bytes if the format flag " + "is present"); + return -1; + } + /* product(shape) * itemsize = len and calcsize(format) = itemsize + do _not_ hold from here on! */ + view->ndim = 1; + view->shape = NULL; } - _PyObject_GC_TRACK(mem); - return (PyObject *)mem; -} -static PyObject * -memory_format_get(PyMemoryViewObject *self) + view->obj = (PyObject *)self; + Py_INCREF(view->obj); + self->exports++; + + return 0; +} + +static void +memory_releasebuf(PyMemoryViewObject *self, Py_buffer *view) { - CHECK_RELEASED(self); - return PyUnicode_FromString(self->view.format); + self->exports--; + return; + /* PyBuffer_Release() decrements view->obj after this function returns. */ } -static PyObject * -memory_itemsize_get(PyMemoryViewObject *self) +/* Buffer methods */ +static PyBufferProcs memory_as_buffer = { + (getbufferproc)memory_getbuf, /* bf_getbuffer */ + (releasebufferproc)memory_releasebuf, /* bf_releasebuffer */ +}; + + +/****************************************************************************/ +/* Optimized pack/unpack for all native format specifiers */ +/****************************************************************************/ + +/* + Fix exceptions: + 1) Include format string in the error message. + 2) OverflowError -> ValueError. + 3) The error message from PyNumber_Index() is not ideal. +*/ +static int +type_error_int(const char *fmt) { - CHECK_RELEASED(self); - return PyLong_FromSsize_t(self->view.itemsize); + PyErr_Format(PyExc_TypeError, + "memoryview: invalid type for format '%s'", fmt); + return -1; } -static PyObject * -_IntTupleFromSsizet(int len, Py_ssize_t *vals) +static int +value_error_int(const char *fmt) { - int i; - PyObject *o; - PyObject *intTuple; + PyErr_Format(PyExc_ValueError, + "memoryview: invalid value for format '%s'", fmt); + return -1; +} - if (vals == NULL) { - Py_INCREF(Py_None); - return Py_None; +static int +fix_error_int(const char *fmt) +{ + assert(PyErr_Occurred()); + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + return type_error_int(fmt); } - intTuple = PyTuple_New(len); - if (!intTuple) - return NULL; - for (i=0; i<len; i++) { - o = PyLong_FromSsize_t(vals[i]); - if (!o) { - Py_DECREF(intTuple); - return NULL; - } - PyTuple_SET_ITEM(intTuple, i, o); + else if (PyErr_ExceptionMatches(PyExc_OverflowError) || + PyErr_ExceptionMatches(PyExc_ValueError)) { + PyErr_Clear(); + return value_error_int(fmt); } - return intTuple; + + return -1; } -static PyObject * -memory_shape_get(PyMemoryViewObject *self) +/* Accept integer objects or objects with an __index__() method. */ +static long +pylong_as_ld(PyObject *item) { - CHECK_RELEASED(self); - return _IntTupleFromSsizet(self->view.ndim, self->view.shape); + PyObject *tmp; + long ld; + + tmp = PyNumber_Index(item); + if (tmp == NULL) + return -1; + + ld = PyLong_AsLong(tmp); + Py_DECREF(tmp); + return ld; } -static PyObject * -memory_strides_get(PyMemoryViewObject *self) +static unsigned long +pylong_as_lu(PyObject *item) { - CHECK_RELEASED(self); - return _IntTupleFromSsizet(self->view.ndim, self->view.strides); + PyObject *tmp; + unsigned long lu; + + tmp = PyNumber_Index(item); + if (tmp == NULL) + return (unsigned long)-1; + + lu = PyLong_AsUnsignedLong(tmp); + Py_DECREF(tmp); + return lu; } -static PyObject * -memory_suboffsets_get(PyMemoryViewObject *self) +#ifdef HAVE_LONG_LONG +static PY_LONG_LONG +pylong_as_lld(PyObject *item) { - CHECK_RELEASED(self); - return _IntTupleFromSsizet(self->view.ndim, self->view.suboffsets); + PyObject *tmp; + PY_LONG_LONG lld; + + tmp = PyNumber_Index(item); + if (tmp == NULL) + return -1; + + lld = PyLong_AsLongLong(tmp); + Py_DECREF(tmp); + return lld; } -static PyObject * -memory_readonly_get(PyMemoryViewObject *self) +static unsigned PY_LONG_LONG +pylong_as_llu(PyObject *item) { - CHECK_RELEASED(self); - return PyBool_FromLong(self->view.readonly); + PyObject *tmp; + unsigned PY_LONG_LONG llu; + + tmp = PyNumber_Index(item); + if (tmp == NULL) + return (unsigned PY_LONG_LONG)-1; + + llu = PyLong_AsUnsignedLongLong(tmp); + Py_DECREF(tmp); + return llu; } +#endif -static PyObject * -memory_ndim_get(PyMemoryViewObject *self) +static Py_ssize_t +pylong_as_zd(PyObject *item) { - CHECK_RELEASED(self); - return PyLong_FromLong(self->view.ndim); + PyObject *tmp; + Py_ssize_t zd; + + tmp = PyNumber_Index(item); + if (tmp == NULL) + return -1; + + zd = PyLong_AsSsize_t(tmp); + Py_DECREF(tmp); + return zd; } -PyDoc_STRVAR(memory_format_doc, - "A string containing the format (in struct module style)\n" - " for each element in the view."); -PyDoc_STRVAR(memory_itemsize_doc, - "The size in bytes of each element of the memoryview."); -PyDoc_STRVAR(memory_shape_doc, - "A tuple of ndim integers giving the shape of the memory\n" - " as an N-dimensional array."); -PyDoc_STRVAR(memory_strides_doc, - "A tuple of ndim integers giving the size in bytes to access\n" - " each element for each dimension of the array."); -PyDoc_STRVAR(memory_suboffsets_doc, - "A tuple of integers used internally for PIL-style arrays."); -PyDoc_STRVAR(memory_readonly_doc, - "A bool indicating whether the memory is read only."); -PyDoc_STRVAR(memory_ndim_doc, - "An integer indicating how many dimensions of a multi-dimensional\n" - " array the memory represents."); +static size_t +pylong_as_zu(PyObject *item) +{ + PyObject *tmp; + size_t zu; -static PyGetSetDef memory_getsetlist[] = { - {"format", (getter)memory_format_get, NULL, memory_format_doc}, - {"itemsize", (getter)memory_itemsize_get, NULL, memory_itemsize_doc}, - {"shape", (getter)memory_shape_get, NULL, memory_shape_doc}, - {"strides", (getter)memory_strides_get, NULL, memory_strides_doc}, - {"suboffsets", (getter)memory_suboffsets_get, NULL, memory_suboffsets_doc}, - {"readonly", (getter)memory_readonly_get, NULL, memory_readonly_doc}, - {"ndim", (getter)memory_ndim_get, NULL, memory_ndim_doc}, - {NULL, NULL, NULL, NULL}, -}; + tmp = PyNumber_Index(item); + if (tmp == NULL) + return (size_t)-1; + zu = PyLong_AsSize_t(tmp); + Py_DECREF(tmp); + return zu; +} -static PyObject * -memory_tobytes(PyMemoryViewObject *mem, PyObject *noargs) +/* Timings with the ndarray from _testbuffer.c indicate that using the + struct module is around 15x slower than the two functions below. */ + +#define UNPACK_SINGLE(dest, ptr, type) \ + do { \ + type x; \ + memcpy((char *)&x, ptr, sizeof x); \ + dest = x; \ + } while (0) + +/* Unpack a single item. 'fmt' can be any native format character in struct + module syntax. This function is very sensitive to small changes. With this + layout gcc automatically generates a fast jump table. */ +Py_LOCAL_INLINE(PyObject *) +unpack_single(const char *ptr, const char *fmt) { - CHECK_RELEASED(mem); - return PyObject_CallFunctionObjArgs( - (PyObject *) &PyBytes_Type, mem, NULL); + unsigned PY_LONG_LONG llu; + unsigned long lu; + size_t zu; + PY_LONG_LONG lld; + long ld; + Py_ssize_t zd; + double d; + unsigned char uc; + void *p; + + switch (fmt[0]) { + + /* signed integers and fast path for 'B' */ + case 'B': uc = *((unsigned char *)ptr); goto convert_uc; + case 'b': ld = *((signed char *)ptr); goto convert_ld; + case 'h': UNPACK_SINGLE(ld, ptr, short); goto convert_ld; + case 'i': UNPACK_SINGLE(ld, ptr, int); goto convert_ld; + case 'l': UNPACK_SINGLE(ld, ptr, long); goto convert_ld; + + /* boolean */ + #ifdef HAVE_C99_BOOL + case '?': UNPACK_SINGLE(ld, ptr, _Bool); goto convert_bool; + #else + case '?': UNPACK_SINGLE(ld, ptr, char); goto convert_bool; + #endif + + /* unsigned integers */ + case 'H': UNPACK_SINGLE(lu, ptr, unsigned short); goto convert_lu; + case 'I': UNPACK_SINGLE(lu, ptr, unsigned int); goto convert_lu; + case 'L': UNPACK_SINGLE(lu, ptr, unsigned long); goto convert_lu; + + /* native 64-bit */ + #ifdef HAVE_LONG_LONG + case 'q': UNPACK_SINGLE(lld, ptr, PY_LONG_LONG); goto convert_lld; + case 'Q': UNPACK_SINGLE(llu, ptr, unsigned PY_LONG_LONG); goto convert_llu; + #endif + + /* ssize_t and size_t */ + case 'n': UNPACK_SINGLE(zd, ptr, Py_ssize_t); goto convert_zd; + case 'N': UNPACK_SINGLE(zu, ptr, size_t); goto convert_zu; + + /* floats */ + case 'f': UNPACK_SINGLE(d, ptr, float); goto convert_double; + case 'd': UNPACK_SINGLE(d, ptr, double); goto convert_double; + + /* bytes object */ + case 'c': goto convert_bytes; + + /* pointer */ + case 'P': UNPACK_SINGLE(p, ptr, void *); goto convert_pointer; + + /* default */ + default: goto err_format; + } + +convert_uc: + /* PyLong_FromUnsignedLong() is slower */ + return PyLong_FromLong(uc); +convert_ld: + return PyLong_FromLong(ld); +convert_lu: + return PyLong_FromUnsignedLong(lu); +convert_lld: + return PyLong_FromLongLong(lld); +convert_llu: + return PyLong_FromUnsignedLongLong(llu); +convert_zd: + return PyLong_FromSsize_t(zd); +convert_zu: + return PyLong_FromSize_t(zu); +convert_double: + return PyFloat_FromDouble(d); +convert_bool: + return PyBool_FromLong(ld); +convert_bytes: + return PyBytes_FromStringAndSize(ptr, 1); +convert_pointer: + return PyLong_FromVoidPtr(p); +err_format: + PyErr_Format(PyExc_NotImplementedError, + "memoryview: format %s not supported", fmt); + return NULL; } -/* TODO: rewrite this function using the struct module to unpack - each buffer item */ +#define PACK_SINGLE(ptr, src, type) \ + do { \ + type x; \ + x = (type)src; \ + memcpy(ptr, (char *)&x, sizeof x); \ + } while (0) -static PyObject * -memory_tolist(PyMemoryViewObject *mem, PyObject *noargs) +/* Pack a single item. 'fmt' can be any native format character in + struct module syntax. */ +static int +pack_single(char *ptr, PyObject *item, const char *fmt) { - Py_buffer *view = &(mem->view); - Py_ssize_t i; - PyObject *res, *item; - char *buf; + unsigned PY_LONG_LONG llu; + unsigned long lu; + size_t zu; + PY_LONG_LONG lld; + long ld; + Py_ssize_t zd; + double d; + void *p; + + switch (fmt[0]) { + /* signed integers */ + case 'b': case 'h': case 'i': case 'l': + ld = pylong_as_ld(item); + if (ld == -1 && PyErr_Occurred()) + goto err_occurred; + switch (fmt[0]) { + case 'b': + if (ld < SCHAR_MIN || ld > SCHAR_MAX) goto err_range; + *((signed char *)ptr) = (signed char)ld; break; + case 'h': + if (ld < SHRT_MIN || ld > SHRT_MAX) goto err_range; + PACK_SINGLE(ptr, ld, short); break; + case 'i': + if (ld < INT_MIN || ld > INT_MAX) goto err_range; + PACK_SINGLE(ptr, ld, int); break; + default: /* 'l' */ + PACK_SINGLE(ptr, ld, long); break; + } + break; + + /* unsigned integers */ + case 'B': case 'H': case 'I': case 'L': + lu = pylong_as_lu(item); + if (lu == (unsigned long)-1 && PyErr_Occurred()) + goto err_occurred; + switch (fmt[0]) { + case 'B': + if (lu > UCHAR_MAX) goto err_range; + *((unsigned char *)ptr) = (unsigned char)lu; break; + case 'H': + if (lu > USHRT_MAX) goto err_range; + PACK_SINGLE(ptr, lu, unsigned short); break; + case 'I': + if (lu > UINT_MAX) goto err_range; + PACK_SINGLE(ptr, lu, unsigned int); break; + default: /* 'L' */ + PACK_SINGLE(ptr, lu, unsigned long); break; + } + break; + + /* native 64-bit */ + #ifdef HAVE_LONG_LONG + case 'q': + lld = pylong_as_lld(item); + if (lld == -1 && PyErr_Occurred()) + goto err_occurred; + PACK_SINGLE(ptr, lld, PY_LONG_LONG); + break; + case 'Q': + llu = pylong_as_llu(item); + if (llu == (unsigned PY_LONG_LONG)-1 && PyErr_Occurred()) + goto err_occurred; + PACK_SINGLE(ptr, llu, unsigned PY_LONG_LONG); + break; + #endif + + /* ssize_t and size_t */ + case 'n': + zd = pylong_as_zd(item); + if (zd == -1 && PyErr_Occurred()) + goto err_occurred; + PACK_SINGLE(ptr, zd, Py_ssize_t); + break; + case 'N': + zu = pylong_as_zu(item); + if (zu == (size_t)-1 && PyErr_Occurred()) + goto err_occurred; + PACK_SINGLE(ptr, zu, size_t); + break; + + /* floats */ + case 'f': case 'd': + d = PyFloat_AsDouble(item); + if (d == -1.0 && PyErr_Occurred()) + goto err_occurred; + if (fmt[0] == 'f') { + PACK_SINGLE(ptr, d, float); + } + else { + PACK_SINGLE(ptr, d, double); + } + break; + + /* bool */ + case '?': + ld = PyObject_IsTrue(item); + if (ld < 0) + return -1; /* preserve original error */ + #ifdef HAVE_C99_BOOL + PACK_SINGLE(ptr, ld, _Bool); + #else + PACK_SINGLE(ptr, ld, char); + #endif + break; + + /* bytes object */ + case 'c': + if (!PyBytes_Check(item)) + return type_error_int(fmt); + if (PyBytes_GET_SIZE(item) != 1) + return value_error_int(fmt); + *ptr = PyBytes_AS_STRING(item)[0]; + break; + + /* pointer */ + case 'P': + p = PyLong_AsVoidPtr(item); + if (p == NULL && PyErr_Occurred()) + goto err_occurred; + PACK_SINGLE(ptr, p, void *); + break; + + /* default */ + default: goto err_format; + } - CHECK_RELEASED(mem); - if (strcmp(view->format, "B") || view->itemsize != 1) { - PyErr_SetString(PyExc_NotImplementedError, - "tolist() only supports byte views"); + return 0; + +err_occurred: + return fix_error_int(fmt); +err_range: + return value_error_int(fmt); +err_format: + PyErr_Format(PyExc_NotImplementedError, + "memoryview: format %s not supported", fmt); + return -1; +} + + +/****************************************************************************/ +/* unpack using the struct module */ +/****************************************************************************/ + +/* For reasonable performance it is necessary to cache all objects required + for unpacking. An unpacker can handle the format passed to unpack_from(). + Invariant: All pointer fields of the struct should either be NULL or valid + pointers. */ +struct unpacker { + PyObject *unpack_from; /* Struct.unpack_from(format) */ + PyObject *mview; /* cached memoryview */ + char *item; /* buffer for mview */ + Py_ssize_t itemsize; /* len(item) */ +}; + +static struct unpacker * +unpacker_new(void) +{ + struct unpacker *x = PyMem_Malloc(sizeof *x); + + if (x == NULL) { + PyErr_NoMemory(); return NULL; } - if (view->ndim != 1) { - PyErr_SetString(PyExc_NotImplementedError, - "tolist() only supports one-dimensional objects"); + + x->unpack_from = NULL; + x->mview = NULL; + x->item = NULL; + x->itemsize = 0; + + return x; +} + +static void +unpacker_free(struct unpacker *x) +{ + if (x) { + Py_XDECREF(x->unpack_from); + Py_XDECREF(x->mview); + PyMem_Free(x->item); + PyMem_Free(x); + } +} + +/* Return a new unpacker for the given format. */ +static struct unpacker * +struct_get_unpacker(const char *fmt, Py_ssize_t itemsize) +{ + PyObject *structmodule; /* XXX cache these two */ + PyObject *Struct = NULL; /* XXX in globals? */ + PyObject *structobj = NULL; + PyObject *format = NULL; + struct unpacker *x = NULL; + + structmodule = PyImport_ImportModule("struct"); + if (structmodule == NULL) + return NULL; + + Struct = PyObject_GetAttrString(structmodule, "Struct"); + Py_DECREF(structmodule); + if (Struct == NULL) + return NULL; + + x = unpacker_new(); + if (x == NULL) + goto error; + + format = PyBytes_FromString(fmt); + if (format == NULL) + goto error; + + structobj = PyObject_CallFunctionObjArgs(Struct, format, NULL); + if (structobj == NULL) + goto error; + + x->unpack_from = PyObject_GetAttrString(structobj, "unpack_from"); + if (x->unpack_from == NULL) + goto error; + + x->item = PyMem_Malloc(itemsize); + if (x->item == NULL) { + PyErr_NoMemory(); + goto error; + } + x->itemsize = itemsize; + + x->mview = PyMemoryView_FromMemory(x->item, itemsize, PyBUF_WRITE); + if (x->mview == NULL) + goto error; + + +out: + Py_XDECREF(Struct); + Py_XDECREF(format); + Py_XDECREF(structobj); + return x; + +error: + unpacker_free(x); + x = NULL; + goto out; +} + +/* unpack a single item */ +static PyObject * +struct_unpack_single(const char *ptr, struct unpacker *x) +{ + PyObject *v; + + memcpy(x->item, ptr, x->itemsize); + v = PyObject_CallFunctionObjArgs(x->unpack_from, x->mview, NULL); + if (v == NULL) return NULL; + + if (PyTuple_GET_SIZE(v) == 1) { + PyObject *tmp = PyTuple_GET_ITEM(v, 0); + Py_INCREF(tmp); + Py_DECREF(v); + return tmp; } - res = PyList_New(view->len); - if (res == NULL) + + return v; +} + + +/****************************************************************************/ +/* Representations */ +/****************************************************************************/ + +/* allow explicit form of native format */ +Py_LOCAL_INLINE(const char *) +adjust_fmt(const Py_buffer *view) +{ + const char *fmt; + + fmt = (view->format[0] == '@') ? view->format+1 : view->format; + if (fmt[0] && fmt[1] == '\0') + return fmt; + + PyErr_Format(PyExc_NotImplementedError, + "memoryview: unsupported format %s", view->format); + return NULL; +} + +/* Base case for multi-dimensional unpacking. Assumption: ndim == 1. */ +static PyObject * +tolist_base(const char *ptr, const Py_ssize_t *shape, + const Py_ssize_t *strides, const Py_ssize_t *suboffsets, + const char *fmt) +{ + PyObject *lst, *item; + Py_ssize_t i; + + lst = PyList_New(shape[0]); + if (lst == NULL) return NULL; - buf = view->buf; - for (i = 0; i < view->len; i++) { - item = PyLong_FromUnsignedLong((unsigned char) *buf); + + for (i = 0; i < shape[0]; ptr+=strides[0], i++) { + const char *xptr = ADJUST_PTR(ptr, suboffsets); + item = unpack_single(xptr, fmt); if (item == NULL) { - Py_DECREF(res); + Py_DECREF(lst); return NULL; } - PyList_SET_ITEM(res, i, item); - buf++; + PyList_SET_ITEM(lst, i, item); } - return res; + + return lst; } -static void -do_release(PyMemoryViewObject *self) +/* Unpack a multi-dimensional array into a nested list. + Assumption: ndim >= 1. */ +static PyObject * +tolist_rec(const char *ptr, Py_ssize_t ndim, const Py_ssize_t *shape, + const Py_ssize_t *strides, const Py_ssize_t *suboffsets, + const char *fmt) { - if (self->view.obj != NULL) { - PyBuffer_Release(&(self->view)); + PyObject *lst, *item; + Py_ssize_t i; + + assert(ndim >= 1); + assert(shape != NULL); + assert(strides != NULL); + + if (ndim == 1) + return tolist_base(ptr, shape, strides, suboffsets, fmt); + + lst = PyList_New(shape[0]); + if (lst == NULL) + return NULL; + + for (i = 0; i < shape[0]; ptr+=strides[0], i++) { + const char *xptr = ADJUST_PTR(ptr, suboffsets); + item = tolist_rec(xptr, ndim-1, shape+1, + strides+1, suboffsets ? suboffsets+1 : NULL, + fmt); + if (item == NULL) { + Py_DECREF(lst); + return NULL; + } + PyList_SET_ITEM(lst, i, item); } - self->view.obj = NULL; - self->view.buf = NULL; + + return lst; } +/* Return a list representation of the memoryview. Currently only buffers + with native format strings are supported. */ static PyObject * -memory_enter(PyObject *self, PyObject *args) +memory_tolist(PyMemoryViewObject *mv, PyObject *noargs) { - CHECK_RELEASED(self); - Py_INCREF(self); - return self; + const Py_buffer *view = &(mv->view); + const char *fmt; + + CHECK_RELEASED(mv); + + fmt = adjust_fmt(view); + if (fmt == NULL) + return NULL; + if (view->ndim == 0) { + return unpack_single(view->buf, fmt); + } + else if (view->ndim == 1) { + return tolist_base(view->buf, view->shape, + view->strides, view->suboffsets, + fmt); + } + else { + return tolist_rec(view->buf, view->ndim, view->shape, + view->strides, view->suboffsets, + fmt); + } } static PyObject * -memory_exit(PyObject *self, PyObject *args) +memory_tobytes(PyMemoryViewObject *self, PyObject *dummy) { - do_release((PyMemoryViewObject *) self); - Py_RETURN_NONE; -} + Py_buffer *src = VIEW_ADDR(self); + PyObject *bytes = NULL; -PyDoc_STRVAR(memory_release_doc, -"M.release() -> None\n\ -\n\ -Release the underlying buffer exposed by the memoryview object."); -PyDoc_STRVAR(memory_tobytes_doc, -"M.tobytes() -> bytes\n\ -\n\ -Return the data in the buffer as a byte string."); -PyDoc_STRVAR(memory_tolist_doc, -"M.tolist() -> list\n\ -\n\ -Return the data in the buffer as a list of elements."); + CHECK_RELEASED(self); -static PyMethodDef memory_methods[] = { - {"release", memory_exit, METH_NOARGS, memory_release_doc}, - {"tobytes", (PyCFunction)memory_tobytes, METH_NOARGS, memory_tobytes_doc}, - {"tolist", (PyCFunction)memory_tolist, METH_NOARGS, memory_tolist_doc}, - {"__enter__", memory_enter, METH_NOARGS}, - {"__exit__", memory_exit, METH_VARARGS}, - {NULL, NULL} /* sentinel */ -}; + if (MV_C_CONTIGUOUS(self->flags)) { + return PyBytes_FromStringAndSize(src->buf, src->len); + } + bytes = PyBytes_FromStringAndSize(NULL, src->len); + if (bytes == NULL) + return NULL; -static void -memory_dealloc(PyMemoryViewObject *self) -{ - _PyObject_GC_UNTRACK(self); - do_release(self); - PyObject_GC_Del(self); + if (buffer_to_contiguous(PyBytes_AS_STRING(bytes), src, 'C') < 0) { + Py_DECREF(bytes); + return NULL; + } + + return bytes; } static PyObject * memory_repr(PyMemoryViewObject *self) { - if (IS_RELEASED(self)) + if (self->flags & _Py_MEMORYVIEW_RELEASED) return PyUnicode_FromFormat("<released memory at %p>", self); else return PyUnicode_FromFormat("<memory at %p>", self); } -/* Sequence methods */ -static Py_ssize_t -memory_length(PyMemoryViewObject *self) + +/**************************************************************************/ +/* Indexing and slicing */ +/**************************************************************************/ + +/* Get the pointer to the item at index. */ +static char * +ptr_from_index(Py_buffer *view, Py_ssize_t index) { - CHECK_RELEASED_INT(self); - return get_shape0(&self->view); + char *ptr; + Py_ssize_t nitems; /* items in the first dimension */ + + assert(view->shape); + assert(view->strides); + + nitems = view->shape[0]; + if (index < 0) { + index += nitems; + } + if (index < 0 || index >= nitems) { + PyErr_SetString(PyExc_IndexError, "index out of bounds"); + return NULL; + } + + ptr = (char *)view->buf; + ptr += view->strides[0] * index; + + ptr = ADJUST_PTR(ptr, view->suboffsets); + + return ptr; } -/* Alternate version of memory_subcript that only accepts indices. - Used by PySeqIter_New(). -*/ +/* Return the item at index. In a one-dimensional view, this is an object + with the type specified by view->format. Otherwise, the item is a sub-view. + The function is used in memory_subscript() and memory_as_sequence. */ static PyObject * -memory_item(PyMemoryViewObject *self, Py_ssize_t result) +memory_item(PyMemoryViewObject *self, Py_ssize_t index) { Py_buffer *view = &(self->view); + const char *fmt; CHECK_RELEASED(self); + + fmt = adjust_fmt(view); + if (fmt == NULL) + return NULL; + if (view->ndim == 0) { - PyErr_SetString(PyExc_IndexError, - "invalid indexing of 0-dim memory"); + PyErr_SetString(PyExc_TypeError, "invalid indexing of 0-dim memory"); return NULL; } if (view->ndim == 1) { - /* Return a bytes object */ - char *ptr; - ptr = (char *)view->buf; - if (result < 0) { - result += get_shape0(view); - } - if ((result < 0) || (result >= get_shape0(view))) { - PyErr_SetString(PyExc_IndexError, - "index out of bounds"); + char *ptr = ptr_from_index(view, index); + if (ptr == NULL) return NULL; - } - if (view->strides == NULL) - ptr += view->itemsize * result; - else - ptr += view->strides[0] * result; - if (view->suboffsets != NULL && - view->suboffsets[0] >= 0) { - ptr = *((char **)ptr) + view->suboffsets[0]; - } - return PyBytes_FromStringAndSize(ptr, view->itemsize); - } else { - /* Return a new memory-view object */ - Py_buffer newview; - memset(&newview, 0, sizeof(newview)); - /* XXX: This needs to be fixed so it actually returns a sub-view */ - return PyMemoryView_FromBuffer(&newview); + return unpack_single(ptr, fmt); } + + PyErr_SetString(PyExc_NotImplementedError, + "multi-dimensional sub-views are not implemented"); + return NULL; } -/* - mem[obj] returns a bytes object holding the data for one element if - obj fully indexes the memory view or another memory-view object - if it does not. +Py_LOCAL_INLINE(int) +init_slice(Py_buffer *base, PyObject *key, int dim) +{ + Py_ssize_t start, stop, step, slicelength; - 0-d memory-view objects can be referenced using ... or () but - not with anything else. - */ + if (PySlice_GetIndicesEx(key, base->shape[dim], + &start, &stop, &step, &slicelength) < 0) { + return -1; + } + + + if (base->suboffsets == NULL || dim == 0) { + adjust_buf: + base->buf = (char *)base->buf + base->strides[dim] * start; + } + else { + Py_ssize_t n = dim-1; + while (n >= 0 && base->suboffsets[n] < 0) + n--; + if (n < 0) + goto adjust_buf; /* all suboffsets are negative */ + base->suboffsets[n] = base->suboffsets[n] + base->strides[dim] * start; + } + base->shape[dim] = slicelength; + base->strides[dim] = base->strides[dim] * step; + + return 0; +} + +static int +is_multislice(PyObject *key) +{ + Py_ssize_t size, i; + + if (!PyTuple_Check(key)) + return 0; + size = PyTuple_GET_SIZE(key); + if (size == 0) + return 0; + + for (i = 0; i < size; i++) { + PyObject *x = PyTuple_GET_ITEM(key, i); + if (!PySlice_Check(x)) + return 0; + } + return 1; +} + +/* mv[obj] returns an object holding the data for one element if obj + fully indexes the memoryview or another memoryview object if it + does not. + + 0-d memoryview objects can be referenced using mv[...] or mv[()] + but not with anything else. */ static PyObject * memory_subscript(PyMemoryViewObject *self, PyObject *key) { @@ -611,248 +2244,702 @@ memory_subscript(PyMemoryViewObject *self, PyObject *key) view = &(self->view); CHECK_RELEASED(self); + if (view->ndim == 0) { - if (key == Py_Ellipsis || - (PyTuple_Check(key) && PyTuple_GET_SIZE(key)==0)) { + if (PyTuple_Check(key) && PyTuple_GET_SIZE(key) == 0) { + const char *fmt = adjust_fmt(view); + if (fmt == NULL) + return NULL; + return unpack_single(view->buf, fmt); + } + else if (key == Py_Ellipsis) { Py_INCREF(self); return (PyObject *)self; } else { - PyErr_SetString(PyExc_IndexError, - "invalid indexing of 0-dim memory"); + PyErr_SetString(PyExc_TypeError, + "invalid indexing of 0-dim memory"); return NULL; } } + if (PyIndex_Check(key)) { - Py_ssize_t result; - result = PyNumber_AsSsize_t(key, NULL); - if (result == -1 && PyErr_Occurred()) - return NULL; - return memory_item(self, result); + Py_ssize_t index; + index = PyNumber_AsSsize_t(key, PyExc_IndexError); + if (index == -1 && PyErr_Occurred()) + return NULL; + return memory_item(self, index); } else if (PySlice_Check(key)) { - Py_ssize_t start, stop, step, slicelength; + PyMemoryViewObject *sliced; - if (PySlice_GetIndicesEx(key, get_shape0(view), - &start, &stop, &step, &slicelength) < 0) { + sliced = (PyMemoryViewObject *)mbuf_add_view(self->mbuf, view); + if (sliced == NULL) + return NULL; + + if (init_slice(&sliced->view, key, 0) < 0) { + Py_DECREF(sliced); return NULL; } - - if (step == 1 && view->ndim == 1) { - Py_buffer newview; - void *newbuf = (char *) view->buf - + start * view->itemsize; - int newflags = view->readonly - ? PyBUF_CONTIG_RO : PyBUF_CONTIG; - - /* XXX There should be an API to create a subbuffer */ - if (view->obj != NULL) { - if (PyObject_GetBuffer(view->obj, &newview, newflags) == -1) - return NULL; - } - else { - newview = *view; - } - newview.buf = newbuf; - newview.len = slicelength * newview.itemsize; - newview.format = view->format; - newview.shape = &(newview.smalltable[0]); - newview.shape[0] = slicelength; - newview.strides = &(newview.itemsize); - return PyMemoryView_FromBuffer(&newview); - } - PyErr_SetNone(PyExc_NotImplementedError); + init_len(&sliced->view); + init_flags(sliced); + + return (PyObject *)sliced; + } + else if (is_multislice(key)) { + PyErr_SetString(PyExc_NotImplementedError, + "multi-dimensional slicing is not implemented"); return NULL; } - PyErr_Format(PyExc_TypeError, - "cannot index memory using \"%.200s\"", - key->ob_type->tp_name); + + PyErr_SetString(PyExc_TypeError, "memoryview: invalid slice key"); return NULL; } - -/* Need to support assigning memory if we can */ static int memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value) { - Py_ssize_t start, len, bytelen; - Py_buffer srcview; Py_buffer *view = &(self->view); - char *srcbuf, *destbuf; + Py_buffer src; + const char *fmt; + char *ptr; CHECK_RELEASED_INT(self); + + fmt = adjust_fmt(view); + if (fmt == NULL) + return -1; + if (view->readonly) { - PyErr_SetString(PyExc_TypeError, - "cannot modify read-only memory"); + PyErr_SetString(PyExc_TypeError, "cannot modify read-only memory"); return -1; } if (value == NULL) { - PyErr_SetString(PyExc_TypeError, - "cannot delete memory"); + PyErr_SetString(PyExc_TypeError, "cannot delete memory"); return -1; } - if (view->ndim != 1) { - PyErr_SetNone(PyExc_NotImplementedError); - return -1; - } - if (PyIndex_Check(key)) { - start = PyNumber_AsSsize_t(key, NULL); - if (start == -1 && PyErr_Occurred()) - return -1; - if (start < 0) { - start += get_shape0(view); + if (view->ndim == 0) { + if (key == Py_Ellipsis || + (PyTuple_Check(key) && PyTuple_GET_SIZE(key)==0)) { + ptr = (char *)view->buf; + return pack_single(ptr, value, fmt); } - if ((start < 0) || (start >= get_shape0(view))) { - PyErr_SetString(PyExc_IndexError, - "index out of bounds"); + else { + PyErr_SetString(PyExc_TypeError, + "invalid indexing of 0-dim memory"); return -1; } - len = 1; } - else if (PySlice_Check(key)) { - Py_ssize_t stop, step; + if (view->ndim != 1) { + PyErr_SetString(PyExc_NotImplementedError, + "memoryview assignments are currently restricted to ndim = 1"); + return -1; + } - if (PySlice_GetIndicesEx(key, get_shape0(view), - &start, &stop, &step, &len) < 0) { + if (PyIndex_Check(key)) { + Py_ssize_t index = PyNumber_AsSsize_t(key, PyExc_IndexError); + if (index == -1 && PyErr_Occurred()) return -1; - } - if (step != 1) { - PyErr_SetNone(PyExc_NotImplementedError); + ptr = ptr_from_index(view, index); + if (ptr == NULL) return -1; - } + return pack_single(ptr, value, fmt); } - else { - PyErr_Format(PyExc_TypeError, - "cannot index memory using \"%.200s\"", - key->ob_type->tp_name); - return -1; + /* one-dimensional: fast path */ + if (PySlice_Check(key) && view->ndim == 1) { + Py_buffer dest; /* sliced view */ + Py_ssize_t arrays[3]; + int ret = -1; + + /* rvalue must be an exporter */ + if (PyObject_GetBuffer(value, &src, PyBUF_FULL_RO) < 0) + return ret; + + dest = *view; + dest.shape = &arrays[0]; dest.shape[0] = view->shape[0]; + dest.strides = &arrays[1]; dest.strides[0] = view->strides[0]; + if (view->suboffsets) { + dest.suboffsets = &arrays[2]; dest.suboffsets[0] = view->suboffsets[0]; + } + + if (init_slice(&dest, key, 0) < 0) + goto end_block; + dest.len = dest.shape[0] * dest.itemsize; + + ret = copy_single(&dest, &src); + + end_block: + PyBuffer_Release(&src); + return ret; } - if (PyObject_GetBuffer(value, &srcview, PyBUF_CONTIG_RO) == -1) { + else if (PySlice_Check(key) || is_multislice(key)) { + /* Call memory_subscript() to produce a sliced lvalue, then copy + rvalue into lvalue. This is already implemented in _testbuffer.c. */ + PyErr_SetString(PyExc_NotImplementedError, + "memoryview slice assignments are currently restricted " + "to ndim = 1"); return -1; } - /* XXX should we allow assignment of different item sizes - as long as the byte length is the same? - (e.g. assign 2 shorts to a 4-byte slice) */ - if (srcview.itemsize != view->itemsize) { - PyErr_Format(PyExc_TypeError, - "mismatching item sizes for \"%.200s\" and \"%.200s\"", - view->obj->ob_type->tp_name, srcview.obj->ob_type->tp_name); - goto _error; - } - bytelen = len * view->itemsize; - if (bytelen != srcview.len) { - PyErr_SetString(PyExc_ValueError, - "cannot modify size of memoryview object"); - goto _error; - } - /* Do the actual copy */ - destbuf = (char *) view->buf + start * view->itemsize; - srcbuf = (char *) srcview.buf; - if (destbuf + bytelen < srcbuf || srcbuf + bytelen < destbuf) - /* No overlapping */ - memcpy(destbuf, srcbuf, bytelen); - else - memmove(destbuf, srcbuf, bytelen); - PyBuffer_Release(&srcview); + PyErr_SetString(PyExc_TypeError, "memoryview: invalid slice key"); + return -1; +} + +static Py_ssize_t +memory_length(PyMemoryViewObject *self) +{ + CHECK_RELEASED_INT(self); + return self->view.ndim == 0 ? 1 : self->view.shape[0]; +} + +/* As mapping */ +static PyMappingMethods memory_as_mapping = { + (lenfunc)memory_length, /* mp_length */ + (binaryfunc)memory_subscript, /* mp_subscript */ + (objobjargproc)memory_ass_sub, /* mp_ass_subscript */ +}; + +/* As sequence */ +static PySequenceMethods memory_as_sequence = { + 0, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + (ssizeargfunc)memory_item, /* sq_item */ +}; + + +/**************************************************************************/ +/* Comparisons */ +/**************************************************************************/ + +#define MV_COMPARE_EX -1 /* exception */ +#define MV_COMPARE_NOT_IMPL -2 /* not implemented */ + +/* Translate a StructError to "not equal". Preserve other exceptions. */ +static int +fix_struct_error_int(void) +{ + assert(PyErr_Occurred()); + /* XXX Cannot get at StructError directly? */ + if (PyErr_ExceptionMatches(PyExc_ImportError) || + PyErr_ExceptionMatches(PyExc_MemoryError)) { + return MV_COMPARE_EX; + } + /* StructError: invalid or unknown format -> not equal */ + PyErr_Clear(); return 0; +} -_error: - PyBuffer_Release(&srcview); - return -1; +/* Unpack and compare single items of p and q using the struct module. */ +static int +struct_unpack_cmp(const char *p, const char *q, + struct unpacker *unpack_p, struct unpacker *unpack_q) +{ + PyObject *v, *w; + int ret; + + /* At this point any exception from the struct module should not be + StructError, since both formats have been accepted already. */ + v = struct_unpack_single(p, unpack_p); + if (v == NULL) + return MV_COMPARE_EX; + + w = struct_unpack_single(q, unpack_q); + if (w == NULL) { + Py_DECREF(v); + return MV_COMPARE_EX; + } + + /* MV_COMPARE_EX == -1: exceptions are preserved */ + ret = PyObject_RichCompareBool(v, w, Py_EQ); + Py_DECREF(v); + Py_DECREF(w); + + return ret; +} + +/* Unpack and compare single items of p and q. If both p and q have the same + single element native format, the comparison uses a fast path (gcc creates + a jump table and converts memcpy into simple assignments on x86/x64). + + Otherwise, the comparison is delegated to the struct module, which is + 30-60x slower. */ +#define CMP_SINGLE(p, q, type) \ + do { \ + type x; \ + type y; \ + memcpy((char *)&x, p, sizeof x); \ + memcpy((char *)&y, q, sizeof y); \ + equal = (x == y); \ + } while (0) + +Py_LOCAL_INLINE(int) +unpack_cmp(const char *p, const char *q, char fmt, + struct unpacker *unpack_p, struct unpacker *unpack_q) +{ + int equal; + + switch (fmt) { + + /* signed integers and fast path for 'B' */ + case 'B': return *((unsigned char *)p) == *((unsigned char *)q); + case 'b': return *((signed char *)p) == *((signed char *)q); + case 'h': CMP_SINGLE(p, q, short); return equal; + case 'i': CMP_SINGLE(p, q, int); return equal; + case 'l': CMP_SINGLE(p, q, long); return equal; + + /* boolean */ + #ifdef HAVE_C99_BOOL + case '?': CMP_SINGLE(p, q, _Bool); return equal; + #else + case '?': CMP_SINGLE(p, q, char); return equal; + #endif + + /* unsigned integers */ + case 'H': CMP_SINGLE(p, q, unsigned short); return equal; + case 'I': CMP_SINGLE(p, q, unsigned int); return equal; + case 'L': CMP_SINGLE(p, q, unsigned long); return equal; + + /* native 64-bit */ + #ifdef HAVE_LONG_LONG + case 'q': CMP_SINGLE(p, q, PY_LONG_LONG); return equal; + case 'Q': CMP_SINGLE(p, q, unsigned PY_LONG_LONG); return equal; + #endif + + /* ssize_t and size_t */ + case 'n': CMP_SINGLE(p, q, Py_ssize_t); return equal; + case 'N': CMP_SINGLE(p, q, size_t); return equal; + + /* floats */ + /* XXX DBL_EPSILON? */ + case 'f': CMP_SINGLE(p, q, float); return equal; + case 'd': CMP_SINGLE(p, q, double); return equal; + + /* bytes object */ + case 'c': return *p == *q; + + /* pointer */ + case 'P': CMP_SINGLE(p, q, void *); return equal; + + /* use the struct module */ + case '_': + assert(unpack_p); + assert(unpack_q); + return struct_unpack_cmp(p, q, unpack_p, unpack_q); + } + + /* NOT REACHED */ + PyErr_SetString(PyExc_RuntimeError, + "memoryview: internal error in richcompare"); + return MV_COMPARE_EX; +} + +/* Base case for recursive array comparisons. Assumption: ndim == 1. */ +static int +cmp_base(const char *p, const char *q, const Py_ssize_t *shape, + const Py_ssize_t *pstrides, const Py_ssize_t *psuboffsets, + const Py_ssize_t *qstrides, const Py_ssize_t *qsuboffsets, + char fmt, struct unpacker *unpack_p, struct unpacker *unpack_q) +{ + Py_ssize_t i; + int equal; + + for (i = 0; i < shape[0]; p+=pstrides[0], q+=qstrides[0], i++) { + const char *xp = ADJUST_PTR(p, psuboffsets); + const char *xq = ADJUST_PTR(q, qsuboffsets); + equal = unpack_cmp(xp, xq, fmt, unpack_p, unpack_q); + if (equal <= 0) + return equal; + } + + return 1; +} + +/* Recursively compare two multi-dimensional arrays that have the same + logical structure. Assumption: ndim >= 1. */ +static int +cmp_rec(const char *p, const char *q, + Py_ssize_t ndim, const Py_ssize_t *shape, + const Py_ssize_t *pstrides, const Py_ssize_t *psuboffsets, + const Py_ssize_t *qstrides, const Py_ssize_t *qsuboffsets, + char fmt, struct unpacker *unpack_p, struct unpacker *unpack_q) +{ + Py_ssize_t i; + int equal; + + assert(ndim >= 1); + assert(shape != NULL); + assert(pstrides != NULL); + assert(qstrides != NULL); + + if (ndim == 1) { + return cmp_base(p, q, shape, + pstrides, psuboffsets, + qstrides, qsuboffsets, + fmt, unpack_p, unpack_q); + } + + for (i = 0; i < shape[0]; p+=pstrides[0], q+=qstrides[0], i++) { + const char *xp = ADJUST_PTR(p, psuboffsets); + const char *xq = ADJUST_PTR(q, qsuboffsets); + equal = cmp_rec(xp, xq, ndim-1, shape+1, + pstrides+1, psuboffsets ? psuboffsets+1 : NULL, + qstrides+1, qsuboffsets ? qsuboffsets+1 : NULL, + fmt, unpack_p, unpack_q); + if (equal <= 0) + return equal; + } + + return 1; } static PyObject * memory_richcompare(PyObject *v, PyObject *w, int op) { - Py_buffer vv, ww; - int equal = 0; PyObject *res; + Py_buffer wbuf, *vv; + Py_buffer *ww = NULL; + struct unpacker *unpack_v = NULL; + struct unpacker *unpack_w = NULL; + char vfmt, wfmt; + int equal = MV_COMPARE_NOT_IMPL; - vv.obj = NULL; - ww.obj = NULL; if (op != Py_EQ && op != Py_NE) - goto _notimpl; - if ((PyMemoryView_Check(v) && IS_RELEASED(v)) || - (PyMemoryView_Check(w) && IS_RELEASED(w))) { + goto result; /* Py_NotImplemented */ + + assert(PyMemoryView_Check(v)); + if (BASE_INACCESSIBLE(v)) { equal = (v == w); - goto _end; + goto result; } - if (PyObject_GetBuffer(v, &vv, PyBUF_CONTIG_RO) == -1) { - PyErr_Clear(); - goto _notimpl; + vv = VIEW_ADDR(v); + + if (PyMemoryView_Check(w)) { + if (BASE_INACCESSIBLE(w)) { + equal = (v == w); + goto result; + } + ww = VIEW_ADDR(w); + } + else { + if (PyObject_GetBuffer(w, &wbuf, PyBUF_FULL_RO) < 0) { + PyErr_Clear(); + goto result; /* Py_NotImplemented */ + } + ww = &wbuf; } - if (PyObject_GetBuffer(w, &ww, PyBUF_CONTIG_RO) == -1) { + + if (!equiv_shape(vv, ww)) { PyErr_Clear(); - goto _notimpl; + equal = 0; + goto result; } - if (vv.itemsize != ww.itemsize || vv.len != ww.len) - goto _end; + /* Use fast unpacking for identical primitive C type formats. */ + if (get_native_fmtchar(&vfmt, vv->format) < 0) + vfmt = '_'; + if (get_native_fmtchar(&wfmt, ww->format) < 0) + wfmt = '_'; + if (vfmt == '_' || wfmt == '_' || vfmt != wfmt) { + /* Use struct module unpacking. NOTE: Even for equal format strings, + memcmp() cannot be used for item comparison since it would give + incorrect results in the case of NaNs or uninitialized padding + bytes. */ + vfmt = '_'; + unpack_v = struct_get_unpacker(vv->format, vv->itemsize); + if (unpack_v == NULL) { + equal = fix_struct_error_int(); + goto result; + } + unpack_w = struct_get_unpacker(ww->format, ww->itemsize); + if (unpack_w == NULL) { + equal = fix_struct_error_int(); + goto result; + } + } - equal = !memcmp(vv.buf, ww.buf, vv.len); + if (vv->ndim == 0) { + equal = unpack_cmp(vv->buf, ww->buf, + vfmt, unpack_v, unpack_w); + } + else if (vv->ndim == 1) { + equal = cmp_base(vv->buf, ww->buf, vv->shape, + vv->strides, vv->suboffsets, + ww->strides, ww->suboffsets, + vfmt, unpack_v, unpack_w); + } + else { + equal = cmp_rec(vv->buf, ww->buf, vv->ndim, vv->shape, + vv->strides, vv->suboffsets, + ww->strides, ww->suboffsets, + vfmt, unpack_v, unpack_w); + } -_end: - PyBuffer_Release(&vv); - PyBuffer_Release(&ww); - if ((equal && op == Py_EQ) || (!equal && op == Py_NE)) +result: + if (equal < 0) { + if (equal == MV_COMPARE_NOT_IMPL) + res = Py_NotImplemented; + else /* exception */ + res = NULL; + } + else if ((equal && op == Py_EQ) || (!equal && op == Py_NE)) res = Py_True; else res = Py_False; - Py_INCREF(res); + + if (ww == &wbuf) + PyBuffer_Release(ww); + + unpacker_free(unpack_v); + unpacker_free(unpack_w); + + Py_XINCREF(res); return res; +} + +/**************************************************************************/ +/* Hash */ +/**************************************************************************/ + +static Py_hash_t +memory_hash(PyMemoryViewObject *self) +{ + if (self->hash == -1) { + Py_buffer *view = &self->view; + char *mem = view->buf; + + CHECK_RELEASED_INT(self); + + if (!view->readonly) { + PyErr_SetString(PyExc_ValueError, + "cannot hash writable memoryview object"); + return -1; + } + if (view->obj != NULL && PyObject_Hash(view->obj) == -1) { + /* Keep the original error message */ + return -1; + } + + if (!MV_C_CONTIGUOUS(self->flags)) { + mem = PyMem_Malloc(view->len); + if (mem == NULL) { + PyErr_NoMemory(); + return -1; + } + if (buffer_to_contiguous(mem, view, 'C') < 0) { + PyMem_Free(mem); + return -1; + } + } + + /* Can't fail */ + self->hash = _Py_HashBytes((unsigned char *)mem, view->len); + + if (mem != view->buf) + PyMem_Free(mem); + } -_notimpl: - PyBuffer_Release(&vv); - PyBuffer_Release(&ww); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + return self->hash; } -static int -memory_traverse(PyMemoryViewObject *self, visitproc visit, void *arg) +/**************************************************************************/ +/* getters */ +/**************************************************************************/ + +static PyObject * +_IntTupleFromSsizet(int len, Py_ssize_t *vals) { - if (self->view.obj != NULL) - Py_VISIT(self->view.obj); - return 0; + int i; + PyObject *o; + PyObject *intTuple; + + if (vals == NULL) + return PyTuple_New(0); + + intTuple = PyTuple_New(len); + if (!intTuple) + return NULL; + for (i=0; i<len; i++) { + o = PyLong_FromSsize_t(vals[i]); + if (!o) { + Py_DECREF(intTuple); + return NULL; + } + PyTuple_SET_ITEM(intTuple, i, o); + } + return intTuple; } -static int -memory_clear(PyMemoryViewObject *self) +static PyObject * +memory_obj_get(PyMemoryViewObject *self) { - PyBuffer_Release(&self->view); - return 0; + Py_buffer *view = &self->view; + + CHECK_RELEASED(self); + if (view->obj == NULL) { + Py_RETURN_NONE; + } + Py_INCREF(view->obj); + return view->obj; } +static PyObject * +memory_nbytes_get(PyMemoryViewObject *self) +{ + CHECK_RELEASED(self); + return PyLong_FromSsize_t(self->view.len); +} -/* As mapping */ -static PyMappingMethods memory_as_mapping = { - (lenfunc)memory_length, /* mp_length */ - (binaryfunc)memory_subscript, /* mp_subscript */ - (objobjargproc)memory_ass_sub, /* mp_ass_subscript */ -}; +static PyObject * +memory_format_get(PyMemoryViewObject *self) +{ + CHECK_RELEASED(self); + return PyUnicode_FromString(self->view.format); +} -static PySequenceMethods memory_as_sequence = { - 0, /* sq_length */ - 0, /* sq_concat */ - 0, /* sq_repeat */ - (ssizeargfunc)memory_item, /* sq_item */ +static PyObject * +memory_itemsize_get(PyMemoryViewObject *self) +{ + CHECK_RELEASED(self); + return PyLong_FromSsize_t(self->view.itemsize); +} + +static PyObject * +memory_shape_get(PyMemoryViewObject *self) +{ + CHECK_RELEASED(self); + return _IntTupleFromSsizet(self->view.ndim, self->view.shape); +} + +static PyObject * +memory_strides_get(PyMemoryViewObject *self) +{ + CHECK_RELEASED(self); + return _IntTupleFromSsizet(self->view.ndim, self->view.strides); +} + +static PyObject * +memory_suboffsets_get(PyMemoryViewObject *self) +{ + CHECK_RELEASED(self); + return _IntTupleFromSsizet(self->view.ndim, self->view.suboffsets); +} + +static PyObject * +memory_readonly_get(PyMemoryViewObject *self) +{ + CHECK_RELEASED(self); + return PyBool_FromLong(self->view.readonly); +} + +static PyObject * +memory_ndim_get(PyMemoryViewObject *self) +{ + CHECK_RELEASED(self); + return PyLong_FromLong(self->view.ndim); +} + +static PyObject * +memory_c_contiguous(PyMemoryViewObject *self, PyObject *dummy) +{ + CHECK_RELEASED(self); + return PyBool_FromLong(MV_C_CONTIGUOUS(self->flags)); +} + +static PyObject * +memory_f_contiguous(PyMemoryViewObject *self, PyObject *dummy) +{ + CHECK_RELEASED(self); + return PyBool_FromLong(MV_F_CONTIGUOUS(self->flags)); +} + +static PyObject * +memory_contiguous(PyMemoryViewObject *self, PyObject *dummy) +{ + CHECK_RELEASED(self); + return PyBool_FromLong(MV_ANY_CONTIGUOUS(self->flags)); +} + +PyDoc_STRVAR(memory_obj_doc, + "The underlying object of the memoryview."); +PyDoc_STRVAR(memory_nbytes_doc, + "The amount of space in bytes that the array would use in\n" + " a contiguous representation."); +PyDoc_STRVAR(memory_readonly_doc, + "A bool indicating whether the memory is read only."); +PyDoc_STRVAR(memory_itemsize_doc, + "The size in bytes of each element of the memoryview."); +PyDoc_STRVAR(memory_format_doc, + "A string containing the format (in struct module style)\n" + " for each element in the view."); +PyDoc_STRVAR(memory_ndim_doc, + "An integer indicating how many dimensions of a multi-dimensional\n" + " array the memory represents."); +PyDoc_STRVAR(memory_shape_doc, + "A tuple of ndim integers giving the shape of the memory\n" + " as an N-dimensional array."); +PyDoc_STRVAR(memory_strides_doc, + "A tuple of ndim integers giving the size in bytes to access\n" + " each element for each dimension of the array."); +PyDoc_STRVAR(memory_suboffsets_doc, + "A tuple of integers used internally for PIL-style arrays."); +PyDoc_STRVAR(memory_c_contiguous_doc, + "A bool indicating whether the memory is C contiguous."); +PyDoc_STRVAR(memory_f_contiguous_doc, + "A bool indicating whether the memory is Fortran contiguous."); +PyDoc_STRVAR(memory_contiguous_doc, + "A bool indicating whether the memory is contiguous."); + +static PyGetSetDef memory_getsetlist[] = { + {"obj", (getter)memory_obj_get, NULL, memory_obj_doc}, + {"nbytes", (getter)memory_nbytes_get, NULL, memory_nbytes_doc}, + {"readonly", (getter)memory_readonly_get, NULL, memory_readonly_doc}, + {"itemsize", (getter)memory_itemsize_get, NULL, memory_itemsize_doc}, + {"format", (getter)memory_format_get, NULL, memory_format_doc}, + {"ndim", (getter)memory_ndim_get, NULL, memory_ndim_doc}, + {"shape", (getter)memory_shape_get, NULL, memory_shape_doc}, + {"strides", (getter)memory_strides_get, NULL, memory_strides_doc}, + {"suboffsets", (getter)memory_suboffsets_get, NULL, memory_suboffsets_doc}, + {"c_contiguous", (getter)memory_c_contiguous, NULL, memory_c_contiguous_doc}, + {"f_contiguous", (getter)memory_f_contiguous, NULL, memory_f_contiguous_doc}, + {"contiguous", (getter)memory_contiguous, NULL, memory_contiguous_doc}, + {NULL, NULL, NULL, NULL}, }; -/* Buffer methods */ +PyDoc_STRVAR(memory_release_doc, +"M.release() -> None\n\ +\n\ +Release the underlying buffer exposed by the memoryview object."); +PyDoc_STRVAR(memory_tobytes_doc, +"M.tobytes() -> bytes\n\ +\n\ +Return the data in the buffer as a byte string."); +PyDoc_STRVAR(memory_tolist_doc, +"M.tolist() -> list\n\ +\n\ +Return the data in the buffer as a list of elements."); +PyDoc_STRVAR(memory_cast_doc, +"M.cast(format[, shape]) -> memoryview\n\ +\n\ +Cast a memoryview to a new format or shape."); -static PyBufferProcs memory_as_buffer = { - (getbufferproc)memory_getbuf, /* bf_getbuffer */ - (releasebufferproc)memory_releasebuf, /* bf_releasebuffer */ +static PyMethodDef memory_methods[] = { + {"release", (PyCFunction)memory_release, METH_NOARGS, memory_release_doc}, + {"tobytes", (PyCFunction)memory_tobytes, METH_NOARGS, memory_tobytes_doc}, + {"tolist", (PyCFunction)memory_tolist, METH_NOARGS, memory_tolist_doc}, + {"cast", (PyCFunction)memory_cast, METH_VARARGS|METH_KEYWORDS, memory_cast_doc}, + {"__enter__", memory_enter, METH_NOARGS, NULL}, + {"__exit__", memory_exit, METH_VARARGS, NULL}, + {NULL, NULL} }; PyTypeObject PyMemoryView_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "memoryview", - sizeof(PyMemoryViewObject), - 0, + "memoryview", /* tp_name */ + offsetof(PyMemoryViewObject, ob_array), /* tp_basicsize */ + sizeof(Py_ssize_t), /* tp_itemsize */ (destructor)memory_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ @@ -862,7 +2949,7 @@ PyTypeObject PyMemoryView_Type = { 0, /* tp_as_number */ &memory_as_sequence, /* tp_as_sequence */ &memory_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ + (hashfunc)memory_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -873,7 +2960,7 @@ PyTypeObject PyMemoryView_Type = { (traverseproc)memory_traverse, /* tp_traverse */ (inquiry)memory_clear, /* tp_clear */ memory_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ + offsetof(PyMemoryViewObject, weakreflist),/* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ memory_methods, /* tp_methods */ diff --git a/Objects/methodobject.c b/Objects/methodobject.c index f1a9f4b..1d143f9 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -44,7 +44,7 @@ PyCFunction_GetFunction(PyObject *op) PyErr_BadInternalCall(); return NULL; } - return ((PyCFunctionObject *)op) -> m_ml -> ml_meth; + return PyCFunction_GET_FUNCTION(op); } PyObject * @@ -54,7 +54,7 @@ PyCFunction_GetSelf(PyObject *op) PyErr_BadInternalCall(); return NULL; } - return ((PyCFunctionObject *)op) -> m_self; + return PyCFunction_GET_SELF(op); } int @@ -64,7 +64,7 @@ PyCFunction_GetFlags(PyObject *op) PyErr_BadInternalCall(); return -1; } - return ((PyCFunctionObject *)op) -> m_ml -> ml_flags; + return PyCFunction_GET_FLAGS(op); } PyObject * @@ -151,6 +151,41 @@ meth_get__name__(PyCFunctionObject *m, void *closure) return PyUnicode_FromString(m->m_ml->ml_name); } +static PyObject * +meth_get__qualname__(PyCFunctionObject *m, void *closure) +{ + /* If __self__ is a module or NULL, return m.__name__ + (e.g. len.__qualname__ == 'len') + + If __self__ is a type, return m.__self__.__qualname__ + '.' + m.__name__ + (e.g. dict.fromkeys.__qualname__ == 'dict.fromkeys') + + Otherwise return type(m.__self__).__qualname__ + '.' + m.__name__ + (e.g. [].append.__qualname__ == 'list.append') */ + PyObject *type, *type_qualname, *res; + _Py_IDENTIFIER(__qualname__); + + if (m->m_self == NULL || PyModule_Check(m->m_self)) + return PyUnicode_FromString(m->m_ml->ml_name); + + type = PyType_Check(m->m_self) ? m->m_self : (PyObject*)Py_TYPE(m->m_self); + + type_qualname = _PyObject_GetAttrId(type, &PyId___qualname__); + if (type_qualname == NULL) + return NULL; + + if (!PyUnicode_Check(type_qualname)) { + PyErr_SetString(PyExc_TypeError, "<method>.__class__." + "__qualname__ is not a unicode object"); + Py_XDECREF(type_qualname); + return NULL; + } + + res = PyUnicode_FromFormat("%S.%s", type_qualname, m->m_ml->ml_name); + Py_DECREF(type_qualname); + return res; +} + static int meth_traverse(PyCFunctionObject *m, visitproc visit, void *arg) { @@ -164,7 +199,7 @@ meth_get__self__(PyCFunctionObject *m, void *closure) { PyObject *self; - self = m->m_self; + self = PyCFunction_GET_SELF(m); if (self == NULL) self = Py_None; Py_INCREF(self); @@ -174,6 +209,7 @@ meth_get__self__(PyCFunctionObject *m, void *closure) static PyGetSetDef meth_getsets [] = { {"__doc__", (getter)meth_get__doc__, NULL, NULL}, {"__name__", (getter)meth_get__name__, NULL, NULL}, + {"__qualname__", (getter)meth_get__qualname__, NULL, NULL}, {"__self__", (getter)meth_get__self__, NULL, NULL}, {0} }; @@ -208,8 +244,7 @@ meth_richcompare(PyObject *self, PyObject *other, int op) !PyCFunction_Check(self) || !PyCFunction_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } a = (PyCFunctionObject *)self; b = (PyCFunctionObject *)other; @@ -303,6 +338,15 @@ PyCFunction_Fini(void) (void)PyCFunction_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyCFunction_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyCFunction", + numfree, sizeof(PyCFunction)); +} + /* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(), but it's part of the API so we need to keep a function around that existing C extensions can call. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 533db46..2f2bd36 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -27,36 +27,45 @@ static PyTypeObject moduledef_type = { PyObject * -PyModule_New(const char *name) +PyModule_NewObject(PyObject *name) { PyModuleObject *m; - PyObject *nameobj; m = PyObject_GC_New(PyModuleObject, &PyModule_Type); if (m == NULL) return NULL; m->md_def = NULL; m->md_state = NULL; - nameobj = PyUnicode_FromString(name); m->md_dict = PyDict_New(); - if (m->md_dict == NULL || nameobj == NULL) + if (m->md_dict == NULL) goto fail; - if (PyDict_SetItemString(m->md_dict, "__name__", nameobj) != 0) + if (PyDict_SetItemString(m->md_dict, "__name__", name) != 0) goto fail; if (PyDict_SetItemString(m->md_dict, "__doc__", Py_None) != 0) goto fail; if (PyDict_SetItemString(m->md_dict, "__package__", Py_None) != 0) goto fail; - Py_DECREF(nameobj); PyObject_GC_Track(m); return (PyObject *)m; fail: - Py_XDECREF(nameobj); Py_DECREF(m); return NULL; } PyObject * +PyModule_New(const char *name) +{ + PyObject *nameobj, *module; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + module = PyModule_NewObject(nameobj); + Py_DECREF(nameobj); + return module; +} + + +PyObject * PyModule_Create2(struct PyModuleDef* module, int module_api_version) { PyObject *d, *v, *n; @@ -175,24 +184,35 @@ PyModule_GetDict(PyObject *m) return d; } -const char * -PyModule_GetName(PyObject *m) +PyObject* +PyModule_GetNameObject(PyObject *m) { PyObject *d; - PyObject *nameobj; + PyObject *name; if (!PyModule_Check(m)) { PyErr_BadArgument(); return NULL; } d = ((PyModuleObject *)m)->md_dict; if (d == NULL || - (nameobj = PyDict_GetItemString(d, "__name__")) == NULL || - !PyUnicode_Check(nameobj)) + (name = PyDict_GetItemString(d, "__name__")) == NULL || + !PyUnicode_Check(name)) { PyErr_SetString(PyExc_SystemError, "nameless module"); return NULL; } - return _PyUnicode_AsString(nameobj); + Py_INCREF(name); + return name; +} + +const char * +PyModule_GetName(PyObject *m) +{ + PyObject *name = PyModule_GetNameObject(m); + if (name == NULL) + return NULL; + Py_DECREF(name); /* module dict has still a reference */ + return _PyUnicode_AsString(name); } PyObject* @@ -225,7 +245,7 @@ PyModule_GetFilename(PyObject *m) if (fileobj == NULL) return NULL; utf8 = _PyUnicode_AsString(fileobj); - Py_DECREF(fileobj); + Py_DECREF(fileobj); /* module dict has still a reference */ return utf8; } @@ -271,8 +291,8 @@ _PyModule_Clear(PyObject *m) pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { if (value != Py_None && PyUnicode_Check(key)) { - Py_UNICODE *u = PyUnicode_AS_UNICODE(key); - if (u[0] == '_' && u[1] != '_') { + if (PyUnicode_READ_CHAR(key, 0) == '_' && + PyUnicode_READ_CHAR(key, 1) != '_') { if (Py_VerboseFlag > 1) { const char *s = _PyUnicode_AsString(key); if (s != NULL) @@ -289,9 +309,8 @@ _PyModule_Clear(PyObject *m) pos = 0; while (PyDict_Next(d, &pos, &key, &value)) { if (value != Py_None && PyUnicode_Check(key)) { - Py_UNICODE *u = PyUnicode_AS_UNICODE(key); - if (u[0] != '_' - || PyUnicode_CompareWithASCIIString(key, "__builtins__") != 0) + if (PyUnicode_READ_CHAR(key, 0) != '_' || + PyUnicode_CompareWithASCIIString(key, "__builtins__") != 0) { if (Py_VerboseFlag > 1) { const char *s = _PyUnicode_AsString(key); @@ -353,21 +372,54 @@ module_dealloc(PyModuleObject *m) static PyObject * module_repr(PyModuleObject *m) { - const char *name; - PyObject *filename, *repr; + PyObject *name, *filename, *repr, *loader = NULL; - name = PyModule_GetName((PyObject *)m); + /* See if the module has an __loader__. If it does, give the loader the + * first shot at producing a repr for the module. + */ + if (m->md_dict != NULL) { + loader = PyDict_GetItemString(m->md_dict, "__loader__"); + } + if (loader != NULL) { + repr = PyObject_CallMethod(loader, "module_repr", "(O)", + (PyObject *)m, NULL); + if (repr == NULL) { + PyErr_Clear(); + } + else { + return repr; + } + } + /* __loader__.module_repr(m) did not provide us with a repr. Next, see if + * the module has an __file__. If it doesn't then use repr(__loader__) if + * it exists, otherwise, just use module.__name__. + */ + name = PyModule_GetNameObject((PyObject *)m); if (name == NULL) { PyErr_Clear(); - name = "?"; + name = PyUnicode_FromStringAndSize("?", 1); + if (name == NULL) + return NULL; } filename = PyModule_GetFilenameObject((PyObject *)m); if (filename == NULL) { PyErr_Clear(); - return PyUnicode_FromFormat("<module '%s' (built-in)>", name); + /* There's no m.__file__, so if there was an __loader__, use that in + * the repr, otherwise, the only thing you can use is m.__name__ + */ + if (loader == NULL) { + repr = PyUnicode_FromFormat("<module %R>", name); + } + else { + repr = PyUnicode_FromFormat("<module %R (%R)>", name, loader); + } + } + /* Finally, use m.__file__ */ + else { + repr = PyUnicode_FromFormat("<module %R from %R>", name, filename); + Py_DECREF(filename); } - repr = PyUnicode_FromFormat("<module '%s' from '%U'>", name, filename); - Py_DECREF(filename); + Py_DECREF(name); return repr; } @@ -395,6 +447,35 @@ module_clear(PyModuleObject *m) return 0; } +static PyObject * +module_dir(PyObject *self, PyObject *args) +{ + _Py_IDENTIFIER(__dict__); + PyObject *result = NULL; + PyObject *dict = _PyObject_GetAttrId(self, &PyId___dict__); + + if (dict != NULL) { + if (PyDict_Check(dict)) + result = PyDict_Keys(dict); + else { + const char *name = PyModule_GetName(self); + if (name) + PyErr_Format(PyExc_TypeError, + "%.200s.__dict__ is not a dictionary", + name); + } + } + + Py_XDECREF(dict); + return result; +} + +static PyMethodDef module_methods[] = { + {"__dir__", module_dir, METH_NOARGS, + PyDoc_STR("__dir__() -> list\nspecialized dir() implementation")}, + {0} +}; + PyDoc_STRVAR(module_doc, "module(name[, doc])\n\ @@ -431,7 +512,7 @@ PyTypeObject PyModule_Type = { 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ - 0, /* tp_methods */ + module_methods, /* tp_methods */ module_members, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c new file mode 100644 index 0000000..ff278d3 --- /dev/null +++ b/Objects/namespaceobject.c @@ -0,0 +1,225 @@ +/* namespace object implementation */ + +#include "Python.h" +#include "structmember.h" + + +typedef struct { + PyObject_HEAD + PyObject *ns_dict; +} _PyNamespaceObject; + + +static PyMemberDef namespace_members[] = { + {"__dict__", T_OBJECT, offsetof(_PyNamespaceObject, ns_dict), READONLY}, + {NULL} +}; + + +/* Methods */ + +static PyObject * +namespace_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *self; + + assert(type != NULL && type->tp_alloc != NULL); + self = type->tp_alloc(type, 0); + if (self != NULL) { + _PyNamespaceObject *ns = (_PyNamespaceObject *)self; + ns->ns_dict = PyDict_New(); + if (ns->ns_dict == NULL) { + Py_DECREF(ns); + return NULL; + } + } + return self; +} + + +static int +namespace_init(_PyNamespaceObject *ns, PyObject *args, PyObject *kwds) +{ + /* ignore args if it's NULL or empty */ + if (args != NULL) { + Py_ssize_t argcount = PyObject_Size(args); + if (argcount < 0) + return argcount; + else if (argcount > 0) { + PyErr_Format(PyExc_TypeError, "no positional arguments expected"); + return -1; + } + } + if (kwds == NULL) + return 0; + return PyDict_Update(ns->ns_dict, kwds); +} + + +static void +namespace_dealloc(_PyNamespaceObject *ns) +{ + PyObject_GC_UnTrack(ns); + Py_CLEAR(ns->ns_dict); + Py_TYPE(ns)->tp_free((PyObject *)ns); +} + + +static PyObject * +namespace_repr(_PyNamespaceObject *ns) +{ + int i, loop_error = 0; + PyObject *pairs = NULL, *d = NULL, *keys = NULL, *keys_iter = NULL; + PyObject *key; + PyObject *separator, *pairsrepr, *repr = NULL; + + i = Py_ReprEnter((PyObject *)ns); + if (i != 0) { + return i > 0 ? PyUnicode_FromString("namespace(...)") : NULL; + } + + pairs = PyList_New(0); + if (pairs == NULL) + goto error; + + d = ((_PyNamespaceObject *)ns)->ns_dict; + assert(d != NULL); + Py_INCREF(d); + + keys = PyDict_Keys(d); + if (keys == NULL) + goto error; + if (PyList_Sort(keys) != 0) + goto error; + + keys_iter = PyObject_GetIter(keys); + if (keys_iter == NULL) + goto error; + + while ((key = PyIter_Next(keys_iter)) != NULL) { + if (PyUnicode_Check(key) && PyUnicode_GET_SIZE(key) > 0) { + PyObject *value, *item; + + value = PyDict_GetItem(d, key); + assert(value != NULL); + + item = PyUnicode_FromFormat("%S=%R", key, value); + if (item == NULL) { + loop_error = 1; + } + else { + loop_error = PyList_Append(pairs, item); + Py_DECREF(item); + } + } + + Py_DECREF(key); + if (loop_error) + goto error; + } + + separator = PyUnicode_FromString(", "); + if (separator == NULL) + goto error; + + pairsrepr = PyUnicode_Join(separator, pairs); + Py_DECREF(separator); + if (pairsrepr == NULL) + goto error; + + repr = PyUnicode_FromFormat("%s(%S)", + ((PyObject *)ns)->ob_type->tp_name, pairsrepr); + Py_DECREF(pairsrepr); + +error: + Py_XDECREF(pairs); + Py_XDECREF(d); + Py_XDECREF(keys); + Py_XDECREF(keys_iter); + Py_ReprLeave((PyObject *)ns); + + return repr; +} + + +static int +namespace_traverse(_PyNamespaceObject *ns, visitproc visit, void *arg) +{ + Py_VISIT(ns->ns_dict); + return 0; +} + + +static int +namespace_clear(_PyNamespaceObject *ns) +{ + Py_CLEAR(ns->ns_dict); + return 0; +} + + +PyDoc_STRVAR(namespace_doc, +"A simple attribute-based namespace.\n\ +\n\ +namespace(**kwargs)"); + +PyTypeObject _PyNamespace_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "namespace", /* tp_name */ + sizeof(_PyNamespaceObject), /* tp_size */ + 0, /* tp_itemsize */ + (destructor)namespace_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)namespace_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + PyObject_GenericSetAttr, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE, /* tp_flags */ + namespace_doc, /* tp_doc */ + (traverseproc)namespace_traverse, /* tp_traverse */ + (inquiry)namespace_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + namespace_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + offsetof(_PyNamespaceObject, ns_dict), /* tp_dictoffset */ + (initproc)namespace_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + (newfunc)namespace_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + + +PyObject * +_PyNamespace_New(PyObject *kwds) +{ + PyObject *ns = namespace_new(&_PyNamespace_Type, NULL, NULL); + if (ns == NULL) + return NULL; + + if (kwds == NULL) + return ns; + if (PyDict_Update(((_PyNamespaceObject *)ns)->ns_dict, kwds) != 0) { + Py_DECREF(ns); + return NULL; + } + + return (PyObject *)ns; +} diff --git a/Objects/object.c b/Objects/object.c index 28bb9c1..949e7dc 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1,5 +1,5 @@ -/* Generic object operations; and implementation of None (NoObject) */ +/* Generic object operations; and implementation of None */ #include "Python.h" #include "frameobject.h" @@ -29,8 +29,6 @@ _Py_GetRefTotal(void) } #endif /* Py_REF_DEBUG */ -int Py_DivisionWarningFlag; - /* Object allocation routines used by NEWOBJ and NEWVAROBJ macros. These are used by the individual routines for object creation. Do not call them otherwise, they do not initialize the object! */ @@ -297,9 +295,7 @@ PyObject_Print(PyObject *op, FILE *fp, int flags) } else if (PyUnicode_Check(s)) { PyObject *t; - t = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(s), - PyUnicode_GET_SIZE(s), - "backslashreplace"); + t = PyUnicode_AsEncodedString(s, "utf-8", "backslashreplace"); if (t == NULL) ret = 0; else { @@ -382,13 +378,19 @@ PyObject_Repr(PyObject *v) return PyUnicode_FromFormat("<%s object at %p>", v->ob_type->tp_name, v); res = (*v->ob_type->tp_repr)(v); - if (res != NULL && !PyUnicode_Check(res)) { + if (res == NULL) + return NULL; + if (!PyUnicode_Check(res)) { PyErr_Format(PyExc_TypeError, "__repr__ returned non-string (type %.200s)", res->ob_type->tp_name); Py_DECREF(res); return NULL; } +#ifndef Py_DEBUG + if (PyUnicode_READY(res) < 0) + return NULL; +#endif return res; } @@ -407,6 +409,10 @@ PyObject_Str(PyObject *v) if (v == NULL) return PyUnicode_FromString("<NULL>"); if (PyUnicode_CheckExact(v)) { +#ifndef Py_DEBUG + if (PyUnicode_READY(v) < 0) + return NULL; +#endif Py_INCREF(v); return v; } @@ -428,6 +434,11 @@ PyObject_Str(PyObject *v) Py_DECREF(res); return NULL; } +#ifndef Py_DEBUG + if (PyUnicode_READY(res) < 0) + return NULL; +#endif + assert(_PyUnicode_CheckConsistency(res, 1)); return res; } @@ -441,11 +452,7 @@ PyObject_ASCII(PyObject *v) return NULL; /* repr is guaranteed to be a PyUnicode object by PyObject_Repr */ - ascii = PyUnicode_EncodeASCII( - PyUnicode_AS_UNICODE(repr), - PyUnicode_GET_SIZE(repr), - "backslashreplace"); - + ascii = _PyUnicode_AsASCIIString(repr, "backslashreplace"); Py_DECREF(repr); if (ascii == NULL) return NULL; @@ -463,7 +470,7 @@ PyObject * PyObject_Bytes(PyObject *v) { PyObject *result, *func; - static PyObject *bytesstring = NULL; + _Py_IDENTIFIER(__bytes__); if (v == NULL) return PyBytes_FromString("<NULL>"); @@ -473,7 +480,7 @@ PyObject_Bytes(PyObject *v) return v; } - func = _PyObject_LookupSpecial(v, "__bytes__", &bytesstring); + func = _PyObject_LookupSpecial(v, &PyId___bytes__); if (func != NULL) { result = PyObject_CallFunctionObjArgs(func, NULL); Py_DECREF(func); @@ -747,6 +754,33 @@ _Py_HashPointer(void *p) } Py_hash_t +_Py_HashBytes(unsigned char *p, Py_ssize_t len) +{ + Py_uhash_t x; + Py_ssize_t i; + + /* + We make the hash of the empty string be 0, rather than using + (prefix ^ suffix), since this slightly obfuscates the hash secret + */ +#ifdef Py_DEBUG + assert(_Py_HashSecret_Initialized); +#endif + if (len == 0) { + return 0; + } + x = (Py_uhash_t) _Py_HashSecret.prefix; + x ^= (Py_uhash_t) *p << 7; + for (i = 0; i < len; i++) + x = (_PyHASH_MULTIPLIER * x) ^ (Py_uhash_t) *p++; + x ^= (Py_uhash_t) len; + x ^= (Py_uhash_t) _Py_HashSecret.suffix; + if (x == -1) + x = -2; + return x; +} + +Py_hash_t PyObject_HashNotImplemented(PyObject *v) { PyErr_Format(PyExc_TypeError, "unhashable type: '%.200s'", @@ -788,7 +822,7 @@ PyObject_GetAttrString(PyObject *v, const char *name) if (w == NULL) return NULL; res = PyObject_GetAttr(v, w); - Py_XDECREF(w); + Py_DECREF(w); return res; } @@ -820,6 +854,62 @@ PyObject_SetAttrString(PyObject *v, const char *name, PyObject *w) return res; } +int +_PyObject_IsAbstract(PyObject *obj) +{ + int res; + PyObject* isabstract; + _Py_IDENTIFIER(__isabstractmethod__); + + if (obj == NULL) + return 0; + + isabstract = _PyObject_GetAttrId(obj, &PyId___isabstractmethod__); + if (isabstract == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + res = PyObject_IsTrue(isabstract); + Py_DECREF(isabstract); + return res; +} + +PyObject * +_PyObject_GetAttrId(PyObject *v, _Py_Identifier *name) +{ + PyObject *result; + PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ + if (!oname) + return NULL; + result = PyObject_GetAttr(v, oname); + return result; +} + +int +_PyObject_HasAttrId(PyObject *v, _Py_Identifier *name) +{ + int result; + PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ + if (!oname) + return -1; + result = PyObject_HasAttr(v, oname); + return result; +} + +int +_PyObject_SetAttrId(PyObject *v, _Py_Identifier *name, PyObject *w) +{ + int result; + PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ + if (!oname) + return -1; + result = PyObject_SetAttr(v, oname, w); + return result; +} + PyObject * PyObject_GetAttr(PyObject *v, PyObject *name) { @@ -938,6 +1028,19 @@ PyObject_SelfIter(PyObject *obj) return obj; } +/* Convenience function to get a builtin from its name */ +PyObject * +_PyObject_GetBuiltin(const char *name) +{ + PyObject *mod, *attr; + mod = PyImport_ImportModule("builtins"); + if (mod == NULL) + return NULL; + attr = PyObject_GetAttrString(mod, name); + Py_DECREF(mod); + return attr; +} + /* Helper used when the __next__ method is removed from a type: tp_iternext is never NULL and can be safely called without checking on every iteration. @@ -986,7 +1089,6 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, PyObject *dict) f = descr->ob_type->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { res = f(descr, obj, (PyObject *)obj->ob_type); - Py_DECREF(descr); goto done; } } @@ -1017,7 +1119,6 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, PyObject *dict) res = PyDict_GetItem(dict, name); if (res != NULL) { Py_INCREF(res); - Py_XDECREF(descr); Py_DECREF(dict); goto done; } @@ -1026,13 +1127,12 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, PyObject *dict) if (f != NULL) { res = f(descr, obj, (PyObject *)Py_TYPE(obj)); - Py_DECREF(descr); goto done; } if (descr != NULL) { res = descr; - /* descr was already increfed above */ + descr = NULL; goto done; } @@ -1040,6 +1140,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, PyObject *dict) "'%.50s' object has no attribute '%U'", tp->tp_name, name); done: + Py_XDECREF(descr); Py_DECREF(name); return res; } @@ -1066,15 +1167,15 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, name->ob_type->tp_name); return -1; } - else - Py_INCREF(name); - if (tp->tp_dict == NULL) { - if (PyType_Ready(tp) < 0) - goto done; - } + if (tp->tp_dict == NULL && PyType_Ready(tp) < 0) + return -1; + + Py_INCREF(name); descr = _PyType_Lookup(tp, name); + Py_XINCREF(descr); + f = NULL; if (descr != NULL) { f = descr->ob_type->tp_descr_set; @@ -1087,13 +1188,10 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, if (dict == NULL) { dictptr = _PyObject_GetDictPtr(obj); if (dictptr != NULL) { - dict = *dictptr; - if (dict == NULL && value != NULL) { - dict = PyDict_New(); - if (dict == NULL) - goto done; - *dictptr = dict; - } + res = _PyObjectDict_SetItem(Py_TYPE(obj), dictptr, name, value); + if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) + PyErr_SetObject(PyExc_AttributeError, name); + goto done; } } if (dict != NULL) { @@ -1102,9 +1200,9 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, res = PyDict_DelItem(dict, name); else res = PyDict_SetItem(dict, name, value); + Py_DECREF(dict); if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) PyErr_SetObject(PyExc_AttributeError, name); - Py_DECREF(dict); goto done; } @@ -1124,6 +1222,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, "'%.50s' object attribute '%U' is read-only", tp->tp_name, name); done: + Py_XDECREF(descr); Py_DECREF(name); return res; } @@ -1134,6 +1233,32 @@ PyObject_GenericSetAttr(PyObject *obj, PyObject *name, PyObject *value) return _PyObject_GenericSetAttrWithDict(obj, name, value, NULL); } +int +PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) +{ + PyObject *dict, **dictptr = _PyObject_GetDictPtr(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return -1; + } + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete __dict__"); + return -1; + } + if (!PyDict_Check(value)) { + PyErr_Format(PyExc_TypeError, + "__dict__ must be set to a dictionary, " + "not a '%.200s'", Py_TYPE(value)->tp_name); + return -1; + } + dict = *dictptr; + Py_XINCREF(value); + *dictptr = value; + Py_XDECREF(dict); + return 0; +} + /* Test a value used as condition, e.g., in a for or if statement. Return -1 if an error occurred */ @@ -1186,66 +1311,6 @@ PyCallable_Check(PyObject *x) return x->ob_type->tp_call != NULL; } -/* ------------------------- PyObject_Dir() helpers ------------------------- */ - -/* Helper for PyObject_Dir. - Merge the __dict__ of aclass into dict, and recursively also all - the __dict__s of aclass's base classes. The order of merging isn't - defined, as it's expected that only the final set of dict keys is - interesting. - Return 0 on success, -1 on error. -*/ - -static int -merge_class_dict(PyObject* dict, PyObject* aclass) -{ - PyObject *classdict; - PyObject *bases; - - assert(PyDict_Check(dict)); - assert(aclass); - - /* Merge in the type's dict (if any). */ - classdict = PyObject_GetAttrString(aclass, "__dict__"); - if (classdict == NULL) - PyErr_Clear(); - else { - int status = PyDict_Update(dict, classdict); - Py_DECREF(classdict); - if (status < 0) - return -1; - } - - /* Recursively merge in the base types' (if any) dicts. */ - bases = PyObject_GetAttrString(aclass, "__bases__"); - if (bases == NULL) - PyErr_Clear(); - else { - /* We have no guarantee that bases is a real tuple */ - Py_ssize_t i, n; - n = PySequence_Size(bases); /* This better be right */ - if (n < 0) - PyErr_Clear(); - else { - for (i = 0; i < n; i++) { - int status; - PyObject *base = PySequence_GetItem(bases, i); - if (base == NULL) { - Py_DECREF(bases); - return -1; - } - status = merge_class_dict(dict, base); - Py_DECREF(base); - if (status < 0) { - Py_DECREF(bases); - return -1; - } - } - } - Py_DECREF(bases); - } - return 0; -} /* Helper for PyObject_Dir without arguments: returns the local scope. */ static PyObject * @@ -1269,140 +1334,43 @@ _dir_locals(void) Py_DECREF(names); return NULL; } + if (PyList_Sort(names)) { + Py_DECREF(names); + return NULL; + } /* the locals don't need to be DECREF'd */ return names; } -/* Helper for PyObject_Dir of type objects: returns __dict__ and __bases__. - We deliberately don't suck up its __class__, as methods belonging to the - metaclass would probably be more confusing than helpful. -*/ -static PyObject * -_specialized_dir_type(PyObject *obj) -{ - PyObject *result = NULL; - PyObject *dict = PyDict_New(); - - if (dict != NULL && merge_class_dict(dict, obj) == 0) - result = PyDict_Keys(dict); - - Py_XDECREF(dict); - return result; -} - -/* Helper for PyObject_Dir of module objects: returns the module's __dict__. */ -static PyObject * -_specialized_dir_module(PyObject *obj) -{ - PyObject *result = NULL; - PyObject *dict = PyObject_GetAttrString(obj, "__dict__"); - - if (dict != NULL) { - if (PyDict_Check(dict)) - result = PyDict_Keys(dict); - else { - const char *name = PyModule_GetName(obj); - if (name) - PyErr_Format(PyExc_TypeError, - "%.200s.__dict__ is not a dictionary", - name); - } - } - - Py_XDECREF(dict); - return result; -} - -/* Helper for PyObject_Dir of generic objects: returns __dict__, __class__, - and recursively up the __class__.__bases__ chain. -*/ -static PyObject * -_generic_dir(PyObject *obj) -{ - PyObject *result = NULL; - PyObject *dict = NULL; - PyObject *itsclass = NULL; - - /* Get __dict__ (which may or may not be a real dict...) */ - dict = PyObject_GetAttrString(obj, "__dict__"); - if (dict == NULL) { - PyErr_Clear(); - dict = PyDict_New(); - } - else if (!PyDict_Check(dict)) { - Py_DECREF(dict); - dict = PyDict_New(); - } - else { - /* Copy __dict__ to avoid mutating it. */ - PyObject *temp = PyDict_Copy(dict); - Py_DECREF(dict); - dict = temp; - } - - if (dict == NULL) - goto error; - - /* Merge in attrs reachable from its class. */ - itsclass = PyObject_GetAttrString(obj, "__class__"); - if (itsclass == NULL) - /* XXX(tomer): Perhaps fall back to obj->ob_type if no - __class__ exists? */ - PyErr_Clear(); - else { - if (merge_class_dict(dict, itsclass) != 0) - goto error; - } - - result = PyDict_Keys(dict); - /* fall through */ -error: - Py_XDECREF(itsclass); - Py_XDECREF(dict); - return result; -} - -/* Helper for PyObject_Dir: object introspection. - This calls one of the above specialized versions if no __dir__ method - exists. */ +/* Helper for PyObject_Dir: object introspection. */ static PyObject * _dir_object(PyObject *obj) { - PyObject *result = NULL; - static PyObject *dir_str = NULL; - PyObject *dirfunc = _PyObject_LookupSpecial(obj, "__dir__", &dir_str); + PyObject *result, *sorted; + _Py_IDENTIFIER(__dir__); + PyObject *dirfunc = _PyObject_LookupSpecial(obj, &PyId___dir__); assert(obj); if (dirfunc == NULL) { - if (PyErr_Occurred()) - return NULL; - /* use default implementation */ - if (PyModule_Check(obj)) - result = _specialized_dir_module(obj); - else if (PyType_Check(obj)) - result = _specialized_dir_type(obj); - else - result = _generic_dir(obj); + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, "object does not provide __dir__"); + return NULL; } - else { - /* use __dir__ */ - result = PyObject_CallFunctionObjArgs(dirfunc, NULL); - Py_DECREF(dirfunc); - if (result == NULL) - return NULL; - - /* result must be a list */ - /* XXX(gbrandl): could also check if all items are strings */ - if (!PyList_Check(result)) { - PyErr_Format(PyExc_TypeError, - "__dir__() must return a list, not %.200s", - Py_TYPE(result)->tp_name); - Py_DECREF(result); - result = NULL; - } + /* use __dir__ */ + result = PyObject_CallFunctionObjArgs(dirfunc, NULL); + Py_DECREF(dirfunc); + if (result == NULL) + return NULL; + /* return sorted(result) */ + sorted = PySequence_List(result); + Py_DECREF(result); + if (sorted == NULL) + return NULL; + if (PyList_Sort(sorted)) { + Py_DECREF(sorted); + return NULL; } - - return result; + return sorted; } /* Implementation of dir() -- if obj is NULL, returns the names in the current @@ -1412,31 +1380,13 @@ _dir_object(PyObject *obj) PyObject * PyObject_Dir(PyObject *obj) { - PyObject * result; - - if (obj == NULL) - /* no object -- introspect the locals */ - result = _dir_locals(); - else - /* object -- introspect the object */ - result = _dir_object(obj); - - assert(result == NULL || PyList_Check(result)); - - if (result != NULL && PyList_Sort(result) != 0) { - /* sorting the list failed */ - Py_DECREF(result); - result = NULL; - } - - return result; + return (obj == NULL) ? _dir_locals() : _dir_object(obj); } /* -NoObject is usable as a non-NULL undefined value, used by the macro None. +None is a non-NULL undefined value. There is (and should be!) no way to create other objects of this type, so there is exactly one (which is indestructible, by the way). -(XXX This type and the type of NotImplemented below should be unified.) */ /* ARGSUSED */ @@ -1456,6 +1406,58 @@ none_dealloc(PyObject* ignore) Py_FatalError("deallocating None"); } +static PyObject * +none_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_Size(kwargs))) { + PyErr_SetString(PyExc_TypeError, "NoneType takes no arguments"); + return NULL; + } + Py_RETURN_NONE; +} + +static int +none_bool(PyObject *v) +{ + return 0; +} + +static PyNumberMethods none_as_number = { + 0, /* nb_add */ + 0, /* nb_subtract */ + 0, /* nb_multiply */ + 0, /* nb_remainder */ + 0, /* nb_divmod */ + 0, /* nb_power */ + 0, /* nb_negative */ + 0, /* nb_positive */ + 0, /* nb_absolute */ + (inquiry)none_bool, /* nb_bool */ + 0, /* nb_invert */ + 0, /* nb_lshift */ + 0, /* nb_rshift */ + 0, /* nb_and */ + 0, /* nb_xor */ + 0, /* nb_or */ + 0, /* nb_int */ + 0, /* nb_reserved */ + 0, /* nb_float */ + 0, /* nb_inplace_add */ + 0, /* nb_inplace_subtract */ + 0, /* nb_inplace_multiply */ + 0, /* nb_inplace_remainder */ + 0, /* nb_inplace_power */ + 0, /* nb_inplace_lshift */ + 0, /* nb_inplace_rshift */ + 0, /* nb_inplace_and */ + 0, /* nb_inplace_xor */ + 0, /* nb_inplace_or */ + 0, /* nb_floor_divide */ + 0, /* nb_true_divide */ + 0, /* nb_inplace_floor_divide */ + 0, /* nb_inplace_true_divide */ + 0, /* nb_index */ +}; static PyTypeObject PyNone_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -1468,10 +1470,34 @@ static PyTypeObject PyNone_Type = { 0, /*tp_setattr*/ 0, /*tp_reserved*/ none_repr, /*tp_repr*/ - 0, /*tp_as_number*/ + &none_as_number, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ + 0, /*tp_call */ + 0, /*tp_str */ + 0, /*tp_getattro */ + 0, /*tp_setattro */ + 0, /*tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /*tp_flags */ + 0, /*tp_doc */ + 0, /*tp_traverse */ + 0, /*tp_clear */ + 0, /*tp_richcompare */ + 0, /*tp_weaklistoffset */ + 0, /*tp_iter */ + 0, /*tp_iternext */ + 0, /*tp_methods */ + 0, /*tp_members */ + 0, /*tp_getset */ + 0, /*tp_base */ + 0, /*tp_dict */ + 0, /*tp_descr_get */ + 0, /*tp_descr_set */ + 0, /*tp_dictoffset */ + 0, /*tp_init */ + 0, /*tp_alloc */ + none_new, /*tp_new */ }; PyObject _Py_NoneStruct = { @@ -1488,6 +1514,16 @@ NotImplemented_repr(PyObject *op) return PyUnicode_FromString("NotImplemented"); } +static PyObject * +notimplemented_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_Size(kwargs))) { + PyErr_SetString(PyExc_TypeError, "NotImplementedType takes no arguments"); + return NULL; + } + Py_RETURN_NOTIMPLEMENTED; +} + static PyTypeObject PyNotImplemented_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "NotImplementedType", @@ -1503,6 +1539,30 @@ static PyTypeObject PyNotImplemented_Type = { 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ + 0, /*tp_call */ + 0, /*tp_str */ + 0, /*tp_getattro */ + 0, /*tp_setattro */ + 0, /*tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /*tp_flags */ + 0, /*tp_doc */ + 0, /*tp_traverse */ + 0, /*tp_clear */ + 0, /*tp_richcompare */ + 0, /*tp_weaklistoffset */ + 0, /*tp_iter */ + 0, /*tp_iternext */ + 0, /*tp_methods */ + 0, /*tp_members */ + 0, /*tp_getset */ + 0, /*tp_base */ + 0, /*tp_dict */ + 0, /*tp_descr_get */ + 0, /*tp_descr_set */ + 0, /*tp_dictoffset */ + 0, /*tp_init */ + 0, /*tp_alloc */ + notimplemented_new, /*tp_new */ }; PyObject _Py_NotImplementedStruct = { @@ -1585,6 +1645,9 @@ _Py_ReadyTypes(void) if (PyType_Ready(&PyProperty_Type) < 0) Py_FatalError("Can't initialize property type"); + if (PyType_Ready(&_PyManagedBuffer_Type) < 0) + Py_FatalError("Can't initialize managed buffer type"); + if (PyType_Ready(&PyMemoryView_Type) < 0) Py_FatalError("Can't initialize memoryview type"); @@ -1645,6 +1708,9 @@ _Py_ReadyTypes(void) if (PyType_Ready(&PyZip_Type) < 0) Py_FatalError("Can't initialize zip type"); + if (PyType_Ready(&_PyNamespace_Type) < 0) + Py_FatalError("Can't initialize namespace type"); + if (PyType_Ready(&PyCapsule_Type) < 0) Py_FatalError("Can't initialize capsule type"); @@ -1810,6 +1876,18 @@ PyMem_Free(void *p) PyMem_FREE(p); } +void +_PyObject_DebugTypeStats(FILE *out) +{ + _PyCFunction_DebugMallocStats(out); + _PyDict_DebugMallocStats(out); + _PyFloat_DebugMallocStats(out); + _PyFrame_DebugMallocStats(out); + _PyList_DebugMallocStats(out); + _PyMethod_DebugMallocStats(out); + _PySet_DebugMallocStats(out); + _PyTuple_DebugMallocStats(out); +} /* These methods are used to control infinite recursion in repr, str, print, etc. Container objects that may recursively contain themselves, diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 3916262..6225ebb 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2,6 +2,13 @@ #ifdef WITH_PYMALLOC +#ifdef HAVE_MMAP + #include <sys/mman.h> + #ifdef MAP_ANONYMOUS + #define ARENAS_USE_MMAP + #endif +#endif + #ifdef WITH_VALGRIND #include <valgrind/valgrind.h> @@ -75,7 +82,8 @@ static int running_on_valgrind = -1; * Allocation strategy abstract: * * For small requests, the allocator sub-allocates <Big> blocks of memory. - * Requests greater than 256 bytes are routed to the system's allocator. + * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the + * system's allocator. * * Small requests are grouped in size classes spaced 8 bytes apart, due * to the required valid alignment of the returned address. Requests of @@ -107,10 +115,11 @@ static int running_on_valgrind = -1; * 57-64 64 7 * 65-72 72 8 * ... ... ... - * 241-248 248 30 - * 249-256 256 31 + * 497-504 504 62 + * 505-512 512 63 * - * 0, 257 and up: routed to the underlying allocator. + * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying + * allocator. */ /*==========================================================================*/ @@ -129,7 +138,6 @@ static int running_on_valgrind = -1; */ #define ALIGNMENT 8 /* must be 2^N */ #define ALIGNMENT_SHIFT 3 -#define ALIGNMENT_MASK (ALIGNMENT - 1) /* Return the number of bytes in size class I, as a uint. */ #define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT) @@ -139,14 +147,17 @@ static int running_on_valgrind = -1; * small enough in order to use preallocated memory pools. You can tune * this value according to your application behaviour and memory needs. * + * Note: a size threshold of 512 guarantees that newly created dictionaries + * will be allocated from preallocated memory pools on 64-bit. + * * The following invariants must hold: - * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 256 + * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT * * Although not required, for better performance and space efficiency, * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. */ -#define SMALL_REQUEST_THRESHOLD 256 +#define SMALL_REQUEST_THRESHOLD 512 #define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) /* @@ -174,15 +185,15 @@ static int running_on_valgrind = -1; /* * The allocator sub-allocates <Big> blocks of memory (called arenas) aligned * on a page boundary. This is a reserved virtual address space for the - * current process (obtained through a malloc call). In no way this means - * that the memory arenas will be used entirely. A malloc(<Big>) is usually - * an address range reservation for <Big> bytes, unless all pages within this - * space are referenced subsequently. So malloc'ing big blocks and not using - * them does not mean "wasting memory". It's an addressable range wastage... + * current process (obtained through a malloc()/mmap() call). In no way this + * means that the memory arenas will be used entirely. A malloc(<Big>) is + * usually an address range reservation for <Big> bytes, unless all pages within + * this space are referenced subsequently. So malloc'ing big blocks and not + * using them does not mean "wasting memory". It's an addressable range + * wastage... * - * Therefore, allocating arenas with malloc is not optimal, because there is - * some address space wastage, but this is the most portable way to request - * memory from the system across various platforms. + * Arenas are allocated with mmap() on systems supporting anonymous memory + * mappings to reduce heap fragmentation. */ #define ARENA_SIZE (256 << 10) /* 256KB */ @@ -302,14 +313,12 @@ struct arena_object { struct arena_object* prevarena; }; -#undef ROUNDUP -#define ROUNDUP(x) (((x) + ALIGNMENT_MASK) & ~ALIGNMENT_MASK) -#define POOL_OVERHEAD ROUNDUP(sizeof(struct pool_header)) +#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) #define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ /* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ -#define POOL_ADDR(P) ((poolp)((uptr)(P) & ~(uptr)POOL_SIZE_MASK)) +#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) /* Return total number of blocks in pool of size index I, as a uint. */ #define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) @@ -440,6 +449,9 @@ static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = { , PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55) #if NB_SMALL_SIZE_CLASSES > 56 , PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63) +#if NB_SMALL_SIZE_CLASSES > 64 +#error "NB_SMALL_SIZE_CLASSES should be less than 64" +#endif /* NB_SMALL_SIZE_CLASSES > 64 */ #endif /* NB_SMALL_SIZE_CLASSES > 56 */ #endif /* NB_SMALL_SIZE_CLASSES > 48 */ #endif /* NB_SMALL_SIZE_CLASSES > 40 */ @@ -508,12 +520,10 @@ static struct arena_object* usable_arenas = NULL; /* Number of arenas allocated that haven't been free()'d. */ static size_t narenas_currently_allocated = 0; -#ifdef PYMALLOC_DEBUG /* Total number of times malloc() called to allocate an arena. */ static size_t ntimes_arena_allocated = 0; /* High water mark (max value ever seen) for narenas_currently_allocated. */ static size_t narenas_highwater = 0; -#endif /* Allocate a new arena. If we run out of memory, return NULL. Else * allocate a new arena, and return the address of an arena_object @@ -525,10 +535,12 @@ new_arena(void) { struct arena_object* arenaobj; uint excess; /* number of bytes above pool alignment */ + void *address; + int err; #ifdef PYMALLOC_DEBUG if (Py_GETENV("PYTHONMALLOCSTATS")) - _PyObject_DebugMallocStats(); + _PyObject_DebugMallocStats(stderr); #endif if (unused_arena_objects == NULL) { uint i; @@ -577,8 +589,15 @@ new_arena(void) arenaobj = unused_arena_objects; unused_arena_objects = arenaobj->nextarena; assert(arenaobj->address == 0); - arenaobj->address = (uptr)malloc(ARENA_SIZE); - if (arenaobj->address == 0) { +#ifdef ARENAS_USE_MMAP + address = mmap(NULL, ARENA_SIZE, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + err = (address == MAP_FAILED); +#else + address = malloc(ARENA_SIZE); + err = (address == 0); +#endif + if (err) { /* The allocation failed: return NULL after putting the * arenaobj back. */ @@ -586,13 +605,12 @@ new_arena(void) unused_arena_objects = arenaobj; return NULL; } + arenaobj->address = (uptr)address; ++narenas_currently_allocated; -#ifdef PYMALLOC_DEBUG ++ntimes_arena_allocated; if (narenas_currently_allocated > narenas_highwater) narenas_highwater = narenas_currently_allocated; -#endif arenaobj->freepools = NULL; /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ @@ -1054,7 +1072,11 @@ PyObject_Free(void *p) unused_arena_objects = ao; /* Free the entire arena. */ +#ifdef ARENAS_USE_MMAP + munmap((void *)ao->address, ARENA_SIZE); +#else free((void *)ao->address); +#endif ao->address = 0; /* mark unassociated */ --narenas_currently_allocated; @@ -1694,17 +1716,19 @@ _PyObject_DebugDumpAddress(const void *p) } } +#endif /* PYMALLOC_DEBUG */ + static size_t -printone(const char* msg, size_t value) +printone(FILE *out, const char* msg, size_t value) { int i, k; char buf[100]; size_t origvalue = value; - fputs(msg, stderr); + fputs(msg, out); for (i = (int)strlen(msg); i < 35; ++i) - fputc(' ', stderr); - fputc('=', stderr); + fputc(' ', out); + fputc('=', out); /* Write the value with commas. */ i = 22; @@ -1725,17 +1749,33 @@ printone(const char* msg, size_t value) while (i >= 0) buf[i--] = ' '; - fputs(buf, stderr); + fputs(buf, out); return origvalue; } -/* Print summary info to stderr about the state of pymalloc's structures. +void +_PyDebugAllocatorStats(FILE *out, + const char *block_name, int num_blocks, size_t sizeof_block) +{ + char buf1[128]; + char buf2[128]; + PyOS_snprintf(buf1, sizeof(buf1), + "%d %ss * %zd bytes each", + num_blocks, block_name, sizeof_block); + PyOS_snprintf(buf2, sizeof(buf2), + "%48s ", buf1); + (void)printone(out, buf2, num_blocks * sizeof_block); +} + +#ifdef WITH_PYMALLOC + +/* Print summary info to "out" about the state of pymalloc's structures. * In Py_DEBUG mode, also perform some expensive internal consistency * checks. */ void -_PyObject_DebugMallocStats(void) +_PyObject_DebugMallocStats(FILE *out) { uint i; const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT; @@ -1764,7 +1804,7 @@ _PyObject_DebugMallocStats(void) size_t total; char buf[128]; - fprintf(stderr, "Small block threshold = %d, in %u size classes.\n", + fprintf(out, "Small block threshold = %d, in %u size classes.\n", SMALL_REQUEST_THRESHOLD, numclasses); for (i = 0; i < numclasses; ++i) @@ -1775,7 +1815,6 @@ _PyObject_DebugMallocStats(void) * will be living in full pools -- would be a shame to miss them. */ for (i = 0; i < maxarenas; ++i) { - uint poolsinarena; uint j; uptr base = arenas[i].address; @@ -1784,7 +1823,6 @@ _PyObject_DebugMallocStats(void) continue; narenas += 1; - poolsinarena = arenas[i].ntotalpools; numfreepools += arenas[i].nfreepools; /* round up to pool alignment */ @@ -1820,10 +1858,10 @@ _PyObject_DebugMallocStats(void) } assert(narenas == narenas_currently_allocated); - fputc('\n', stderr); + fputc('\n', out); fputs("class size num pools blocks in use avail blocks\n" "----- ---- --------- ------------- ------------\n", - stderr); + out); for (i = 0; i < numclasses; ++i) { size_t p = numpools[i]; @@ -1834,7 +1872,7 @@ _PyObject_DebugMallocStats(void) assert(b == 0 && f == 0); continue; } - fprintf(stderr, "%5u %6u " + fprintf(out, "%5u %6u " "%11" PY_FORMAT_SIZE_T "u " "%15" PY_FORMAT_SIZE_T "u " "%13" PY_FORMAT_SIZE_T "u\n", @@ -1844,35 +1882,36 @@ _PyObject_DebugMallocStats(void) pool_header_bytes += p * POOL_OVERHEAD; quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } - fputc('\n', stderr); - (void)printone("# times object malloc called", serialno); - - (void)printone("# arenas allocated total", ntimes_arena_allocated); - (void)printone("# arenas reclaimed", ntimes_arena_allocated - narenas); - (void)printone("# arenas highwater mark", narenas_highwater); - (void)printone("# arenas allocated current", narenas); + fputc('\n', out); +#ifdef PYMALLOC_DEBUG + (void)printone(out, "# times object malloc called", serialno); +#endif + (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); + (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); + (void)printone(out, "# arenas highwater mark", narenas_highwater); + (void)printone(out, "# arenas allocated current", narenas); PyOS_snprintf(buf, sizeof(buf), "%" PY_FORMAT_SIZE_T "u arenas * %d bytes/arena", narenas, ARENA_SIZE); - (void)printone(buf, narenas * ARENA_SIZE); + (void)printone(out, buf, narenas * ARENA_SIZE); - fputc('\n', stderr); + fputc('\n', out); - total = printone("# bytes in allocated blocks", allocated_bytes); - total += printone("# bytes in available blocks", available_bytes); + total = printone(out, "# bytes in allocated blocks", allocated_bytes); + total += printone(out, "# bytes in available blocks", available_bytes); PyOS_snprintf(buf, sizeof(buf), "%u unused pools * %d bytes", numfreepools, POOL_SIZE); - total += printone(buf, (size_t)numfreepools * POOL_SIZE); + total += printone(out, buf, (size_t)numfreepools * POOL_SIZE); - total += printone("# bytes lost to pool headers", pool_header_bytes); - total += printone("# bytes lost to quantization", quantization); - total += printone("# bytes lost to arena alignment", arena_alignment); - (void)printone("Total", total); + total += printone(out, "# bytes lost to pool headers", pool_header_bytes); + total += printone(out, "# bytes lost to quantization", quantization); + total += printone(out, "# bytes lost to arena alignment", arena_alignment); + (void)printone(out, "Total", total); } -#endif /* PYMALLOC_DEBUG */ +#endif /* #ifdef WITH_PYMALLOC */ #ifdef Py_USING_MEMORY_DEBUGGER /* Make this function last so gcc won't inline it since the definition is diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index b67b969..68d5636 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -1,6 +1,7 @@ /* Range object implementation */ #include "Python.h" +#include "structmember.h" /* Support objects whose length is > PY_SSIZE_T_MAX. @@ -610,6 +611,137 @@ range_contains(rangeobject *r, PyObject *ob) PY_ITERSEARCH_CONTAINS); } +/* Compare two range objects. Return 1 for equal, 0 for not equal + and -1 on error. The algorithm is roughly the C equivalent of + + if r0 is r1: + return True + if len(r0) != len(r1): + return False + if not len(r0): + return True + if r0.start != r1.start: + return False + if len(r0) == 1: + return True + return r0.step == r1.step +*/ +static int +range_equals(rangeobject *r0, rangeobject *r1) +{ + int cmp_result; + PyObject *one; + + if (r0 == r1) + return 1; + cmp_result = PyObject_RichCompareBool(r0->length, r1->length, Py_EQ); + /* Return False or error to the caller. */ + if (cmp_result != 1) + return cmp_result; + cmp_result = PyObject_Not(r0->length); + /* Return True or error to the caller. */ + if (cmp_result != 0) + return cmp_result; + cmp_result = PyObject_RichCompareBool(r0->start, r1->start, Py_EQ); + /* Return False or error to the caller. */ + if (cmp_result != 1) + return cmp_result; + one = PyLong_FromLong(1); + if (!one) + return -1; + cmp_result = PyObject_RichCompareBool(r0->length, one, Py_EQ); + Py_DECREF(one); + /* Return True or error to the caller. */ + if (cmp_result != 0) + return cmp_result; + return PyObject_RichCompareBool(r0->step, r1->step, Py_EQ); +} + +static PyObject * +range_richcompare(PyObject *self, PyObject *other, int op) +{ + int result; + + if (!PyRange_Check(other)) + Py_RETURN_NOTIMPLEMENTED; + switch (op) { + case Py_NE: + case Py_EQ: + result = range_equals((rangeobject*)self, (rangeobject*)other); + if (result == -1) + return NULL; + if (op == Py_NE) + result = !result; + if (result) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; + case Py_LE: + case Py_GE: + case Py_LT: + case Py_GT: + Py_RETURN_NOTIMPLEMENTED; + default: + PyErr_BadArgument(); + return NULL; + } +} + +/* Hash function for range objects. Rough C equivalent of + + if not len(r): + return hash((len(r), None, None)) + if len(r) == 1: + return hash((len(r), r.start, None)) + return hash((len(r), r.start, r.step)) +*/ +static Py_hash_t +range_hash(rangeobject *r) +{ + PyObject *t; + Py_hash_t result = -1; + int cmp_result; + + t = PyTuple_New(3); + if (!t) + return -1; + Py_INCREF(r->length); + PyTuple_SET_ITEM(t, 0, r->length); + cmp_result = PyObject_Not(r->length); + if (cmp_result == -1) + goto end; + if (cmp_result == 1) { + Py_INCREF(Py_None); + Py_INCREF(Py_None); + PyTuple_SET_ITEM(t, 1, Py_None); + PyTuple_SET_ITEM(t, 2, Py_None); + } + else { + PyObject *one; + Py_INCREF(r->start); + PyTuple_SET_ITEM(t, 1, r->start); + one = PyLong_FromLong(1); + if (!one) + goto end; + cmp_result = PyObject_RichCompareBool(r->length, one, Py_EQ); + Py_DECREF(one); + if (cmp_result == -1) + goto end; + if (cmp_result == 1) { + Py_INCREF(Py_None); + PyTuple_SET_ITEM(t, 2, Py_None); + } + else { + Py_INCREF(r->step); + PyTuple_SET_ITEM(t, 2, r->step); + } + } + result = PyObject_Hash(t); + end: + Py_DECREF(t); + return result; +} + static PyObject * range_count(rangeobject *r, PyObject *ob) { @@ -750,6 +882,13 @@ static PyMethodDef range_methods[] = { {NULL, NULL} /* sentinel */ }; +static PyMemberDef range_members[] = { + {"start", T_OBJECT_EX, offsetof(rangeobject, start), READONLY}, + {"stop", T_OBJECT_EX, offsetof(rangeobject, stop), READONLY}, + {"step", T_OBJECT_EX, offsetof(rangeobject, step), READONLY}, + {0} +}; + PyTypeObject PyRange_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "range", /* Name of this type */ @@ -764,7 +903,7 @@ PyTypeObject PyRange_Type = { 0, /* tp_as_number */ &range_as_sequence, /* tp_as_sequence */ &range_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ + (hashfunc)range_hash, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ @@ -774,12 +913,12 @@ PyTypeObject PyRange_Type = { range_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ - 0, /* tp_richcompare */ + range_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ range_iter, /* tp_iter */ 0, /* tp_iternext */ range_methods, /* tp_methods */ - 0, /* tp_members */ + range_members, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ @@ -826,9 +965,59 @@ rangeiter_len(rangeiterobject *r) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +static PyObject * +rangeiter_reduce(rangeiterobject *r) +{ + PyObject *start=NULL, *stop=NULL, *step=NULL; + PyObject *range; + + /* create a range object for pickling */ + start = PyLong_FromLong(r->start); + if (start == NULL) + goto err; + stop = PyLong_FromLong(r->start + r->len * r->step); + if (stop == NULL) + goto err; + step = PyLong_FromLong(r->step); + if (step == NULL) + goto err; + range = (PyObject*)make_range_object(&PyRange_Type, + start, stop, step); + if (range == NULL) + goto err; + /* return the result */ + return Py_BuildValue("N(N)i", _PyObject_GetBuiltin("iter"), range, r->index); +err: + Py_XDECREF(start); + Py_XDECREF(stop); + Py_XDECREF(step); + return NULL; +} + +static PyObject * +rangeiter_setstate(rangeiterobject *r, PyObject *state) +{ + long index = PyLong_AsLong(state); + if (index == -1 && PyErr_Occurred()) + return NULL; + if (index < 0 || index >= r->len) { + PyErr_SetString(PyExc_ValueError, "index out of range"); + return NULL; + } + r->index = index; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); +PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); + static PyMethodDef rangeiter_methods[] = { {"__length_hint__", (PyCFunction)rangeiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)rangeiter_reduce, METH_NOARGS, + reduce_doc}, + {"__setstate__", (PyCFunction)rangeiter_setstate, METH_O, + setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -957,9 +1146,51 @@ longrangeiter_len(longrangeiterobject *r, PyObject *no_args) return PyNumber_Subtract(r->len, r->index); } +static PyObject * +longrangeiter_reduce(longrangeiterobject *r) +{ + PyObject *product, *stop=NULL; + PyObject *range; + + /* create a range object for pickling. Must calculate the "stop" value */ + product = PyNumber_Multiply(r->len, r->step); + if (product == NULL) + return NULL; + stop = PyNumber_Add(r->start, product); + Py_DECREF(product); + if (stop == NULL) + return NULL; + Py_INCREF(r->start); + Py_INCREF(r->step); + range = (PyObject*)make_range_object(&PyRange_Type, + r->start, stop, r->step); + if (range == NULL) { + Py_DECREF(r->start); + Py_DECREF(stop); + Py_DECREF(r->step); + return NULL; + } + + /* return the result */ + return Py_BuildValue("N(N)O", _PyObject_GetBuiltin("iter"), range, r->index); +} + +static PyObject * +longrangeiter_setstate(longrangeiterobject *r, PyObject *state) +{ + Py_CLEAR(r->index); + r->index = state; + Py_INCREF(r->index); + Py_RETURN_NONE; +} + static PyMethodDef longrangeiter_methods[] = { {"__length_hint__", (PyCFunction)longrangeiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)longrangeiter_reduce, METH_NOARGS, + reduce_doc}, + {"__setstate__", (PyCFunction)longrangeiter_setstate, METH_O, + setstate_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/setobject.c b/Objects/setobject.c index 3abeefb..723679a 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -77,7 +77,7 @@ NULL if the rich comparison returns an error. static setentry * set_lookkey(PySetObject *so, PyObject *key, register Py_hash_t hash) { - register Py_ssize_t i; + register size_t i; register size_t perturb; register setentry *freeslot; register size_t mask = so->mask; @@ -86,7 +86,7 @@ set_lookkey(PySetObject *so, PyObject *key, register Py_hash_t hash) register int cmp; PyObject *startkey; - i = hash & mask; + i = (size_t)hash & mask; entry = &table[i]; if (entry->key == NULL || entry->key == key) return entry; @@ -159,7 +159,7 @@ set_lookkey(PySetObject *so, PyObject *key, register Py_hash_t hash) static setentry * set_lookkey_unicode(PySetObject *so, PyObject *key, register Py_hash_t hash) { - register Py_ssize_t i; + register size_t i; register size_t perturb; register setentry *freeslot; register size_t mask = so->mask; @@ -174,7 +174,7 @@ set_lookkey_unicode(PySetObject *so, PyObject *key, register Py_hash_t hash) so->lookup = set_lookkey; return set_lookkey(so, key, hash); } - i = hash & mask; + i = (size_t)hash & mask; entry = &table[i]; if (entry->key == NULL || entry->key == key) return entry; @@ -256,7 +256,7 @@ set_insert_clean(register PySetObject *so, PyObject *key, Py_hash_t hash) setentry *table = so->table; register setentry *entry; - i = hash & mask; + i = (size_t)hash & mask; entry = &table[i]; for (perturb = hash; entry->key != NULL; perturb >>= PERTURB_SHIFT) { i = (i << 2) + i + perturb + 1; @@ -386,7 +386,7 @@ set_add_key(register PySetObject *so, PyObject *key) register Py_ssize_t n_used; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; @@ -434,7 +434,7 @@ set_discard_key(PySetObject *so, PyObject *key) assert (PyAnySet_Check(so)); if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; @@ -579,11 +579,8 @@ set_dealloc(PySetObject *so) static PyObject * set_repr(PySetObject *so) { - PyObject *keys, *result=NULL; - Py_UNICODE *u; + PyObject *result=NULL, *keys, *listrepr, *tmp; int status = Py_ReprEnter((PyObject*)so); - PyObject *listrepr; - Py_ssize_t newsize; if (status != 0) { if (status < 0) @@ -601,29 +598,24 @@ set_repr(PySetObject *so) if (keys == NULL) goto done; + /* repr(keys)[1:-1] */ listrepr = PyObject_Repr(keys); Py_DECREF(keys); if (listrepr == NULL) goto done; - newsize = PyUnicode_GET_SIZE(listrepr); - result = PyUnicode_FromUnicode(NULL, newsize); - if (result) { - u = PyUnicode_AS_UNICODE(result); - *u++ = '{'; - /* Omit the brackets from the listrepr */ - Py_UNICODE_COPY(u, PyUnicode_AS_UNICODE(listrepr)+1, - PyUnicode_GET_SIZE(listrepr)-2); - u += newsize-2; - *u++ = '}'; - } + tmp = PyUnicode_Substring(listrepr, 1, PyUnicode_GET_LENGTH(listrepr)-1); + Py_DECREF(listrepr); + if (tmp == NULL) + goto done; + listrepr = tmp; + + if (Py_TYPE(so) != &PySet_Type) + result = PyUnicode_FromFormat("%s({%U})", + Py_TYPE(so)->tp_name, + listrepr); + else + result = PyUnicode_FromFormat("{%U}", listrepr); Py_DECREF(listrepr); - if (Py_TYPE(so) != &PySet_Type) { - PyObject *tmp = PyUnicode_FromFormat("%s(%U)", - Py_TYPE(so)->tp_name, - result); - Py_DECREF(result); - result = tmp; - } done: Py_ReprLeave((PyObject*)so); return result; @@ -682,7 +674,7 @@ set_contains_key(PySetObject *so, PyObject *key) setentry *entry; if (!PyUnicode_CheckExact(key) || - (hash = ((PyUnicodeObject *) key)->hash) == -1) { + (hash = ((PyASCIIObject *) key)->hash) == -1) { hash = PyObject_Hash(key); if (hash == -1) return -1; @@ -768,14 +760,14 @@ static Py_hash_t frozenset_hash(PyObject *self) { PySetObject *so = (PySetObject *)self; - Py_hash_t h, hash = 1927868237L; + Py_uhash_t h, hash = 1927868237U; setentry *entry; Py_ssize_t pos = 0; if (so->hash != -1) return so->hash; - hash *= PySet_GET_SIZE(self) + 1; + hash *= (Py_uhash_t)PySet_GET_SIZE(self) + 1; while (set_next(so, &pos, &entry)) { /* Work to increase the bit dispersion for closely spaced hash values. The is important because some use cases have many @@ -783,11 +775,11 @@ frozenset_hash(PyObject *self) hashes so that many distinct combinations collapse to only a handful of distinct hash values. */ h = entry->hash; - hash ^= (h ^ (h << 16) ^ 89869747L) * 3644798167u; + hash ^= (h ^ (h << 16) ^ 89869747U) * 3644798167U; } - hash = hash * 69069L + 907133923L; + hash = hash * 69069U + 907133923U; if (hash == -1) - hash = 590923713L; + hash = 590923713U; so->hash = hash; return hash; } @@ -827,8 +819,51 @@ setiter_len(setiterobject *si) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +static PyObject *setiter_iternext(setiterobject *si); + +static PyObject * +setiter_reduce(setiterobject *si) +{ + PyObject *list; + setiterobject tmp; + + list = PyList_New(0); + if (!list) + return NULL; + + /* copy the iterator state */ + tmp = *si; + Py_XINCREF(tmp.si_set); + + /* iterate the temporary into a list */ + for(;;) { + PyObject *element = setiter_iternext(&tmp); + if (element) { + if (PyList_Append(list, element)) { + Py_DECREF(element); + Py_DECREF(list); + Py_XDECREF(tmp.si_set); + return NULL; + } + Py_DECREF(element); + } else + break; + } + Py_XDECREF(tmp.si_set); + /* check for error */ + if (tmp.si_set != NULL) { + /* we have an error */ + Py_DECREF(list); + return NULL; + } + return Py_BuildValue("N(N)", _PyObject_GetBuiltin("iter"), list); +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); + static PyMethodDef setiter_methods[] = { {"__length_hint__", (PyCFunction)setiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)setiter_reduce, METH_NOARGS, reduce_doc}, {NULL, NULL} /* sentinel */ }; @@ -1076,9 +1111,10 @@ frozenset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return emptyfrozenset; } -void -PySet_Fini(void) +int +PySet_ClearFreeList(void) { + int freelist_size = numfree; PySetObject *so; while (numfree) { @@ -1086,10 +1122,27 @@ PySet_Fini(void) so = free_list[numfree]; PyObject_GC_Del(so); } + return freelist_size; +} + +void +PySet_Fini(void) +{ + PySet_ClearFreeList(); Py_CLEAR(dummy); Py_CLEAR(emptyfrozenset); } +/* Print summary info about the state of the optimized allocator */ +void +_PySet_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PySetObject", + numfree, sizeof(PySetObject)); +} + + static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -1210,10 +1263,8 @@ set_or(PySetObject *so, PyObject *other) { PySetObject *result; - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + Py_RETURN_NOTIMPLEMENTED; result = (PySetObject *)set_copy(so); if (result == NULL) @@ -1230,10 +1281,9 @@ set_or(PySetObject *so, PyObject *other) static PyObject * set_ior(PySetObject *so, PyObject *other) { - if (!PyAnySet_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyAnySet_Check(other)) + Py_RETURN_NOTIMPLEMENTED; + if (set_update_internal(so, other) == -1) return NULL; Py_INCREF(so); @@ -1383,10 +1433,8 @@ PyDoc_STRVAR(intersection_update_doc, static PyObject * set_and(PySetObject *so, PyObject *other) { - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + Py_RETURN_NOTIMPLEMENTED; return set_intersection(so, other); } @@ -1395,10 +1443,8 @@ set_iand(PySetObject *so, PyObject *other) { PyObject *result; - if (!PyAnySet_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyAnySet_Check(other)) + Py_RETURN_NOTIMPLEMENTED; result = set_intersection_update(so, other); if (result == NULL) return NULL; @@ -1625,20 +1671,16 @@ PyDoc_STRVAR(difference_doc, static PyObject * set_sub(PySetObject *so, PyObject *other) { - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + Py_RETURN_NOTIMPLEMENTED; return set_difference(so, other); } static PyObject * set_isub(PySetObject *so, PyObject *other) { - if (!PyAnySet_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyAnySet_Check(other)) + Py_RETURN_NOTIMPLEMENTED; if (set_difference_update_internal(so, other) == -1) return NULL; Py_INCREF(so); @@ -1736,10 +1778,8 @@ PyDoc_STRVAR(symmetric_difference_doc, static PyObject * set_xor(PySetObject *so, PyObject *other) { - if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyAnySet_Check(so) || !PyAnySet_Check(other)) + Py_RETURN_NOTIMPLEMENTED; return set_symmetric_difference(so, other); } @@ -1748,10 +1788,8 @@ set_ixor(PySetObject *so, PyObject *other) { PyObject *result; - if (!PyAnySet_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyAnySet_Check(other)) + Py_RETURN_NOTIMPLEMENTED; result = set_symmetric_difference_update(so, other); if (result == NULL) return NULL; @@ -1813,10 +1851,9 @@ set_richcompare(PySetObject *v, PyObject *w, int op) { PyObject *r1, *r2; - if(!PyAnySet_Check(w)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if(!PyAnySet_Check(w)) + Py_RETURN_NOTIMPLEMENTED; + switch (op) { case Py_EQ: if (PySet_GET_SIZE(v) != PySet_GET_SIZE(w)) @@ -1846,8 +1883,7 @@ set_richcompare(PySetObject *v, PyObject *w, int op) Py_RETURN_FALSE; return set_issuperset(v, w); } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static PyObject * @@ -1959,6 +1995,7 @@ static PyObject * set_reduce(PySetObject *so) { PyObject *keys=NULL, *args=NULL, *result=NULL, *dict=NULL; + _Py_IDENTIFIER(__dict__); keys = PySequence_List((PyObject *)so); if (keys == NULL) @@ -1966,7 +2003,7 @@ set_reduce(PySetObject *so) args = PyTuple_Pack(1, keys); if (args == NULL) goto done; - dict = PyObject_GetAttrString((PyObject *)so, "__dict__"); + dict = _PyObject_GetAttrId((PyObject *)so, &PyId___dict__); if (dict == NULL) { PyErr_Clear(); dict = Py_None; @@ -1980,8 +2017,6 @@ done: return result; } -PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); - static PyObject * set_sizeof(PySetObject *so) { diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 0f4b647..1593335 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -17,6 +17,17 @@ this type and there is exactly one in existence. #include "structmember.h" static PyObject * +ellipsis_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_Size(kwargs))) { + PyErr_SetString(PyExc_TypeError, "EllipsisType takes no arguments"); + return NULL; + } + Py_INCREF(Py_Ellipsis); + return Py_Ellipsis; +} + +static PyObject * ellipsis_repr(PyObject *op) { return PyUnicode_FromString("Ellipsis"); @@ -43,6 +54,24 @@ PyTypeObject PyEllipsis_Type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + ellipsis_new, /* tp_new */ }; PyObject _Py_EllipsisObject = { @@ -51,19 +80,38 @@ PyObject _Py_EllipsisObject = { }; -/* Slice object implementation +/* Slice object implementation */ - start, stop, and step are python objects with None indicating no +/* Using a cache is very effective since typically only a single slice is + * created and then deleted again + */ +static PySliceObject *slice_cache = NULL; +void PySlice_Fini(void) +{ + PySliceObject *obj = slice_cache; + if (obj != NULL) { + slice_cache = NULL; + PyObject_Del(obj); + } +} + +/* start, stop, and step are python objects with None indicating no index is present. */ PyObject * PySlice_New(PyObject *start, PyObject *stop, PyObject *step) { - PySliceObject *obj = PyObject_New(PySliceObject, &PySlice_Type); - - if (obj == NULL) - return NULL; + PySliceObject *obj; + if (slice_cache != NULL) { + obj = slice_cache; + slice_cache = NULL; + _Py_NewReference((PyObject *)obj); + } else { + obj = PyObject_New(PySliceObject, &PySlice_Type); + if (obj == NULL) + return NULL; + } if (step == NULL) step = Py_None; Py_INCREF(step); @@ -232,7 +280,10 @@ slice_dealloc(PySliceObject *r) Py_DECREF(r->step); Py_DECREF(r->start); Py_DECREF(r->stop); - PyObject_Del(r); + if (slice_cache == NULL) + slice_cache = r; + else + PyObject_Del(r); } static PyObject * @@ -298,10 +349,8 @@ slice_richcompare(PyObject *v, PyObject *w, int op) PyObject *t2; PyObject *res; - if (!PySlice_Check(v) || !PySlice_Check(w)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PySlice_Check(v) || !PySlice_Check(w)) + Py_RETURN_NOTIMPLEMENTED; if (v == w) { /* XXX Do we really need this shortcut? diff --git a/Objects/stringlib/asciilib.h b/Objects/stringlib/asciilib.h new file mode 100644 index 0000000..f62813d --- /dev/null +++ b/Objects/stringlib/asciilib.h @@ -0,0 +1,30 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH asciilib_fastsearch +#define STRINGLIB(F) asciilib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR 1 +#define STRINGLIB_MAX_CHAR 0x7Fu +#define STRINGLIB_CHAR Py_UCS1 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_STR PyUnicode_1BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW(STR,LEN) _PyUnicode_FromASCII((char*)(STR),(LEN)) +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ascii_InsertThousandsGrouping + diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h new file mode 100644 index 0000000..2a01089 --- /dev/null +++ b/Objects/stringlib/codecs.h @@ -0,0 +1,629 @@ +/* stringlib: codec implementations */ + +#if STRINGLIB_IS_UNICODE + +/* Mask to quickly check whether a C 'long' contains a + non-ASCII, UTF8-encoded char. */ +#if (SIZEOF_LONG == 8) +# define ASCII_CHAR_MASK 0x8080808080808080UL +#elif (SIZEOF_LONG == 4) +# define ASCII_CHAR_MASK 0x80808080UL +#else +# error C 'long' size should be either 4 or 8! +#endif + +/* 10xxxxxx */ +#define IS_CONTINUATION_BYTE(ch) ((ch) >= 0x80 && (ch) < 0xC0) + +Py_LOCAL_INLINE(Py_UCS4) +STRINGLIB(utf8_decode)(const char **inptr, const char *end, + STRINGLIB_CHAR *dest, + Py_ssize_t *outpos) +{ + Py_UCS4 ch; + const char *s = *inptr; + const char *aligned_end = (const char *) _Py_ALIGN_DOWN(end, SIZEOF_LONG); + STRINGLIB_CHAR *p = dest + *outpos; + + while (s < end) { + ch = (unsigned char)*s; + + if (ch < 0x80) { + /* Fast path for runs of ASCII characters. Given that common UTF-8 + input will consist of an overwhelming majority of ASCII + characters, we try to optimize for this case by checking + as many characters as a C 'long' can contain. + First, check if we can do an aligned read, as most CPUs have + a penalty for unaligned reads. + */ + if (_Py_IS_ALIGNED(s, SIZEOF_LONG)) { + /* Help register allocation */ + register const char *_s = s; + register STRINGLIB_CHAR *_p = p; + while (_s < aligned_end) { + /* Read a whole long at a time (either 4 or 8 bytes), + and do a fast unrolled copy if it only contains ASCII + characters. */ + unsigned long value = *(unsigned long *) _s; + if (value & ASCII_CHAR_MASK) + break; +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + _p[0] = (STRINGLIB_CHAR)(value & 0xFFu); + _p[1] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); + _p[2] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); + _p[3] = (STRINGLIB_CHAR)((value >> 24) & 0xFFu); +# if SIZEOF_LONG == 8 + _p[4] = (STRINGLIB_CHAR)((value >> 32) & 0xFFu); + _p[5] = (STRINGLIB_CHAR)((value >> 40) & 0xFFu); + _p[6] = (STRINGLIB_CHAR)((value >> 48) & 0xFFu); + _p[7] = (STRINGLIB_CHAR)((value >> 56) & 0xFFu); +# endif +#else +# if SIZEOF_LONG == 8 + _p[0] = (STRINGLIB_CHAR)((value >> 56) & 0xFFu); + _p[1] = (STRINGLIB_CHAR)((value >> 48) & 0xFFu); + _p[2] = (STRINGLIB_CHAR)((value >> 40) & 0xFFu); + _p[3] = (STRINGLIB_CHAR)((value >> 32) & 0xFFu); + _p[4] = (STRINGLIB_CHAR)((value >> 24) & 0xFFu); + _p[5] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); + _p[6] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); + _p[7] = (STRINGLIB_CHAR)(value & 0xFFu); +# else + _p[0] = (STRINGLIB_CHAR)((value >> 24) & 0xFFu); + _p[1] = (STRINGLIB_CHAR)((value >> 16) & 0xFFu); + _p[2] = (STRINGLIB_CHAR)((value >> 8) & 0xFFu); + _p[3] = (STRINGLIB_CHAR)(value & 0xFFu); +# endif +#endif + _s += SIZEOF_LONG; + _p += SIZEOF_LONG; + } + s = _s; + p = _p; + if (s == end) + break; + ch = (unsigned char)*s; + } + if (ch < 0x80) { + s++; + *p++ = ch; + continue; + } + } + + if (ch < 0xC2) { + /* invalid sequence + \x80-\xBF -- continuation byte + \xC0-\xC1 -- fake 0000-007F */ + goto InvalidStart; + } + + if (ch < 0xE0) { + /* \xC2\x80-\xDF\xBF -- 0080-07FF */ + Py_UCS4 ch2; + if (end - s < 2) { + /* unexpected end of data: the caller will decide whether + it's an error or not */ + break; + } + ch2 = (unsigned char)s[1]; + if (!IS_CONTINUATION_BYTE(ch2)) + /* invalid continuation byte */ + goto InvalidContinuation; + ch = (ch << 6) + ch2 - + ((0xC0 << 6) + 0x80); + assert ((ch > 0x007F) && (ch <= 0x07FF)); + s += 2; + if (STRINGLIB_MAX_CHAR <= 0x007F || + (STRINGLIB_MAX_CHAR < 0x07FF && ch > STRINGLIB_MAX_CHAR)) + goto Overflow; + *p++ = ch; + continue; + } + + if (ch < 0xF0) { + /* \xE0\xA0\x80-\xEF\xBF\xBF -- 0800-FFFF */ + Py_UCS4 ch2, ch3; + if (end - s < 3) { + /* unexpected end of data: the caller will decide whether + it's an error or not */ + break; + } + ch2 = (unsigned char)s[1]; + ch3 = (unsigned char)s[2]; + if (!IS_CONTINUATION_BYTE(ch2) || + !IS_CONTINUATION_BYTE(ch3)) { + /* invalid continuation byte */ + goto InvalidContinuation; + } + if (ch == 0xE0) { + if (ch2 < 0xA0) + /* invalid sequence + \xE0\x80\x80-\xE0\x9F\xBF -- fake 0000-0800 */ + goto InvalidContinuation; + } + else if (ch == 0xED && ch2 > 0x9F) { + /* Decoding UTF-8 sequences in range \xED\xA0\x80-\xED\xBF\xBF + will result in surrogates in range D800-DFFF. Surrogates are + not valid UTF-8 so they are rejected. + See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf + (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */ + goto InvalidContinuation; + } + ch = (ch << 12) + (ch2 << 6) + ch3 - + ((0xE0 << 12) + (0x80 << 6) + 0x80); + assert ((ch > 0x07FF) && (ch <= 0xFFFF)); + s += 3; + if (STRINGLIB_MAX_CHAR <= 0x07FF || + (STRINGLIB_MAX_CHAR < 0xFFFF && ch > STRINGLIB_MAX_CHAR)) + goto Overflow; + *p++ = ch; + continue; + } + + if (ch < 0xF5) { + /* \xF0\x90\x80\x80-\xF4\x8F\xBF\xBF -- 10000-10FFFF */ + Py_UCS4 ch2, ch3, ch4; + if (end - s < 4) { + /* unexpected end of data: the caller will decide whether + it's an error or not */ + break; + } + ch2 = (unsigned char)s[1]; + ch3 = (unsigned char)s[2]; + ch4 = (unsigned char)s[3]; + if (!IS_CONTINUATION_BYTE(ch2) || + !IS_CONTINUATION_BYTE(ch3) || + !IS_CONTINUATION_BYTE(ch4)) { + /* invalid continuation byte */ + goto InvalidContinuation; + } + if (ch == 0xF0) { + if (ch2 < 0x90) + /* invalid sequence + \xF0\x80\x80\x80-\xF0\x80\xBF\xBF -- fake 0000-FFFF */ + goto InvalidContinuation; + } + else if (ch == 0xF4 && ch2 > 0x8F) { + /* invalid sequence + \xF4\x90\x80\80- -- 110000- overflow */ + goto InvalidContinuation; + } + ch = (ch << 18) + (ch2 << 12) + (ch3 << 6) + ch4 - + ((0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80); + assert ((ch > 0xFFFF) && (ch <= 0x10FFFF)); + s += 4; + if (STRINGLIB_MAX_CHAR <= 0xFFFF || + (STRINGLIB_MAX_CHAR < 0x10FFFF && ch > STRINGLIB_MAX_CHAR)) + goto Overflow; + *p++ = ch; + continue; + } + goto InvalidStart; + } + ch = 0; +Overflow: +Return: + *inptr = s; + *outpos = p - dest; + return ch; +InvalidStart: + ch = 1; + goto Return; +InvalidContinuation: + ch = 2; + goto Return; +} + +#undef ASCII_CHAR_MASK +#undef IS_CONTINUATION_BYTE + + +/* UTF-8 encoder specialized for a Unicode kind to avoid the slow + PyUnicode_READ() macro. Delete some parts of the code depending on the kind: + UCS-1 strings don't need to handle surrogates for example. */ +Py_LOCAL_INLINE(PyObject *) +STRINGLIB(utf8_encoder)(PyObject *unicode, + STRINGLIB_CHAR *data, + Py_ssize_t size, + const char *errors) +{ +#define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */ + + Py_ssize_t i; /* index into s of next input byte */ + PyObject *result; /* result string object */ + char *p; /* next free byte in output buffer */ + Py_ssize_t nallocated; /* number of result bytes allocated */ + Py_ssize_t nneeded; /* number of result bytes needed */ +#if STRINGLIB_SIZEOF_CHAR > 1 + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + PyObject *rep = NULL; +#endif +#if STRINGLIB_SIZEOF_CHAR == 1 + const Py_ssize_t max_char_size = 2; + char stackbuf[MAX_SHORT_UNICHARS * 2]; +#elif STRINGLIB_SIZEOF_CHAR == 2 + const Py_ssize_t max_char_size = 3; + char stackbuf[MAX_SHORT_UNICHARS * 3]; +#else /* STRINGLIB_SIZEOF_CHAR == 4 */ + const Py_ssize_t max_char_size = 4; + char stackbuf[MAX_SHORT_UNICHARS * 4]; +#endif + + assert(size >= 0); + + if (size <= MAX_SHORT_UNICHARS) { + /* Write into the stack buffer; nallocated can't overflow. + * At the end, we'll allocate exactly as much heap space as it + * turns out we need. + */ + nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int); + result = NULL; /* will allocate after we're done */ + p = stackbuf; + } + else { + if (size > PY_SSIZE_T_MAX / max_char_size) { + /* integer overflow */ + return PyErr_NoMemory(); + } + /* Overallocate on the heap, and give the excess back at the end. */ + nallocated = size * max_char_size; + result = PyBytes_FromStringAndSize(NULL, nallocated); + if (result == NULL) + return NULL; + p = PyBytes_AS_STRING(result); + } + + for (i = 0; i < size;) { + Py_UCS4 ch = data[i++]; + + if (ch < 0x80) { + /* Encode ASCII */ + *p++ = (char) ch; + + } + else +#if STRINGLIB_SIZEOF_CHAR > 1 + if (ch < 0x0800) +#endif + { + /* Encode Latin-1 */ + *p++ = (char)(0xc0 | (ch >> 6)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } +#if STRINGLIB_SIZEOF_CHAR > 1 + else if (Py_UNICODE_IS_SURROGATE(ch)) { + Py_ssize_t newpos; + Py_ssize_t repsize, k, startpos; + startpos = i-1; + rep = unicode_encode_call_errorhandler( + errors, &errorHandler, "utf-8", "surrogates not allowed", + unicode, &exc, startpos, startpos+1, &newpos); + if (!rep) + goto error; + + if (PyBytes_Check(rep)) + repsize = PyBytes_GET_SIZE(rep); + else + repsize = PyUnicode_GET_LENGTH(rep); + + if (repsize > max_char_size) { + Py_ssize_t offset; + + if (result == NULL) + offset = p - stackbuf; + else + offset = p - PyBytes_AS_STRING(result); + + if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) { + /* integer overflow */ + PyErr_NoMemory(); + goto error; + } + nallocated += repsize - max_char_size; + if (result != NULL) { + if (_PyBytes_Resize(&result, nallocated) < 0) + goto error; + } else { + result = PyBytes_FromStringAndSize(NULL, nallocated); + if (result == NULL) + goto error; + Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); + } + p = PyBytes_AS_STRING(result) + offset; + } + + if (PyBytes_Check(rep)) { + char *prep = PyBytes_AS_STRING(rep); + for(k = repsize; k > 0; k--) + *p++ = *prep++; + } else /* rep is unicode */ { + enum PyUnicode_Kind repkind; + void *repdata; + + if (PyUnicode_READY(rep) < 0) + goto error; + repkind = PyUnicode_KIND(rep); + repdata = PyUnicode_DATA(rep); + + for(k=0; k<repsize; k++) { + Py_UCS4 c = PyUnicode_READ(repkind, repdata, k); + if (0x80 <= c) { + raise_encode_exception(&exc, "utf-8", + unicode, + i-1, i, + "surrogates not allowed"); + goto error; + } + *p++ = (char)c; + } + } + Py_CLEAR(rep); + } + else +#if STRINGLIB_SIZEOF_CHAR > 2 + if (ch < 0x10000) +#endif + { + *p++ = (char)(0xe0 | (ch >> 12)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } +#if STRINGLIB_SIZEOF_CHAR > 2 + else /* ch >= 0x10000 */ + { + assert(ch <= MAX_UNICODE); + /* Encode UCS4 Unicode ordinals */ + *p++ = (char)(0xf0 | (ch >> 18)); + *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } +#endif /* STRINGLIB_SIZEOF_CHAR > 2 */ +#endif /* STRINGLIB_SIZEOF_CHAR > 1 */ + } + + if (result == NULL) { + /* This was stack allocated. */ + nneeded = p - stackbuf; + assert(nneeded <= nallocated); + result = PyBytes_FromStringAndSize(stackbuf, nneeded); + } + else { + /* Cut back to size actually needed. */ + nneeded = p - PyBytes_AS_STRING(result); + assert(nneeded <= nallocated); + _PyBytes_Resize(&result, nneeded); + } + +#if STRINGLIB_SIZEOF_CHAR > 1 + Py_XDECREF(errorHandler); + Py_XDECREF(exc); +#endif + return result; + +#if STRINGLIB_SIZEOF_CHAR > 1 + error: + Py_XDECREF(rep); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + Py_XDECREF(result); + return NULL; +#endif + +#undef MAX_SHORT_UNICHARS +} + +/* The pattern for constructing UCS2-repeated masks. */ +#if SIZEOF_LONG == 8 +# define UCS2_REPEAT_MASK 0x0001000100010001ul +#elif SIZEOF_LONG == 4 +# define UCS2_REPEAT_MASK 0x00010001ul +#else +# error C 'long' size should be either 4 or 8! +#endif + +/* The mask for fast checking. */ +#if STRINGLIB_SIZEOF_CHAR == 1 +/* The mask for fast checking of whether a C 'long' contains a + non-ASCII or non-Latin1 UTF16-encoded characters. */ +# define FAST_CHAR_MASK (UCS2_REPEAT_MASK * (0xFFFFu & ~STRINGLIB_MAX_CHAR)) +#else +/* The mask for fast checking of whether a C 'long' may contain + UTF16-encoded surrogate characters. This is an efficient heuristic, + assuming that non-surrogate characters with a code point >= 0x8000 are + rare in most input. +*/ +# define FAST_CHAR_MASK (UCS2_REPEAT_MASK * 0x8000u) +#endif +/* The mask for fast byte-swapping. */ +#define STRIPPED_MASK (UCS2_REPEAT_MASK * 0x00FFu) +/* Swap bytes. */ +#define SWAB(value) ((((value) >> 8) & STRIPPED_MASK) | \ + (((value) & STRIPPED_MASK) << 8)) + +Py_LOCAL_INLINE(Py_UCS4) +STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e, + STRINGLIB_CHAR *dest, Py_ssize_t *outpos, + int native_ordering) +{ + Py_UCS4 ch; + const unsigned char *aligned_end = + (const unsigned char *) _Py_ALIGN_DOWN(e, SIZEOF_LONG); + const unsigned char *q = *inptr; + STRINGLIB_CHAR *p = dest + *outpos; + /* Offsets from q for retrieving byte pairs in the right order. */ +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + int ihi = !!native_ordering, ilo = !native_ordering; +#else + int ihi = !native_ordering, ilo = !!native_ordering; +#endif + --e; + + while (q < e) { + Py_UCS4 ch2; + /* First check for possible aligned read of a C 'long'. Unaligned + reads are more expensive, better to defer to another iteration. */ + if (_Py_IS_ALIGNED(q, SIZEOF_LONG)) { + /* Fast path for runs of in-range non-surrogate chars. */ + register const unsigned char *_q = q; + while (_q < aligned_end) { + unsigned long block = * (unsigned long *) _q; + if (native_ordering) { + /* Can use buffer directly */ + if (block & FAST_CHAR_MASK) + break; + } + else { + /* Need to byte-swap */ + if (block & SWAB(FAST_CHAR_MASK)) + break; +#if STRINGLIB_SIZEOF_CHAR == 1 + block >>= 8; +#else + block = SWAB(block); +#endif + } +#ifdef BYTEORDER_IS_LITTLE_ENDIAN +# if SIZEOF_LONG == 4 + p[0] = (STRINGLIB_CHAR)(block & 0xFFFFu); + p[1] = (STRINGLIB_CHAR)(block >> 16); +# elif SIZEOF_LONG == 8 + p[0] = (STRINGLIB_CHAR)(block & 0xFFFFu); + p[1] = (STRINGLIB_CHAR)((block >> 16) & 0xFFFFu); + p[2] = (STRINGLIB_CHAR)((block >> 32) & 0xFFFFu); + p[3] = (STRINGLIB_CHAR)(block >> 48); +# endif +#else +# if SIZEOF_LONG == 4 + p[0] = (STRINGLIB_CHAR)(block >> 16); + p[1] = (STRINGLIB_CHAR)(block & 0xFFFFu); +# elif SIZEOF_LONG == 8 + p[0] = (STRINGLIB_CHAR)(block >> 48); + p[1] = (STRINGLIB_CHAR)((block >> 32) & 0xFFFFu); + p[2] = (STRINGLIB_CHAR)((block >> 16) & 0xFFFFu); + p[3] = (STRINGLIB_CHAR)(block & 0xFFFFu); +# endif +#endif + _q += SIZEOF_LONG; + p += SIZEOF_LONG / 2; + } + q = _q; + if (q >= e) + break; + } + + ch = (q[ihi] << 8) | q[ilo]; + q += 2; + if (!Py_UNICODE_IS_SURROGATE(ch)) { +#if STRINGLIB_SIZEOF_CHAR < 2 + if (ch > STRINGLIB_MAX_CHAR) + /* Out-of-range */ + goto Return; +#endif + *p++ = (STRINGLIB_CHAR)ch; + continue; + } + + /* UTF-16 code pair: */ + if (q >= e) + goto UnexpectedEnd; + if (!Py_UNICODE_IS_HIGH_SURROGATE(ch)) + goto IllegalEncoding; + ch2 = (q[ihi] << 8) | q[ilo]; + q += 2; + if (!Py_UNICODE_IS_LOW_SURROGATE(ch2)) + goto IllegalSurrogate; + ch = Py_UNICODE_JOIN_SURROGATES(ch, ch2); +#if STRINGLIB_SIZEOF_CHAR < 4 + /* Out-of-range */ + goto Return; +#else + *p++ = (STRINGLIB_CHAR)ch; +#endif + } + ch = 0; +Return: + *inptr = q; + *outpos = p - dest; + return ch; +UnexpectedEnd: + ch = 1; + goto Return; +IllegalEncoding: + ch = 2; + goto Return; +IllegalSurrogate: + ch = 3; + goto Return; +} +#undef UCS2_REPEAT_MASK +#undef FAST_CHAR_MASK +#undef STRIPPED_MASK +#undef SWAB + + +Py_LOCAL_INLINE(void) +STRINGLIB(utf16_encode)(unsigned short *out, + const STRINGLIB_CHAR *in, + Py_ssize_t len, + int native_ordering) +{ + const STRINGLIB_CHAR *end = in + len; +#if STRINGLIB_SIZEOF_CHAR == 1 +# define SWAB2(CH) ((CH) << 8) +#else +# define SWAB2(CH) (((CH) << 8) | ((CH) >> 8)) +#endif +#if STRINGLIB_MAX_CHAR < 0x10000 + if (native_ordering) { +# if STRINGLIB_SIZEOF_CHAR == 2 + Py_MEMCPY(out, in, 2 * len); +# else + _PyUnicode_CONVERT_BYTES(STRINGLIB_CHAR, unsigned short, in, end, out); +# endif + } else { + const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4); + while (in < unrolled_end) { + out[0] = SWAB2(in[0]); + out[1] = SWAB2(in[1]); + out[2] = SWAB2(in[2]); + out[3] = SWAB2(in[3]); + in += 4; out += 4; + } + while (in < end) { + *out++ = SWAB2(*in); + ++in; + } + } +#else + if (native_ordering) { + while (in < end) { + Py_UCS4 ch = *in++; + if (ch < 0x10000) + *out++ = ch; + else { + out[0] = Py_UNICODE_HIGH_SURROGATE(ch); + out[1] = Py_UNICODE_LOW_SURROGATE(ch); + out += 2; + } + } + } else { + while (in < end) { + Py_UCS4 ch = *in++; + if (ch < 0x10000) + *out++ = SWAB2((Py_UCS2)ch); + else { + Py_UCS2 ch1 = Py_UNICODE_HIGH_SURROGATE(ch); + Py_UCS2 ch2 = Py_UNICODE_LOW_SURROGATE(ch); + out[0] = SWAB2(ch1); + out[1] = SWAB2(ch2); + out += 2; + } + } + } +#endif +#undef SWAB2 +} +#endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/stringlib/count.h b/Objects/stringlib/count.h index de34f96..f48500b 100644 --- a/Objects/stringlib/count.h +++ b/Objects/stringlib/count.h @@ -1,14 +1,11 @@ /* stringlib: count implementation */ -#ifndef STRINGLIB_COUNT_H -#define STRINGLIB_COUNT_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif Py_LOCAL_INLINE(Py_ssize_t) -stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(count)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t maxcount) { @@ -19,7 +16,7 @@ stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len, if (sub_len == 0) return (str_len < maxcount) ? str_len + 1 : maxcount; - count = fastsearch(str, str_len, sub, sub_len, maxcount, FAST_COUNT); + count = FASTSEARCH(str, str_len, sub, sub_len, maxcount, FAST_COUNT); if (count < 0) return 0; /* no match */ @@ -27,4 +24,4 @@ stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len, return count; } -#endif + diff --git a/Objects/stringlib/eq.h b/Objects/stringlib/eq.h index 3e7f5e8..8e79a43 100644 --- a/Objects/stringlib/eq.h +++ b/Objects/stringlib/eq.h @@ -9,13 +9,26 @@ unicode_eq(PyObject *aa, PyObject *bb) register PyUnicodeObject *a = (PyUnicodeObject *)aa; register PyUnicodeObject *b = (PyUnicodeObject *)bb; - if (a->length != b->length) + if (PyUnicode_READY(a) == -1 || PyUnicode_READY(b) == -1) { + assert(0 && "unicode_eq ready fail"); return 0; - if (a->length == 0) + } + + if (PyUnicode_GET_LENGTH(a) != PyUnicode_GET_LENGTH(b)) + return 0; + if (PyUnicode_GET_LENGTH(a) == 0) return 1; - if (a->str[0] != b->str[0]) + if (PyUnicode_KIND(a) != PyUnicode_KIND(b)) + return 0; + /* Just comparing the first byte is enough to see if a and b differ. + * If they are 2 byte or 4 byte character most differences will happen in + * the lower bytes anyways. + */ + if (PyUnicode_1BYTE_DATA(a)[0] != PyUnicode_1BYTE_DATA(b)[0]) return 0; - if (a->length == 1) + if (PyUnicode_KIND(a) == PyUnicode_1BYTE_KIND && + PyUnicode_GET_LENGTH(a) == 1) return 1; - return memcmp(a->str, b->str, a->length * sizeof(Py_UNICODE)) == 0; + return memcmp(PyUnicode_1BYTE_DATA(a), PyUnicode_1BYTE_DATA(b), + PyUnicode_GET_LENGTH(a) * PyUnicode_KIND(a)) == 0; } diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index e231c58..ecf885e 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -1,6 +1,5 @@ /* stringlib: fastsearch implementation */ -#ifndef STRINGLIB_FASTSEARCH_H #define STRINGLIB_FASTSEARCH_H /* fast search/count implementation, based on a mix between boyer- @@ -33,8 +32,61 @@ #define STRINGLIB_BLOOM(mask, ch) \ ((mask & (1UL << ((ch) & (STRINGLIB_BLOOM_WIDTH -1))))) + +Py_LOCAL_INLINE(Py_ssize_t) +STRINGLIB(fastsearch_memchr_1char)(const STRINGLIB_CHAR* s, Py_ssize_t n, + STRINGLIB_CHAR ch, unsigned char needle, + Py_ssize_t maxcount, int mode) +{ + void *candidate; + const STRINGLIB_CHAR *found; + +#define DO_MEMCHR(memchr, s, needle, nchars) do { \ + candidate = memchr((const void *) (s), (needle), (nchars) * sizeof(STRINGLIB_CHAR)); \ + found = (const STRINGLIB_CHAR *) _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); \ + } while (0) + + if (mode == FAST_SEARCH) { + const STRINGLIB_CHAR *ptr = s; + const STRINGLIB_CHAR *e = s + n; + while (ptr < e) { + DO_MEMCHR(memchr, ptr, needle, e - ptr); + if (found == NULL) + return -1; + if (sizeof(STRINGLIB_CHAR) == 1 || *found == ch) + return (found - s); + /* False positive */ + ptr = found + 1; + } + return -1; + } +#ifdef HAVE_MEMRCHR + /* memrchr() is a GNU extension, available since glibc 2.1.91. + it doesn't seem as optimized as memchr(), but is still quite + faster than our hand-written loop in FASTSEARCH below */ + else if (mode == FAST_RSEARCH) { + while (n > 0) { + DO_MEMCHR(memrchr, s, needle, n); + if (found == NULL) + return -1; + n = found - s; + if (sizeof(STRINGLIB_CHAR) == 1 || *found == ch) + return n; + /* False positive */ + } + return -1; + } +#endif + else { + assert(0); /* Should never get here */ + return 0; + } + +#undef DO_MEMCHR +} + Py_LOCAL_INLINE(Py_ssize_t) -fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n, +FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, const STRINGLIB_CHAR* p, Py_ssize_t m, Py_ssize_t maxcount, int mode) { @@ -52,6 +104,24 @@ fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n, if (m <= 0) return -1; /* use special case for 1-character strings */ + if (n > 10 && (mode == FAST_SEARCH +#ifdef HAVE_MEMRCHR + || mode == FAST_RSEARCH +#endif + )) { + /* use memchr if we can choose a needle without two many likely + false positives */ + unsigned char needle; + needle = p[0] & 0xff; +#if STRINGLIB_SIZEOF_CHAR > 1 + /* If looking for a multiple of 256, we'd have too + many false positives looking for the '\0' byte in UCS2 + and UCS4 representations. */ + if (needle != 0) +#endif + return STRINGLIB(fastsearch_memchr_1char) + (s, n, p[0], needle, maxcount, mode); + } if (mode == FAST_COUNT) { for (i = 0; i < n; i++) if (s[i] == p[0]) { @@ -157,4 +227,3 @@ fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n, return count; } -#endif diff --git a/Objects/stringlib/find.h b/Objects/stringlib/find.h index ce615dc..518e012 100644 --- a/Objects/stringlib/find.h +++ b/Objects/stringlib/find.h @@ -1,14 +1,11 @@ /* stringlib: find/index implementation */ -#ifndef STRINGLIB_FIND_H -#define STRINGLIB_FIND_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif Py_LOCAL_INLINE(Py_ssize_t) -stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(find)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t offset) { @@ -19,7 +16,7 @@ stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len, if (sub_len == 0) return offset; - pos = fastsearch(str, str_len, sub, sub_len, -1, FAST_SEARCH); + pos = FASTSEARCH(str, str_len, sub, sub_len, -1, FAST_SEARCH); if (pos >= 0) pos += offset; @@ -28,7 +25,7 @@ stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len, } Py_LOCAL_INLINE(Py_ssize_t) -stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(rfind)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t offset) { @@ -39,7 +36,7 @@ stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len, if (sub_len == 0) return str_len + offset; - pos = fastsearch(str, str_len, sub, sub_len, -1, FAST_RSEARCH); + pos = FASTSEARCH(str, str_len, sub, sub_len, -1, FAST_RSEARCH); if (pos >= 0) pos += offset; @@ -63,29 +60,29 @@ stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len, } Py_LOCAL_INLINE(Py_ssize_t) -stringlib_find_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(find_slice)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t start, Py_ssize_t end) { ADJUST_INDICES(start, end, str_len); - return stringlib_find(str + start, end - start, sub, sub_len, start); + return STRINGLIB(find)(str + start, end - start, sub, sub_len, start); } Py_LOCAL_INLINE(Py_ssize_t) -stringlib_rfind_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len, +STRINGLIB(rfind_slice)(const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sub, Py_ssize_t sub_len, Py_ssize_t start, Py_ssize_t end) { ADJUST_INDICES(start, end, str_len); - return stringlib_rfind(str + start, end - start, sub, sub_len, start); + return STRINGLIB(rfind)(str + start, end - start, sub, sub_len, start); } #ifdef STRINGLIB_WANT_CONTAINS_OBJ Py_LOCAL_INLINE(int) -stringlib_contains_obj(PyObject* str, PyObject* sub) +STRINGLIB(contains_obj)(PyObject* str, PyObject* sub) { - return stringlib_find( + return STRINGLIB(find)( STRINGLIB_STR(str), STRINGLIB_LEN(str), STRINGLIB_STR(sub), STRINGLIB_LEN(sub), 0 ) != -1; @@ -98,14 +95,14 @@ This function is a helper for the "find" family (find, rfind, index, rindex) and for count, startswith and endswith, because they all have the same behaviour for the arguments. -It does not touch the variables received until it knows everything +It does not touch the variables received until it knows everything is ok. */ #define FORMAT_BUFFER_SIZE 50 Py_LOCAL_INLINE(int) -stringlib_parse_args_finds(const char * function_name, PyObject *args, +STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args, PyObject **subobj, Py_ssize_t *start, Py_ssize_t *end) { @@ -148,28 +145,76 @@ first argument is a unicode object. Note that we receive a pointer to the pointer of the substring object, so when we create that object in this function we don't DECREF it, -because it continues living in the caller functions (those functions, +because it continues living in the caller functions (those functions, after finishing using the substring, must DECREF it). */ Py_LOCAL_INLINE(int) -stringlib_parse_args_finds_unicode(const char * function_name, PyObject *args, - PyUnicodeObject **substring, +STRINGLIB(parse_args_finds_unicode)(const char * function_name, PyObject *args, + PyObject **substring, Py_ssize_t *start, Py_ssize_t *end) { PyObject *tmp_substring; - if(stringlib_parse_args_finds(function_name, args, &tmp_substring, + if(STRINGLIB(parse_args_finds)(function_name, args, &tmp_substring, start, end)) { tmp_substring = PyUnicode_FromObject(tmp_substring); if (!tmp_substring) return 0; - *substring = (PyUnicodeObject *)tmp_substring; + *substring = tmp_substring; return 1; } return 0; } -#endif /* STRINGLIB_IS_UNICODE */ +#else /* !STRINGLIB_IS_UNICODE */ + +/* +Wraps stringlib_parse_args_finds() and additionally checks whether the +first argument is an integer in range(0, 256). + +If this is the case, writes the integer value to the byte parameter +and sets subobj to NULL. Otherwise, sets the first argument to subobj +and doesn't touch byte. The other parameters are similar to those of +stringlib_parse_args_finds(). +*/ + +Py_LOCAL_INLINE(int) +STRINGLIB(parse_args_finds_byte)(const char *function_name, PyObject *args, + PyObject **subobj, char *byte, + Py_ssize_t *start, Py_ssize_t *end) +{ + PyObject *tmp_subobj; + Py_ssize_t ival; + PyObject *err; + + if(!STRINGLIB(parse_args_finds)(function_name, args, &tmp_subobj, + start, end)) + return 0; -#endif /* STRINGLIB_FIND_H */ + if (!PyNumber_Check(tmp_subobj)) { + *subobj = tmp_subobj; + return 1; + } + + ival = PyNumber_AsSsize_t(tmp_subobj, PyExc_OverflowError); + if (ival == -1) { + err = PyErr_Occurred(); + if (err && !PyErr_GivenExceptionMatches(err, PyExc_OverflowError)) { + PyErr_Clear(); + *subobj = tmp_subobj; + return 1; + } + } + + if (ival < 0 || ival > 255) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return 0; + } + + *subobj = NULL; + *byte = (char)ival; + return 1; +} + +#endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/stringlib/find_max_char.h b/Objects/stringlib/find_max_char.h new file mode 100644 index 0000000..06559c8 --- /dev/null +++ b/Objects/stringlib/find_max_char.h @@ -0,0 +1,133 @@ +/* Finding the optimal width of unicode characters in a buffer */ + +#if STRINGLIB_IS_UNICODE + +/* Mask to quickly check whether a C 'long' contains a + non-ASCII, UTF8-encoded char. */ +#if (SIZEOF_LONG == 8) +# define UCS1_ASCII_CHAR_MASK 0x8080808080808080UL +#elif (SIZEOF_LONG == 4) +# define UCS1_ASCII_CHAR_MASK 0x80808080UL +#else +# error C 'long' size should be either 4 or 8! +#endif + +#if STRINGLIB_SIZEOF_CHAR == 1 + +Py_LOCAL_INLINE(Py_UCS4) +STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) +{ + const unsigned char *p = (const unsigned char *) begin; + const unsigned char *aligned_end = + (const unsigned char *) _Py_ALIGN_DOWN(end, SIZEOF_LONG); + + while (p < end) { + if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) { + /* Help register allocation */ + register const unsigned char *_p = p; + while (_p < aligned_end) { + unsigned long value = *(unsigned long *) _p; + if (value & UCS1_ASCII_CHAR_MASK) + return 255; + _p += SIZEOF_LONG; + } + p = _p; + if (p == end) + break; + } + if (*p++ & 0x80) + return 255; + } + return 127; +} + +#undef ASCII_CHAR_MASK + +#else /* STRINGLIB_SIZEOF_CHAR == 1 */ + +#define MASK_ASCII 0xFFFFFF80 +#define MASK_UCS1 0xFFFFFF00 +#define MASK_UCS2 0xFFFF0000 + +#define MAX_CHAR_ASCII 0x7f +#define MAX_CHAR_UCS1 0xff +#define MAX_CHAR_UCS2 0xffff +#define MAX_CHAR_UCS4 0x10ffff + +Py_LOCAL_INLINE(Py_UCS4) +STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) +{ +#if STRINGLIB_SIZEOF_CHAR == 2 + const Py_UCS4 mask_limit = MASK_UCS1; + const Py_UCS4 max_char_limit = MAX_CHAR_UCS2; +#elif STRINGLIB_SIZEOF_CHAR == 4 + const Py_UCS4 mask_limit = MASK_UCS2; + const Py_UCS4 max_char_limit = MAX_CHAR_UCS4; +#else +#error Invalid STRINGLIB_SIZEOF_CHAR (must be 1, 2 or 4) +#endif + register Py_UCS4 mask; + Py_ssize_t n = end - begin; + const STRINGLIB_CHAR *p = begin; + const STRINGLIB_CHAR *unrolled_end = begin + _Py_SIZE_ROUND_DOWN(n, 4); + Py_UCS4 max_char; + + max_char = MAX_CHAR_ASCII; + mask = MASK_ASCII; + while (p < unrolled_end) { + STRINGLIB_CHAR bits = p[0] | p[1] | p[2] | p[3]; + if (bits & mask) { + if (mask == mask_limit) { + /* Limit reached */ + return max_char_limit; + } + if (mask == MASK_ASCII) { + max_char = MAX_CHAR_UCS1; + mask = MASK_UCS1; + } + else { + /* mask can't be MASK_UCS2 because of mask_limit above */ + assert(mask == MASK_UCS1); + max_char = MAX_CHAR_UCS2; + mask = MASK_UCS2; + } + /* We check the new mask on the same chars in the next iteration */ + continue; + } + p += 4; + } + while (p < end) { + if (p[0] & mask) { + if (mask == mask_limit) { + /* Limit reached */ + return max_char_limit; + } + if (mask == MASK_ASCII) { + max_char = MAX_CHAR_UCS1; + mask = MASK_UCS1; + } + else { + /* mask can't be MASK_UCS2 because of mask_limit above */ + assert(mask == MASK_UCS1); + max_char = MAX_CHAR_UCS2; + mask = MASK_UCS2; + } + /* We check the new mask on the same chars in the next iteration */ + continue; + } + p++; + } + return max_char; +} + +#undef MASK_ASCII +#undef MASK_UCS1 +#undef MASK_UCS2 +#undef MAX_CHAR_ASCII +#undef MAX_CHAR_UCS1 +#undef MAX_CHAR_UCS2 +#undef MAX_CHAR_UCS4 + +#endif /* STRINGLIB_SIZEOF_CHAR == 1 */ +#endif /* STRINGLIB_IS_UNICODE */ + diff --git a/Objects/stringlib/formatter.h b/Objects/stringlib/formatter.h deleted file mode 100644 index 139b56c..0000000 --- a/Objects/stringlib/formatter.h +++ /dev/null @@ -1,1516 +0,0 @@ -/* implements the string, long, and float formatters. that is, - string.__format__, etc. */ - -#include <locale.h> - -/* Before including this, you must include either: - stringlib/unicodedefs.h - stringlib/stringdefs.h - - Also, you should define the names: - FORMAT_STRING - FORMAT_LONG - FORMAT_FLOAT - FORMAT_COMPLEX - to be whatever you want the public names of these functions to - be. These are the only non-static functions defined here. -*/ - -/* Raises an exception about an unknown presentation type for this - * type. */ - -static void -unknown_presentation_type(STRINGLIB_CHAR presentation_type, - const char* type_name) -{ -#if STRINGLIB_IS_UNICODE - /* If STRINGLIB_CHAR is Py_UNICODE, %c might be out-of-range, - hence the two cases. If it is char, gcc complains that the - condition below is always true, hence the ifdef. */ - if (presentation_type > 32 && presentation_type < 128) -#endif - PyErr_Format(PyExc_ValueError, - "Unknown format code '%c' " - "for object of type '%.200s'", - (char)presentation_type, - type_name); -#if STRINGLIB_IS_UNICODE - else - PyErr_Format(PyExc_ValueError, - "Unknown format code '\\x%x' " - "for object of type '%.200s'", - (unsigned int)presentation_type, - type_name); -#endif -} - -static void -invalid_comma_type(STRINGLIB_CHAR presentation_type) -{ -#if STRINGLIB_IS_UNICODE - /* See comment in unknown_presentation_type */ - if (presentation_type > 32 && presentation_type < 128) -#endif - PyErr_Format(PyExc_ValueError, - "Cannot specify ',' with '%c'.", - (char)presentation_type); -#if STRINGLIB_IS_UNICODE - else - PyErr_Format(PyExc_ValueError, - "Cannot specify ',' with '\\x%x'.", - (unsigned int)presentation_type); -#endif -} - -/* - get_integer consumes 0 or more decimal digit characters from an - input string, updates *result with the corresponding positive - integer, and returns the number of digits consumed. - - returns -1 on error. -*/ -static int -get_integer(STRINGLIB_CHAR **ptr, STRINGLIB_CHAR *end, - Py_ssize_t *result) -{ - Py_ssize_t accumulator, digitval; - int numdigits; - accumulator = numdigits = 0; - for (;;(*ptr)++, numdigits++) { - if (*ptr >= end) - break; - digitval = STRINGLIB_TODECIMAL(**ptr); - if (digitval < 0) - break; - /* - Detect possible overflow before it happens: - - accumulator * 10 + digitval > PY_SSIZE_T_MAX if and only if - accumulator > (PY_SSIZE_T_MAX - digitval) / 10. - */ - if (accumulator > (PY_SSIZE_T_MAX - digitval) / 10) { - PyErr_Format(PyExc_ValueError, - "Too many decimal digits in format string"); - return -1; - } - accumulator = accumulator * 10 + digitval; - } - *result = accumulator; - return numdigits; -} - -/************************************************************************/ -/*********** standard format specifier parsing **************************/ -/************************************************************************/ - -/* returns true if this character is a specifier alignment token */ -Py_LOCAL_INLINE(int) -is_alignment_token(STRINGLIB_CHAR c) -{ - switch (c) { - case '<': case '>': case '=': case '^': - return 1; - default: - return 0; - } -} - -/* returns true if this character is a sign element */ -Py_LOCAL_INLINE(int) -is_sign_element(STRINGLIB_CHAR c) -{ - switch (c) { - case ' ': case '+': case '-': - return 1; - default: - return 0; - } -} - - -typedef struct { - STRINGLIB_CHAR fill_char; - STRINGLIB_CHAR align; - int alternate; - STRINGLIB_CHAR sign; - Py_ssize_t width; - int thousands_separators; - Py_ssize_t precision; - STRINGLIB_CHAR type; -} InternalFormatSpec; - - -#if 0 -/* Occassionally useful for debugging. Should normally be commented out. */ -static void -DEBUG_PRINT_FORMAT_SPEC(InternalFormatSpec *format) -{ - printf("internal format spec: fill_char %d\n", format->fill_char); - printf("internal format spec: align %d\n", format->align); - printf("internal format spec: alternate %d\n", format->alternate); - printf("internal format spec: sign %d\n", format->sign); - printf("internal format spec: width %zd\n", format->width); - printf("internal format spec: thousands_separators %d\n", - format->thousands_separators); - printf("internal format spec: precision %zd\n", format->precision); - printf("internal format spec: type %c\n", format->type); - printf("\n"); -} -#endif - - -/* - ptr points to the start of the format_spec, end points just past its end. - fills in format with the parsed information. - returns 1 on success, 0 on failure. - if failure, sets the exception -*/ -static int -parse_internal_render_format_spec(STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len, - InternalFormatSpec *format, - char default_type, - char default_align) -{ - STRINGLIB_CHAR *ptr = format_spec; - STRINGLIB_CHAR *end = format_spec + format_spec_len; - - /* end-ptr is used throughout this code to specify the length of - the input string */ - - Py_ssize_t consumed; - int align_specified = 0; - - format->fill_char = '\0'; - format->align = default_align; - format->alternate = 0; - format->sign = '\0'; - format->width = -1; - format->thousands_separators = 0; - format->precision = -1; - format->type = default_type; - - /* If the second char is an alignment token, - then parse the fill char */ - if (end-ptr >= 2 && is_alignment_token(ptr[1])) { - format->align = ptr[1]; - format->fill_char = ptr[0]; - align_specified = 1; - ptr += 2; - } - else if (end-ptr >= 1 && is_alignment_token(ptr[0])) { - format->align = ptr[0]; - align_specified = 1; - ++ptr; - } - - /* Parse the various sign options */ - if (end-ptr >= 1 && is_sign_element(ptr[0])) { - format->sign = ptr[0]; - ++ptr; - } - - /* If the next character is #, we're in alternate mode. This only - applies to integers. */ - if (end-ptr >= 1 && ptr[0] == '#') { - format->alternate = 1; - ++ptr; - } - - /* The special case for 0-padding (backwards compat) */ - if (format->fill_char == '\0' && end-ptr >= 1 && ptr[0] == '0') { - format->fill_char = '0'; - if (!align_specified) { - format->align = '='; - } - ++ptr; - } - - consumed = get_integer(&ptr, end, &format->width); - if (consumed == -1) - /* Overflow error. Exception already set. */ - return 0; - - /* If consumed is 0, we didn't consume any characters for the - width. In that case, reset the width to -1, because - get_integer() will have set it to zero. -1 is how we record - that the width wasn't specified. */ - if (consumed == 0) - format->width = -1; - - /* Comma signifies add thousands separators */ - if (end-ptr && ptr[0] == ',') { - format->thousands_separators = 1; - ++ptr; - } - - /* Parse field precision */ - if (end-ptr && ptr[0] == '.') { - ++ptr; - - consumed = get_integer(&ptr, end, &format->precision); - if (consumed == -1) - /* Overflow error. Exception already set. */ - return 0; - - /* Not having a precision after a dot is an error. */ - if (consumed == 0) { - PyErr_Format(PyExc_ValueError, - "Format specifier missing precision"); - return 0; - } - - } - - /* Finally, parse the type field. */ - - if (end-ptr > 1) { - /* More than one char remain, invalid conversion spec. */ - PyErr_Format(PyExc_ValueError, "Invalid conversion specification"); - return 0; - } - - if (end-ptr == 1) { - format->type = ptr[0]; - ++ptr; - } - - /* Do as much validating as we can, just by looking at the format - specifier. Do not take into account what type of formatting - we're doing (int, float, string). */ - - if (format->thousands_separators) { - switch (format->type) { - case 'd': - case 'e': - case 'f': - case 'g': - case 'E': - case 'G': - case '%': - case 'F': - case '\0': - /* These are allowed. See PEP 378.*/ - break; - default: - invalid_comma_type(format->type); - return 0; - } - } - - return 1; -} - -/* Calculate the padding needed. */ -static void -calc_padding(Py_ssize_t nchars, Py_ssize_t width, STRINGLIB_CHAR align, - Py_ssize_t *n_lpadding, Py_ssize_t *n_rpadding, - Py_ssize_t *n_total) -{ - if (width >= 0) { - if (nchars > width) - *n_total = nchars; - else - *n_total = width; - } - else { - /* not specified, use all of the chars and no more */ - *n_total = nchars; - } - - /* Figure out how much leading space we need, based on the - aligning */ - if (align == '>') - *n_lpadding = *n_total - nchars; - else if (align == '^') - *n_lpadding = (*n_total - nchars) / 2; - else if (align == '<' || align == '=') - *n_lpadding = 0; - else { - /* We should never have an unspecified alignment. */ - *n_lpadding = 0; - assert(0); - } - - *n_rpadding = *n_total - nchars - *n_lpadding; -} - -/* Do the padding, and return a pointer to where the caller-supplied - content goes. */ -static STRINGLIB_CHAR * -fill_padding(STRINGLIB_CHAR *p, Py_ssize_t nchars, STRINGLIB_CHAR fill_char, - Py_ssize_t n_lpadding, Py_ssize_t n_rpadding) -{ - /* Pad on left. */ - if (n_lpadding) - STRINGLIB_FILL(p, fill_char, n_lpadding); - - /* Pad on right. */ - if (n_rpadding) - STRINGLIB_FILL(p + nchars + n_lpadding, fill_char, n_rpadding); - - /* Pointer to the user content. */ - return p + n_lpadding; -} - -#if defined FORMAT_FLOAT || defined FORMAT_LONG || defined FORMAT_COMPLEX -/************************************************************************/ -/*********** common routines for numeric formatting *********************/ -/************************************************************************/ - -/* Locale type codes. */ -#define LT_CURRENT_LOCALE 0 -#define LT_DEFAULT_LOCALE 1 -#define LT_NO_LOCALE 2 - -/* Locale info needed for formatting integers and the part of floats - before and including the decimal. Note that locales only support - 8-bit chars, not unicode. */ -typedef struct { - char *decimal_point; - char *thousands_sep; - char *grouping; -} LocaleInfo; - -/* describes the layout for an integer, see the comment in - calc_number_widths() for details */ -typedef struct { - Py_ssize_t n_lpadding; - Py_ssize_t n_prefix; - Py_ssize_t n_spadding; - Py_ssize_t n_rpadding; - char sign; - Py_ssize_t n_sign; /* number of digits needed for sign (0/1) */ - Py_ssize_t n_grouped_digits; /* Space taken up by the digits, including - any grouping chars. */ - Py_ssize_t n_decimal; /* 0 if only an integer */ - Py_ssize_t n_remainder; /* Digits in decimal and/or exponent part, - excluding the decimal itself, if - present. */ - - /* These 2 are not the widths of fields, but are needed by - STRINGLIB_GROUPING. */ - Py_ssize_t n_digits; /* The number of digits before a decimal - or exponent. */ - Py_ssize_t n_min_width; /* The min_width we used when we computed - the n_grouped_digits width. */ -} NumberFieldWidths; - - -/* Given a number of the form: - digits[remainder] - where ptr points to the start and end points to the end, find where - the integer part ends. This could be a decimal, an exponent, both, - or neither. - If a decimal point is present, set *has_decimal and increment - remainder beyond it. - Results are undefined (but shouldn't crash) for improperly - formatted strings. -*/ -static void -parse_number(STRINGLIB_CHAR *ptr, Py_ssize_t len, - Py_ssize_t *n_remainder, int *has_decimal) -{ - STRINGLIB_CHAR *end = ptr + len; - STRINGLIB_CHAR *remainder; - - while (ptr<end && isdigit(*ptr)) - ++ptr; - remainder = ptr; - - /* Does remainder start with a decimal point? */ - *has_decimal = ptr<end && *remainder == '.'; - - /* Skip the decimal point. */ - if (*has_decimal) - remainder++; - - *n_remainder = end - remainder; -} - -/* not all fields of format are used. for example, precision is - unused. should this take discrete params in order to be more clear - about what it does? or is passing a single format parameter easier - and more efficient enough to justify a little obfuscation? */ -static Py_ssize_t -calc_number_widths(NumberFieldWidths *spec, Py_ssize_t n_prefix, - STRINGLIB_CHAR sign_char, STRINGLIB_CHAR *number, - Py_ssize_t n_number, Py_ssize_t n_remainder, - int has_decimal, const LocaleInfo *locale, - const InternalFormatSpec *format) -{ - Py_ssize_t n_non_digit_non_padding; - Py_ssize_t n_padding; - - spec->n_digits = n_number - n_remainder - (has_decimal?1:0); - spec->n_lpadding = 0; - spec->n_prefix = n_prefix; - spec->n_decimal = has_decimal ? strlen(locale->decimal_point) : 0; - spec->n_remainder = n_remainder; - spec->n_spadding = 0; - spec->n_rpadding = 0; - spec->sign = '\0'; - spec->n_sign = 0; - - /* the output will look like: - | | - | <lpadding> <sign> <prefix> <spadding> <grouped_digits> <decimal> <remainder> <rpadding> | - | | - - sign is computed from format->sign and the actual - sign of the number - - prefix is given (it's for the '0x' prefix) - - digits is already known - - the total width is either given, or computed from the - actual digits - - only one of lpadding, spadding, and rpadding can be non-zero, - and it's calculated from the width and other fields - */ - - /* compute the various parts we're going to write */ - switch (format->sign) { - case '+': - /* always put a + or - */ - spec->n_sign = 1; - spec->sign = (sign_char == '-' ? '-' : '+'); - break; - case ' ': - spec->n_sign = 1; - spec->sign = (sign_char == '-' ? '-' : ' '); - break; - default: - /* Not specified, or the default (-) */ - if (sign_char == '-') { - spec->n_sign = 1; - spec->sign = '-'; - } - } - - /* The number of chars used for non-digits and non-padding. */ - n_non_digit_non_padding = spec->n_sign + spec->n_prefix + spec->n_decimal + - spec->n_remainder; - - /* min_width can go negative, that's okay. format->width == -1 means - we don't care. */ - if (format->fill_char == '0' && format->align == '=') - spec->n_min_width = format->width - n_non_digit_non_padding; - else - spec->n_min_width = 0; - - if (spec->n_digits == 0) - /* This case only occurs when using 'c' formatting, we need - to special case it because the grouping code always wants - to have at least one character. */ - spec->n_grouped_digits = 0; - else - spec->n_grouped_digits = STRINGLIB_GROUPING(NULL, 0, NULL, - spec->n_digits, - spec->n_min_width, - locale->grouping, - locale->thousands_sep); - - /* Given the desired width and the total of digit and non-digit - space we consume, see if we need any padding. format->width can - be negative (meaning no padding), but this code still works in - that case. */ - n_padding = format->width - - (n_non_digit_non_padding + spec->n_grouped_digits); - if (n_padding > 0) { - /* Some padding is needed. Determine if it's left, space, or right. */ - switch (format->align) { - case '<': - spec->n_rpadding = n_padding; - break; - case '^': - spec->n_lpadding = n_padding / 2; - spec->n_rpadding = n_padding - spec->n_lpadding; - break; - case '=': - spec->n_spadding = n_padding; - break; - case '>': - spec->n_lpadding = n_padding; - break; - default: - /* Shouldn't get here, but treat it as '>' */ - spec->n_lpadding = n_padding; - assert(0); - break; - } - } - return spec->n_lpadding + spec->n_sign + spec->n_prefix + - spec->n_spadding + spec->n_grouped_digits + spec->n_decimal + - spec->n_remainder + spec->n_rpadding; -} - -/* Fill in the digit parts of a numbers's string representation, - as determined in calc_number_widths(). - No error checking, since we know the buffer is the correct size. */ -static void -fill_number(STRINGLIB_CHAR *buf, const NumberFieldWidths *spec, - STRINGLIB_CHAR *digits, Py_ssize_t n_digits, - STRINGLIB_CHAR *prefix, STRINGLIB_CHAR fill_char, - LocaleInfo *locale, int toupper) -{ - /* Used to keep track of digits, decimal, and remainder. */ - STRINGLIB_CHAR *p = digits; - -#ifndef NDEBUG - Py_ssize_t r; -#endif - - if (spec->n_lpadding) { - STRINGLIB_FILL(buf, fill_char, spec->n_lpadding); - buf += spec->n_lpadding; - } - if (spec->n_sign == 1) { - *buf++ = spec->sign; - } - if (spec->n_prefix) { - memmove(buf, - prefix, - spec->n_prefix * sizeof(STRINGLIB_CHAR)); - if (toupper) { - Py_ssize_t t; - for (t = 0; t < spec->n_prefix; ++t) - buf[t] = STRINGLIB_TOUPPER(buf[t]); - } - buf += spec->n_prefix; - } - if (spec->n_spadding) { - STRINGLIB_FILL(buf, fill_char, spec->n_spadding); - buf += spec->n_spadding; - } - - /* Only for type 'c' special case, it has no digits. */ - if (spec->n_digits != 0) { - /* Fill the digits with InsertThousandsGrouping. */ -#ifndef NDEBUG - r = -#endif - STRINGLIB_GROUPING(buf, spec->n_grouped_digits, digits, - spec->n_digits, spec->n_min_width, - locale->grouping, locale->thousands_sep); -#ifndef NDEBUG - assert(r == spec->n_grouped_digits); -#endif - p += spec->n_digits; - } - if (toupper) { - Py_ssize_t t; - for (t = 0; t < spec->n_grouped_digits; ++t) - buf[t] = STRINGLIB_TOUPPER(buf[t]); - } - buf += spec->n_grouped_digits; - - if (spec->n_decimal) { - Py_ssize_t t; - for (t = 0; t < spec->n_decimal; ++t) - buf[t] = locale->decimal_point[t]; - buf += spec->n_decimal; - p += 1; - } - - if (spec->n_remainder) { - memcpy(buf, p, spec->n_remainder * sizeof(STRINGLIB_CHAR)); - buf += spec->n_remainder; - p += spec->n_remainder; - } - - if (spec->n_rpadding) { - STRINGLIB_FILL(buf, fill_char, spec->n_rpadding); - buf += spec->n_rpadding; - } -} - -static char no_grouping[1] = {CHAR_MAX}; - -/* Find the decimal point character(s?), thousands_separator(s?), and - grouping description, either for the current locale if type is - LT_CURRENT_LOCALE, a hard-coded locale if LT_DEFAULT_LOCALE, or - none if LT_NO_LOCALE. */ -static void -get_locale_info(int type, LocaleInfo *locale_info) -{ - switch (type) { - case LT_CURRENT_LOCALE: { - struct lconv *locale_data = localeconv(); - locale_info->decimal_point = locale_data->decimal_point; - locale_info->thousands_sep = locale_data->thousands_sep; - locale_info->grouping = locale_data->grouping; - break; - } - case LT_DEFAULT_LOCALE: - locale_info->decimal_point = "."; - locale_info->thousands_sep = ","; - locale_info->grouping = "\3"; /* Group every 3 characters. The - (implicit) trailing 0 means repeat - infinitely. */ - break; - case LT_NO_LOCALE: - locale_info->decimal_point = "."; - locale_info->thousands_sep = ""; - locale_info->grouping = no_grouping; - break; - default: - assert(0); - } -} - -#endif /* FORMAT_FLOAT || FORMAT_LONG || FORMAT_COMPLEX */ - -/************************************************************************/ -/*********** string formatting ******************************************/ -/************************************************************************/ - -static PyObject * -format_string_internal(PyObject *value, const InternalFormatSpec *format) -{ - Py_ssize_t lpad; - Py_ssize_t rpad; - Py_ssize_t total; - STRINGLIB_CHAR *p; - Py_ssize_t len = STRINGLIB_LEN(value); - PyObject *result = NULL; - - /* sign is not allowed on strings */ - if (format->sign != '\0') { - PyErr_SetString(PyExc_ValueError, - "Sign not allowed in string format specifier"); - goto done; - } - - /* alternate is not allowed on strings */ - if (format->alternate) { - PyErr_SetString(PyExc_ValueError, - "Alternate form (#) not allowed in string format " - "specifier"); - goto done; - } - - /* '=' alignment not allowed on strings */ - if (format->align == '=') { - PyErr_SetString(PyExc_ValueError, - "'=' alignment not allowed " - "in string format specifier"); - goto done; - } - - /* if precision is specified, output no more that format.precision - characters */ - if (format->precision >= 0 && len >= format->precision) { - len = format->precision; - } - - calc_padding(len, format->width, format->align, &lpad, &rpad, &total); - - /* allocate the resulting string */ - result = STRINGLIB_NEW(NULL, total); - if (result == NULL) - goto done; - - /* Write into that space. First the padding. */ - p = fill_padding(STRINGLIB_STR(result), len, - format->fill_char=='\0'?' ':format->fill_char, - lpad, rpad); - - /* Then the source string. */ - memcpy(p, STRINGLIB_STR(value), len * sizeof(STRINGLIB_CHAR)); - -done: - return result; -} - - -/************************************************************************/ -/*********** long formatting ********************************************/ -/************************************************************************/ - -#if defined FORMAT_LONG || defined FORMAT_INT -typedef PyObject* -(*IntOrLongToString)(PyObject *value, int base); - -static PyObject * -format_int_or_long_internal(PyObject *value, const InternalFormatSpec *format, - IntOrLongToString tostring) -{ - PyObject *result = NULL; - PyObject *tmp = NULL; - STRINGLIB_CHAR *pnumeric_chars; - STRINGLIB_CHAR numeric_char; - STRINGLIB_CHAR sign_char = '\0'; - Py_ssize_t n_digits; /* count of digits need from the computed - string */ - Py_ssize_t n_remainder = 0; /* Used only for 'c' formatting, which - produces non-digits */ - Py_ssize_t n_prefix = 0; /* Count of prefix chars, (e.g., '0x') */ - Py_ssize_t n_total; - STRINGLIB_CHAR *prefix = NULL; - NumberFieldWidths spec; - long x; - - /* Locale settings, either from the actual locale or - from a hard-code pseudo-locale */ - LocaleInfo locale; - - /* no precision allowed on integers */ - if (format->precision != -1) { - PyErr_SetString(PyExc_ValueError, - "Precision not allowed in integer format specifier"); - goto done; - } - - /* special case for character formatting */ - if (format->type == 'c') { - /* error to specify a sign */ - if (format->sign != '\0') { - PyErr_SetString(PyExc_ValueError, - "Sign not allowed with integer" - " format specifier 'c'"); - goto done; - } - - /* taken from unicodeobject.c formatchar() */ - /* Integer input truncated to a character */ -/* XXX: won't work for int */ - x = PyLong_AsLong(value); - if (x == -1 && PyErr_Occurred()) - goto done; -#ifdef Py_UNICODE_WIDE - if (x < 0 || x > 0x10ffff) { - PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x110000) " - "(wide Python build)"); - goto done; - } -#else - if (x < 0 || x > 0xffff) { - PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x10000) " - "(narrow Python build)"); - goto done; - } -#endif - numeric_char = (STRINGLIB_CHAR)x; - pnumeric_chars = &numeric_char; - n_digits = 1; - - /* As a sort-of hack, we tell calc_number_widths that we only - have "remainder" characters. calc_number_widths thinks - these are characters that don't get formatted, only copied - into the output string. We do this for 'c' formatting, - because the characters are likely to be non-digits. */ - n_remainder = 1; - } - else { - int base; - int leading_chars_to_skip = 0; /* Number of characters added by - PyNumber_ToBase that we want to - skip over. */ - - /* Compute the base and how many characters will be added by - PyNumber_ToBase */ - switch (format->type) { - case 'b': - base = 2; - leading_chars_to_skip = 2; /* 0b */ - break; - case 'o': - base = 8; - leading_chars_to_skip = 2; /* 0o */ - break; - case 'x': - case 'X': - base = 16; - leading_chars_to_skip = 2; /* 0x */ - break; - default: /* shouldn't be needed, but stops a compiler warning */ - case 'd': - case 'n': - base = 10; - break; - } - - /* The number of prefix chars is the same as the leading - chars to skip */ - if (format->alternate) - n_prefix = leading_chars_to_skip; - - /* Do the hard part, converting to a string in a given base */ - tmp = tostring(value, base); - if (tmp == NULL) - goto done; - - pnumeric_chars = STRINGLIB_STR(tmp); - n_digits = STRINGLIB_LEN(tmp); - - prefix = pnumeric_chars; - - /* Remember not to modify what pnumeric_chars points to. it - might be interned. Only modify it after we copy it into a - newly allocated output buffer. */ - - /* Is a sign character present in the output? If so, remember it - and skip it */ - if (pnumeric_chars[0] == '-') { - sign_char = pnumeric_chars[0]; - ++prefix; - ++leading_chars_to_skip; - } - - /* Skip over the leading chars (0x, 0b, etc.) */ - n_digits -= leading_chars_to_skip; - pnumeric_chars += leading_chars_to_skip; - } - - /* Determine the grouping, separator, and decimal point, if any. */ - get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : - (format->thousands_separators ? - LT_DEFAULT_LOCALE : - LT_NO_LOCALE), - &locale); - - /* Calculate how much memory we'll need. */ - n_total = calc_number_widths(&spec, n_prefix, sign_char, pnumeric_chars, - n_digits, n_remainder, 0, &locale, format); - - /* Allocate the memory. */ - result = STRINGLIB_NEW(NULL, n_total); - if (!result) - goto done; - - /* Populate the memory. */ - fill_number(STRINGLIB_STR(result), &spec, pnumeric_chars, n_digits, - prefix, format->fill_char == '\0' ? ' ' : format->fill_char, - &locale, format->type == 'X'); - -done: - Py_XDECREF(tmp); - return result; -} -#endif /* defined FORMAT_LONG || defined FORMAT_INT */ - -/************************************************************************/ -/*********** float formatting *******************************************/ -/************************************************************************/ - -#ifdef FORMAT_FLOAT -#if STRINGLIB_IS_UNICODE -static void -strtounicode(Py_UNICODE *buffer, const char *charbuffer, Py_ssize_t len) -{ - Py_ssize_t i; - for (i = 0; i < len; ++i) - buffer[i] = (Py_UNICODE)charbuffer[i]; -} -#endif - -/* much of this is taken from unicodeobject.c */ -static PyObject * -format_float_internal(PyObject *value, - const InternalFormatSpec *format) -{ - char *buf = NULL; /* buffer returned from PyOS_double_to_string */ - Py_ssize_t n_digits; - Py_ssize_t n_remainder; - Py_ssize_t n_total; - int has_decimal; - double val; - Py_ssize_t precision = format->precision; - Py_ssize_t default_precision = 6; - STRINGLIB_CHAR type = format->type; - int add_pct = 0; - STRINGLIB_CHAR *p; - NumberFieldWidths spec; - int flags = 0; - PyObject *result = NULL; - STRINGLIB_CHAR sign_char = '\0'; - int float_type; /* Used to see if we have a nan, inf, or regular float. */ - -#if STRINGLIB_IS_UNICODE - Py_UNICODE *unicode_tmp = NULL; -#endif - - /* Locale settings, either from the actual locale or - from a hard-code pseudo-locale */ - LocaleInfo locale; - - if (format->alternate) - flags |= Py_DTSF_ALT; - - if (type == '\0') { - /* Omitted type specifier. Behaves in the same way as repr(x) - and str(x) if no precision is given, else like 'g', but with - at least one digit after the decimal point. */ - flags |= Py_DTSF_ADD_DOT_0; - type = 'r'; - default_precision = 0; - } - - if (type == 'n') - /* 'n' is the same as 'g', except for the locale used to - format the result. We take care of that later. */ - type = 'g'; - - val = PyFloat_AsDouble(value); - if (val == -1.0 && PyErr_Occurred()) - goto done; - - if (type == '%') { - type = 'f'; - val *= 100; - add_pct = 1; - } - - if (precision < 0) - precision = default_precision; - else if (type == 'r') - type = 'g'; - - /* Cast "type", because if we're in unicode we need to pass a - 8-bit char. This is safe, because we've restricted what "type" - can be. */ - buf = PyOS_double_to_string(val, (char)type, precision, flags, - &float_type); - if (buf == NULL) - goto done; - n_digits = strlen(buf); - - if (add_pct) { - /* We know that buf has a trailing zero (since we just called - strlen() on it), and we don't use that fact any more. So we - can just write over the trailing zero. */ - buf[n_digits] = '%'; - n_digits += 1; - } - - /* Since there is no unicode version of PyOS_double_to_string, - just use the 8 bit version and then convert to unicode. */ -#if STRINGLIB_IS_UNICODE - unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_digits)*sizeof(Py_UNICODE)); - if (unicode_tmp == NULL) { - PyErr_NoMemory(); - goto done; - } - strtounicode(unicode_tmp, buf, n_digits); - p = unicode_tmp; -#else - p = buf; -#endif - - /* Is a sign character present in the output? If so, remember it - and skip it */ - if (*p == '-') { - sign_char = *p; - ++p; - --n_digits; - } - - /* Determine if we have any "remainder" (after the digits, might include - decimal or exponent or both (or neither)) */ - parse_number(p, n_digits, &n_remainder, &has_decimal); - - /* Determine the grouping, separator, and decimal point, if any. */ - get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : - (format->thousands_separators ? - LT_DEFAULT_LOCALE : - LT_NO_LOCALE), - &locale); - - /* Calculate how much memory we'll need. */ - n_total = calc_number_widths(&spec, 0, sign_char, p, n_digits, - n_remainder, has_decimal, &locale, format); - - /* Allocate the memory. */ - result = STRINGLIB_NEW(NULL, n_total); - if (result == NULL) - goto done; - - /* Populate the memory. */ - fill_number(STRINGLIB_STR(result), &spec, p, n_digits, NULL, - format->fill_char == '\0' ? ' ' : format->fill_char, &locale, - 0); - -done: - PyMem_Free(buf); -#if STRINGLIB_IS_UNICODE - PyMem_Free(unicode_tmp); -#endif - return result; -} -#endif /* FORMAT_FLOAT */ - -/************************************************************************/ -/*********** complex formatting *****************************************/ -/************************************************************************/ - -#ifdef FORMAT_COMPLEX - -static PyObject * -format_complex_internal(PyObject *value, - const InternalFormatSpec *format) -{ - double re; - double im; - char *re_buf = NULL; /* buffer returned from PyOS_double_to_string */ - char *im_buf = NULL; /* buffer returned from PyOS_double_to_string */ - - InternalFormatSpec tmp_format = *format; - Py_ssize_t n_re_digits; - Py_ssize_t n_im_digits; - Py_ssize_t n_re_remainder; - Py_ssize_t n_im_remainder; - Py_ssize_t n_re_total; - Py_ssize_t n_im_total; - int re_has_decimal; - int im_has_decimal; - Py_ssize_t precision = format->precision; - Py_ssize_t default_precision = 6; - STRINGLIB_CHAR type = format->type; - STRINGLIB_CHAR *p_re; - STRINGLIB_CHAR *p_im; - NumberFieldWidths re_spec; - NumberFieldWidths im_spec; - int flags = 0; - PyObject *result = NULL; - STRINGLIB_CHAR *p; - STRINGLIB_CHAR re_sign_char = '\0'; - STRINGLIB_CHAR im_sign_char = '\0'; - int re_float_type; /* Used to see if we have a nan, inf, or regular float. */ - int im_float_type; - int add_parens = 0; - int skip_re = 0; - Py_ssize_t lpad; - Py_ssize_t rpad; - Py_ssize_t total; - -#if STRINGLIB_IS_UNICODE - Py_UNICODE *re_unicode_tmp = NULL; - Py_UNICODE *im_unicode_tmp = NULL; -#endif - - /* Locale settings, either from the actual locale or - from a hard-code pseudo-locale */ - LocaleInfo locale; - - /* Zero padding is not allowed. */ - if (format->fill_char == '0') { - PyErr_SetString(PyExc_ValueError, - "Zero padding is not allowed in complex format " - "specifier"); - goto done; - } - - /* Neither is '=' alignment . */ - if (format->align == '=') { - PyErr_SetString(PyExc_ValueError, - "'=' alignment flag is not allowed in complex format " - "specifier"); - goto done; - } - - re = PyComplex_RealAsDouble(value); - if (re == -1.0 && PyErr_Occurred()) - goto done; - im = PyComplex_ImagAsDouble(value); - if (im == -1.0 && PyErr_Occurred()) - goto done; - - if (format->alternate) - flags |= Py_DTSF_ALT; - - if (type == '\0') { - /* Omitted type specifier. Should be like str(self). */ - type = 'r'; - default_precision = 0; - if (re == 0.0 && copysign(1.0, re) == 1.0) - skip_re = 1; - else - add_parens = 1; - } - - if (type == 'n') - /* 'n' is the same as 'g', except for the locale used to - format the result. We take care of that later. */ - type = 'g'; - - if (precision < 0) - precision = default_precision; - else if (type == 'r') - type = 'g'; - - /* Cast "type", because if we're in unicode we need to pass a - 8-bit char. This is safe, because we've restricted what "type" - can be. */ - re_buf = PyOS_double_to_string(re, (char)type, precision, flags, - &re_float_type); - if (re_buf == NULL) - goto done; - im_buf = PyOS_double_to_string(im, (char)type, precision, flags, - &im_float_type); - if (im_buf == NULL) - goto done; - - n_re_digits = strlen(re_buf); - n_im_digits = strlen(im_buf); - - /* Since there is no unicode version of PyOS_double_to_string, - just use the 8 bit version and then convert to unicode. */ -#if STRINGLIB_IS_UNICODE - re_unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_re_digits)*sizeof(Py_UNICODE)); - if (re_unicode_tmp == NULL) { - PyErr_NoMemory(); - goto done; - } - strtounicode(re_unicode_tmp, re_buf, n_re_digits); - p_re = re_unicode_tmp; - - im_unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_im_digits)*sizeof(Py_UNICODE)); - if (im_unicode_tmp == NULL) { - PyErr_NoMemory(); - goto done; - } - strtounicode(im_unicode_tmp, im_buf, n_im_digits); - p_im = im_unicode_tmp; -#else - p_re = re_buf; - p_im = im_buf; -#endif - - /* Is a sign character present in the output? If so, remember it - and skip it */ - if (*p_re == '-') { - re_sign_char = *p_re; - ++p_re; - --n_re_digits; - } - if (*p_im == '-') { - im_sign_char = *p_im; - ++p_im; - --n_im_digits; - } - - /* Determine if we have any "remainder" (after the digits, might include - decimal or exponent or both (or neither)) */ - parse_number(p_re, n_re_digits, &n_re_remainder, &re_has_decimal); - parse_number(p_im, n_im_digits, &n_im_remainder, &im_has_decimal); - - /* Determine the grouping, separator, and decimal point, if any. */ - get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE : - (format->thousands_separators ? - LT_DEFAULT_LOCALE : - LT_NO_LOCALE), - &locale); - - /* Turn off any padding. We'll do it later after we've composed - the numbers without padding. */ - tmp_format.fill_char = '\0'; - tmp_format.align = '<'; - tmp_format.width = -1; - - /* Calculate how much memory we'll need. */ - n_re_total = calc_number_widths(&re_spec, 0, re_sign_char, p_re, - n_re_digits, n_re_remainder, - re_has_decimal, &locale, &tmp_format); - - /* Same formatting, but always include a sign, unless the real part is - * going to be omitted, in which case we use whatever sign convention was - * requested by the original format. */ - if (!skip_re) - tmp_format.sign = '+'; - n_im_total = calc_number_widths(&im_spec, 0, im_sign_char, p_im, - n_im_digits, n_im_remainder, - im_has_decimal, &locale, &tmp_format); - - if (skip_re) - n_re_total = 0; - - /* Add 1 for the 'j', and optionally 2 for parens. */ - calc_padding(n_re_total + n_im_total + 1 + add_parens * 2, - format->width, format->align, &lpad, &rpad, &total); - - result = STRINGLIB_NEW(NULL, total); - if (result == NULL) - goto done; - - /* Populate the memory. First, the padding. */ - p = fill_padding(STRINGLIB_STR(result), - n_re_total + n_im_total + 1 + add_parens * 2, - format->fill_char=='\0' ? ' ' : format->fill_char, - lpad, rpad); - - if (add_parens) - *p++ = '('; - - if (!skip_re) { - fill_number(p, &re_spec, p_re, n_re_digits, NULL, 0, &locale, 0); - p += n_re_total; - } - fill_number(p, &im_spec, p_im, n_im_digits, NULL, 0, &locale, 0); - p += n_im_total; - *p++ = 'j'; - - if (add_parens) - *p++ = ')'; - -done: - PyMem_Free(re_buf); - PyMem_Free(im_buf); -#if STRINGLIB_IS_UNICODE - PyMem_Free(re_unicode_tmp); - PyMem_Free(im_unicode_tmp); -#endif - return result; -} -#endif /* FORMAT_COMPLEX */ - -/************************************************************************/ -/*********** built in formatters ****************************************/ -/************************************************************************/ -PyObject * -FORMAT_STRING(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - InternalFormatSpec format; - PyObject *result = NULL; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, format_spec_len, - &format, 's', '<')) - goto done; - - /* type conversion? */ - switch (format.type) { - case 's': - /* no type conversion needed, already a string. do the formatting */ - result = format_string_internal(obj, &format); - break; - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - return result; -} - -#if defined FORMAT_LONG || defined FORMAT_INT -static PyObject* -format_int_or_long(PyObject* obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len, - IntOrLongToString tostring) -{ - PyObject *result = NULL; - PyObject *tmp = NULL; - InternalFormatSpec format; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, - format_spec_len, - &format, 'd', '>')) - goto done; - - /* type conversion? */ - switch (format.type) { - case 'b': - case 'c': - case 'd': - case 'o': - case 'x': - case 'X': - case 'n': - /* no type conversion needed, already an int (or long). do - the formatting */ - result = format_int_or_long_internal(obj, &format, tostring); - break; - - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case '%': - /* convert to float */ - tmp = PyNumber_Float(obj); - if (tmp == NULL) - goto done; - result = format_float_internal(tmp, &format); - break; - - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - Py_XDECREF(tmp); - return result; -} -#endif /* FORMAT_LONG || defined FORMAT_INT */ - -#ifdef FORMAT_LONG -/* Need to define long_format as a function that will convert a long - to a string. In 3.0, _PyLong_Format has the correct signature. In - 2.x, we need to fudge a few parameters */ -#if PY_VERSION_HEX >= 0x03000000 -#define long_format _PyLong_Format -#else -static PyObject* -long_format(PyObject* value, int base) -{ - /* Convert to base, don't add trailing 'L', and use the new octal - format. We already know this is a long object */ - assert(PyLong_Check(value)); - /* convert to base, don't add 'L', and use the new octal format */ - return _PyLong_Format(value, base, 0, 1); -} -#endif - -PyObject * -FORMAT_LONG(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - return format_int_or_long(obj, format_spec, format_spec_len, - long_format); -} -#endif /* FORMAT_LONG */ - -#ifdef FORMAT_INT -/* this is only used for 2.x, not 3.0 */ -static PyObject* -int_format(PyObject* value, int base) -{ - /* Convert to base, and use the new octal format. We already - know this is an int object */ - assert(PyInt_Check(value)); - return _PyInt_Format((PyIntObject*)value, base, 1); -} - -PyObject * -FORMAT_INT(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - return format_int_or_long(obj, format_spec, format_spec_len, - int_format); -} -#endif /* FORMAT_INT */ - -#ifdef FORMAT_FLOAT -PyObject * -FORMAT_FLOAT(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - PyObject *result = NULL; - InternalFormatSpec format; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, - format_spec_len, - &format, '\0', '>')) - goto done; - - /* type conversion? */ - switch (format.type) { - case '\0': /* No format code: like 'g', but with at least one decimal. */ - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case 'n': - case '%': - /* no conversion, already a float. do the formatting */ - result = format_float_internal(obj, &format); - break; - - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - return result; -} -#endif /* FORMAT_FLOAT */ - -#ifdef FORMAT_COMPLEX -PyObject * -FORMAT_COMPLEX(PyObject *obj, - STRINGLIB_CHAR *format_spec, - Py_ssize_t format_spec_len) -{ - PyObject *result = NULL; - InternalFormatSpec format; - - /* check for the special case of zero length format spec, make - it equivalent to str(obj) */ - if (format_spec_len == 0) { - result = STRINGLIB_TOSTR(obj); - goto done; - } - - /* parse the format_spec */ - if (!parse_internal_render_format_spec(format_spec, - format_spec_len, - &format, '\0', '>')) - goto done; - - /* type conversion? */ - switch (format.type) { - case '\0': /* No format code: like 'g', but with at least one decimal. */ - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - case 'n': - /* no conversion, already a complex. do the formatting */ - result = format_complex_internal(obj, &format); - break; - - default: - /* unknown */ - unknown_presentation_type(format.type, obj->ob_type->tp_name); - goto done; - } - -done: - return result; -} -#endif /* FORMAT_COMPLEX */ diff --git a/Objects/stringlib/localeutil.h b/Objects/stringlib/localeutil.h index f548133..6e2f073 100644 --- a/Objects/stringlib/localeutil.h +++ b/Objects/stringlib/localeutil.h @@ -1,21 +1,19 @@ /* stringlib: locale related helpers implementation */ -#ifndef STRINGLIB_LOCALEUTIL_H -#define STRINGLIB_LOCALEUTIL_H - #include <locale.h> -#define MAX(x, y) ((x) < (y) ? (y) : (x)) -#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#ifndef STRINGLIB_IS_UNICODE +# error "localeutil is specific to Unicode" +#endif typedef struct { const char *grouping; char previous; Py_ssize_t i; /* Where we're currently pointing in grouping. */ -} GroupGenerator; +} STRINGLIB(GroupGenerator); static void -_GroupGenerator_init(GroupGenerator *self, const char *grouping) +STRINGLIB(GroupGenerator_init)(STRINGLIB(GroupGenerator) *self, const char *grouping) { self->grouping = grouping; self->i = 0; @@ -24,7 +22,7 @@ _GroupGenerator_init(GroupGenerator *self, const char *grouping) /* Returns the next grouping, or 0 to signify end. */ static Py_ssize_t -_GroupGenerator_next(GroupGenerator *self) +STRINGLIB(GroupGenerator_next)(STRINGLIB(GroupGenerator) *self) { /* Note that we don't really do much error checking here. If a grouping string contains just CHAR_MAX, for example, then just @@ -48,27 +46,18 @@ _GroupGenerator_next(GroupGenerator *self) /* Fill in some digits, leading zeros, and thousands separator. All are optional, depending on when we're called. */ static void -fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, - Py_ssize_t n_chars, Py_ssize_t n_zeros, const char* thousands_sep, +STRINGLIB(fill)(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, + Py_ssize_t n_chars, Py_ssize_t n_zeros, STRINGLIB_CHAR* thousands_sep, Py_ssize_t thousands_sep_len) { -#if STRINGLIB_IS_UNICODE Py_ssize_t i; -#endif if (thousands_sep) { *buffer_end -= thousands_sep_len; /* Copy the thousands_sep chars into the buffer. */ -#if STRINGLIB_IS_UNICODE - /* Convert from the char's of the thousands_sep from - the locale into unicode. */ - for (i = 0; i < thousands_sep_len; ++i) - (*buffer_end)[i] = thousands_sep[i]; -#else - /* No conversion, just memcpy the thousands_sep. */ - memcpy(*buffer_end, thousands_sep, thousands_sep_len); -#endif + memcpy(*buffer_end, thousands_sep, + thousands_sep_len * STRINGLIB_SIZEOF_CHAR); } *buffer_end -= n_chars; @@ -76,11 +65,12 @@ fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, memcpy(*buffer_end, *digits_end, n_chars * sizeof(STRINGLIB_CHAR)); *buffer_end -= n_zeros; - STRINGLIB_FILL(*buffer_end, '0', n_zeros); + for (i = 0; i < n_zeros; i++) + (*buffer_end)[i] = '0'; } /** - * _Py_InsertThousandsGrouping: + * InsertThousandsGrouping: * @buffer: A pointer to the start of a string. * @n_buffer: Number of characters in @buffer. * @digits: A pointer to the digits we're reading from. If count @@ -109,14 +99,16 @@ fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end, * As closely as possible, this code mimics the logic in decimal.py's _insert_thousands_sep(). **/ -Py_ssize_t -_Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer, - Py_ssize_t n_buffer, - STRINGLIB_CHAR *digits, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - const char *thousands_sep) +static Py_ssize_t +STRINGLIB(InsertThousandsGrouping)( + STRINGLIB_CHAR *buffer, + Py_ssize_t n_buffer, + STRINGLIB_CHAR *digits, + Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + STRINGLIB_CHAR *thousands_sep, + Py_ssize_t thousands_sep_len) { Py_ssize_t count = 0; Py_ssize_t n_zeros; @@ -128,23 +120,22 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer, STRINGLIB_CHAR *digits_end = NULL; Py_ssize_t l; Py_ssize_t n_chars; - Py_ssize_t thousands_sep_len = strlen(thousands_sep); Py_ssize_t remaining = n_digits; /* Number of chars remaining to be looked at */ /* A generator that returns all of the grouping widths, until it returns 0. */ - GroupGenerator groupgen; - _GroupGenerator_init(&groupgen, grouping); + STRINGLIB(GroupGenerator) groupgen; + STRINGLIB(GroupGenerator_init)(&groupgen, grouping); if (buffer) { buffer_end = buffer + n_buffer; digits_end = digits + n_digits; } - while ((l = _GroupGenerator_next(&groupgen)) > 0) { - l = MIN(l, MAX(MAX(remaining, min_width), 1)); - n_zeros = MAX(0, l - remaining); - n_chars = MAX(0, MIN(remaining, l)); + while ((l = STRINGLIB(GroupGenerator_next)(&groupgen)) > 0) { + l = Py_MIN(l, Py_MAX(Py_MAX(remaining, min_width), 1)); + n_zeros = Py_MAX(0, l - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, l)); /* Use n_zero zero's and n_chars chars */ @@ -153,7 +144,7 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer, if (buffer) { /* Copy into the output buffer. */ - fill(&digits_end, &buffer_end, n_chars, n_zeros, + STRINGLIB(fill)(&digits_end, &buffer_end, n_chars, n_zeros, use_separator ? thousands_sep : NULL, thousands_sep_len); } @@ -172,41 +163,18 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer, if (!loop_broken) { /* We left the loop without using a break statement. */ - l = MAX(MAX(remaining, min_width), 1); - n_zeros = MAX(0, l - remaining); - n_chars = MAX(0, MIN(remaining, l)); + l = Py_MAX(Py_MAX(remaining, min_width), 1); + n_zeros = Py_MAX(0, l - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, l)); /* Use n_zero zero's and n_chars chars */ count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; if (buffer) { /* Copy into the output buffer. */ - fill(&digits_end, &buffer_end, n_chars, n_zeros, + STRINGLIB(fill)(&digits_end, &buffer_end, n_chars, n_zeros, use_separator ? thousands_sep : NULL, thousands_sep_len); } } return count; } -/** - * _Py_InsertThousandsGroupingLocale: - * @buffer: A pointer to the start of a string. - * @n_digits: The number of digits in the string, in which we want - * to put the grouping chars. - * - * Reads thee current locale and calls _Py_InsertThousandsGrouping(). - **/ -Py_ssize_t -_Py_InsertThousandsGroupingLocale(STRINGLIB_CHAR *buffer, - Py_ssize_t n_buffer, - STRINGLIB_CHAR *digits, - Py_ssize_t n_digits, - Py_ssize_t min_width) -{ - struct lconv *locale_data = localeconv(); - const char *grouping = locale_data->grouping; - const char *thousands_sep = locale_data->thousands_sep; - - return _Py_InsertThousandsGrouping(buffer, n_buffer, digits, n_digits, - min_width, grouping, thousands_sep); -} -#endif /* STRINGLIB_LOCALEUTIL_H */ diff --git a/Objects/stringlib/partition.h b/Objects/stringlib/partition.h index 0170bdd..40cb512 100644 --- a/Objects/stringlib/partition.h +++ b/Objects/stringlib/partition.h @@ -1,14 +1,11 @@ /* stringlib: partition implementation */ -#ifndef STRINGLIB_PARTITION_H -#define STRINGLIB_PARTITION_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif Py_LOCAL_INLINE(PyObject*) -stringlib_partition(PyObject* str_obj, +STRINGLIB(partition)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, PyObject* sep_obj, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len) @@ -25,7 +22,7 @@ stringlib_partition(PyObject* str_obj, if (!out) return NULL; - pos = fastsearch(str, str_len, sep, sep_len, -1, FAST_SEARCH); + pos = FASTSEARCH(str, str_len, sep, sep_len, -1, FAST_SEARCH); if (pos < 0) { #if STRINGLIB_MUTABLE @@ -58,7 +55,7 @@ stringlib_partition(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject*) -stringlib_rpartition(PyObject* str_obj, +STRINGLIB(rpartition)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, PyObject* sep_obj, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len) @@ -75,7 +72,7 @@ stringlib_rpartition(PyObject* str_obj, if (!out) return NULL; - pos = fastsearch(str, str_len, sep, sep_len, -1, FAST_RSEARCH); + pos = FASTSEARCH(str, str_len, sep, sep_len, -1, FAST_RSEARCH); if (pos < 0) { #if STRINGLIB_MUTABLE @@ -107,4 +104,3 @@ stringlib_rpartition(PyObject* str_obj, return out; } -#endif diff --git a/Objects/stringlib/split.h b/Objects/stringlib/split.h index 60e7767..947dd28 100644 --- a/Objects/stringlib/split.h +++ b/Objects/stringlib/split.h @@ -1,8 +1,5 @@ /* stringlib: split implementation */ -#ifndef STRINGLIB_SPLIT_H -#define STRINGLIB_SPLIT_H - #ifndef STRINGLIB_FASTSEARCH_H #error must include "stringlib/fastsearch.h" before including this module #endif @@ -54,7 +51,7 @@ #define FIX_PREALLOC_SIZE(list) Py_SIZE(list) = count Py_LOCAL_INLINE(PyObject *) -stringlib_split_whitespace(PyObject* str_obj, +STRINGLIB(split_whitespace)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, Py_ssize_t maxcount) { @@ -102,7 +99,7 @@ stringlib_split_whitespace(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_split_char(PyObject* str_obj, +STRINGLIB(split_char)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR ch, Py_ssize_t maxcount) @@ -145,7 +142,7 @@ stringlib_split_char(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_split(PyObject* str_obj, +STRINGLIB(split)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len, Py_ssize_t maxcount) @@ -158,7 +155,7 @@ stringlib_split(PyObject* str_obj, return NULL; } else if (sep_len == 1) - return stringlib_split_char(str_obj, str, str_len, sep[0], maxcount); + return STRINGLIB(split_char)(str_obj, str, str_len, sep[0], maxcount); list = PyList_New(PREALLOC_SIZE(maxcount)); if (list == NULL) @@ -166,7 +163,7 @@ stringlib_split(PyObject* str_obj, i = j = 0; while (maxcount-- > 0) { - pos = fastsearch(str+i, str_len-i, sep, sep_len, -1, FAST_SEARCH); + pos = FASTSEARCH(str+i, str_len-i, sep, sep_len, -1, FAST_SEARCH); if (pos < 0) break; j = i + pos; @@ -193,7 +190,7 @@ stringlib_split(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_rsplit_whitespace(PyObject* str_obj, +STRINGLIB(rsplit_whitespace)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, Py_ssize_t maxcount) { @@ -243,7 +240,7 @@ stringlib_rsplit_whitespace(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_rsplit_char(PyObject* str_obj, +STRINGLIB(rsplit_char)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR ch, Py_ssize_t maxcount) @@ -287,7 +284,7 @@ stringlib_rsplit_char(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_rsplit(PyObject* str_obj, +STRINGLIB(rsplit)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, const STRINGLIB_CHAR* sep, Py_ssize_t sep_len, Py_ssize_t maxcount) @@ -300,7 +297,7 @@ stringlib_rsplit(PyObject* str_obj, return NULL; } else if (sep_len == 1) - return stringlib_rsplit_char(str_obj, str, str_len, sep[0], maxcount); + return STRINGLIB(rsplit_char)(str_obj, str, str_len, sep[0], maxcount); list = PyList_New(PREALLOC_SIZE(maxcount)); if (list == NULL) @@ -308,7 +305,7 @@ stringlib_rsplit(PyObject* str_obj, j = str_len; while (maxcount-- > 0) { - pos = fastsearch(str, j, sep, sep_len, -1, FAST_RSEARCH); + pos = FASTSEARCH(str, j, sep, sep_len, -1, FAST_RSEARCH); if (pos < 0) break; SPLIT_ADD(str, pos + sep_len, j); @@ -336,7 +333,7 @@ stringlib_rsplit(PyObject* str_obj, } Py_LOCAL_INLINE(PyObject *) -stringlib_splitlines(PyObject* str_obj, +STRINGLIB(splitlines)(PyObject* str_obj, const STRINGLIB_CHAR* str, Py_ssize_t str_len, int keepends) { @@ -391,4 +388,3 @@ stringlib_splitlines(PyObject* str_obj, return NULL; } -#endif diff --git a/Objects/stringlib/stringdefs.h b/Objects/stringlib/stringdefs.h index 1c49426..7bb91a7 100644 --- a/Objects/stringlib/stringdefs.h +++ b/Objects/stringlib/stringdefs.h @@ -6,7 +6,10 @@ compiled as unicode. */ #define STRINGLIB_IS_UNICODE 0 +#define FASTSEARCH fastsearch +#define STRINGLIB(F) stringlib_##F #define STRINGLIB_OBJECT PyBytesObject +#define STRINGLIB_SIZEOF_CHAR 1 #define STRINGLIB_CHAR char #define STRINGLIB_TYPE_NAME "string" #define STRINGLIB_PARSE_CODE "S" @@ -15,9 +18,6 @@ #define STRINGLIB_ISLINEBREAK(x) ((x == '\n') || (x == '\r')) #define STRINGLIB_ISDECIMAL(x) ((x >= '0') && (x <= '9')) #define STRINGLIB_TODECIMAL(x) (STRINGLIB_ISDECIMAL(x) ? (x - '0') : -1) -#define STRINGLIB_TOUPPER Py_TOUPPER -#define STRINGLIB_TOLOWER Py_TOLOWER -#define STRINGLIB_FILL memset #define STRINGLIB_STR PyBytes_AS_STRING #define STRINGLIB_LEN PyBytes_GET_SIZE #define STRINGLIB_NEW PyBytes_FromStringAndSize @@ -25,7 +25,5 @@ #define STRINGLIB_CHECK PyBytes_Check #define STRINGLIB_CHECK_EXACT PyBytes_CheckExact #define STRINGLIB_TOSTR PyObject_Str -#define STRINGLIB_GROUPING _PyBytes_InsertThousandsGrouping -#define STRINGLIB_GROUPING_LOCALE _PyBytes_InsertThousandsGroupingLocale #define STRINGLIB_TOASCII PyObject_Repr #endif /* !STRINGLIB_STRINGDEFS_H */ diff --git a/Objects/stringlib/ucs1lib.h b/Objects/stringlib/ucs1lib.h new file mode 100644 index 0000000..e8c6fcb --- /dev/null +++ b/Objects/stringlib/ucs1lib.h @@ -0,0 +1,31 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH ucs1lib_fastsearch +#define STRINGLIB(F) ucs1lib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR 1 +#define STRINGLIB_MAX_CHAR 0xFFu +#define STRINGLIB_CHAR Py_UCS1 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_STR PyUnicode_1BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW _PyUnicode_FromUCS1 +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ucs1_InsertThousandsGrouping + + diff --git a/Objects/stringlib/ucs2lib.h b/Objects/stringlib/ucs2lib.h new file mode 100644 index 0000000..45e5729 --- /dev/null +++ b/Objects/stringlib/ucs2lib.h @@ -0,0 +1,30 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH ucs2lib_fastsearch +#define STRINGLIB(F) ucs2lib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR 2 +#define STRINGLIB_MAX_CHAR 0xFFFFu +#define STRINGLIB_CHAR Py_UCS2 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_STR PyUnicode_2BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW _PyUnicode_FromUCS2 +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ucs2_InsertThousandsGrouping + diff --git a/Objects/stringlib/ucs4lib.h b/Objects/stringlib/ucs4lib.h new file mode 100644 index 0000000..647a27e --- /dev/null +++ b/Objects/stringlib/ucs4lib.h @@ -0,0 +1,30 @@ +/* this is sort of a hack. there's at least one place (formatting + floats) where some stringlib code takes a different path if it's + compiled as unicode. */ +#define STRINGLIB_IS_UNICODE 1 + +#define FASTSEARCH ucs4lib_fastsearch +#define STRINGLIB(F) ucs4lib_##F +#define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR 4 +#define STRINGLIB_MAX_CHAR 0x10FFFFu +#define STRINGLIB_CHAR Py_UCS4 +#define STRINGLIB_TYPE_NAME "unicode" +#define STRINGLIB_PARSE_CODE "U" +#define STRINGLIB_EMPTY unicode_empty +#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE +#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK +#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL +#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL +#define STRINGLIB_STR PyUnicode_4BYTE_DATA +#define STRINGLIB_LEN PyUnicode_GET_LENGTH +#define STRINGLIB_NEW _PyUnicode_FromUCS4 +#define STRINGLIB_RESIZE not_supported +#define STRINGLIB_CHECK PyUnicode_Check +#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact + +#define STRINGLIB_TOSTR PyObject_Str +#define STRINGLIB_TOASCII PyObject_ASCII + +#define _Py_InsertThousandsGrouping _PyUnicode_ucs4_InsertThousandsGrouping + diff --git a/Objects/stringlib/undef.h b/Objects/stringlib/undef.h new file mode 100644 index 0000000..03117ec --- /dev/null +++ b/Objects/stringlib/undef.h @@ -0,0 +1,12 @@ +#undef FASTSEARCH +#undef STRINGLIB +#undef STRINGLIB_SIZEOF_CHAR +#undef STRINGLIB_MAX_CHAR +#undef STRINGLIB_CHAR +#undef STRINGLIB_STR +#undef STRINGLIB_LEN +#undef STRINGLIB_NEW +#undef STRINGLIB_RESIZE +#undef _Py_InsertThousandsGrouping +#undef STRINGLIB_IS_UNICODE + diff --git a/Objects/stringlib/string_format.h b/Objects/stringlib/unicode_format.h index c46bdc2..be580c6 100644 --- a/Objects/stringlib/string_format.h +++ b/Objects/stringlib/unicode_format.h @@ -1,16 +1,7 @@ /* - string_format.h -- implementation of string.format(). - - It uses the Objects/stringlib conventions, so that it can be - compiled for both unicode and string objects. + unicode_format.h -- implementation of str.format(). */ - -/* Defines for Python 2.6 compatibility */ -#if PY_VERSION_HEX < 0x03000000 -#define PyLong_FromSsize_t _PyLong_FromSsize_t -#endif - /* Defines for more efficiently reallocating the string buffer */ #define INITIAL_SIZE_INCREMENT 100 #define SIZE_MULTIPLIER 2 @@ -26,8 +17,8 @@ unicode pointers. */ typedef struct { - STRINGLIB_CHAR *ptr; - STRINGLIB_CHAR *end; + PyObject *str; /* borrowed reference */ + Py_ssize_t start, end; } SubString; @@ -64,34 +55,32 @@ AutoNumber_Init(AutoNumber *auto_number) /* fill in a SubString from a pointer and length */ Py_LOCAL_INLINE(void) -SubString_init(SubString *str, STRINGLIB_CHAR *p, Py_ssize_t len) +SubString_init(SubString *str, PyObject *s, Py_ssize_t start, Py_ssize_t end) { - str->ptr = p; - if (p == NULL) - str->end = NULL; - else - str->end = str->ptr + len; + str->str = s; + str->start = start; + str->end = end; } -/* return a new string. if str->ptr is NULL, return None */ +/* return a new string. if str->str is NULL, return None */ Py_LOCAL_INLINE(PyObject *) SubString_new_object(SubString *str) { - if (str->ptr == NULL) { + if (str->str == NULL) { Py_INCREF(Py_None); return Py_None; } - return STRINGLIB_NEW(str->ptr, str->end - str->ptr); + return PyUnicode_Substring(str->str, str->start, str->end); } -/* return a new string. if str->ptr is NULL, return None */ +/* return a new string. if str->str is NULL, return None */ Py_LOCAL_INLINE(PyObject *) SubString_new_object_or_empty(SubString *str) { - if (str->ptr == NULL) { - return STRINGLIB_NEW(NULL, 0); + if (str->str == NULL) { + return PyUnicode_New(0, 0); } - return STRINGLIB_NEW(str->ptr, str->end - str->ptr); + return SubString_new_object(str); } /* Return 1 if an error has been detected switching between automatic @@ -121,74 +110,6 @@ autonumber_state_error(AutoNumberState state, int field_name_is_empty) /************************************************************************/ -/*********** Output string management functions ****************/ -/************************************************************************/ - -typedef struct { - STRINGLIB_CHAR *ptr; - STRINGLIB_CHAR *end; - PyObject *obj; - Py_ssize_t size_increment; -} OutputString; - -/* initialize an OutputString object, reserving size characters */ -static int -output_initialize(OutputString *output, Py_ssize_t size) -{ - output->obj = STRINGLIB_NEW(NULL, size); - if (output->obj == NULL) - return 0; - - output->ptr = STRINGLIB_STR(output->obj); - output->end = STRINGLIB_LEN(output->obj) + output->ptr; - output->size_increment = INITIAL_SIZE_INCREMENT; - - return 1; -} - -/* - output_extend reallocates the output string buffer. - It returns a status: 0 for a failed reallocation, - 1 for success. -*/ - -static int -output_extend(OutputString *output, Py_ssize_t count) -{ - STRINGLIB_CHAR *startptr = STRINGLIB_STR(output->obj); - Py_ssize_t curlen = output->ptr - startptr; - Py_ssize_t maxlen = curlen + count + output->size_increment; - - if (STRINGLIB_RESIZE(&output->obj, maxlen) < 0) - return 0; - startptr = STRINGLIB_STR(output->obj); - output->ptr = startptr + curlen; - output->end = startptr + maxlen; - if (output->size_increment < MAX_SIZE_INCREMENT) - output->size_increment *= SIZE_MULTIPLIER; - return 1; -} - -/* - output_data dumps characters into our output string - buffer. - - In some cases, it has to reallocate the string. - - It returns a status: 0 for a failed reallocation, - 1 for success. -*/ -static int -output_data(OutputString *output, const STRINGLIB_CHAR *s, Py_ssize_t count) -{ - if ((count > output->end - output->ptr) && !output_extend(output, count)) - return 0; - memcpy(output->ptr, s, count * sizeof(STRINGLIB_CHAR)); - output->ptr += count; - return 1; -} - -/************************************************************************/ /*********** Format string parsing -- integers and identifiers *********/ /************************************************************************/ @@ -197,14 +118,14 @@ get_integer(const SubString *str) { Py_ssize_t accumulator = 0; Py_ssize_t digitval; - STRINGLIB_CHAR *p; + Py_ssize_t i; /* empty string is an error */ - if (str->ptr >= str->end) + if (str->start >= str->end) return -1; - for (p = str->ptr; p < str->end; p++) { - digitval = STRINGLIB_TODECIMAL(*p); + for (i = str->start; i < str->end; i++) { + digitval = Py_UNICODE_TODECIMAL(PyUnicode_READ_CHAR(str->str, i)); if (digitval < 0) return -1; /* @@ -279,34 +200,36 @@ typedef struct { lifetime of the iterator. can be empty */ SubString str; - /* pointer to where we are inside field_name */ - STRINGLIB_CHAR *ptr; + /* index to where we are inside field_name */ + Py_ssize_t index; } FieldNameIterator; static int -FieldNameIterator_init(FieldNameIterator *self, STRINGLIB_CHAR *ptr, - Py_ssize_t len) +FieldNameIterator_init(FieldNameIterator *self, PyObject *s, + Py_ssize_t start, Py_ssize_t end) { - SubString_init(&self->str, ptr, len); - self->ptr = self->str.ptr; + SubString_init(&self->str, s, start, end); + self->index = start; return 1; } static int _FieldNameIterator_attr(FieldNameIterator *self, SubString *name) { - STRINGLIB_CHAR c; + Py_UCS4 c; - name->ptr = self->ptr; + name->str = self->str.str; + name->start = self->index; /* return everything until '.' or '[' */ - while (self->ptr < self->str.end) { - switch (c = *self->ptr++) { + while (self->index < self->str.end) { + c = PyUnicode_READ_CHAR(self->str.str, self->index++); + switch (c) { case '[': case '.': /* backup so that we this character will be seen next time */ - self->ptr--; + self->index--; break; default: continue; @@ -314,7 +237,7 @@ _FieldNameIterator_attr(FieldNameIterator *self, SubString *name) break; } /* end of string is okay */ - name->end = self->ptr; + name->end = self->index; return 1; } @@ -322,13 +245,15 @@ static int _FieldNameIterator_item(FieldNameIterator *self, SubString *name) { int bracket_seen = 0; - STRINGLIB_CHAR c; + Py_UCS4 c; - name->ptr = self->ptr; + name->str = self->str.str; + name->start = self->index; /* return everything until ']' */ - while (self->ptr < self->str.end) { - switch (c = *self->ptr++) { + while (self->index < self->str.end) { + c = PyUnicode_READ_CHAR(self->str.str, self->index++); + switch (c) { case ']': bracket_seen = 1; break; @@ -345,7 +270,7 @@ _FieldNameIterator_item(FieldNameIterator *self, SubString *name) /* end of string is okay */ /* don't include the ']' */ - name->end = self->ptr-1; + name->end = self->index-1; return 1; } @@ -355,10 +280,10 @@ FieldNameIterator_next(FieldNameIterator *self, int *is_attribute, Py_ssize_t *name_idx, SubString *name) { /* check at end of input */ - if (self->ptr >= self->str.end) + if (self->index >= self->str.end) return 1; - switch (*self->ptr++) { + switch (PyUnicode_READ_CHAR(self->str.str, self->index++)) { case '.': *is_attribute = 1; if (_FieldNameIterator_attr(self, name) == 0) @@ -381,7 +306,7 @@ FieldNameIterator_next(FieldNameIterator *self, int *is_attribute, } /* empty string is an error */ - if (name->ptr == name->end) { + if (name->start == name->end) { PyErr_SetString(PyExc_ValueError, "Empty attribute in format string"); return 0; } @@ -397,24 +322,23 @@ FieldNameIterator_next(FieldNameIterator *self, int *is_attribute, 'rest' is an iterator to return the rest */ static int -field_name_split(STRINGLIB_CHAR *ptr, Py_ssize_t len, SubString *first, +field_name_split(PyObject *str, Py_ssize_t start, Py_ssize_t end, SubString *first, Py_ssize_t *first_idx, FieldNameIterator *rest, AutoNumber *auto_number) { - STRINGLIB_CHAR c; - STRINGLIB_CHAR *p = ptr; - STRINGLIB_CHAR *end = ptr + len; + Py_UCS4 c; + Py_ssize_t i = start; int field_name_is_empty; int using_numeric_index; /* find the part up until the first '.' or '[' */ - while (p < end) { - switch (c = *p++) { + while (i < end) { + switch (c = PyUnicode_READ_CHAR(str, i++)) { case '[': case '.': /* backup so that we this character is available to the "rest" iterator */ - p--; + i--; break; default: continue; @@ -423,15 +347,15 @@ field_name_split(STRINGLIB_CHAR *ptr, Py_ssize_t len, SubString *first, } /* set up the return values */ - SubString_init(first, ptr, p - ptr); - FieldNameIterator_init(rest, p, end - p); + SubString_init(first, str, start, i); + FieldNameIterator_init(rest, str, i, end); /* see if "first" is an integer, in which case it's used as an index */ *first_idx = get_integer(first); if (*first_idx == -1 && PyErr_Occurred()) return 0; - field_name_is_empty = first->ptr >= first->end; + field_name_is_empty = first->start >= first->end; /* If the field name is omitted or if we have a numeric index specified, then we're doing numeric indexing into args. */ @@ -486,7 +410,7 @@ get_field_object(SubString *input, PyObject *args, PyObject *kwargs, Py_ssize_t index; FieldNameIterator rest; - if (!field_name_split(input->ptr, input->end - input->ptr, &first, + if (!field_name_split(input->str, input->start, input->end, &first, &index, &rest, auto_number)) { goto error; } @@ -570,39 +494,41 @@ error: appends to the output. */ static int -render_field(PyObject *fieldobj, SubString *format_spec, OutputString *output) +render_field(PyObject *fieldobj, SubString *format_spec, _PyUnicodeWriter *writer) { int ok = 0; PyObject *result = NULL; PyObject *format_spec_object = NULL; - PyObject *(*formatter)(PyObject *, STRINGLIB_CHAR *, Py_ssize_t) = NULL; - STRINGLIB_CHAR* format_spec_start = format_spec->ptr ? - format_spec->ptr : NULL; - Py_ssize_t format_spec_len = format_spec->ptr ? - format_spec->end - format_spec->ptr : 0; + int (*formatter) (_PyUnicodeWriter*, PyObject *, PyObject *, Py_ssize_t, Py_ssize_t) = NULL; + int err; /* If we know the type exactly, skip the lookup of __format__ and just call the formatter directly. */ if (PyUnicode_CheckExact(fieldobj)) - formatter = _PyUnicode_FormatAdvanced; + formatter = _PyUnicode_FormatAdvancedWriter; else if (PyLong_CheckExact(fieldobj)) - formatter =_PyLong_FormatAdvanced; + formatter = _PyLong_FormatAdvancedWriter; else if (PyFloat_CheckExact(fieldobj)) - formatter = _PyFloat_FormatAdvanced; - - /* XXX: for 2.6, convert format_spec to the appropriate type - (unicode, str) */ + formatter = _PyFloat_FormatAdvancedWriter; + else if (PyComplex_CheckExact(fieldobj)) + formatter = _PyComplex_FormatAdvancedWriter; if (formatter) { /* we know exactly which formatter will be called when __format__ is looked up, so call it directly, instead. */ - result = formatter(fieldobj, format_spec_start, format_spec_len); + err = formatter(writer, fieldobj, format_spec->str, + format_spec->start, format_spec->end); + return (err == 0); } else { /* We need to create an object out of the pointers we have, because __format__ takes a string/unicode object for format_spec. */ - format_spec_object = STRINGLIB_NEW(format_spec_start, - format_spec_len); + if (format_spec->str) + format_spec_object = PyUnicode_Substring(format_spec->str, + format_spec->start, + format_spec->end); + else + format_spec_object = PyUnicode_New(0, 0); if (format_spec_object == NULL) goto done; @@ -611,24 +537,10 @@ render_field(PyObject *fieldobj, SubString *format_spec, OutputString *output) if (result == NULL) goto done; -#if PY_VERSION_HEX >= 0x03000000 - assert(PyUnicode_Check(result)); -#else - assert(PyBytes_Check(result) || PyUnicode_Check(result)); - - /* Convert result to our type. We could be str, and result could - be unicode */ - { - PyObject *tmp = STRINGLIB_TOSTR(result); - if (tmp == NULL) - goto done; - Py_DECREF(result); - result = tmp; - } -#endif + if (_PyUnicodeWriter_WriteStr(writer, result) == -1) + goto done; + ok = 1; - ok = output_data(output, - STRINGLIB_STR(result), STRINGLIB_LEN(result)); done: Py_XDECREF(format_spec_object); Py_XDECREF(result); @@ -637,23 +549,24 @@ done: static int parse_field(SubString *str, SubString *field_name, SubString *format_spec, - STRINGLIB_CHAR *conversion) + Py_UCS4 *conversion) { /* Note this function works if the field name is zero length, which is good. Zero length field names are handled later, in field_name_split. */ - STRINGLIB_CHAR c = 0; + Py_UCS4 c = 0; /* initialize these, as they may be empty */ *conversion = '\0'; - SubString_init(format_spec, NULL, 0); + SubString_init(format_spec, NULL, 0, 0); /* Search for the field name. it's terminated by the end of the string, or a ':' or '!' */ - field_name->ptr = str->ptr; - while (str->ptr < str->end) { - switch (c = *(str->ptr++)) { + field_name->str = str->str; + field_name->start = str->start; + while (str->start < str->end) { + switch ((c = PyUnicode_READ_CHAR(str->str, str->start++))) { case ':': case '!': break; @@ -666,26 +579,27 @@ parse_field(SubString *str, SubString *field_name, SubString *format_spec, if (c == '!' || c == ':') { /* we have a format specifier and/or a conversion */ /* don't include the last character */ - field_name->end = str->ptr-1; + field_name->end = str->start-1; /* the format specifier is the rest of the string */ - format_spec->ptr = str->ptr; + format_spec->str = str->str; + format_spec->start = str->start; format_spec->end = str->end; /* see if there's a conversion specifier */ if (c == '!') { /* there must be another character present */ - if (format_spec->ptr >= format_spec->end) { + if (format_spec->start >= format_spec->end) { PyErr_SetString(PyExc_ValueError, "end of format while looking for conversion " "specifier"); return 0; } - *conversion = *(format_spec->ptr++); + *conversion = PyUnicode_READ_CHAR(format_spec->str, format_spec->start++); /* if there is another character, it must be a colon */ - if (format_spec->ptr < format_spec->end) { - c = *(format_spec->ptr++); + if (format_spec->start < format_spec->end) { + c = PyUnicode_READ_CHAR(format_spec->str, format_spec->start++); if (c != ':') { PyErr_SetString(PyExc_ValueError, "expected ':' after format specifier"); @@ -696,7 +610,7 @@ parse_field(SubString *str, SubString *field_name, SubString *format_spec, } else /* end of string, there's no format_spec or conversion */ - field_name->end = str->ptr; + field_name->end = str->start; return 1; } @@ -715,9 +629,10 @@ typedef struct { } MarkupIterator; static int -MarkupIterator_init(MarkupIterator *self, STRINGLIB_CHAR *ptr, Py_ssize_t len) +MarkupIterator_init(MarkupIterator *self, PyObject *str, + Py_ssize_t start, Py_ssize_t end) { - SubString_init(&self->str, ptr, len); + SubString_init(&self->str, str, start, end); return 1; } @@ -726,30 +641,30 @@ MarkupIterator_init(MarkupIterator *self, STRINGLIB_CHAR *ptr, Py_ssize_t len) static int MarkupIterator_next(MarkupIterator *self, SubString *literal, int *field_present, SubString *field_name, - SubString *format_spec, STRINGLIB_CHAR *conversion, + SubString *format_spec, Py_UCS4 *conversion, int *format_spec_needs_expanding) { int at_end; - STRINGLIB_CHAR c = 0; - STRINGLIB_CHAR *start; + Py_UCS4 c = 0; + Py_ssize_t start; int count; Py_ssize_t len; int markup_follows = 0; /* initialize all of the output variables */ - SubString_init(literal, NULL, 0); - SubString_init(field_name, NULL, 0); - SubString_init(format_spec, NULL, 0); + SubString_init(literal, NULL, 0, 0); + SubString_init(field_name, NULL, 0, 0); + SubString_init(format_spec, NULL, 0, 0); *conversion = '\0'; *format_spec_needs_expanding = 0; *field_present = 0; /* No more input, end of iterator. This is the normal exit path. */ - if (self->str.ptr >= self->str.end) + if (self->str.start >= self->str.end) return 1; - start = self->str.ptr; + start = self->str.start; /* First read any literal text. Read until the end of string, an escaped '{' or '}', or an unescaped '{'. In order to never @@ -758,8 +673,8 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, including the brace, but no format object. The next time through, we'll return the rest of the literal, skipping past the second consecutive brace. */ - while (self->str.ptr < self->str.end) { - switch (c = *(self->str.ptr++)) { + while (self->str.start < self->str.end) { + switch (c = PyUnicode_READ_CHAR(self->str.str, self->str.start++)) { case '{': case '}': markup_follows = 1; @@ -770,10 +685,12 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, break; } - at_end = self->str.ptr >= self->str.end; - len = self->str.ptr - start; + at_end = self->str.start >= self->str.end; + len = self->str.start - start; - if ((c == '}') && (at_end || (c != *self->str.ptr))) { + if ((c == '}') && (at_end || + (c != PyUnicode_READ_CHAR(self->str.str, + self->str.start)))) { PyErr_SetString(PyExc_ValueError, "Single '}' encountered " "in format string"); return 0; @@ -784,10 +701,10 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, return 0; } if (!at_end) { - if (c == *self->str.ptr) { + if (c == PyUnicode_READ_CHAR(self->str.str, self->str.start)) { /* escaped } or {, skip it in the input. there is no markup object following us, just this literal text */ - self->str.ptr++; + self->str.start++; markup_follows = 0; } else @@ -795,7 +712,8 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, } /* record the literal text */ - literal->ptr = start; + literal->str = self->str.str; + literal->start = start; literal->end = start + len; if (!markup_follows) @@ -807,12 +725,12 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, *field_present = 1; count = 1; - start = self->str.ptr; + start = self->str.start; /* we know we can't have a zero length string, so don't worry about that case */ - while (self->str.ptr < self->str.end) { - switch (c = *(self->str.ptr++)) { + while (self->str.start < self->str.end) { + switch (c = PyUnicode_READ_CHAR(self->str.str, self->str.start++)) { case '{': /* the format spec needs to be recursively expanded. this is an optimization, and not strictly needed */ @@ -825,7 +743,7 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, /* we're done. parse and get out */ SubString s; - SubString_init(&s, start, self->str.ptr - 1 - start); + SubString_init(&s, self->str.str, start, self->str.start - 1); if (parse_field(&s, field_name, format_spec, conversion) == 0) return 0; @@ -844,7 +762,7 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal, /* do the !r or !s conversion on obj */ static PyObject * -do_conversion(PyObject *obj, STRINGLIB_CHAR conversion) +do_conversion(PyObject *obj, Py_UCS4 conversion) { /* XXX in pre-3.0, do we need to convert this to unicode, since it might have returned a string? */ @@ -852,11 +770,9 @@ do_conversion(PyObject *obj, STRINGLIB_CHAR conversion) case 'r': return PyObject_Repr(obj); case 's': - return STRINGLIB_TOSTR(obj); -#if PY_VERSION_HEX >= 0x03000000 + return PyObject_Str(obj); case 'a': - return STRINGLIB_TOASCII(obj); -#endif + return PyObject_ASCII(obj); default: if (conversion > 32 && conversion < 127) { /* It's the ASCII subrange; casting to char is safe @@ -888,8 +804,8 @@ do_conversion(PyObject *obj, STRINGLIB_CHAR conversion) static int output_markup(SubString *field_name, SubString *format_spec, - int format_spec_needs_expanding, STRINGLIB_CHAR conversion, - OutputString *output, PyObject *args, PyObject *kwargs, + int format_spec_needs_expanding, Py_UCS4 conversion, + _PyUnicodeWriter *writer, PyObject *args, PyObject *kwargs, int recursion_depth, AutoNumber *auto_number) { PyObject *tmp = NULL; @@ -905,7 +821,7 @@ output_markup(SubString *field_name, SubString *format_spec, if (conversion != '\0') { tmp = do_conversion(fieldobj, conversion); - if (tmp == NULL) + if (tmp == NULL || PyUnicode_READY(tmp) == -1) goto done; /* do the assignment, transferring ownership: fieldobj = tmp */ @@ -918,20 +834,19 @@ output_markup(SubString *field_name, SubString *format_spec, if (format_spec_needs_expanding) { tmp = build_string(format_spec, args, kwargs, recursion_depth-1, auto_number); - if (tmp == NULL) + if (tmp == NULL || PyUnicode_READY(tmp) == -1) goto done; /* note that in the case we're expanding the format string, tmp must be kept around until after the call to render_field. */ - SubString_init(&expanded_format_spec, - STRINGLIB_STR(tmp), STRINGLIB_LEN(tmp)); + SubString_init(&expanded_format_spec, tmp, 0, PyUnicode_GET_LENGTH(tmp)); actual_format_spec = &expanded_format_spec; } else actual_format_spec = format_spec; - if (render_field(fieldobj, actual_format_spec, output) == 0) + if (render_field(fieldobj, actual_format_spec, writer) == 0) goto done; result = 1; @@ -951,7 +866,7 @@ done: */ static int do_markup(SubString *input, PyObject *args, PyObject *kwargs, - OutputString *output, int recursion_depth, AutoNumber *auto_number) + _PyUnicodeWriter *writer, int recursion_depth, AutoNumber *auto_number) { MarkupIterator iter; int format_spec_needs_expanding; @@ -960,20 +875,35 @@ do_markup(SubString *input, PyObject *args, PyObject *kwargs, SubString literal; SubString field_name; SubString format_spec; - STRINGLIB_CHAR conversion; + Py_UCS4 conversion, maxchar; + Py_ssize_t sublen; + int err; - MarkupIterator_init(&iter, input->ptr, input->end - input->ptr); + MarkupIterator_init(&iter, input->str, input->start, input->end); while ((result = MarkupIterator_next(&iter, &literal, &field_present, &field_name, &format_spec, &conversion, &format_spec_needs_expanding)) == 2) { - if (!output_data(output, literal.ptr, literal.end - literal.ptr)) - return 0; - if (field_present) + sublen = literal.end - literal.start; + if (sublen) { + maxchar = _PyUnicode_FindMaxChar(literal.str, + literal.start, literal.end); + err = _PyUnicodeWriter_Prepare(writer, sublen, maxchar); + if (err == -1) + return 0; + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + literal.str, literal.start, sublen); + writer->pos += sublen; + } + + if (field_present) { + if (iter.str.start == iter.str.end) + writer->overallocate = 0; if (!output_markup(&field_name, &format_spec, - format_spec_needs_expanding, conversion, output, + format_spec_needs_expanding, conversion, writer, args, kwargs, recursion_depth, auto_number)) return 0; + } } return result; } @@ -987,43 +917,26 @@ static PyObject * build_string(SubString *input, PyObject *args, PyObject *kwargs, int recursion_depth, AutoNumber *auto_number) { - OutputString output; - PyObject *result = NULL; - Py_ssize_t count; - - output.obj = NULL; /* needed so cleanup code always works */ + _PyUnicodeWriter writer; + Py_ssize_t minlen; /* check the recursion level */ if (recursion_depth <= 0) { PyErr_SetString(PyExc_ValueError, "Max string recursion exceeded"); - goto done; + return NULL; } - /* initial size is the length of the format string, plus the size - increment. seems like a reasonable default */ - if (!output_initialize(&output, - input->end - input->ptr + - INITIAL_SIZE_INCREMENT)) - goto done; + minlen = PyUnicode_GET_LENGTH(input->str) + 100; + _PyUnicodeWriter_Init(&writer, minlen); - if (!do_markup(input, args, kwargs, &output, recursion_depth, + if (!do_markup(input, args, kwargs, &writer, recursion_depth, auto_number)) { - goto done; - } - - count = output.ptr - STRINGLIB_STR(output.obj); - if (STRINGLIB_RESIZE(&output.obj, count) < 0) { - goto done; + _PyUnicodeWriter_Dealloc(&writer); + return NULL; } - /* transfer ownership to result */ - result = output.obj; - output.obj = NULL; - -done: - Py_XDECREF(output.obj); - return result; + return _PyUnicodeWriter_Finish(&writer); } /************************************************************************/ @@ -1044,8 +957,11 @@ do_string_format(PyObject *self, PyObject *args, PyObject *kwargs) AutoNumber auto_number; + if (PyUnicode_READY(self) == -1) + return NULL; + AutoNumber_Init(&auto_number); - SubString_init(&input, STRINGLIB_STR(self), STRINGLIB_LEN(self)); + SubString_init(&input, self, 0, PyUnicode_GET_LENGTH(self)); return build_string(&input, args, kwargs, recursion_depth, &auto_number); } @@ -1067,9 +983,7 @@ do_string_format_map(PyObject *self, PyObject *obj) typedef struct { PyObject_HEAD - - STRINGLIB_OBJECT *str; - + PyObject *str; MarkupIterator it_markup; } formatteriterobject; @@ -1094,7 +1008,7 @@ formatteriter_next(formatteriterobject *it) SubString literal; SubString field_name; SubString format_spec; - STRINGLIB_CHAR conversion; + Py_UCS4 conversion; int format_spec_needs_expanding; int field_present; int result = MarkupIterator_next(&it->it_markup, &literal, &field_present, @@ -1138,7 +1052,8 @@ formatteriter_next(formatteriterobject *it) Py_INCREF(conversion_str); } else - conversion_str = STRINGLIB_NEW(&conversion, 1); + conversion_str = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + &conversion, 1); if (conversion_str == NULL) goto done; @@ -1195,7 +1110,7 @@ static PyTypeObject PyFormatterIter_Type = { describing the parsed elements. It's a wrapper around stringlib/string_format.h's MarkupIterator */ static PyObject * -formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self) +formatter_parser(PyObject *ignored, PyObject *self) { formatteriterobject *it; @@ -1204,6 +1119,9 @@ formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self) return NULL; } + if (PyUnicode_READY(self) == -1) + return NULL; + it = PyObject_New(formatteriterobject, &PyFormatterIter_Type); if (it == NULL) return NULL; @@ -1213,10 +1131,7 @@ formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self) it->str = self; /* initialize the contained MarkupIterator */ - MarkupIterator_init(&it->it_markup, - STRINGLIB_STR(self), - STRINGLIB_LEN(self)); - + MarkupIterator_init(&it->it_markup, (PyObject*)self, 0, PyUnicode_GET_LENGTH(self)); return (PyObject *)it; } @@ -1232,9 +1147,7 @@ formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self) typedef struct { PyObject_HEAD - - STRINGLIB_OBJECT *str; - + PyObject *str; FieldNameIterator it_field; } fieldnameiterobject; @@ -1335,7 +1248,7 @@ static PyTypeObject PyFieldNameIter_Type = { field_name_split. The iterator it returns is a FieldNameIterator */ static PyObject * -formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self) +formatter_field_name_split(PyObject *ignored, PyObject *self) { SubString first; Py_ssize_t first_idx; @@ -1349,6 +1262,9 @@ formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self) return NULL; } + if (PyUnicode_READY(self) == -1) + return NULL; + it = PyObject_New(fieldnameiterobject, &PyFieldNameIter_Type); if (it == NULL) return NULL; @@ -1360,8 +1276,7 @@ formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self) /* Pass in auto_number = NULL. We'll return an empty string for first_obj in that case. */ - if (!field_name_split(STRINGLIB_STR(self), - STRINGLIB_LEN(self), + if (!field_name_split((PyObject*)self, 0, PyUnicode_GET_LENGTH(self), &first, &first_idx, &it->it_field, NULL)) goto done; diff --git a/Objects/stringlib/unicodedefs.h b/Objects/stringlib/unicodedefs.h index 09dae6d..f16f21e 100644 --- a/Objects/stringlib/unicodedefs.h +++ b/Objects/stringlib/unicodedefs.h @@ -6,7 +6,10 @@ compiled as unicode. */ #define STRINGLIB_IS_UNICODE 1 +#define FASTSEARCH fastsearch +#define STRINGLIB(F) stringlib_##F #define STRINGLIB_OBJECT PyUnicodeObject +#define STRINGLIB_SIZEOF_CHAR Py_UNICODE_SIZE #define STRINGLIB_CHAR Py_UNICODE #define STRINGLIB_TYPE_NAME "unicode" #define STRINGLIB_PARSE_CODE "U" @@ -15,17 +18,12 @@ #define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK #define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL #define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL -#define STRINGLIB_TOUPPER Py_UNICODE_TOUPPER -#define STRINGLIB_TOLOWER Py_UNICODE_TOLOWER -#define STRINGLIB_FILL Py_UNICODE_FILL #define STRINGLIB_STR PyUnicode_AS_UNICODE #define STRINGLIB_LEN PyUnicode_GET_SIZE #define STRINGLIB_NEW PyUnicode_FromUnicode #define STRINGLIB_RESIZE PyUnicode_Resize #define STRINGLIB_CHECK PyUnicode_Check #define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact -#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping -#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale #if PY_VERSION_HEX < 0x03000000 #define STRINGLIB_TOSTR PyObject_Unicode diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index b345460..9c843fa 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -45,6 +45,22 @@ show_track(void) } #endif +/* Print summary info about the state of the optimized allocator */ +void +_PyTuple_DebugMallocStats(FILE *out) +{ +#if PyTuple_MAXSAVESIZE > 0 + int i; + char buf[128]; + for (i = 1; i < PyTuple_MAXSAVESIZE; i++) { + PyOS_snprintf(buf, sizeof(buf), + "free %d-sized PyTupleObject", i); + _PyDebugAllocatorStats(out, + buf, + numfree[i], _PyObject_VAR_SIZE(&PyTuple_Type, i)); + } +#endif +} PyObject * PyTuple_New(register Py_ssize_t size) @@ -80,15 +96,11 @@ PyTuple_New(register Py_ssize_t size) else #endif { - Py_ssize_t nbytes = size * sizeof(PyObject *); /* Check for overflow */ - if (nbytes / sizeof(PyObject *) != (size_t)size || - (nbytes > PY_SSIZE_T_MAX - sizeof(PyTupleObject) - sizeof(PyObject *))) - { + if (size > (PY_SSIZE_T_MAX - sizeof(PyTupleObject) - + sizeof(PyObject *)) / sizeof(PyObject *)) { return PyErr_NoMemory(); } - nbytes += sizeof(PyTupleObject) - sizeof(PyObject *); - op = PyObject_GC_NewVar(PyTupleObject, &PyTuple_Type, size); if (op == NULL) return NULL; @@ -315,11 +327,12 @@ error: static Py_hash_t tuplehash(PyTupleObject *v) { - register Py_hash_t x, y; + register Py_uhash_t x; + register Py_hash_t y; register Py_ssize_t len = Py_SIZE(v); register PyObject **p; - Py_hash_t mult = _PyHASH_MULTIPLIER; - x = 0x345678L; + Py_uhash_t mult = _PyHASH_MULTIPLIER; + x = 0x345678; p = v->ob_item; while (--len >= 0) { y = PyObject_Hash(*p++); @@ -330,7 +343,7 @@ tuplehash(PyTupleObject *v) mult += (Py_hash_t)(82520L + len + len); } x += 97531L; - if (x == -1) + if (x == (Py_uhash_t)-1) x = -2; return x; } @@ -464,9 +477,9 @@ tuplerepeat(PyTupleObject *a, Py_ssize_t n) if (Py_SIZE(a) == 0) return PyTuple_New(0); } - size = Py_SIZE(a) * n; - if (size/Py_SIZE(a) != n) + if (n > PY_SSIZE_T_MAX / Py_SIZE(a)) return PyErr_NoMemory(); + size = Py_SIZE(a) * n; np = (PyTupleObject *) PyTuple_New(size); if (np == NULL) return NULL; @@ -546,10 +559,8 @@ tuplerichcompare(PyObject *v, PyObject *w, int op) Py_ssize_t i; Py_ssize_t vlen, wlen; - if (!PyTuple_Check(v) || !PyTuple_Check(w)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyTuple_Check(v) || !PyTuple_Check(w)) + Py_RETURN_NOTIMPLEMENTED; vt = (PyTupleObject *)v; wt = (PyTupleObject *)w; @@ -970,8 +981,39 @@ tupleiter_len(tupleiterobject *it) PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +static PyObject * +tupleiter_reduce(tupleiterobject *it) +{ + if (it->it_seq) + return Py_BuildValue("N(O)l", _PyObject_GetBuiltin("iter"), + it->it_seq, it->it_index); + else + return Py_BuildValue("N(())", _PyObject_GetBuiltin("iter")); +} + +static PyObject * +tupleiter_setstate(tupleiterobject *it, PyObject *state) +{ + long index = PyLong_AsLong(state); + if (index == -1 && PyErr_Occurred()) + return NULL; + if (it->it_seq != NULL) { + if (index < 0) + index = 0; + else if (it->it_seq != NULL && index > PyTuple_GET_SIZE(it->it_seq)) + index = PyTuple_GET_SIZE(it->it_seq); + it->it_index = index; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); +PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); + static PyMethodDef tupleiter_methods[] = { {"__length_hint__", (PyCFunction)tupleiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)tupleiter_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", (PyCFunction)tupleiter_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index fd2ae67..413c7da 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -14,16 +14,17 @@ MCACHE_MAX_ATTR_SIZE, since it might be a problem if very large strings are used as attribute names. */ #define MCACHE_MAX_ATTR_SIZE 100 -#define MCACHE_SIZE_EXP 10 +#define MCACHE_SIZE_EXP 9 #define MCACHE_HASH(version, name_hash) \ (((unsigned int)(version) * (unsigned int)(name_hash)) \ >> (8*sizeof(unsigned int) - MCACHE_SIZE_EXP)) #define MCACHE_HASH_METHOD(type, name) \ MCACHE_HASH((type)->tp_version_tag, \ - ((PyUnicodeObject *)(name))->hash) + ((PyASCIIObject *)(name))->hash) #define MCACHE_CACHEABLE_NAME(name) \ PyUnicode_CheckExact(name) && \ - PyUnicode_GET_SIZE(name) <= MCACHE_MAX_ATTR_SIZE + PyUnicode_READY(name) != -1 && \ + PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE struct method_cache_entry { unsigned int version; @@ -34,6 +35,22 @@ struct method_cache_entry { static struct method_cache_entry method_cache[1 << MCACHE_SIZE_EXP]; static unsigned int next_version_tag = 0; +_Py_IDENTIFIER(__class__); +_Py_IDENTIFIER(__dict__); +_Py_IDENTIFIER(__doc__); +_Py_IDENTIFIER(__getitem__); +_Py_IDENTIFIER(__getattribute__); +_Py_IDENTIFIER(__hash__); +_Py_IDENTIFIER(__module__); +_Py_IDENTIFIER(__name__); +_Py_IDENTIFIER(__new__); + +static PyObject * +_PyType_LookupId(PyTypeObject *type, struct _Py_Identifier *name); + +static PyObject * +slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds); + unsigned int PyType_ClearCache(void) { @@ -96,15 +113,13 @@ PyType_Modified(PyTypeObject *type) static void type_mro_modified(PyTypeObject *type, PyObject *bases) { /* - Check that all base classes or elements of the mro of type are + Check that all base classes or elements of the MRO of type are able to be cached. This function is called after the base classes or mro of the type are altered. Unset HAVE_VERSION_TAG and VALID_VERSION_TAG if the type - inherits from an old-style class, either directly or if it - appears in the MRO of a new-style class. No support either for - custom MROs that include types that are not officially super - types. + has a custom MRO that includes a type which is not officially + super type. Called from mro_internal, which will subsequently be called on each subclass when their mro is recursively updated. @@ -120,11 +135,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { PyObject *b = PyTuple_GET_ITEM(bases, i); PyTypeObject *cls; - if (!PyType_Check(b) ) { - clear = 1; - break; - } - + assert(PyType_Check(b)); cls = (PyTypeObject *)b; if (!PyType_HasFeature(cls, Py_TPFLAGS_HAVE_VERSION_TAG) || @@ -201,6 +212,22 @@ static PyMemberDef type_members[] = { {0} }; +static int +check_set_special_type_attr(PyTypeObject *type, PyObject *value, const char *name) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_TypeError, + "can't set %s.%s", type->tp_name, name); + return 0; + } + if (!value) { + PyErr_Format(PyExc_TypeError, + "can't delete %s.%s", type->tp_name, name); + return 0; + } + return 1; +} + static PyObject * type_name(PyTypeObject *type, void *context) { @@ -222,6 +249,19 @@ type_name(PyTypeObject *type, void *context) } } +static PyObject * +type_qualname(PyTypeObject *type, void *context) +{ + if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { + PyHeapTypeObject* et = (PyHeapTypeObject*)type; + Py_INCREF(et->ht_qualname); + return et->ht_qualname; + } + else { + return type_name(type, context); + } +} + static int type_set_name(PyTypeObject *type, PyObject *value, void *context) { @@ -229,16 +269,8 @@ type_set_name(PyTypeObject *type, PyObject *value, void *context) char *tp_name; PyObject *tmp; - if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - PyErr_Format(PyExc_TypeError, - "can't set %s.__name__", type->tp_name); - return -1; - } - if (!value) { - PyErr_Format(PyExc_TypeError, - "can't delete %s.__name__", type->tp_name); + if (!check_set_special_type_attr(type, value, "__name__")) return -1; - } if (!PyUnicode_Check(value)) { PyErr_Format(PyExc_TypeError, "can only assign string to %s.__name__, not '%s'", @@ -274,6 +306,27 @@ type_set_name(PyTypeObject *type, PyObject *value, void *context) return 0; } +static int +type_set_qualname(PyTypeObject *type, PyObject *value, void *context) +{ + PyHeapTypeObject* et; + + if (!check_set_special_type_attr(type, value, "__qualname__")) + return -1; + if (!PyUnicode_Check(value)) { + PyErr_Format(PyExc_TypeError, + "can only assign string to %s.__qualname__, not '%s'", + type->tp_name, Py_TYPE(value)->tp_name); + return -1; + } + + et = (PyHeapTypeObject*)type; + Py_INCREF(value); + Py_DECREF(et->ht_qualname); + et->ht_qualname = value; + return 0; +} + static PyObject * type_module(PyTypeObject *type, void *context) { @@ -281,7 +334,7 @@ type_module(PyTypeObject *type, void *context) char *s; if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { - mod = PyDict_GetItemString(type->tp_dict, "__module__"); + mod = _PyDict_GetItemId(type->tp_dict, &PyId___module__); if (!mod) { PyErr_Format(PyExc_AttributeError, "__module__"); return 0; @@ -301,20 +354,12 @@ type_module(PyTypeObject *type, void *context) static int type_set_module(PyTypeObject *type, PyObject *value, void *context) { - if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - PyErr_Format(PyExc_TypeError, - "can't set %s.__module__", type->tp_name); - return -1; - } - if (!value) { - PyErr_Format(PyExc_TypeError, - "can't delete %s.__module__", type->tp_name); + if (!check_set_special_type_attr(type, value, "__module__")) return -1; - } PyType_Modified(type); - return PyDict_SetItemString(type->tp_dict, "__module__", value); + return _PyDict_SetItemId(type->tp_dict, &PyId___module__, value); } static PyObject * @@ -435,16 +480,8 @@ type_set_bases(PyTypeObject *type, PyObject *value, void *context) PyTypeObject *new_base, *old_base; PyObject *old_bases, *old_mro; - if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - PyErr_Format(PyExc_TypeError, - "can't set %s.__bases__", type->tp_name); - return -1; - } - if (!value) { - PyErr_Format(PyExc_TypeError, - "can't delete %s.__bases__", type->tp_name); + if (!check_set_special_type_attr(type, value, "__bases__")) return -1; - } if (!PyTuple_Check(value)) { PyErr_Format(PyExc_TypeError, "can only assign tuple to %s.__bases__, not %s", @@ -461,8 +498,7 @@ type_set_bases(PyTypeObject *type, PyObject *value, void *context) ob = PyTuple_GET_ITEM(value, i); if (!PyType_Check(ob)) { PyErr_Format(PyExc_TypeError, - "%s.__bases__ must be tuple of old- or " - "new-style classes, not '%s'", + "%s.__bases__ must be tuple of classes, not '%s'", type->tp_name, Py_TYPE(ob)->tp_name); return -1; } @@ -580,7 +616,7 @@ type_get_doc(PyTypeObject *type, void *context) PyObject *result; if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE) && type->tp_doc != NULL) return PyUnicode_FromString(type->tp_doc); - result = PyDict_GetItemString(type->tp_dict, "__doc__"); + result = _PyDict_GetItemId(type->tp_dict, &PyId___doc__); if (result == NULL) { result = Py_None; Py_INCREF(result); @@ -595,6 +631,15 @@ type_get_doc(PyTypeObject *type, void *context) return result; } +static int +type_set_doc(PyTypeObject *type, PyObject *value, void *context) +{ + if (!check_set_special_type_attr(type, value, "__doc__")) + return -1; + PyType_Modified(type); + return _PyDict_SetItemId(type->tp_dict, &PyId___doc__, value); +} + static PyObject * type___instancecheck__(PyObject *type, PyObject *inst) { @@ -625,12 +670,13 @@ type___subclasscheck__(PyObject *type, PyObject *inst) static PyGetSetDef type_getsets[] = { {"__name__", (getter)type_name, (setter)type_set_name, NULL}, + {"__qualname__", (getter)type_qualname, (setter)type_set_qualname, NULL}, {"__bases__", (getter)type_get_bases, (setter)type_set_bases, NULL}, {"__module__", (getter)type_module, (setter)type_set_module, NULL}, {"__abstractmethods__", (getter)type_abstractmethods, (setter)type_set_abstractmethods, NULL}, {"__dict__", (getter)type_dict, NULL, NULL}, - {"__doc__", (getter)type_get_doc, NULL, NULL}, + {"__doc__", (getter)type_get_doc, (setter)type_set_doc, NULL}, {0} }; @@ -646,7 +692,7 @@ type_repr(PyTypeObject *type) Py_DECREF(mod); mod = NULL; } - name = type_name(type, NULL); + name = type_qualname(type, NULL); if (name == NULL) { Py_XDECREF(mod); return NULL; @@ -911,7 +957,7 @@ subtype_dealloc(PyObject *self) /* Find the nearest base with a different tp_dealloc */ base = type; - while ((basedealloc = base->tp_dealloc) == subtype_dealloc) { + while ((/*basedealloc =*/ base->tp_dealloc) == subtype_dealloc) { base = base->tp_base; assert(base); } @@ -1136,16 +1182,11 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) */ static PyObject * -lookup_maybe(PyObject *self, char *attrstr, PyObject **attrobj) +lookup_maybe(PyObject *self, _Py_Identifier *attrid) { PyObject *res; - if (*attrobj == NULL) { - *attrobj = PyUnicode_InternFromString(attrstr); - if (*attrobj == NULL) - return NULL; - } - res = _PyType_Lookup(Py_TYPE(self), *attrobj); + res = _PyType_LookupId(Py_TYPE(self), attrid); if (res != NULL) { descrgetfunc f; if ((f = Py_TYPE(res)->tp_descr_get) == NULL) @@ -1157,18 +1198,18 @@ lookup_maybe(PyObject *self, char *attrstr, PyObject **attrobj) } static PyObject * -lookup_method(PyObject *self, char *attrstr, PyObject **attrobj) +lookup_method(PyObject *self, _Py_Identifier *attrid) { - PyObject *res = lookup_maybe(self, attrstr, attrobj); + PyObject *res = lookup_maybe(self, attrid); if (res == NULL && !PyErr_Occurred()) - PyErr_SetObject(PyExc_AttributeError, *attrobj); + PyErr_SetObject(PyExc_AttributeError, attrid->object); return res; } PyObject * -_PyObject_LookupSpecial(PyObject *self, char *attrstr, PyObject **attrobj) +_PyObject_LookupSpecial(PyObject *self, _Py_Identifier *attrid) { - return lookup_maybe(self, attrstr, attrobj); + return lookup_maybe(self, attrid); } /* A variation of PyObject_CallMethod that uses lookup_method() @@ -1176,17 +1217,17 @@ _PyObject_LookupSpecial(PyObject *self, char *attrstr, PyObject **attrobj) as lookup_method to cache the interned name string object. */ static PyObject * -call_method(PyObject *o, char *name, PyObject **nameobj, char *format, ...) +call_method(PyObject *o, _Py_Identifier *nameid, char *format, ...) { va_list va; PyObject *args, *func = 0, *retval; va_start(va, format); - func = lookup_maybe(o, name, nameobj); + func = lookup_maybe(o, nameid); if (func == NULL) { va_end(va); if (!PyErr_Occurred()) - PyErr_SetObject(PyExc_AttributeError, *nameobj); + PyErr_SetObject(PyExc_AttributeError, nameid->object); return NULL; } @@ -1212,19 +1253,17 @@ call_method(PyObject *o, char *name, PyObject **nameobj, char *format, ...) /* Clone of call_method() that returns NotImplemented when the lookup fails. */ static PyObject * -call_maybe(PyObject *o, char *name, PyObject **nameobj, char *format, ...) +call_maybe(PyObject *o, _Py_Identifier *nameid, char *format, ...) { va_list va; PyObject *args, *func = 0, *retval; va_start(va, format); - func = lookup_maybe(o, name, nameobj); + func = lookup_maybe(o, nameid); if (func == NULL) { va_end(va); - if (!PyErr_Occurred()) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyErr_Occurred()) + Py_RETURN_NOTIMPLEMENTED; return NULL; } @@ -1290,7 +1329,7 @@ tail_contains(PyObject *list, int whence, PyObject *o) { static PyObject * class_name(PyObject *cls) { - PyObject *name = PyObject_GetAttrString(cls, "__name__"); + PyObject *name = _PyObject_GetAttrId(cls, &PyId___name__); if (name == NULL) { PyErr_Clear(); Py_XDECREF(name); @@ -1545,9 +1584,9 @@ mro_internal(PyTypeObject *type) result = mro_implementation(type); } else { - static PyObject *mro_str; + _Py_IDENTIFIER(mro); checkit = 1; - mro = lookup_method((PyObject *)type, "mro", &mro_str); + mro = lookup_method((PyObject *)type, &PyId_mro); if (mro == NULL) return -1; result = PyObject_CallObject(mro, NULL); @@ -1591,7 +1630,7 @@ mro_internal(PyTypeObject *type) type->tp_mro = tuple; type_mro_modified(type, type->tp_mro); - /* corner case: the old-style super class might have been hidden + /* corner case: the super class might have been hidden from the custom MRO */ type_mro_modified(type, type->tp_bases); @@ -1648,9 +1687,8 @@ best_base(PyObject *bases) return NULL; } } - if (base == NULL) - PyErr_SetString(PyExc_TypeError, - "a new-style class can't have only classic bases"); + assert (base != NULL); + return base; } @@ -1718,15 +1756,9 @@ get_builtin_base_with_dict(PyTypeObject *type) static PyObject * get_dict_descriptor(PyTypeObject *type) { - static PyObject *dict_str; PyObject *descr; - if (dict_str == NULL) { - dict_str = PyUnicode_InternFromString("__dict__"); - if (dict_str == NULL) - return NULL; - } - descr = _PyType_Lookup(type, dict_str); + descr = _PyType_LookupId(type, &PyId___dict__); if (descr == NULL || !PyDescr_IsData(descr)) return NULL; @@ -1744,8 +1776,6 @@ raise_dict_descr_error(PyObject *obj) static PyObject * subtype_dict(PyObject *obj, void *context) { - PyObject **dictptr; - PyObject *dict; PyTypeObject *base; base = get_builtin_base_with_dict(Py_TYPE(obj)); @@ -1763,25 +1793,13 @@ subtype_dict(PyObject *obj, void *context) } return func(descr, obj, (PyObject *)(Py_TYPE(obj))); } - - dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return NULL; - } - dict = *dictptr; - if (dict == NULL) - *dictptr = dict = PyDict_New(); - Py_XINCREF(dict); - return dict; + return PyObject_GenericGetDict(obj, context); } static int subtype_setdict(PyObject *obj, PyObject *value, void *context) { - PyObject **dictptr; - PyObject *dict; + PyObject *dict, **dictptr; PyTypeObject *base; base = get_builtin_base_with_dict(Py_TYPE(obj)); @@ -1799,7 +1817,7 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) } return func(descr, obj, value); } - + /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */ dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { PyErr_SetString(PyExc_AttributeError, @@ -1960,14 +1978,16 @@ _PyType_CalculateMetaclass(PyTypeObject *metatype, PyObject *bases) static PyObject * type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) { - PyObject *name, *bases, *dict; + PyObject *name, *bases = NULL, *orig_dict, *dict = NULL; static char *kwlist[] = {"name", "bases", "dict", 0}; - PyObject *slots, *tmp, *newslots; - PyTypeObject *type, *base, *tmptype, *winner; + PyObject *qualname, *slots = NULL, *tmp, *newslots; + PyTypeObject *type = NULL, *base, *tmptype, *winner; PyHeapTypeObject *et; PyMemberDef *mp; Py_ssize_t i, nbases, nslots, slotoffset, add_dict, add_weak; int j, may_add_dict, may_add_weak; + _Py_IDENTIFIER(__qualname__); + _Py_IDENTIFIER(__slots__); assert(args != NULL && PyTuple_Check(args)); assert(kwds == NULL || PyDict_Check(kwds)); @@ -1997,7 +2017,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "UO!O!:type", kwlist, &name, &PyTuple_Type, &bases, - &PyDict_Type, &dict)) + &PyDict_Type, &orig_dict)) return NULL; /* Determine the proper metatype to deal with this: */ @@ -2017,30 +2037,30 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) if (nbases == 0) { bases = PyTuple_Pack(1, &PyBaseObject_Type); if (bases == NULL) - return NULL; + goto error; nbases = 1; } else Py_INCREF(bases); - /* XXX From here until type is allocated, "return NULL" leaks bases! */ - /* Calculate best base, and check that all bases are type objects */ base = best_base(bases); if (base == NULL) { - Py_DECREF(bases); - return NULL; + goto error; } if (!PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)) { PyErr_Format(PyExc_TypeError, "type '%.100s' is not an acceptable base type", base->tp_name); - Py_DECREF(bases); - return NULL; + goto error; } + dict = PyDict_Copy(orig_dict); + if (dict == NULL) + goto error; + /* Check for a __slots__ sequence variable in dict, and count it */ - slots = PyDict_GetItemString(dict, "__slots__"); + slots = _PyDict_GetItemId(dict, &PyId___slots__); nslots = 0; add_dict = 0; add_weak = 0; @@ -2062,10 +2082,8 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) slots = PyTuple_Pack(1, slots); else slots = PySequence_Tuple(slots); - if (slots == NULL) { - Py_DECREF(bases); - return NULL; - } + if (slots == NULL) + goto error; assert(PyTuple_Check(slots)); /* Are slots allowed? */ @@ -2075,24 +2093,21 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) "nonempty __slots__ " "not supported for subtype of '%s'", base->tp_name); - bad_slots: - Py_DECREF(bases); - Py_DECREF(slots); - return NULL; + goto error; } /* Check for valid slot names and two special cases */ for (i = 0; i < nslots; i++) { PyObject *tmp = PyTuple_GET_ITEM(slots, i); if (!valid_identifier(tmp)) - goto bad_slots; + goto error; assert(PyUnicode_Check(tmp)); if (PyUnicode_CompareWithASCIIString(tmp, "__dict__") == 0) { if (!may_add_dict || add_dict) { PyErr_SetString(PyExc_TypeError, "__dict__ slot disallowed: " "we already got one"); - goto bad_slots; + goto error; } add_dict++; } @@ -2102,7 +2117,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) "__weakref__ slot disallowed: " "either we already got one, " "or __itemsize__ != 0"); - goto bad_slots; + goto error; } add_weak++; } @@ -2114,7 +2129,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) */ newslots = PyList_New(nslots - add_dict - add_weak); if (newslots == NULL) - goto bad_slots; + goto error; for (i = j = 0; i < nslots; i++) { tmp = PyTuple_GET_ITEM(slots, i); if ((add_dict && @@ -2125,25 +2140,29 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) tmp =_Py_Mangle(name, tmp); if (!tmp) { Py_DECREF(newslots); - goto bad_slots; + goto error; } PyList_SET_ITEM(newslots, j, tmp); + if (PyDict_GetItem(dict, tmp)) { + PyErr_Format(PyExc_ValueError, + "%R in __slots__ conflicts with class variable", + tmp); + Py_DECREF(newslots); + goto error; + } j++; } assert(j == nslots - add_dict - add_weak); nslots = j; - Py_DECREF(slots); + Py_CLEAR(slots); if (PyList_Sort(newslots) == -1) { - Py_DECREF(bases); Py_DECREF(newslots); - return NULL; + goto error; } slots = PyList_AsTuple(newslots); Py_DECREF(newslots); - if (slots == NULL) { - Py_DECREF(bases); - return NULL; - } + if (slots == NULL) + goto error; /* Secondary bases may provide weakrefs or dict */ if (nbases > 1 && @@ -2171,22 +2190,17 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) } } - /* XXX From here until type is safely allocated, - "return NULL" may leak slots! */ - /* Allocate the type object */ type = (PyTypeObject *)metatype->tp_alloc(metatype, nslots); - if (type == NULL) { - Py_XDECREF(slots); - Py_DECREF(bases); - return NULL; - } + if (type == NULL) + goto error; /* Keep name and slots alive in the extended type object */ et = (PyHeapTypeObject *)type; Py_INCREF(name); et->ht_name = name; et->ht_slots = slots; + slots = NULL; /* Initialize tp_flags */ type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE | @@ -2200,59 +2214,68 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; type->tp_name = _PyUnicode_AsString(name); - if (!type->tp_name) { - Py_DECREF(type); - return NULL; - } + if (!type->tp_name) + goto error; /* Set tp_base and tp_bases */ type->tp_bases = bases; + bases = NULL; Py_INCREF(base); type->tp_base = base; /* Initialize tp_dict from passed-in dict */ - type->tp_dict = dict = PyDict_Copy(dict); - if (dict == NULL) { - Py_DECREF(type); - return NULL; - } + Py_INCREF(dict); + type->tp_dict = dict; /* Set __module__ in the dict */ - if (PyDict_GetItemString(dict, "__module__") == NULL) { + if (_PyDict_GetItemId(dict, &PyId___module__) == NULL) { tmp = PyEval_GetGlobals(); if (tmp != NULL) { - tmp = PyDict_GetItemString(tmp, "__name__"); + tmp = _PyDict_GetItemId(tmp, &PyId___name__); if (tmp != NULL) { - if (PyDict_SetItemString(dict, "__module__", - tmp) < 0) - return NULL; + if (_PyDict_SetItemId(dict, &PyId___module__, + tmp) < 0) + goto error; } } } + /* Set ht_qualname to dict['__qualname__'] if available, else to + __name__. The __qualname__ accessor will look for ht_qualname. + */ + qualname = _PyDict_GetItemId(dict, &PyId___qualname__); + if (qualname != NULL) { + if (!PyUnicode_Check(qualname)) { + PyErr_Format(PyExc_TypeError, + "type __qualname__ must be a str, not %s", + Py_TYPE(qualname)->tp_name); + goto error; + } + } + et->ht_qualname = qualname ? qualname : et->ht_name; + Py_INCREF(et->ht_qualname); + if (qualname != NULL && PyDict_DelItem(dict, PyId___qualname__.object) < 0) + goto error; + /* Set tp_doc to a copy of dict['__doc__'], if the latter is there and is a string. The __doc__ accessor will first look for tp_doc; if that fails, it will still look into __dict__. */ { - PyObject *doc = PyDict_GetItemString(dict, "__doc__"); + PyObject *doc = _PyDict_GetItemId(dict, &PyId___doc__); if (doc != NULL && PyUnicode_Check(doc)) { Py_ssize_t len; char *doc_str; char *tp_doc; doc_str = _PyUnicode_AsString(doc); - if (doc_str == NULL) { - Py_DECREF(type); - return NULL; - } + if (doc_str == NULL) + goto error; /* Silently truncate the docstring if it contains null bytes. */ len = strlen(doc_str); tp_doc = (char *)PyObject_MALLOC(len + 1); - if (tp_doc == NULL) { - Py_DECREF(type); - return NULL; - } + if (tp_doc == NULL) + goto error; memcpy(tp_doc, doc_str, len + 1); type->tp_doc = tp_doc; } @@ -2260,28 +2283,25 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) /* Special-case __new__: if it's a plain function, make it a static function */ - tmp = PyDict_GetItemString(dict, "__new__"); + tmp = _PyDict_GetItemId(dict, &PyId___new__); if (tmp != NULL && PyFunction_Check(tmp)) { tmp = PyStaticMethod_New(tmp); - if (tmp == NULL) { - Py_DECREF(type); - return NULL; - } - PyDict_SetItemString(dict, "__new__", tmp); + if (tmp == NULL) + goto error; + if (_PyDict_SetItemId(dict, &PyId___new__, tmp) < 0) + goto error; Py_DECREF(tmp); } /* Add descriptors for custom slots from __slots__, or for __dict__ */ mp = PyHeapType_GET_MEMBERS(et); slotoffset = base->tp_basicsize; - if (slots != NULL) { + if (et->ht_slots != NULL) { for (i = 0; i < nslots; i++, mp++) { mp->name = _PyUnicode_AsString( - PyTuple_GET_ITEM(slots, i)); - if (mp->name == NULL) { - Py_DECREF(type); - return NULL; - } + PyTuple_GET_ITEM(et->ht_slots, i)); + if (mp->name == NULL) + goto error; mp->type = T_OBJECT_EX; mp->offset = slotoffset; @@ -2299,6 +2319,9 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) type->tp_dictoffset = slotoffset; slotoffset += sizeof(PyObject *); } + if (type->tp_dictoffset) { + et->ht_cached_keys = _PyDict_NewKeysForClass(); + } if (add_weak) { assert(!base->tp_itemsize); type->tp_weaklistoffset = slotoffset; @@ -2342,15 +2365,21 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) type->tp_free = PyObject_Del; /* Initialize the rest */ - if (PyType_Ready(type) < 0) { - Py_DECREF(type); - return NULL; - } + if (PyType_Ready(type) < 0) + goto error; /* Put the proper slots in place */ fixup_slot_dispatchers(type); + Py_DECREF(dict); return (PyObject *)type; + +error: + Py_XDECREF(dict); + Py_XDECREF(bases); + Py_XDECREF(slots); + Py_XDECREF(type); + return NULL; } static short slotoffsets[] = { @@ -2359,30 +2388,89 @@ static short slotoffsets[] = { }; PyObject * -PyType_FromSpec(PyType_Spec *spec) +PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { PyHeapTypeObject *res = (PyHeapTypeObject*)PyType_GenericAlloc(&PyType_Type, 0); + PyTypeObject *type, *base; + char *s; char *res_start = (char*)res; PyType_Slot *slot; + + /* Set the type name and qualname */ + s = strrchr(spec->name, '.'); + if (s == NULL) + s = (char*)spec->name; + else + s++; if (res == NULL) - return NULL; - res->ht_name = PyUnicode_FromString(spec->name); + return NULL; + type = &res->ht_type; + /* The flags must be initialized early, before the GC traverses us */ + type->tp_flags = spec->flags | Py_TPFLAGS_HEAPTYPE; + res->ht_name = PyUnicode_FromString(s); if (!res->ht_name) goto fail; - res->ht_type.tp_name = _PyUnicode_AsString(res->ht_name); - if (!res->ht_type.tp_name) + res->ht_qualname = res->ht_name; + Py_INCREF(res->ht_qualname); + type->tp_name = spec->name; + if (!type->tp_name) + goto fail; + + /* Adjust for empty tuple bases */ + if (!bases) { + base = &PyBaseObject_Type; + /* See whether Py_tp_base(s) was specified */ + for (slot = spec->slots; slot->slot; slot++) { + if (slot->slot == Py_tp_base) + base = slot->pfunc; + else if (slot->slot == Py_tp_bases) { + bases = slot->pfunc; + Py_INCREF(bases); + } + } + if (!bases) + bases = PyTuple_Pack(1, base); + if (!bases) + goto fail; + } + else + Py_INCREF(bases); + + /* Calculate best base, and check that all bases are type objects */ + base = best_base(bases); + if (base == NULL) { + goto fail; + } + if (!PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)) { + PyErr_Format(PyExc_TypeError, + "type '%.100s' is not an acceptable base type", + base->tp_name); goto fail; + } + + /* Initialize essential fields */ + type->tp_as_number = &res->as_number; + type->tp_as_sequence = &res->as_sequence; + type->tp_as_mapping = &res->as_mapping; + type->tp_as_buffer = &res->as_buffer; + /* Set tp_base and tp_bases */ + type->tp_bases = bases; + bases = NULL; + Py_INCREF(base); + type->tp_base = base; - res->ht_type.tp_basicsize = spec->basicsize; - res->ht_type.tp_itemsize = spec->itemsize; - res->ht_type.tp_flags = spec->flags | Py_TPFLAGS_HEAPTYPE; + type->tp_basicsize = spec->basicsize; + type->tp_itemsize = spec->itemsize; for (slot = spec->slots; slot->slot; slot++) { - if (slot->slot >= sizeof(slotoffsets)/sizeof(slotoffsets[0])) { + if (slot->slot >= Py_ARRAY_LENGTH(slotoffsets)) { PyErr_SetString(PyExc_RuntimeError, "invalid slot offset"); goto fail; } + if (slot->slot == Py_tp_base || slot->slot == Py_tp_bases) + /* Processed above */ + continue; *(void**)(res_start + slotoffsets[slot->slot]) = slot->pfunc; /* need to make a copy of the docstring slot, which usually @@ -2393,19 +2481,29 @@ PyType_FromSpec(PyType_Spec *spec) if (tp_doc == NULL) goto fail; memcpy(tp_doc, slot->pfunc, len); - res->ht_type.tp_doc = tp_doc; + type->tp_doc = tp_doc; } } - if (res->ht_type.tp_dealloc == NULL) { + if (type->tp_dictoffset) { + res->ht_cached_keys = _PyDict_NewKeysForClass(); + } + if (type->tp_dealloc == NULL) { /* It's a heap type, so needs the heap types' dealloc. subtype_dealloc will call the base type's tp_dealloc, if necessary. */ - res->ht_type.tp_dealloc = subtype_dealloc; + type->tp_dealloc = subtype_dealloc; } - if (PyType_Ready(&res->ht_type) < 0) + if (PyType_Ready(type) < 0) goto fail; + /* Set type.__module__ */ + s = strrchr(spec->name, '.'); + if (s != NULL) + _PyDict_SetItemId(type->tp_dict, &PyId___module__, + PyUnicode_FromStringAndSize( + spec->name, (Py_ssize_t)(s - spec->name))); + return (PyObject*)res; fail: @@ -2413,6 +2511,12 @@ PyType_FromSpec(PyType_Spec *spec) return NULL; } +PyObject * +PyType_FromSpec(PyType_Spec *spec) +{ + return PyType_FromSpecWithBases(spec, NULL); +} + /* Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception! */ @@ -2442,6 +2546,9 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) return NULL; res = NULL; + /* keep a strong reference to mro because type->tp_mro can be replaced + during PyDict_GetItem(dict, name) */ + Py_INCREF(mro); assert(PyTuple_Check(mro)); n = PyTuple_GET_SIZE(mro); for (i = 0; i < n; i++) { @@ -2453,6 +2560,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) if (res != NULL) break; } + Py_DECREF(mro); if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(type)) { h = MCACHE_HASH_METHOD(type, name); @@ -2465,6 +2573,16 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) return res; } +static PyObject * +_PyType_LookupId(PyTypeObject *type, struct _Py_Identifier *name) +{ + PyObject *oname; + oname = _PyUnicode_FromId(name); /* borrowed */ + if (oname == NULL) + return NULL; + return _PyType_Lookup(type, oname); +} + /* This is similar to PyObject_GenericGetAttr(), but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ static PyObject * @@ -2564,6 +2682,9 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) return update_slot(type, name); } +extern void +_PyDictKeys_DecRef(PyDictKeysObject *keys); + static void type_dealloc(PyTypeObject *type) { @@ -2585,7 +2706,10 @@ type_dealloc(PyTypeObject *type) */ PyObject_Free((char *)type->tp_doc); Py_XDECREF(et->ht_name); + Py_XDECREF(et->ht_qualname); Py_XDECREF(et->ht_slots); + if (et->ht_cached_keys) + _PyDictKeys_DecRef(et->ht_cached_keys); Py_TYPE(type)->tp_free((PyObject *)type); } @@ -2623,6 +2747,99 @@ type_prepare(PyObject *self, PyObject *args, PyObject *kwds) return PyDict_New(); } +/* + Merge the __dict__ of aclass into dict, and recursively also all + the __dict__s of aclass's base classes. The order of merging isn't + defined, as it's expected that only the final set of dict keys is + interesting. + Return 0 on success, -1 on error. +*/ + +static int +merge_class_dict(PyObject *dict, PyObject *aclass) +{ + PyObject *classdict; + PyObject *bases; + _Py_IDENTIFIER(__bases__); + + assert(PyDict_Check(dict)); + assert(aclass); + + /* Merge in the type's dict (if any). */ + classdict = _PyObject_GetAttrId(aclass, &PyId___dict__); + if (classdict == NULL) + PyErr_Clear(); + else { + int status = PyDict_Update(dict, classdict); + Py_DECREF(classdict); + if (status < 0) + return -1; + } + + /* Recursively merge in the base types' (if any) dicts. */ + bases = _PyObject_GetAttrId(aclass, &PyId___bases__); + if (bases == NULL) + PyErr_Clear(); + else { + /* We have no guarantee that bases is a real tuple */ + Py_ssize_t i, n; + n = PySequence_Size(bases); /* This better be right */ + if (n < 0) + PyErr_Clear(); + else { + for (i = 0; i < n; i++) { + int status; + PyObject *base = PySequence_GetItem(bases, i); + if (base == NULL) { + Py_DECREF(bases); + return -1; + } + status = merge_class_dict(dict, base); + Py_DECREF(base); + if (status < 0) { + Py_DECREF(bases); + return -1; + } + } + } + Py_DECREF(bases); + } + return 0; +} + +/* __dir__ for type objects: returns __dict__ and __bases__. + We deliberately don't suck up its __class__, as methods belonging to the + metaclass would probably be more confusing than helpful. +*/ +static PyObject * +type_dir(PyObject *self, PyObject *args) +{ + PyObject *result = NULL; + PyObject *dict = PyDict_New(); + + if (dict != NULL && merge_class_dict(dict, self) == 0) + result = PyDict_Keys(dict); + + Py_XDECREF(dict); + return result; +} + +static PyObject* +type_sizeof(PyObject *self, PyObject *args_unused) +{ + Py_ssize_t size; + PyTypeObject *type = (PyTypeObject*)self; + if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { + PyHeapTypeObject* et = (PyHeapTypeObject*)type; + size = sizeof(PyHeapTypeObject); + if (et->ht_cached_keys) + size += _PyDict_KeysSize(et->ht_cached_keys); + } + else + size = sizeof(PyTypeObject); + return PyLong_FromSsize_t(size); +} + static PyMethodDef type_methods[] = { {"mro", (PyCFunction)mro_external, METH_NOARGS, PyDoc_STR("mro() -> list\nreturn a type's method resolution order")}, @@ -2636,6 +2853,10 @@ static PyMethodDef type_methods[] = { PyDoc_STR("__instancecheck__() -> bool\ncheck if an object is an instance")}, {"__subclasscheck__", type___subclasscheck__, METH_O, PyDoc_STR("__subclasscheck__() -> bool\ncheck if a class is a subclass")}, + {"__dir__", type_dir, METH_NOARGS, + PyDoc_STR("__dir__() -> list\nspecialized __dir__ implementation for types")}, + {"__sizeof__", type_sizeof, METH_NOARGS, + "__sizeof__() -> int\nreturn memory consumption of the type object"}, {0} }; @@ -2648,7 +2869,12 @@ type_traverse(PyTypeObject *type, visitproc visit, void *arg) { /* Because of type_is_gc(), the collector only calls this for heaptypes. */ - assert(type->tp_flags & Py_TPFLAGS_HEAPTYPE); + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + char msg[200]; + sprintf(msg, "type_traverse() called for non-heap type '%.100s'", + type->tp_name); + Py_FatalError(msg); + } Py_VISIT(type->tp_dict); Py_VISIT(type->tp_cache); @@ -2667,6 +2893,7 @@ type_traverse(PyTypeObject *type, visitproc visit, void *arg) static int type_clear(PyTypeObject *type) { + PyDictKeysObject *cached_keys; /* Because of type_is_gc(), the collector only calls this for heaptypes. */ assert(type->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -2698,6 +2925,11 @@ type_clear(PyTypeObject *type) */ PyType_Modified(type); + cached_keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + if (cached_keys != NULL) { + ((PyHeapTypeObject *)type)->ht_cached_keys = NULL; + _PyDictKeys_DecRef(cached_keys); + } if (type->tp_dict) PyDict_Clear(type->tp_dict); Py_CLEAR(type->tp_mro); @@ -2813,22 +3045,11 @@ static int object_init(PyObject *self, PyObject *args, PyObject *kwds) { int err = 0; - if (excess_args(args, kwds)) { - PyTypeObject *type = Py_TYPE(self); - if (type->tp_init != object_init && - type->tp_new != object_new) - { - err = PyErr_WarnEx(PyExc_DeprecationWarning, - "object.__init__() takes no parameters", - 1); - } - else if (type->tp_init != object_init || - type->tp_new == object_new) - { - PyErr_SetString(PyExc_TypeError, - "object.__init__() takes no parameters"); - err = -1; - } + PyTypeObject *type = Py_TYPE(self); + if (excess_args(args, kwds) && + (type->tp_new == object_new || type->tp_init != object_init)) { + PyErr_SetString(PyExc_TypeError, "object.__init__() takes no parameters"); + err = -1; } return err; } @@ -2836,33 +3057,21 @@ object_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - int err = 0; - if (excess_args(args, kwds)) { - if (type->tp_new != object_new && - type->tp_init != object_init) - { - err = PyErr_WarnEx(PyExc_DeprecationWarning, - "object.__new__() takes no parameters", - 1); - } - else if (type->tp_new != object_new || - type->tp_init == object_init) - { - PyErr_SetString(PyExc_TypeError, - "object.__new__() takes no parameters"); - err = -1; - } - } - if (err < 0) + if (excess_args(args, kwds) && + (type->tp_init == object_init || type->tp_new != object_new)) { + PyErr_SetString(PyExc_TypeError, "object.__new__() takes no parameters"); return NULL; + } if (type->tp_flags & Py_TPFLAGS_IS_ABSTRACT) { - static PyObject *comma = NULL; PyObject *abstract_methods = NULL; PyObject *builtins; PyObject *sorted; PyObject *sorted_methods = NULL; PyObject *joined = NULL; + PyObject *comma; + _Py_static_string(comma_id, ", "); + _Py_IDENTIFIER(sorted); /* Compute ", ".join(sorted(type.__abstractmethods__)) into joined. */ @@ -2872,7 +3081,7 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) builtins = PyEval_GetBuiltins(); if (builtins == NULL) goto error; - sorted = PyDict_GetItemString(builtins, "sorted"); + sorted = _PyDict_GetItemId(builtins, &PyId_sorted); if (sorted == NULL) goto error; sorted_methods = PyObject_CallFunctionObjArgs(sorted, @@ -2880,13 +3089,10 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) NULL); if (sorted_methods == NULL) goto error; - if (comma == NULL) { - comma = PyUnicode_InternFromString(", "); - if (comma == NULL) - goto error; - } - joined = PyObject_CallMethod(comma, "join", - "O", sorted_methods); + comma = _PyUnicode_FromId(&comma_id); + if (comma == NULL) + goto error; + joined = PyUnicode_Join(comma, sorted_methods); if (joined == NULL) goto error; @@ -2924,7 +3130,7 @@ object_repr(PyObject *self) Py_DECREF(mod); mod = NULL; } - name = type_name(type, NULL); + name = type_qualname(type, NULL); if (name == NULL) { Py_XDECREF(mod); return NULL; @@ -3089,7 +3295,7 @@ object_set_class(PyObject *self, PyObject *value, void *closure) } if (!PyType_Check(value)) { PyErr_Format(PyExc_TypeError, - "__class__ must be set to new-style class, not '%s' object", + "__class__ must be set to a class, not '%s' object", Py_TYPE(value)->tp_name); return -1; } @@ -3130,14 +3336,19 @@ static PyObject * import_copyreg(void) { static PyObject *copyreg_str; + static PyObject *mod_copyreg = NULL; if (!copyreg_str) { copyreg_str = PyUnicode_InternFromString("copyreg"); if (copyreg_str == NULL) return NULL; } + if (!mod_copyreg) { + mod_copyreg = PyImport_Import(copyreg_str); + } - return PyImport_Import(copyreg_str); + Py_XINCREF(mod_copyreg); + return mod_copyreg; } static PyObject * @@ -3146,14 +3357,11 @@ slotnames(PyObject *cls) PyObject *clsdict; PyObject *copyreg; PyObject *slotnames; - - if (!PyType_Check(cls)) { - Py_INCREF(Py_None); - return Py_None; - } + _Py_IDENTIFIER(__slotnames__); + _Py_IDENTIFIER(_slotnames); clsdict = ((PyTypeObject *)cls)->tp_dict; - slotnames = PyDict_GetItemString(clsdict, "__slotnames__"); + slotnames = _PyDict_GetItemId(clsdict, &PyId___slotnames__); if (slotnames != NULL && PyList_Check(slotnames)) { Py_INCREF(slotnames); return slotnames; @@ -3163,7 +3371,7 @@ slotnames(PyObject *cls) if (copyreg == NULL) return NULL; - slotnames = PyObject_CallMethod(copyreg, "_slotnames", "O", cls); + slotnames = _PyObject_CallMethodId(copyreg, &PyId__slotnames, "O", cls); Py_DECREF(copyreg); if (slotnames != NULL && slotnames != Py_None && @@ -3187,12 +3395,13 @@ reduce_2(PyObject *obj) PyObject *slots = NULL, *listitems = NULL, *dictitems = NULL; PyObject *copyreg = NULL, *newobj = NULL, *res = NULL; Py_ssize_t i, n; + _Py_IDENTIFIER(__getnewargs__); + _Py_IDENTIFIER(__getstate__); + _Py_IDENTIFIER(__newobj__); - cls = PyObject_GetAttrString(obj, "__class__"); - if (cls == NULL) - return NULL; + cls = (PyObject *) Py_TYPE(obj); - getnewargs = PyObject_GetAttrString(obj, "__getnewargs__"); + getnewargs = _PyObject_GetAttrId(obj, &PyId___getnewargs__); if (getnewargs != NULL) { args = PyObject_CallObject(getnewargs, NULL); Py_DECREF(getnewargs); @@ -3210,7 +3419,7 @@ reduce_2(PyObject *obj) if (args == NULL) goto end; - getstate = PyObject_GetAttrString(obj, "__getstate__"); + getstate = _PyObject_GetAttrId(obj, &PyId___getstate__); if (getstate != NULL) { state = PyObject_CallObject(getstate, NULL); Py_DECREF(getstate); @@ -3218,17 +3427,18 @@ reduce_2(PyObject *obj) goto end; } else { + PyObject **dict; PyErr_Clear(); - state = PyObject_GetAttrString(obj, "__dict__"); - if (state == NULL) { - PyErr_Clear(); + dict = _PyObject_GetDictPtr(obj); + if (dict && *dict) + state = *dict; + else state = Py_None; - Py_INCREF(state); - } + Py_INCREF(state); names = slotnames(cls); if (names == NULL) goto end; - if (names != Py_None) { + if (names != Py_None && PyList_GET_SIZE(names) > 0) { assert(PyList_Check(names)); slots = PyDict_New(); if (slots == NULL) @@ -3275,7 +3485,8 @@ reduce_2(PyObject *obj) Py_INCREF(dictitems); } else { - PyObject *items = PyObject_CallMethod(obj, "items", ""); + _Py_IDENTIFIER(items); + PyObject *items = _PyObject_CallMethodId(obj, &PyId_items, ""); if (items == NULL) goto end; dictitems = PyObject_GetIter(items); @@ -3287,7 +3498,7 @@ reduce_2(PyObject *obj) copyreg = import_copyreg(); if (copyreg == NULL) goto end; - newobj = PyObject_GetAttrString(copyreg, "__newobj__"); + newobj = _PyObject_GetAttrId(copyreg, &PyId___newobj__); if (newobj == NULL) goto end; @@ -3295,8 +3506,8 @@ reduce_2(PyObject *obj) args2 = PyTuple_New(n+1); if (args2 == NULL) goto end; + Py_INCREF(cls); PyTuple_SET_ITEM(args2, 0, cls); - cls = NULL; for (i = 0; i < n; i++) { PyObject *v = PyTuple_GET_ITEM(args, i); Py_INCREF(v); @@ -3306,7 +3517,6 @@ reduce_2(PyObject *obj) res = PyTuple_Pack(5, newobj, args2, state, listitems, dictitems); end: - Py_XDECREF(cls); Py_XDECREF(args); Py_XDECREF(args2); Py_XDECREF(slots); @@ -3366,31 +3576,34 @@ object_reduce(PyObject *self, PyObject *args) static PyObject * object_reduce_ex(PyObject *self, PyObject *args) { + static PyObject *objreduce; PyObject *reduce, *res; int proto = 0; + _Py_IDENTIFIER(__reduce__); if (!PyArg_ParseTuple(args, "|i:__reduce_ex__", &proto)) return NULL; - reduce = PyObject_GetAttrString(self, "__reduce__"); + if (objreduce == NULL) { + objreduce = _PyDict_GetItemId(PyBaseObject_Type.tp_dict, + &PyId___reduce__); + if (objreduce == NULL) + return NULL; + } + + reduce = _PyObject_GetAttrId(self, &PyId___reduce__); if (reduce == NULL) PyErr_Clear(); else { - PyObject *cls, *clsreduce, *objreduce; + PyObject *cls, *clsreduce; int override; - cls = PyObject_GetAttrString(self, "__class__"); - if (cls == NULL) { - Py_DECREF(reduce); - return NULL; - } - clsreduce = PyObject_GetAttrString(cls, "__reduce__"); - Py_DECREF(cls); + + cls = (PyObject *) Py_TYPE(self); + clsreduce = _PyObject_GetAttrId(cls, &PyId___reduce__); if (clsreduce == NULL) { Py_DECREF(reduce); return NULL; } - objreduce = PyDict_GetItemString(PyBaseObject_Type.tp_dict, - "__reduce__"); override = (clsreduce != objreduce); Py_DECREF(clsreduce); if (override) { @@ -3408,8 +3621,7 @@ object_reduce_ex(PyObject *self, PyObject *args) static PyObject * object_subclasshook(PyObject *cls, PyObject *args) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } PyDoc_STRVAR(object_subclasshook_doc, @@ -3440,21 +3652,21 @@ object_format(PyObject *self, PyObject *args) self_as_str = PyObject_Str(self); if (self_as_str != NULL) { /* Issue 7994: If we're converting to a string, we - should reject format specifications */ - if (PyUnicode_GET_SIZE(format_spec) > 0) { - if (PyErr_WarnEx(PyExc_PendingDeprecationWarning, - "object.__format__ with a non-empty format " - "string is deprecated", 1) < 0) { - goto done; - } - /* Eventually this will become an error: - PyErr_Format(PyExc_TypeError, - "non-empty format string passed to object.__format__"); - goto done; - */ - } - - result = PyObject_Format(self_as_str, format_spec); + should reject format specifications */ + if (PyUnicode_GET_LENGTH(format_spec) > 0) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "object.__format__ with a non-empty format " + "string is deprecated", 1) < 0) { + goto done; + } + /* Eventually this will become an error: + PyErr_Format(PyExc_TypeError, + "non-empty format string passed to object.__format__"); + goto done; + */ + } + + result = PyObject_Format(self_as_str, format_spec); } done: @@ -3477,6 +3689,53 @@ object_sizeof(PyObject *self, PyObject *args) return PyLong_FromSsize_t(res); } +/* __dir__ for generic objects: returns __dict__, __class__, + and recursively up the __class__.__bases__ chain. +*/ +static PyObject * +object_dir(PyObject *self, PyObject *args) +{ + PyObject *result = NULL; + PyObject *dict = NULL; + PyObject *itsclass = NULL; + + /* Get __dict__ (which may or may not be a real dict...) */ + dict = _PyObject_GetAttrId(self, &PyId___dict__); + if (dict == NULL) { + PyErr_Clear(); + dict = PyDict_New(); + } + else if (!PyDict_Check(dict)) { + Py_DECREF(dict); + dict = PyDict_New(); + } + else { + /* Copy __dict__ to avoid mutating it. */ + PyObject *temp = PyDict_Copy(dict); + Py_DECREF(dict); + dict = temp; + } + + if (dict == NULL) + goto error; + + /* Merge in attrs reachable from its class. */ + itsclass = _PyObject_GetAttrId(self, &PyId___class__); + if (itsclass == NULL) + /* XXX(tomer): Perhaps fall back to obj->ob_type if no + __class__ exists? */ + PyErr_Clear(); + else if (merge_class_dict(dict, itsclass) != 0) + goto error; + + result = PyDict_Keys(dict); + /* fall through */ +error: + Py_XDECREF(itsclass); + Py_XDECREF(dict); + return result; +} + static PyMethodDef object_methods[] = { {"__reduce_ex__", object_reduce_ex, METH_VARARGS, PyDoc_STR("helper for pickle")}, @@ -3488,6 +3747,8 @@ static PyMethodDef object_methods[] = { PyDoc_STR("default object formatter")}, {"__sizeof__", object_sizeof, METH_NOARGS, PyDoc_STR("__sizeof__() -> int\nsize of object in memory, in bytes")}, + {"__dir__", object_dir, METH_NOARGS, + PyDoc_STR("__dir__() -> list\ndefault dir() implementation")}, {0} }; @@ -3557,7 +3818,7 @@ add_methods(PyTypeObject *type, PyMethodDef *meth) descr = PyDescr_NewClassMethod(type, meth); } else if (meth->ml_flags & METH_STATIC) { - PyObject *cfunc = PyCFunction_New(meth, NULL); + PyObject *cfunc = PyCFunction_New(meth, (PyObject*)type); if (cfunc == NULL) return -1; descr = PyStaticMethod_New(cfunc); @@ -3638,8 +3899,8 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) that the extension type's own factory function ensures). Heap types, of course, are under our control, so they do inherit tp_new; static extension types that specify some - other built-in type as the default are considered - new-style-aware so they also inherit object.__new__. */ + other built-in type as the default also + inherit object.__new__. */ if (base != &PyBaseObject_Type || (type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { if (type->tp_new == NULL) @@ -3678,23 +3939,17 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) type->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS; } -static char *hash_name_op[] = { - "__eq__", - "__hash__", - NULL -}; - static int overrides_hash(PyTypeObject *type) { - char **p; PyObject *dict = type->tp_dict; + _Py_IDENTIFIER(__eq__); assert(dict != NULL); - for (p = hash_name_op; *p; p++) { - if (PyDict_GetItemString(dict, *p) != NULL) - return 1; - } + if (_PyDict_GetItemId(dict, &PyId___eq__) != NULL) + return 1; + if (_PyDict_GetItemId(dict, &PyId___hash__) != NULL) + return 1; return 0; } @@ -3980,16 +4235,16 @@ PyType_Ready(PyTypeObject *type) /* if the type dictionary doesn't contain a __doc__, set it from the tp_doc slot. */ - if (PyDict_GetItemString(type->tp_dict, "__doc__") == NULL) { + if (_PyDict_GetItemId(type->tp_dict, &PyId___doc__) == NULL) { if (type->tp_doc != NULL) { PyObject *doc = PyUnicode_FromString(type->tp_doc); if (doc == NULL) goto error; - PyDict_SetItemString(type->tp_dict, "__doc__", doc); + _PyDict_SetItemId(type->tp_dict, &PyId___doc__, doc); Py_DECREF(doc); } else { - PyDict_SetItemString(type->tp_dict, - "__doc__", Py_None); + _PyDict_SetItemId(type->tp_dict, + &PyId___doc__, Py_None); } } @@ -4000,8 +4255,8 @@ PyType_Ready(PyTypeObject *type) This signals that __hash__ is not inherited. */ if (type->tp_hash == NULL) { - if (PyDict_GetItemString(type->tp_dict, "__hash__") == NULL) { - if (PyDict_SetItemString(type->tp_dict, "__hash__", Py_None) < 0) + if (_PyDict_GetItemId(type->tp_dict, &PyId___hash__) == NULL) { + if (_PyDict_SetItemId(type->tp_dict, &PyId___hash__, Py_None) < 0) goto error; type->tp_hash = PyObject_HashNotImplemented; } @@ -4598,7 +4853,7 @@ tp_new_wrapper(PyObject *self, PyObject *args, PyObject *kwds) object.__new__(dict). To do this, we check that the most derived base that's not a heap type is this type. */ staticbase = subtype; - while (staticbase && (staticbase->tp_flags & Py_TPFLAGS_HEAPTYPE)) + while (staticbase && (staticbase->tp_new == slot_tp_new)) staticbase = staticbase->tp_base; /* If staticbase is NULL now, it is a really weird type. In the spirit of backwards compatibility (?), just shut up. */ @@ -4631,12 +4886,12 @@ add_tp_new_wrapper(PyTypeObject *type) { PyObject *func; - if (PyDict_GetItemString(type->tp_dict, "__new__") != NULL) + if (_PyDict_GetItemId(type->tp_dict, &PyId___new__) != NULL) return 0; func = PyCFunction_New(tp_new_methoddef, (PyObject *)type); if (func == NULL) return -1; - if (PyDict_SetItemString(type->tp_dict, "__new__", func)) { + if (_PyDict_SetItemId(type->tp_dict, &PyId___new__, func)) { Py_DECREF(func); return -1; } @@ -4651,34 +4906,34 @@ add_tp_new_wrapper(PyTypeObject *type) static PyObject * \ FUNCNAME(PyObject *self) \ { \ - static PyObject *cache_str; \ - return call_method(self, OPSTR, &cache_str, "()"); \ + _Py_static_string(id, OPSTR); \ + return call_method(self, &id, "()"); \ } #define SLOT1(FUNCNAME, OPSTR, ARG1TYPE, ARGCODES) \ static PyObject * \ FUNCNAME(PyObject *self, ARG1TYPE arg1) \ { \ - static PyObject *cache_str; \ - return call_method(self, OPSTR, &cache_str, "(" ARGCODES ")", arg1); \ + _Py_static_string(id, OPSTR); \ + return call_method(self, &id, "(" ARGCODES ")", arg1); \ } /* Boolean helper for SLOT1BINFULL(). right.__class__ is a nontrivial subclass of left.__class__. */ static int -method_is_overloaded(PyObject *left, PyObject *right, char *name) +method_is_overloaded(PyObject *left, PyObject *right, struct _Py_Identifier *name) { PyObject *a, *b; int ok; - b = PyObject_GetAttrString((PyObject *)(Py_TYPE(right)), name); + b = _PyObject_GetAttrId((PyObject *)(Py_TYPE(right)), name); if (b == NULL) { PyErr_Clear(); /* If right doesn't have it, it's not overloaded */ return 0; } - a = PyObject_GetAttrString((PyObject *)(Py_TYPE(left)), name); + a = _PyObject_GetAttrId((PyObject *)(Py_TYPE(left)), name); if (a == NULL) { PyErr_Clear(); Py_DECREF(b); @@ -4702,7 +4957,8 @@ method_is_overloaded(PyObject *left, PyObject *right, char *name) static PyObject * \ FUNCNAME(PyObject *self, PyObject *other) \ { \ - static PyObject *cache_str, *rcache_str; \ + _Py_static_string(op_id, OPSTR); \ + _Py_static_string(rop_id, ROPSTR); \ int do_other = Py_TYPE(self) != Py_TYPE(other) && \ Py_TYPE(other)->tp_as_number != NULL && \ Py_TYPE(other)->tp_as_number->SLOTNAME == TESTFUNC; \ @@ -4711,27 +4967,23 @@ FUNCNAME(PyObject *self, PyObject *other) \ PyObject *r; \ if (do_other && \ PyType_IsSubtype(Py_TYPE(other), Py_TYPE(self)) && \ - method_is_overloaded(self, other, ROPSTR)) { \ - r = call_maybe( \ - other, ROPSTR, &rcache_str, "(O)", self); \ + method_is_overloaded(self, other, &rop_id)) { \ + r = call_maybe(other, &rop_id, "(O)", self); \ if (r != Py_NotImplemented) \ return r; \ Py_DECREF(r); \ do_other = 0; \ } \ - r = call_maybe( \ - self, OPSTR, &cache_str, "(O)", other); \ + r = call_maybe(self, &op_id, "(O)", other); \ if (r != Py_NotImplemented || \ Py_TYPE(other) == Py_TYPE(self)) \ return r; \ Py_DECREF(r); \ } \ if (do_other) { \ - return call_maybe( \ - other, ROPSTR, &rcache_str, "(O)", self); \ + return call_maybe(other, &rop_id, "(O)", self); \ } \ - Py_INCREF(Py_NotImplemented); \ - return Py_NotImplemented; \ + Py_RETURN_NOTIMPLEMENTED; \ } #define SLOT1BIN(FUNCNAME, SLOTNAME, OPSTR, ROPSTR) \ @@ -4741,16 +4993,15 @@ FUNCNAME(PyObject *self, PyObject *other) \ static PyObject * \ FUNCNAME(PyObject *self, ARG1TYPE arg1, ARG2TYPE arg2) \ { \ - static PyObject *cache_str; \ - return call_method(self, OPSTR, &cache_str, \ - "(" ARGCODES ")", arg1, arg2); \ + _Py_static_string(id, #OPSTR); \ + return call_method(self, &id, "(" ARGCODES ")", arg1, arg2); \ } static Py_ssize_t slot_sq_length(PyObject *self) { - static PyObject *len_str; - PyObject *res = call_method(self, "__len__", &len_str, "()"); + _Py_IDENTIFIER(__len__); + PyObject *res = call_method(self, &PyId___len__, "()"); Py_ssize_t len; if (res == NULL) @@ -4771,16 +5022,10 @@ slot_sq_length(PyObject *self) static PyObject * slot_sq_item(PyObject *self, Py_ssize_t i) { - static PyObject *getitem_str; PyObject *func, *args = NULL, *ival = NULL, *retval = NULL; descrgetfunc f; - if (getitem_str == NULL) { - getitem_str = PyUnicode_InternFromString("__getitem__"); - if (getitem_str == NULL) - return NULL; - } - func = _PyType_Lookup(Py_TYPE(self), getitem_str); + func = _PyType_LookupId(Py_TYPE(self), &PyId___getitem__); if (func != NULL) { if ((f = Py_TYPE(func)->tp_descr_get) == NULL) Py_INCREF(func); @@ -4803,6 +5048,7 @@ slot_sq_item(PyObject *self, Py_ssize_t i) } } else { + PyObject *getitem_str = _PyUnicode_FromId(&PyId___getitem__); PyErr_SetObject(PyExc_AttributeError, getitem_str); } Py_XDECREF(args); @@ -4815,14 +5061,13 @@ static int slot_sq_ass_item(PyObject *self, Py_ssize_t index, PyObject *value) { PyObject *res; - static PyObject *delitem_str, *setitem_str; + _Py_IDENTIFIER(__delitem__); + _Py_IDENTIFIER(__setitem__); if (value == NULL) - res = call_method(self, "__delitem__", &delitem_str, - "(n)", index); + res = call_method(self, &PyId___delitem__, "(n)", index); else - res = call_method(self, "__setitem__", &setitem_str, - "(nO)", index, value); + res = call_method(self, &PyId___setitem__, "(nO)", index, value); if (res == NULL) return -1; Py_DECREF(res); @@ -4834,10 +5079,9 @@ slot_sq_contains(PyObject *self, PyObject *value) { PyObject *func, *res, *args; int result = -1; + _Py_IDENTIFIER(__contains__); - static PyObject *contains_str; - - func = lookup_maybe(self, "__contains__", &contains_str); + func = lookup_maybe(self, &PyId___contains__); if (func != NULL) { args = PyTuple_Pack(1, value); if (args == NULL) @@ -4868,14 +5112,14 @@ static int slot_mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value) { PyObject *res; - static PyObject *delitem_str, *setitem_str; + _Py_IDENTIFIER(__delitem__); + _Py_IDENTIFIER(__setitem__); if (value == NULL) - res = call_method(self, "__delitem__", &delitem_str, - "(O)", key); + res = call_method(self, &PyId___delitem__, "(O)", key); else - res = call_method(self, "__setitem__", &setitem_str, - "(OO)", key, value); + res = call_method(self, &PyId___setitem__, "(OO)", key, value); + if (res == NULL) return -1; Py_DECREF(res); @@ -4896,7 +5140,7 @@ SLOT1BINFULL(slot_nb_power_binary, slot_nb_power, static PyObject * slot_nb_power(PyObject *self, PyObject *other, PyObject *modulus) { - static PyObject *pow_str; + _Py_IDENTIFIER(__pow__); if (modulus == Py_None) return slot_nb_power_binary(self, other); @@ -4905,11 +5149,9 @@ slot_nb_power(PyObject *self, PyObject *other, PyObject *modulus) slot_nb_power, so check before calling self.__pow__. */ if (Py_TYPE(self)->tp_as_number != NULL && Py_TYPE(self)->tp_as_number->nb_power == slot_nb_power) { - return call_method(self, "__pow__", &pow_str, - "(OO)", other, modulus); + return call_method(self, &PyId___pow__, "(OO)", other, modulus); } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } SLOT0(slot_nb_negative, "__neg__") @@ -4920,15 +5162,16 @@ static int slot_nb_bool(PyObject *self) { PyObject *func, *args; - static PyObject *bool_str, *len_str; int result = -1; int using_len = 0; + _Py_IDENTIFIER(__len__); + _Py_IDENTIFIER(__bool__); - func = lookup_maybe(self, "__bool__", &bool_str); + func = lookup_maybe(self, &PyId___bool__); if (func == NULL) { if (PyErr_Occurred()) return -1; - func = lookup_maybe(self, "__len__", &len_str); + func = lookup_maybe(self, &PyId___len__); if (func == NULL) return PyErr_Occurred() ? -1 : 1; using_len = 1; @@ -4963,8 +5206,8 @@ slot_nb_bool(PyObject *self) static PyObject * slot_nb_index(PyObject *self) { - static PyObject *index_str; - return call_method(self, "__index__", &index_str, "()"); + _Py_IDENTIFIER(__index__); + return call_method(self, &PyId___index__, "()"); } @@ -4985,8 +5228,8 @@ SLOT1(slot_nb_inplace_remainder, "__imod__", PyObject *, "O") static PyObject * slot_nb_inplace_power(PyObject *self, PyObject * arg1, PyObject *arg2) { - static PyObject *cache_str; - return call_method(self, "__ipow__", &cache_str, "(" "O" ")", arg1); + _Py_IDENTIFIER(__ipow__); + return call_method(self, &PyId___ipow__, "(" "O" ")", arg1); } SLOT1(slot_nb_inplace_lshift, "__ilshift__", PyObject *, "O") SLOT1(slot_nb_inplace_rshift, "__irshift__", PyObject *, "O") @@ -5003,9 +5246,9 @@ static PyObject * slot_tp_repr(PyObject *self) { PyObject *func, *res; - static PyObject *repr_str; + _Py_IDENTIFIER(__repr__); - func = lookup_method(self, "__repr__", &repr_str); + func = lookup_method(self, &PyId___repr__); if (func != NULL) { res = PyEval_CallObject(func, NULL); Py_DECREF(func); @@ -5020,23 +5263,30 @@ static PyObject * slot_tp_str(PyObject *self) { PyObject *func, *res; - static PyObject *str_str; + _Py_IDENTIFIER(__str__); - func = lookup_method(self, "__str__", &str_str); + func = lookup_method(self, &PyId___str__); if (func != NULL) { res = PyEval_CallObject(func, NULL); Py_DECREF(func); return res; } else { - PyObject *ress; + /* PyObject *ress; */ PyErr_Clear(); res = slot_tp_repr(self); if (!res) return NULL; - ress = _PyUnicode_AsDefaultEncodedString(res, NULL); + /* XXX this is non-sensical. Why should we return + a bytes object from __str__. Is this code even + used? - mvl */ + assert(0); + return res; + /* + ress = _PyUnicode_AsDefaultEncodedString(res); Py_DECREF(res); return ress; + */ } } @@ -5044,10 +5294,9 @@ static Py_hash_t slot_tp_hash(PyObject *self) { PyObject *func, *res; - static PyObject *hash_str; Py_ssize_t h; - func = lookup_method(self, "__hash__", &hash_str); + func = lookup_method(self, &PyId___hash__); if (func == Py_None) { Py_DECREF(func); @@ -5091,8 +5340,8 @@ slot_tp_hash(PyObject *self) static PyObject * slot_tp_call(PyObject *self, PyObject *args, PyObject *kwds) { - static PyObject *call_str; - PyObject *meth = lookup_method(self, "__call__", &call_str); + _Py_IDENTIFIER(__call__); + PyObject *meth = lookup_method(self, &PyId___call__); PyObject *res; if (meth == NULL) @@ -5118,9 +5367,7 @@ slot_tp_call(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * slot_tp_getattro(PyObject *self, PyObject *name) { - static PyObject *getattribute_str = NULL; - return call_method(self, "__getattribute__", &getattribute_str, - "(O)", name); + return call_method(self, &PyId___getattribute__, "(O)", name); } static PyObject * @@ -5146,26 +5393,14 @@ slot_tp_getattr_hook(PyObject *self, PyObject *name) { PyTypeObject *tp = Py_TYPE(self); PyObject *getattr, *getattribute, *res; - static PyObject *getattribute_str = NULL; - static PyObject *getattr_str = NULL; + _Py_IDENTIFIER(__getattr__); - if (getattr_str == NULL) { - getattr_str = PyUnicode_InternFromString("__getattr__"); - if (getattr_str == NULL) - return NULL; - } - if (getattribute_str == NULL) { - getattribute_str = - PyUnicode_InternFromString("__getattribute__"); - if (getattribute_str == NULL) - return NULL; - } /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when the attribute is present. So we use _PyType_Lookup and create the method only when needed, with call_attribute. */ - getattr = _PyType_Lookup(tp, getattr_str); + getattr = _PyType_LookupId(tp, &PyId___getattr__); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ tp->tp_getattro = slot_tp_getattro; @@ -5177,7 +5412,7 @@ slot_tp_getattr_hook(PyObject *self, PyObject *name) __getattr__, even when self has the default __getattribute__ method. So we use _PyType_Lookup and create the method only when needed, with call_attribute. */ - getattribute = _PyType_Lookup(tp, getattribute_str); + getattribute = _PyType_LookupId(tp, &PyId___getattribute__); if (getattribute == NULL || (Py_TYPE(getattribute) == &PyWrapperDescr_Type && ((PyWrapperDescrObject *)getattribute)->d_wrapped == @@ -5200,40 +5435,37 @@ static int slot_tp_setattro(PyObject *self, PyObject *name, PyObject *value) { PyObject *res; - static PyObject *delattr_str, *setattr_str; + _Py_IDENTIFIER(__delattr__); + _Py_IDENTIFIER(__setattr__); if (value == NULL) - res = call_method(self, "__delattr__", &delattr_str, - "(O)", name); + res = call_method(self, &PyId___delattr__, "(O)", name); else - res = call_method(self, "__setattr__", &setattr_str, - "(OO)", name, value); + res = call_method(self, &PyId___setattr__, "(OO)", name, value); if (res == NULL) return -1; Py_DECREF(res); return 0; } -static char *name_op[] = { - "__lt__", - "__le__", - "__eq__", - "__ne__", - "__gt__", - "__ge__", +static _Py_Identifier name_op[] = { + {0, "__lt__", 0}, + {0, "__le__", 0}, + {0, "__eq__", 0}, + {0, "__ne__", 0}, + {0, "__gt__", 0}, + {0, "__ge__", 0} }; static PyObject * slot_tp_richcompare(PyObject *self, PyObject *other, int op) { PyObject *func, *args, *res; - static PyObject *op_str[6]; - func = lookup_method(self, name_op[op], &op_str[op]); + func = lookup_method(self, &name_op[op]); if (func == NULL) { PyErr_Clear(); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } args = PyTuple_Pack(1, other); if (args == NULL) @@ -5250,9 +5482,9 @@ static PyObject * slot_tp_iter(PyObject *self) { PyObject *func, *res; - static PyObject *iter_str, *getitem_str; + _Py_IDENTIFIER(__iter__); - func = lookup_method(self, "__iter__", &iter_str); + func = lookup_method(self, &PyId___iter__); if (func != NULL) { PyObject *args; args = res = PyTuple_New(0); @@ -5264,7 +5496,7 @@ slot_tp_iter(PyObject *self) return res; } PyErr_Clear(); - func = lookup_method(self, "__getitem__", &getitem_str); + func = lookup_method(self, &PyId___getitem__); if (func == NULL) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not iterable", @@ -5278,8 +5510,8 @@ slot_tp_iter(PyObject *self) static PyObject * slot_tp_iternext(PyObject *self) { - static PyObject *next_str; - return call_method(self, "__next__", &next_str, "()"); + _Py_IDENTIFIER(__next__); + return call_method(self, &PyId___next__, "()"); } static PyObject * @@ -5287,14 +5519,9 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) { PyTypeObject *tp = Py_TYPE(self); PyObject *get; - static PyObject *get_str = NULL; + _Py_IDENTIFIER(__get__); - if (get_str == NULL) { - get_str = PyUnicode_InternFromString("__get__"); - if (get_str == NULL) - return NULL; - } - get = _PyType_Lookup(tp, get_str); + get = _PyType_LookupId(tp, &PyId___get__); if (get == NULL) { /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -5313,14 +5540,13 @@ static int slot_tp_descr_set(PyObject *self, PyObject *target, PyObject *value) { PyObject *res; - static PyObject *del_str, *set_str; + _Py_IDENTIFIER(__delete__); + _Py_IDENTIFIER(__set__); if (value == NULL) - res = call_method(self, "__delete__", &del_str, - "(O)", target); + res = call_method(self, &PyId___delete__, "(O)", target); else - res = call_method(self, "__set__", &set_str, - "(OO)", target, value); + res = call_method(self, &PyId___set__, "(OO)", target, value); if (res == NULL) return -1; Py_DECREF(res); @@ -5330,8 +5556,8 @@ slot_tp_descr_set(PyObject *self, PyObject *target, PyObject *value) static int slot_tp_init(PyObject *self, PyObject *args, PyObject *kwds) { - static PyObject *init_str; - PyObject *meth = lookup_method(self, "__init__", &init_str); + _Py_IDENTIFIER(__init__); + PyObject *meth = lookup_method(self, &PyId___init__); PyObject *res; if (meth == NULL) @@ -5354,17 +5580,12 @@ slot_tp_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - static PyObject *new_str; PyObject *func; PyObject *newargs, *x; Py_ssize_t i, n; + _Py_IDENTIFIER(__new__); - if (new_str == NULL) { - new_str = PyUnicode_InternFromString("__new__"); - if (new_str == NULL) - return NULL; - } - func = PyObject_GetAttr((PyObject *)type, new_str); + func = _PyObject_GetAttrId((PyObject *)type, &PyId___new__); if (func == NULL) return NULL; assert(PyTuple_Check(args)); @@ -5388,7 +5609,7 @@ slot_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static void slot_tp_del(PyObject *self) { - static PyObject *del_str = NULL; + _Py_IDENTIFIER(__del__); PyObject *del, *res; PyObject *error_type, *error_value, *error_traceback; @@ -5400,7 +5621,7 @@ slot_tp_del(PyObject *self) PyErr_Fetch(&error_type, &error_value, &error_traceback); /* Execute __del__ method, if any. */ - del = lookup_maybe(self, "__del__", &del_str); + del = lookup_maybe(self, &PyId___del__); if (del != NULL) { res = PyEval_CallObject(del, NULL); if (res == NULL) @@ -6115,7 +6336,7 @@ super_getattro(PyObject *self, PyObject *name) /* We want __class__ to return the class of the super object (i.e. super, or a subclass), not the class of su->obj. */ skip = (PyUnicode_Check(name) && - PyUnicode_GET_SIZE(name) == 9 && + PyUnicode_GET_LENGTH(name) == 9 && PyUnicode_CompareWithASCIIString(name, "__class__") == 0); } @@ -6140,6 +6361,9 @@ super_getattro(PyObject *self, PyObject *name) } i++; res = NULL; + /* keep a strong reference to mro because starttype->tp_mro can be + replaced during PyDict_GetItem(dict, name) */ + Py_INCREF(mro); for (; i < n; i++) { tmp = PyTuple_GET_ITEM(mro, i); if (PyType_Check(tmp)) @@ -6164,9 +6388,11 @@ super_getattro(PyObject *self, PyObject *name) Py_DECREF(res); res = tmp; } + Py_DECREF(mro); return res; } } + Py_DECREF(mro); } return PyObject_GenericGetAttr(self, name); } @@ -6176,7 +6402,7 @@ supercheck(PyTypeObject *type, PyObject *obj) { /* Check that a super() call makes sense. Return a type object. - obj can be a new-style class, or an instance of one: + obj can be a class, or an instance of one: - If it is a class, it must be a subclass of 'type'. This case is used for class methods; the return value is obj. @@ -6202,17 +6428,9 @@ supercheck(PyTypeObject *type, PyObject *obj) } else { /* Try the slow way */ - static PyObject *class_str = NULL; PyObject *class_attr; - if (class_str == NULL) { - class_str = PyUnicode_FromString("__class__"); - if (class_str == NULL) - return NULL; - } - - class_attr = PyObject_GetAttr(obj, class_str); - + class_attr = _PyObject_GetAttrId(obj, &PyId___class__); if (class_attr != NULL && PyType_Check(class_attr) && (PyTypeObject *)class_attr != Py_TYPE(obj)) @@ -6289,18 +6507,18 @@ super_init(PyObject *self, PyObject *args, PyObject *kwds) PyCodeObject *co = f->f_code; Py_ssize_t i, n; if (co == NULL) { - PyErr_SetString(PyExc_SystemError, + PyErr_SetString(PyExc_RuntimeError, "super(): no code object"); return -1; } if (co->co_argcount == 0) { - PyErr_SetString(PyExc_SystemError, + PyErr_SetString(PyExc_RuntimeError, "super(): no arguments"); return -1; } obj = f->f_localsplus[0]; if (obj == NULL) { - PyErr_SetString(PyExc_SystemError, + PyErr_SetString(PyExc_RuntimeError, "super(): arg[0] deleted"); return -1; } @@ -6319,18 +6537,18 @@ super_init(PyObject *self, PyObject *args, PyObject *kwds) PyTuple_GET_SIZE(co->co_cellvars) + i; PyObject *cell = f->f_localsplus[index]; if (cell == NULL || !PyCell_Check(cell)) { - PyErr_SetString(PyExc_SystemError, + PyErr_SetString(PyExc_RuntimeError, "super(): bad __class__ cell"); return -1; } type = (PyTypeObject *) PyCell_GET(cell); if (type == NULL) { - PyErr_SetString(PyExc_SystemError, + PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); return -1; } if (!PyType_Check(type)) { - PyErr_Format(PyExc_SystemError, + PyErr_Format(PyExc_RuntimeError, "super(): __class__ is not a type (%s)", Py_TYPE(type)->tp_name); return -1; @@ -6339,7 +6557,7 @@ super_init(PyObject *self, PyObject *args, PyObject *kwds) } } if (type == NULL) { - PyErr_SetString(PyExc_SystemError, + PyErr_SetString(PyExc_RuntimeError, "super(): __class__ cell not found"); return -1; } diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index 5186dcf..caa1e03 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -1,4 +1,4 @@ -/* Generated by typeslots.py $Revision$ */ +/* Generated by typeslots.py */ 0, 0, offsetof(PyHeapTypeObject, as_mapping.mp_ass_subscript), diff --git a/Objects/typeslots.py b/Objects/typeslots.py index 2e00c80..b24c7f4 100644 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -3,7 +3,7 @@ import sys, re -print("/* Generated by typeslots.py $Revision$ */") +print("/* Generated by typeslots.py */") res = {} for line in sys.stdin: m = re.match("#define Py_([a-z_]+) ([0-9]+)", line) diff --git a/Objects/unicodectype.c b/Objects/unicodectype.c index 9f6ac89..a572c12 100644 --- a/Objects/unicodectype.c +++ b/Objects/unicodectype.c @@ -21,13 +21,20 @@ #define XID_START_MASK 0x100 #define XID_CONTINUE_MASK 0x200 #define PRINTABLE_MASK 0x400 -#define NODELTA_MASK 0x800 -#define NUMERIC_MASK 0x1000 +#define NUMERIC_MASK 0x800 +#define CASE_IGNORABLE_MASK 0x1000 +#define CASED_MASK 0x2000 +#define EXTENDED_CASE_MASK 0x4000 typedef struct { - const Py_UCS4 upper; - const Py_UCS4 lower; - const Py_UCS4 title; + /* + These are either deltas to the character or offsets in + _PyUnicode_ExtendedCase. + */ + const int upper; + const int lower; + const int title; + /* Note if more flag space is needed, decimal and digit could be unified. */ const unsigned char decimal; const unsigned char digit; const unsigned short flags; @@ -57,15 +64,10 @@ gettyperecord(Py_UCS4 code) Py_UCS4 _PyUnicode_ToTitlecase(register Py_UCS4 ch) { const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); - int delta = ctype->title; - if (ctype->flags & NODELTA_MASK) - return delta; - - if (delta >= 32768) - delta -= 65536; - - return ch + delta; + if (ctype->flags & EXTENDED_CASE_MASK) + return _PyUnicode_ExtendedCase[ctype->title & 0xFFFF]; + return ch + ctype->title; } /* Returns 1 for Unicode characters having the category 'Lt', 0 @@ -188,12 +190,10 @@ int _PyUnicode_IsUppercase(Py_UCS4 ch) Py_UCS4 _PyUnicode_ToUppercase(Py_UCS4 ch) { const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); - int delta = ctype->upper; - if (ctype->flags & NODELTA_MASK) - return delta; - if (delta >= 32768) - delta -= 65536; - return ch + delta; + + if (ctype->flags & EXTENDED_CASE_MASK) + return _PyUnicode_ExtendedCase[ctype->upper & 0xFFFF]; + return ch + ctype->upper; } /* Returns the lowercase Unicode characters corresponding to ch or just @@ -202,12 +202,87 @@ Py_UCS4 _PyUnicode_ToUppercase(Py_UCS4 ch) Py_UCS4 _PyUnicode_ToLowercase(Py_UCS4 ch) { const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); - int delta = ctype->lower; - if (ctype->flags & NODELTA_MASK) - return delta; - if (delta >= 32768) - delta -= 65536; - return ch + delta; + + if (ctype->flags & EXTENDED_CASE_MASK) + return _PyUnicode_ExtendedCase[ctype->lower & 0xFFFF]; + return ch + ctype->lower; +} + +int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res) +{ + const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); + + if (ctype->flags & EXTENDED_CASE_MASK) { + int index = ctype->lower & 0xFFFF; + int n = ctype->lower >> 24; + int i; + for (i = 0; i < n; i++) + res[i] = _PyUnicode_ExtendedCase[index + i]; + return n; + } + res[0] = ch + ctype->lower; + return 1; +} + +int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res) +{ + const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); + + if (ctype->flags & EXTENDED_CASE_MASK) { + int index = ctype->title & 0xFFFF; + int n = ctype->title >> 24; + int i; + for (i = 0; i < n; i++) + res[i] = _PyUnicode_ExtendedCase[index + i]; + return n; + } + res[0] = ch + ctype->title; + return 1; +} + +int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res) +{ + const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); + + if (ctype->flags & EXTENDED_CASE_MASK) { + int index = ctype->upper & 0xFFFF; + int n = ctype->upper >> 24; + int i; + for (i = 0; i < n; i++) + res[i] = _PyUnicode_ExtendedCase[index + i]; + return n; + } + res[0] = ch + ctype->upper; + return 1; +} + +int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res) +{ + const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); + + if (ctype->flags & EXTENDED_CASE_MASK && (ctype->lower >> 20) & 7) { + int index = (ctype->lower & 0xFFFF) + (ctype->lower >> 24); + int n = (ctype->lower >> 20) & 7; + int i; + for (i = 0; i < n; i++) + res[i] = _PyUnicode_ExtendedCase[index + i]; + return n; + } + return _PyUnicode_ToLowerFull(ch, res); +} + +int _PyUnicode_IsCased(Py_UCS4 ch) +{ + const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); + + return (ctype->flags & CASED_MASK) != 0; +} + +int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch) +{ + const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); + + return (ctype->flags & CASE_IGNORABLE_MASK) != 0; } /* Returns 1 for Unicode characters having the category 'Ll', 'Lu', 'Lt', diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 35b424e..f61f9d0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1,8 +1,7 @@ /* Unicode implementation based on original code by Fredrik Lundh, -modified by Marc-Andre Lemburg <mal@lemburg.com> according to the -Unicode Integration Proposal (see file Misc/unicode.txt). +modified by Marc-Andre Lemburg <mal@lemburg.com>. Major speed upgrades to the method implementations at the Reykjavik NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke. @@ -42,34 +41,12 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #define PY_SSIZE_T_CLEAN #include "Python.h" #include "ucnhash.h" +#include "bytes_methods.h" #ifdef MS_WINDOWS #include <windows.h> #endif -/* Limit for the Unicode object free list */ - -#define PyUnicode_MAXFREELIST 1024 - -/* Limit for the Unicode object free list stay alive optimization. - - The implementation will keep allocated Unicode memory intact for - all objects on the free list having a size less than this - limit. This reduces malloc() overhead for small Unicode objects. - - At worst this will result in PyUnicode_MAXFREELIST * - (sizeof(PyUnicodeObject) + KEEPALIVE_SIZE_LIMIT + - malloc()-overhead) bytes of unused garbage. - - Setting the limit to 0 effectively turns the feature off. - - Note: This is an experimental feature ! If you get core dumps when - using Unicode objects, turn this feature off. - -*/ - -#define KEEPALIVE_SIZE_LIMIT 9 - /* Endianness switches; defaults to little endian */ #ifdef WORDS_BIGENDIAN @@ -90,6 +67,110 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. extern "C" { #endif +/* Maximum code point of Unicode 6.0: 0x10ffff (1,114,111) */ +#define MAX_UNICODE 0x10ffff + +#ifdef Py_DEBUG +# define _PyUnicode_CHECK(op) _PyUnicode_CheckConsistency(op, 0) +#else +# define _PyUnicode_CHECK(op) PyUnicode_Check(op) +#endif + +#define _PyUnicode_UTF8(op) \ + (((PyCompactUnicodeObject*)(op))->utf8) +#define PyUnicode_UTF8(op) \ + (assert(_PyUnicode_CHECK(op)), \ + assert(PyUnicode_IS_READY(op)), \ + PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((char*)((PyASCIIObject*)(op) + 1)) : \ + _PyUnicode_UTF8(op)) +#define _PyUnicode_UTF8_LENGTH(op) \ + (((PyCompactUnicodeObject*)(op))->utf8_length) +#define PyUnicode_UTF8_LENGTH(op) \ + (assert(_PyUnicode_CHECK(op)), \ + assert(PyUnicode_IS_READY(op)), \ + PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((PyASCIIObject*)(op))->length : \ + _PyUnicode_UTF8_LENGTH(op)) +#define _PyUnicode_WSTR(op) \ + (((PyASCIIObject*)(op))->wstr) +#define _PyUnicode_WSTR_LENGTH(op) \ + (((PyCompactUnicodeObject*)(op))->wstr_length) +#define _PyUnicode_LENGTH(op) \ + (((PyASCIIObject *)(op))->length) +#define _PyUnicode_STATE(op) \ + (((PyASCIIObject *)(op))->state) +#define _PyUnicode_HASH(op) \ + (((PyASCIIObject *)(op))->hash) +#define _PyUnicode_KIND(op) \ + (assert(_PyUnicode_CHECK(op)), \ + ((PyASCIIObject *)(op))->state.kind) +#define _PyUnicode_GET_LENGTH(op) \ + (assert(_PyUnicode_CHECK(op)), \ + ((PyASCIIObject *)(op))->length) +#define _PyUnicode_DATA_ANY(op) \ + (((PyUnicodeObject*)(op))->data.any) + +/* Optimized version of Py_MAX() to compute the maximum character: + use it when your are computing the second argument of PyUnicode_New() */ +#define MAX_MAXCHAR(maxchar1, maxchar2) \ + ((maxchar1) | (maxchar2)) + +#undef PyUnicode_READY +#define PyUnicode_READY(op) \ + (assert(_PyUnicode_CHECK(op)), \ + (PyUnicode_IS_READY(op) ? \ + 0 : \ + _PyUnicode_Ready(op))) + +#define _PyUnicode_SHARE_UTF8(op) \ + (assert(_PyUnicode_CHECK(op)), \ + assert(!PyUnicode_IS_COMPACT_ASCII(op)), \ + (_PyUnicode_UTF8(op) == PyUnicode_DATA(op))) +#define _PyUnicode_SHARE_WSTR(op) \ + (assert(_PyUnicode_CHECK(op)), \ + (_PyUnicode_WSTR(unicode) == PyUnicode_DATA(op))) + +/* true if the Unicode object has an allocated UTF-8 memory block + (not shared with other data) */ +#define _PyUnicode_HAS_UTF8_MEMORY(op) \ + (assert(_PyUnicode_CHECK(op)), \ + (!PyUnicode_IS_COMPACT_ASCII(op) \ + && _PyUnicode_UTF8(op) \ + && _PyUnicode_UTF8(op) != PyUnicode_DATA(op))) + +/* true if the Unicode object has an allocated wstr memory block + (not shared with other data) */ +#define _PyUnicode_HAS_WSTR_MEMORY(op) \ + (assert(_PyUnicode_CHECK(op)), \ + (_PyUnicode_WSTR(op) && \ + (!PyUnicode_IS_READY(op) || \ + _PyUnicode_WSTR(op) != PyUnicode_DATA(op)))) + +/* Generic helper macro to convert characters of different types. + from_type and to_type have to be valid type names, begin and end + are pointers to the source characters which should be of type + "from_type *". to is a pointer of type "to_type *" and points to the + buffer where the result characters are written to. */ +#define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ + do { \ + to_type *_to = (to_type *) to; \ + const from_type *_iter = (begin); \ + const from_type *_end = (end); \ + Py_ssize_t n = (_end) - (_iter); \ + const from_type *_unrolled_end = \ + _iter + _Py_SIZE_ROUND_DOWN(n, 4); \ + while (_iter < (_unrolled_end)) { \ + _to[0] = (to_type) _iter[0]; \ + _to[1] = (to_type) _iter[1]; \ + _to[2] = (to_type) _iter[2]; \ + _to[3] = (to_type) _iter[3]; \ + _iter += 4; _to += 4; \ + } \ + while (_iter < (_end)) \ + *_to++ = (to_type) *_iter++; \ + } while (0) + /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -100,16 +181,15 @@ extern "C" { */ static PyObject *interned; -/* Free list for Unicode objects */ -static PyUnicodeObject *free_list; -static int numfree; - /* The empty Unicode object is shared to improve performance. */ -static PyUnicodeObject *unicode_empty; +static PyObject *unicode_empty; + +/* List of static strings. */ +static _Py_Identifier *static_strings; /* Single character Unicode strings in the Latin-1 range are being shared as well. */ -static PyUnicodeObject *unicode_latin1[256]; +static PyObject *unicode_latin1[256]; /* Fast detection of the most frequent whitespace characters */ const unsigned char _Py_ascii_whitespace[] = { @@ -142,16 +222,31 @@ const unsigned char _Py_ascii_whitespace[] = { 0, 0, 0, 0, 0, 0, 0, 0 }; -static PyObject *unicode_encode_call_errorhandler(const char *errors, +/* forward */ +static PyUnicodeObject *_PyUnicode_New(Py_ssize_t length); +static PyObject* get_latin1_char(unsigned char ch); +static int unicode_modifiable(PyObject *unicode); + + +static PyObject * +_PyUnicode_FromUCS1(const unsigned char *s, Py_ssize_t size); +static PyObject * +_PyUnicode_FromUCS2(const Py_UCS2 *s, Py_ssize_t size); +static PyObject * +_PyUnicode_FromUCS4(const Py_UCS4 *s, Py_ssize_t size); + +static PyObject * +unicode_encode_call_errorhandler(const char *errors, PyObject **errorHandler,const char *encoding, const char *reason, - const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, + PyObject *unicode, PyObject **exceptionObject, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos); -static void raise_encode_exception(PyObject **exceptionObject, - const char *encoding, - const Py_UNICODE *unicode, Py_ssize_t size, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason); +static void +raise_encode_exception(PyObject **exceptionObject, + const char *encoding, + PyObject *unicode, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason); /* Same for linebreaks */ static unsigned char ascii_linebreak[] = { @@ -181,7 +276,8 @@ static unsigned char ascii_linebreak[] = { 0, 0, 0, 0, 0, 0, 0, 0 }; - +/* The max unicode value is always 0x10FFFF while using the PEP-393 API. + This function is kept for backward compatibility with the old API. */ Py_UNICODE PyUnicode_GetMax(void) { @@ -194,6 +290,224 @@ PyUnicode_GetMax(void) #endif } +#ifdef Py_DEBUG +int +_PyUnicode_CheckConsistency(PyObject *op, int check_content) +{ + PyASCIIObject *ascii; + unsigned int kind; + + assert(PyUnicode_Check(op)); + + ascii = (PyASCIIObject *)op; + kind = ascii->state.kind; + + if (ascii->state.ascii == 1 && ascii->state.compact == 1) { + assert(kind == PyUnicode_1BYTE_KIND); + assert(ascii->state.ready == 1); + } + else { + PyCompactUnicodeObject *compact = (PyCompactUnicodeObject *)op; + void *data; + + if (ascii->state.compact == 1) { + data = compact + 1; + assert(kind == PyUnicode_1BYTE_KIND + || kind == PyUnicode_2BYTE_KIND + || kind == PyUnicode_4BYTE_KIND); + assert(ascii->state.ascii == 0); + assert(ascii->state.ready == 1); + assert (compact->utf8 != data); + } + else { + PyUnicodeObject *unicode = (PyUnicodeObject *)op; + + data = unicode->data.any; + if (kind == PyUnicode_WCHAR_KIND) { + assert(ascii->length == 0); + assert(ascii->hash == -1); + assert(ascii->state.compact == 0); + assert(ascii->state.ascii == 0); + assert(ascii->state.ready == 0); + assert(ascii->state.interned == SSTATE_NOT_INTERNED); + assert(ascii->wstr != NULL); + assert(data == NULL); + assert(compact->utf8 == NULL); + } + else { + assert(kind == PyUnicode_1BYTE_KIND + || kind == PyUnicode_2BYTE_KIND + || kind == PyUnicode_4BYTE_KIND); + assert(ascii->state.compact == 0); + assert(ascii->state.ready == 1); + assert(data != NULL); + if (ascii->state.ascii) { + assert (compact->utf8 == data); + assert (compact->utf8_length == ascii->length); + } + else + assert (compact->utf8 != data); + } + } + if (kind != PyUnicode_WCHAR_KIND) { + if ( +#if SIZEOF_WCHAR_T == 2 + kind == PyUnicode_2BYTE_KIND +#else + kind == PyUnicode_4BYTE_KIND +#endif + ) + { + assert(ascii->wstr == data); + assert(compact->wstr_length == ascii->length); + } else + assert(ascii->wstr != data); + } + + if (compact->utf8 == NULL) + assert(compact->utf8_length == 0); + if (ascii->wstr == NULL) + assert(compact->wstr_length == 0); + } + /* check that the best kind is used */ + if (check_content && kind != PyUnicode_WCHAR_KIND) + { + Py_ssize_t i; + Py_UCS4 maxchar = 0; + void *data; + Py_UCS4 ch; + + data = PyUnicode_DATA(ascii); + for (i=0; i < ascii->length; i++) + { + ch = PyUnicode_READ(kind, data, i); + if (ch > maxchar) + maxchar = ch; + } + if (kind == PyUnicode_1BYTE_KIND) { + if (ascii->state.ascii == 0) { + assert(maxchar >= 128); + assert(maxchar <= 255); + } + else + assert(maxchar < 128); + } + else if (kind == PyUnicode_2BYTE_KIND) { + assert(maxchar >= 0x100); + assert(maxchar <= 0xFFFF); + } + else { + assert(maxchar >= 0x10000); + assert(maxchar <= MAX_UNICODE); + } + assert(PyUnicode_READ(kind, data, ascii->length) == 0); + } + return 1; +} +#endif + +static PyObject* +unicode_result_wchar(PyObject *unicode) +{ +#ifndef Py_DEBUG + Py_ssize_t len; + + assert(Py_REFCNT(unicode) == 1); + + len = _PyUnicode_WSTR_LENGTH(unicode); + if (len == 0) { + Py_INCREF(unicode_empty); + Py_DECREF(unicode); + return unicode_empty; + } + + if (len == 1) { + wchar_t ch = _PyUnicode_WSTR(unicode)[0]; + if (ch < 256) { + PyObject *latin1_char = get_latin1_char((unsigned char)ch); + Py_DECREF(unicode); + return latin1_char; + } + } + + if (_PyUnicode_Ready(unicode) < 0) { + Py_XDECREF(unicode); + return NULL; + } +#else + /* don't make the result ready in debug mode to ensure that the caller + makes the string ready before using it */ + assert(_PyUnicode_CheckConsistency(unicode, 1)); +#endif + return unicode; +} + +static PyObject* +unicode_result_ready(PyObject *unicode) +{ + Py_ssize_t length; + + length = PyUnicode_GET_LENGTH(unicode); + if (length == 0) { + if (unicode != unicode_empty) { + Py_INCREF(unicode_empty); + Py_DECREF(unicode); + } + return unicode_empty; + } + + if (length == 1) { + Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, 0); + if (ch < 256) { + PyObject *latin1_char = unicode_latin1[ch]; + if (latin1_char != NULL) { + if (unicode != latin1_char) { + Py_INCREF(latin1_char); + Py_DECREF(unicode); + } + return latin1_char; + } + else { + assert(_PyUnicode_CheckConsistency(unicode, 1)); + Py_INCREF(unicode); + unicode_latin1[ch] = unicode; + return unicode; + } + } + } + + assert(_PyUnicode_CheckConsistency(unicode, 1)); + return unicode; +} + +static PyObject* +unicode_result(PyObject *unicode) +{ + assert(_PyUnicode_CHECK(unicode)); + if (PyUnicode_IS_READY(unicode)) + return unicode_result_ready(unicode); + else + return unicode_result_wchar(unicode); +} + +static PyObject* +unicode_result_unchanged(PyObject *unicode) +{ + if (PyUnicode_CheckExact(unicode)) { + if (PyUnicode_READY(unicode) == -1) + return NULL; + Py_INCREF(unicode); + return unicode; + } + else + /* Subtype -- return genuine unicode string with the same value. */ + return _PyUnicode_Copy(unicode); +} + +#ifdef HAVE_MBCS +static OSVERSIONINFOEX winver; +#endif + /* --- Bloom Filters ----------------------------------------------------- */ /* stuff to implement simple "bloom filters" for Unicode characters. @@ -223,7 +537,8 @@ static BLOOM_MASK bloom_linebreak; ((ch) < 128U ? ascii_linebreak[(ch)] : \ (BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK(ch))) -Py_LOCAL_INLINE(BLOOM_MASK) make_bloom_mask(Py_UNICODE* ptr, Py_ssize_t len) +Py_LOCAL_INLINE(BLOOM_MASK) +make_bloom_mask(int kind, void* ptr, Py_ssize_t len) { /* calculate simple bloom-style bitmask for a given unicode string */ @@ -232,76 +547,253 @@ Py_LOCAL_INLINE(BLOOM_MASK) make_bloom_mask(Py_UNICODE* ptr, Py_ssize_t len) mask = 0; for (i = 0; i < len; i++) - BLOOM_ADD(mask, ptr[i]); + BLOOM_ADD(mask, PyUnicode_READ(kind, ptr, i)); return mask; } -Py_LOCAL_INLINE(int) unicode_member(Py_UNICODE chr, Py_UNICODE* set, Py_ssize_t setlen) -{ - Py_ssize_t i; +#define BLOOM_MEMBER(mask, chr, str) \ + (BLOOM(mask, chr) \ + && (PyUnicode_FindChar(str, chr, 0, PyUnicode_GET_LENGTH(str), 1) >= 0)) - for (i = 0; i < setlen; i++) - if (set[i] == chr) - return 1; +/* Compilation of templated routines */ - return 0; -} +#include "stringlib/asciilib.h" +#include "stringlib/fastsearch.h" +#include "stringlib/partition.h" +#include "stringlib/split.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/find_max_char.h" +#include "stringlib/localeutil.h" +#include "stringlib/undef.h" -#define BLOOM_MEMBER(mask, chr, set, setlen) \ - BLOOM(mask, chr) && unicode_member(chr, set, setlen) +#include "stringlib/ucs1lib.h" +#include "stringlib/fastsearch.h" +#include "stringlib/partition.h" +#include "stringlib/split.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/find_max_char.h" +#include "stringlib/localeutil.h" +#include "stringlib/undef.h" + +#include "stringlib/ucs2lib.h" +#include "stringlib/fastsearch.h" +#include "stringlib/partition.h" +#include "stringlib/split.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/find_max_char.h" +#include "stringlib/localeutil.h" +#include "stringlib/undef.h" + +#include "stringlib/ucs4lib.h" +#include "stringlib/fastsearch.h" +#include "stringlib/partition.h" +#include "stringlib/split.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/find_max_char.h" +#include "stringlib/localeutil.h" +#include "stringlib/undef.h" + +#include "stringlib/unicodedefs.h" +#include "stringlib/fastsearch.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/undef.h" /* --- Unicode Object ----------------------------------------------------- */ -static -int unicode_resize(register PyUnicodeObject *unicode, - Py_ssize_t length) +static PyObject * +fixup(PyObject *self, Py_UCS4 (*fixfct)(PyObject *s)); + +Py_LOCAL_INLINE(Py_ssize_t) findchar(void *s, int kind, + Py_ssize_t size, Py_UCS4 ch, + int direction) { - void *oldstr; + int mode = (direction == 1) ? FAST_SEARCH : FAST_RSEARCH; - /* Shortcut if there's nothing much to do. */ - if (unicode->length == length) - goto reset; + switch (kind) { + case PyUnicode_1BYTE_KIND: + { + Py_UCS1 ch1 = (Py_UCS1) ch; + if (ch1 == ch) + return ucs1lib_fastsearch((Py_UCS1 *) s, size, &ch1, 1, 0, mode); + else + return -1; + } + case PyUnicode_2BYTE_KIND: + { + Py_UCS2 ch2 = (Py_UCS2) ch; + if (ch2 == ch) + return ucs2lib_fastsearch((Py_UCS2 *) s, size, &ch2, 1, 0, mode); + else + return -1; + } + case PyUnicode_4BYTE_KIND: + return ucs4lib_fastsearch((Py_UCS4 *) s, size, &ch, 1, 0, mode); + default: + assert(0); + return -1; + } +} - /* Resizing shared object (unicode_empty or single character - objects) in-place is not allowed. Use PyUnicode_Resize() - instead ! */ +static PyObject* +resize_compact(PyObject *unicode, Py_ssize_t length) +{ + Py_ssize_t char_size; + Py_ssize_t struct_size; + Py_ssize_t new_size; + int share_wstr; + PyObject *new_unicode; + assert(unicode_modifiable(unicode)); + assert(PyUnicode_IS_READY(unicode)); + assert(PyUnicode_IS_COMPACT(unicode)); + + char_size = PyUnicode_KIND(unicode); + if (PyUnicode_IS_ASCII(unicode)) + struct_size = sizeof(PyASCIIObject); + else + struct_size = sizeof(PyCompactUnicodeObject); + share_wstr = _PyUnicode_SHARE_WSTR(unicode); - if (unicode == unicode_empty || - (unicode->length == 1 && - unicode->str[0] < 256U && - unicode_latin1[unicode->str[0]] == unicode)) { - PyErr_SetString(PyExc_SystemError, - "can't resize shared str objects"); - return -1; + if (length > ((PY_SSIZE_T_MAX - struct_size) / char_size - 1)) { + PyErr_NoMemory(); + return NULL; } + new_size = (struct_size + (length + 1) * char_size); - /* We allocate one more byte to make sure the string is Ux0000 terminated. - The overallocation is also used by fastsearch, which assumes that it's - safe to look at str[length] (without making any assumptions about what - it contains). */ + _Py_DEC_REFTOTAL; + _Py_ForgetReference(unicode); - oldstr = unicode->str; - unicode->str = PyObject_REALLOC(unicode->str, - sizeof(Py_UNICODE) * (length + 1)); - if (!unicode->str) { - unicode->str = (Py_UNICODE *)oldstr; + new_unicode = (PyObject *)PyObject_REALLOC((char *)unicode, new_size); + if (new_unicode == NULL) { + _Py_NewReference(unicode); PyErr_NoMemory(); - return -1; + return NULL; + } + unicode = new_unicode; + _Py_NewReference(unicode); + + _PyUnicode_LENGTH(unicode) = length; + if (share_wstr) { + _PyUnicode_WSTR(unicode) = PyUnicode_DATA(unicode); + if (!PyUnicode_IS_ASCII(unicode)) + _PyUnicode_WSTR_LENGTH(unicode) = length; } - unicode->str[length] = 0; - unicode->length = length; + PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode), + length, 0); + assert(_PyUnicode_CheckConsistency(unicode, 0)); + return unicode; +} + +static int +resize_inplace(PyObject *unicode, Py_ssize_t length) +{ + wchar_t *wstr; + Py_ssize_t new_size; + assert(!PyUnicode_IS_COMPACT(unicode)); + assert(Py_REFCNT(unicode) == 1); + + if (PyUnicode_IS_READY(unicode)) { + Py_ssize_t char_size; + int share_wstr, share_utf8; + void *data; + + data = _PyUnicode_DATA_ANY(unicode); + char_size = PyUnicode_KIND(unicode); + share_wstr = _PyUnicode_SHARE_WSTR(unicode); + share_utf8 = _PyUnicode_SHARE_UTF8(unicode); + + if (length > (PY_SSIZE_T_MAX / char_size - 1)) { + PyErr_NoMemory(); + return -1; + } + new_size = (length + 1) * char_size; + + if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) + { + PyObject_DEL(_PyUnicode_UTF8(unicode)); + _PyUnicode_UTF8(unicode) = NULL; + _PyUnicode_UTF8_LENGTH(unicode) = 0; + } - reset: - /* Reset the object caches */ - if (unicode->defenc) { - Py_CLEAR(unicode->defenc); + data = (PyObject *)PyObject_REALLOC(data, new_size); + if (data == NULL) { + PyErr_NoMemory(); + return -1; + } + _PyUnicode_DATA_ANY(unicode) = data; + if (share_wstr) { + _PyUnicode_WSTR(unicode) = data; + _PyUnicode_WSTR_LENGTH(unicode) = length; + } + if (share_utf8) { + _PyUnicode_UTF8(unicode) = data; + _PyUnicode_UTF8_LENGTH(unicode) = length; + } + _PyUnicode_LENGTH(unicode) = length; + PyUnicode_WRITE(PyUnicode_KIND(unicode), data, length, 0); + if (share_wstr || _PyUnicode_WSTR(unicode) == NULL) { + assert(_PyUnicode_CheckConsistency(unicode, 0)); + return 0; + } } - unicode->hash = -1; + assert(_PyUnicode_WSTR(unicode) != NULL); + /* check for integer overflow */ + if (length > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { + PyErr_NoMemory(); + return -1; + } + new_size = sizeof(wchar_t) * (length + 1); + wstr = _PyUnicode_WSTR(unicode); + wstr = PyObject_REALLOC(wstr, new_size); + if (!wstr) { + PyErr_NoMemory(); + return -1; + } + _PyUnicode_WSTR(unicode) = wstr; + _PyUnicode_WSTR(unicode)[length] = 0; + _PyUnicode_WSTR_LENGTH(unicode) = length; + assert(_PyUnicode_CheckConsistency(unicode, 0)); return 0; } +static PyObject* +resize_copy(PyObject *unicode, Py_ssize_t length) +{ + Py_ssize_t copy_length; + if (_PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND) { + PyObject *copy; + + if (PyUnicode_READY(unicode) == -1) + return NULL; + + copy = PyUnicode_New(length, PyUnicode_MAX_CHAR_VALUE(unicode)); + if (copy == NULL) + return NULL; + + copy_length = Py_MIN(length, PyUnicode_GET_LENGTH(unicode)); + _PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, copy_length); + return copy; + } + else { + PyObject *w; + + w = (PyObject*)_PyUnicode_New(length); + if (w == NULL) + return NULL; + copy_length = _PyUnicode_WSTR_LENGTH(unicode); + copy_length = Py_MIN(copy_length, length); + Py_UNICODE_COPY(_PyUnicode_WSTR(w), _PyUnicode_WSTR(unicode), + copy_length); + return w; + } +} + /* We allocate one more byte to make sure the string is Ux0000 terminated; some code (e.g. new_identifier) relies on that. @@ -311,55 +803,39 @@ int unicode_resize(register PyUnicodeObject *unicode, */ -static -PyUnicodeObject *_PyUnicode_New(Py_ssize_t length) +static PyUnicodeObject * +_PyUnicode_New(Py_ssize_t length) { register PyUnicodeObject *unicode; + size_t new_size; /* Optimization for empty strings */ if (length == 0 && unicode_empty != NULL) { Py_INCREF(unicode_empty); - return unicode_empty; + return (PyUnicodeObject*)unicode_empty; } /* Ensure we won't overflow the size. */ if (length > ((PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) - 1)) { return (PyUnicodeObject *)PyErr_NoMemory(); } - - /* Unicode freelist & memory allocation */ - if (free_list) { - unicode = free_list; - free_list = *(PyUnicodeObject **)unicode; - numfree--; - if (unicode->str) { - /* Keep-Alive optimization: we only upsize the buffer, - never downsize it. */ - if ((unicode->length < length) && - unicode_resize(unicode, length) < 0) { - PyObject_DEL(unicode->str); - unicode->str = NULL; - } - } - else { - size_t new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); - unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size); - } - PyObject_INIT(unicode, &PyUnicode_Type); - } - else { - size_t new_size; - unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type); - if (unicode == NULL) - return NULL; - new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); - unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size); + if (length < 0) { + PyErr_SetString(PyExc_SystemError, + "Negative size passed to _PyUnicode_New"); + return NULL; } - if (!unicode->str) { + unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type); + if (unicode == NULL) + return NULL; + new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); + _PyUnicode_WSTR(unicode) = (Py_UNICODE*) PyObject_MALLOC(new_size); + if (!_PyUnicode_WSTR(unicode)) { + Py_DECREF(unicode); PyErr_NoMemory(); - goto onError; + return NULL; } + /* Initialize the first element to guard against cases where * the caller fails before initializing str -- unicode_resize() * reads str[0], and the Keep-Alive optimization can keep memory @@ -367,24 +843,658 @@ PyUnicodeObject *_PyUnicode_New(Py_ssize_t length) * We don't want unicode_resize to read uninitialized memory in * that case. */ - unicode->str[0] = 0; - unicode->str[length] = 0; - unicode->length = length; - unicode->hash = -1; - unicode->state = 0; - unicode->defenc = NULL; + _PyUnicode_WSTR(unicode)[0] = 0; + _PyUnicode_WSTR(unicode)[length] = 0; + _PyUnicode_WSTR_LENGTH(unicode) = length; + _PyUnicode_HASH(unicode) = -1; + _PyUnicode_STATE(unicode).interned = 0; + _PyUnicode_STATE(unicode).kind = 0; + _PyUnicode_STATE(unicode).compact = 0; + _PyUnicode_STATE(unicode).ready = 0; + _PyUnicode_STATE(unicode).ascii = 0; + _PyUnicode_DATA_ANY(unicode) = NULL; + _PyUnicode_LENGTH(unicode) = 0; + _PyUnicode_UTF8(unicode) = NULL; + _PyUnicode_UTF8_LENGTH(unicode) = 0; + assert(_PyUnicode_CheckConsistency((PyObject *)unicode, 0)); return unicode; +} - onError: - /* XXX UNREF/NEWREF interface should be more symmetrical */ - _Py_DEC_REFTOTAL; - _Py_ForgetReference((PyObject *)unicode); - PyObject_Del(unicode); - return NULL; +static const char* +unicode_kind_name(PyObject *unicode) +{ + /* don't check consistency: unicode_kind_name() is called from + _PyUnicode_Dump() */ + if (!PyUnicode_IS_COMPACT(unicode)) + { + if (!PyUnicode_IS_READY(unicode)) + return "wstr"; + switch (PyUnicode_KIND(unicode)) + { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(unicode)) + return "legacy ascii"; + else + return "legacy latin1"; + case PyUnicode_2BYTE_KIND: + return "legacy UCS2"; + case PyUnicode_4BYTE_KIND: + return "legacy UCS4"; + default: + return "<legacy invalid kind>"; + } + } + assert(PyUnicode_IS_READY(unicode)); + switch (PyUnicode_KIND(unicode)) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(unicode)) + return "ascii"; + else + return "latin1"; + case PyUnicode_2BYTE_KIND: + return "UCS2"; + case PyUnicode_4BYTE_KIND: + return "UCS4"; + default: + return "<invalid compact kind>"; + } } -static -void unicode_dealloc(register PyUnicodeObject *unicode) +#ifdef Py_DEBUG +/* Functions wrapping macros for use in debugger */ +char *_PyUnicode_utf8(void *unicode){ + return PyUnicode_UTF8(unicode); +} + +void *_PyUnicode_compact_data(void *unicode) { + return _PyUnicode_COMPACT_DATA(unicode); +} +void *_PyUnicode_data(void *unicode){ + printf("obj %p\n", unicode); + printf("compact %d\n", PyUnicode_IS_COMPACT(unicode)); + printf("compact ascii %d\n", PyUnicode_IS_COMPACT_ASCII(unicode)); + printf("ascii op %p\n", ((void*)((PyASCIIObject*)(unicode) + 1))); + printf("compact op %p\n", ((void*)((PyCompactUnicodeObject*)(unicode) + 1))); + printf("compact data %p\n", _PyUnicode_COMPACT_DATA(unicode)); + return PyUnicode_DATA(unicode); +} + +void +_PyUnicode_Dump(PyObject *op) +{ + PyASCIIObject *ascii = (PyASCIIObject *)op; + PyCompactUnicodeObject *compact = (PyCompactUnicodeObject *)op; + PyUnicodeObject *unicode = (PyUnicodeObject *)op; + void *data; + + if (ascii->state.compact) + { + if (ascii->state.ascii) + data = (ascii + 1); + else + data = (compact + 1); + } + else + data = unicode->data.any; + printf("%s: len=%zu, ",unicode_kind_name(op), ascii->length); + + if (ascii->wstr == data) + printf("shared "); + printf("wstr=%p", ascii->wstr); + + if (!(ascii->state.ascii == 1 && ascii->state.compact == 1)) { + printf(" (%zu), ", compact->wstr_length); + if (!ascii->state.compact && compact->utf8 == unicode->data.any) + printf("shared "); + printf("utf8=%p (%zu)", compact->utf8, compact->utf8_length); + } + printf(", data=%p\n", data); +} +#endif + +PyObject * +PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) +{ + PyObject *obj; + PyCompactUnicodeObject *unicode; + void *data; + enum PyUnicode_Kind kind; + int is_sharing, is_ascii; + Py_ssize_t char_size; + Py_ssize_t struct_size; + + /* Optimization for empty strings */ + if (size == 0 && unicode_empty != NULL) { + Py_INCREF(unicode_empty); + return unicode_empty; + } + + is_ascii = 0; + is_sharing = 0; + struct_size = sizeof(PyCompactUnicodeObject); + if (maxchar < 128) { + kind = PyUnicode_1BYTE_KIND; + char_size = 1; + is_ascii = 1; + struct_size = sizeof(PyASCIIObject); + } + else if (maxchar < 256) { + kind = PyUnicode_1BYTE_KIND; + char_size = 1; + } + else if (maxchar < 65536) { + kind = PyUnicode_2BYTE_KIND; + char_size = 2; + if (sizeof(wchar_t) == 2) + is_sharing = 1; + } + else { + if (maxchar > MAX_UNICODE) { + PyErr_SetString(PyExc_SystemError, + "invalid maximum character passed to PyUnicode_New"); + return NULL; + } + kind = PyUnicode_4BYTE_KIND; + char_size = 4; + if (sizeof(wchar_t) == 4) + is_sharing = 1; + } + + /* Ensure we won't overflow the size. */ + if (size < 0) { + PyErr_SetString(PyExc_SystemError, + "Negative size passed to PyUnicode_New"); + return NULL; + } + if (size > ((PY_SSIZE_T_MAX - struct_size) / char_size - 1)) + return PyErr_NoMemory(); + + /* Duplicated allocation code from _PyObject_New() instead of a call to + * PyObject_New() so we are able to allocate space for the object and + * it's data buffer. + */ + obj = (PyObject *) PyObject_MALLOC(struct_size + (size + 1) * char_size); + if (obj == NULL) + return PyErr_NoMemory(); + obj = PyObject_INIT(obj, &PyUnicode_Type); + if (obj == NULL) + return NULL; + + unicode = (PyCompactUnicodeObject *)obj; + if (is_ascii) + data = ((PyASCIIObject*)obj) + 1; + else + data = unicode + 1; + _PyUnicode_LENGTH(unicode) = size; + _PyUnicode_HASH(unicode) = -1; + _PyUnicode_STATE(unicode).interned = 0; + _PyUnicode_STATE(unicode).kind = kind; + _PyUnicode_STATE(unicode).compact = 1; + _PyUnicode_STATE(unicode).ready = 1; + _PyUnicode_STATE(unicode).ascii = is_ascii; + if (is_ascii) { + ((char*)data)[size] = 0; + _PyUnicode_WSTR(unicode) = NULL; + } + else if (kind == PyUnicode_1BYTE_KIND) { + ((char*)data)[size] = 0; + _PyUnicode_WSTR(unicode) = NULL; + _PyUnicode_WSTR_LENGTH(unicode) = 0; + unicode->utf8 = NULL; + unicode->utf8_length = 0; + } + else { + unicode->utf8 = NULL; + unicode->utf8_length = 0; + if (kind == PyUnicode_2BYTE_KIND) + ((Py_UCS2*)data)[size] = 0; + else /* kind == PyUnicode_4BYTE_KIND */ + ((Py_UCS4*)data)[size] = 0; + if (is_sharing) { + _PyUnicode_WSTR_LENGTH(unicode) = size; + _PyUnicode_WSTR(unicode) = (wchar_t *)data; + } + else { + _PyUnicode_WSTR_LENGTH(unicode) = 0; + _PyUnicode_WSTR(unicode) = NULL; + } + } +#ifdef Py_DEBUG + /* Fill the data with invalid characters to detect bugs earlier. + _PyUnicode_CheckConsistency(str, 1) detects invalid characters, + at least for ASCII and UCS-4 strings. U+00FF is invalid in ASCII + and U+FFFFFFFF is an invalid character in Unicode 6.0. */ + memset(data, 0xff, size * kind); +#endif + assert(_PyUnicode_CheckConsistency((PyObject*)unicode, 0)); + return obj; +} + +#if SIZEOF_WCHAR_T == 2 +/* Helper function to convert a 16-bits wchar_t representation to UCS4, this + will decode surrogate pairs, the other conversions are implemented as macros + for efficiency. + + This function assumes that unicode can hold one more code point than wstr + characters for a terminating null character. */ +static void +unicode_convert_wchar_to_ucs4(const wchar_t *begin, const wchar_t *end, + PyObject *unicode) +{ + const wchar_t *iter; + Py_UCS4 *ucs4_out; + + assert(unicode != NULL); + assert(_PyUnicode_CHECK(unicode)); + assert(_PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); + ucs4_out = PyUnicode_4BYTE_DATA(unicode); + + for (iter = begin; iter < end; ) { + assert(ucs4_out < (PyUnicode_4BYTE_DATA(unicode) + + _PyUnicode_GET_LENGTH(unicode))); + if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) + && (iter+1) < end + && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) + { + *ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); + iter += 2; + } + else { + *ucs4_out++ = *iter; + iter++; + } + } + assert(ucs4_out == (PyUnicode_4BYTE_DATA(unicode) + + _PyUnicode_GET_LENGTH(unicode))); + +} +#endif + +static int +unicode_check_modifiable(PyObject *unicode) +{ + if (!unicode_modifiable(unicode)) { + PyErr_SetString(PyExc_SystemError, + "Cannot modify a string currently used"); + return -1; + } + return 0; +} + +static int +_copy_characters(PyObject *to, Py_ssize_t to_start, + PyObject *from, Py_ssize_t from_start, + Py_ssize_t how_many, int check_maxchar) +{ + unsigned int from_kind, to_kind; + void *from_data, *to_data; + + assert(0 <= how_many); + assert(0 <= from_start); + assert(0 <= to_start); + assert(PyUnicode_Check(from)); + assert(PyUnicode_IS_READY(from)); + assert(from_start + how_many <= PyUnicode_GET_LENGTH(from)); + + assert(PyUnicode_Check(to)); + assert(PyUnicode_IS_READY(to)); + assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); + + if (how_many == 0) + return 0; + + from_kind = PyUnicode_KIND(from); + from_data = PyUnicode_DATA(from); + to_kind = PyUnicode_KIND(to); + to_data = PyUnicode_DATA(to); + +#ifdef Py_DEBUG + if (!check_maxchar + && PyUnicode_MAX_CHAR_VALUE(from) > PyUnicode_MAX_CHAR_VALUE(to)) + { + const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to); + Py_UCS4 ch; + Py_ssize_t i; + for (i=0; i < how_many; i++) { + ch = PyUnicode_READ(from_kind, from_data, from_start + i); + assert(ch <= to_maxchar); + } + } +#endif + + if (from_kind == to_kind) { + if (check_maxchar + && !PyUnicode_IS_ASCII(from) && PyUnicode_IS_ASCII(to)) + { + /* Writing Latin-1 characters into an ASCII string requires to + check that all written characters are pure ASCII */ + Py_UCS4 max_char; + max_char = ucs1lib_find_max_char(from_data, + (Py_UCS1*)from_data + how_many); + if (max_char >= 128) + return -1; + } + Py_MEMCPY((char*)to_data + to_kind * to_start, + (char*)from_data + from_kind * from_start, + to_kind * how_many); + } + else if (from_kind == PyUnicode_1BYTE_KIND + && to_kind == PyUnicode_2BYTE_KIND) + { + _PyUnicode_CONVERT_BYTES( + Py_UCS1, Py_UCS2, + PyUnicode_1BYTE_DATA(from) + from_start, + PyUnicode_1BYTE_DATA(from) + from_start + how_many, + PyUnicode_2BYTE_DATA(to) + to_start + ); + } + else if (from_kind == PyUnicode_1BYTE_KIND + && to_kind == PyUnicode_4BYTE_KIND) + { + _PyUnicode_CONVERT_BYTES( + Py_UCS1, Py_UCS4, + PyUnicode_1BYTE_DATA(from) + from_start, + PyUnicode_1BYTE_DATA(from) + from_start + how_many, + PyUnicode_4BYTE_DATA(to) + to_start + ); + } + else if (from_kind == PyUnicode_2BYTE_KIND + && to_kind == PyUnicode_4BYTE_KIND) + { + _PyUnicode_CONVERT_BYTES( + Py_UCS2, Py_UCS4, + PyUnicode_2BYTE_DATA(from) + from_start, + PyUnicode_2BYTE_DATA(from) + from_start + how_many, + PyUnicode_4BYTE_DATA(to) + to_start + ); + } + else { + assert (PyUnicode_MAX_CHAR_VALUE(from) > PyUnicode_MAX_CHAR_VALUE(to)); + + if (!check_maxchar) { + if (from_kind == PyUnicode_2BYTE_KIND + && to_kind == PyUnicode_1BYTE_KIND) + { + _PyUnicode_CONVERT_BYTES( + Py_UCS2, Py_UCS1, + PyUnicode_2BYTE_DATA(from) + from_start, + PyUnicode_2BYTE_DATA(from) + from_start + how_many, + PyUnicode_1BYTE_DATA(to) + to_start + ); + } + else if (from_kind == PyUnicode_4BYTE_KIND + && to_kind == PyUnicode_1BYTE_KIND) + { + _PyUnicode_CONVERT_BYTES( + Py_UCS4, Py_UCS1, + PyUnicode_4BYTE_DATA(from) + from_start, + PyUnicode_4BYTE_DATA(from) + from_start + how_many, + PyUnicode_1BYTE_DATA(to) + to_start + ); + } + else if (from_kind == PyUnicode_4BYTE_KIND + && to_kind == PyUnicode_2BYTE_KIND) + { + _PyUnicode_CONVERT_BYTES( + Py_UCS4, Py_UCS2, + PyUnicode_4BYTE_DATA(from) + from_start, + PyUnicode_4BYTE_DATA(from) + from_start + how_many, + PyUnicode_2BYTE_DATA(to) + to_start + ); + } + else { + assert(0); + return -1; + } + } + else { + const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to); + Py_UCS4 ch; + Py_ssize_t i; + + for (i=0; i < how_many; i++) { + ch = PyUnicode_READ(from_kind, from_data, from_start + i); + if (ch > to_maxchar) + return -1; + PyUnicode_WRITE(to_kind, to_data, to_start + i, ch); + } + } + } + return 0; +} + +void +_PyUnicode_FastCopyCharacters( + PyObject *to, Py_ssize_t to_start, + PyObject *from, Py_ssize_t from_start, Py_ssize_t how_many) +{ + (void)_copy_characters(to, to_start, from, from_start, how_many, 0); +} + +Py_ssize_t +PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start, + PyObject *from, Py_ssize_t from_start, + Py_ssize_t how_many) +{ + int err; + + if (!PyUnicode_Check(from) || !PyUnicode_Check(to)) { + PyErr_BadInternalCall(); + return -1; + } + + if (PyUnicode_READY(from) == -1) + return -1; + if (PyUnicode_READY(to) == -1) + return -1; + + if (from_start < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } + if (to_start < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } + how_many = Py_MIN(PyUnicode_GET_LENGTH(from), how_many); + if (to_start + how_many > PyUnicode_GET_LENGTH(to)) { + PyErr_Format(PyExc_SystemError, + "Cannot write %zi characters at %zi " + "in a string of %zi characters", + how_many, to_start, PyUnicode_GET_LENGTH(to)); + return -1; + } + + if (how_many == 0) + return 0; + + if (unicode_check_modifiable(to)) + return -1; + + err = _copy_characters(to, to_start, from, from_start, how_many, 1); + if (err) { + PyErr_Format(PyExc_SystemError, + "Cannot copy %s characters " + "into a string of %s characters", + unicode_kind_name(from), + unicode_kind_name(to)); + return -1; + } + return how_many; +} + +/* Find the maximum code point and count the number of surrogate pairs so a + correct string length can be computed before converting a string to UCS4. + This function counts single surrogates as a character and not as a pair. + + Return 0 on success, or -1 on error. */ +static int +find_maxchar_surrogates(const wchar_t *begin, const wchar_t *end, + Py_UCS4 *maxchar, Py_ssize_t *num_surrogates) +{ + const wchar_t *iter; + Py_UCS4 ch; + + assert(num_surrogates != NULL && maxchar != NULL); + *num_surrogates = 0; + *maxchar = 0; + + for (iter = begin; iter < end; ) { +#if SIZEOF_WCHAR_T == 2 + if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) + && (iter+1) < end + && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) + { + ch = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); + ++(*num_surrogates); + iter += 2; + } + else +#endif + { + ch = *iter; + iter++; + } + if (ch > *maxchar) { + *maxchar = ch; + if (*maxchar > MAX_UNICODE) { + PyErr_Format(PyExc_ValueError, + "character U+%x is not in range [U+0000; U+10ffff]", + ch); + return -1; + } + } + } + return 0; +} + +int +_PyUnicode_Ready(PyObject *unicode) +{ + wchar_t *end; + Py_UCS4 maxchar = 0; + Py_ssize_t num_surrogates; +#if SIZEOF_WCHAR_T == 2 + Py_ssize_t length_wo_surrogates; +#endif + + /* _PyUnicode_Ready() is only intended for old-style API usage where + strings were created using _PyObject_New() and where no canonical + representation (the str field) has been set yet aka strings + which are not yet ready. */ + assert(_PyUnicode_CHECK(unicode)); + assert(_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND); + assert(_PyUnicode_WSTR(unicode) != NULL); + assert(_PyUnicode_DATA_ANY(unicode) == NULL); + assert(_PyUnicode_UTF8(unicode) == NULL); + /* Actually, it should neither be interned nor be anything else: */ + assert(_PyUnicode_STATE(unicode).interned == SSTATE_NOT_INTERNED); + + end = _PyUnicode_WSTR(unicode) + _PyUnicode_WSTR_LENGTH(unicode); + if (find_maxchar_surrogates(_PyUnicode_WSTR(unicode), end, + &maxchar, &num_surrogates) == -1) + return -1; + + if (maxchar < 256) { + _PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(_PyUnicode_WSTR_LENGTH(unicode) + 1); + if (!_PyUnicode_DATA_ANY(unicode)) { + PyErr_NoMemory(); + return -1; + } + _PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, + _PyUnicode_WSTR(unicode), end, + PyUnicode_1BYTE_DATA(unicode)); + PyUnicode_1BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; + _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); + _PyUnicode_STATE(unicode).kind = PyUnicode_1BYTE_KIND; + if (maxchar < 128) { + _PyUnicode_STATE(unicode).ascii = 1; + _PyUnicode_UTF8(unicode) = _PyUnicode_DATA_ANY(unicode); + _PyUnicode_UTF8_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); + } + else { + _PyUnicode_STATE(unicode).ascii = 0; + _PyUnicode_UTF8(unicode) = NULL; + _PyUnicode_UTF8_LENGTH(unicode) = 0; + } + PyObject_FREE(_PyUnicode_WSTR(unicode)); + _PyUnicode_WSTR(unicode) = NULL; + _PyUnicode_WSTR_LENGTH(unicode) = 0; + } + /* In this case we might have to convert down from 4-byte native + wchar_t to 2-byte unicode. */ + else if (maxchar < 65536) { + assert(num_surrogates == 0 && + "FindMaxCharAndNumSurrogatePairs() messed up"); + +#if SIZEOF_WCHAR_T == 2 + /* We can share representations and are done. */ + _PyUnicode_DATA_ANY(unicode) = _PyUnicode_WSTR(unicode); + PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; + _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); + _PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND; + _PyUnicode_UTF8(unicode) = NULL; + _PyUnicode_UTF8_LENGTH(unicode) = 0; +#else + /* sizeof(wchar_t) == 4 */ + _PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC( + 2 * (_PyUnicode_WSTR_LENGTH(unicode) + 1)); + if (!_PyUnicode_DATA_ANY(unicode)) { + PyErr_NoMemory(); + return -1; + } + _PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, + _PyUnicode_WSTR(unicode), end, + PyUnicode_2BYTE_DATA(unicode)); + PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; + _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); + _PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND; + _PyUnicode_UTF8(unicode) = NULL; + _PyUnicode_UTF8_LENGTH(unicode) = 0; + PyObject_FREE(_PyUnicode_WSTR(unicode)); + _PyUnicode_WSTR(unicode) = NULL; + _PyUnicode_WSTR_LENGTH(unicode) = 0; +#endif + } + /* maxchar exeeds 16 bit, wee need 4 bytes for unicode characters */ + else { +#if SIZEOF_WCHAR_T == 2 + /* in case the native representation is 2-bytes, we need to allocate a + new normalized 4-byte version. */ + length_wo_surrogates = _PyUnicode_WSTR_LENGTH(unicode) - num_surrogates; + _PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(4 * (length_wo_surrogates + 1)); + if (!_PyUnicode_DATA_ANY(unicode)) { + PyErr_NoMemory(); + return -1; + } + _PyUnicode_LENGTH(unicode) = length_wo_surrogates; + _PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND; + _PyUnicode_UTF8(unicode) = NULL; + _PyUnicode_UTF8_LENGTH(unicode) = 0; + /* unicode_convert_wchar_to_ucs4() requires a ready string */ + _PyUnicode_STATE(unicode).ready = 1; + unicode_convert_wchar_to_ucs4(_PyUnicode_WSTR(unicode), end, unicode); + PyObject_FREE(_PyUnicode_WSTR(unicode)); + _PyUnicode_WSTR(unicode) = NULL; + _PyUnicode_WSTR_LENGTH(unicode) = 0; +#else + assert(num_surrogates == 0); + + _PyUnicode_DATA_ANY(unicode) = _PyUnicode_WSTR(unicode); + _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); + _PyUnicode_UTF8(unicode) = NULL; + _PyUnicode_UTF8_LENGTH(unicode) = 0; + _PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND; +#endif + PyUnicode_4BYTE_DATA(unicode)[_PyUnicode_LENGTH(unicode)] = '\0'; + } + _PyUnicode_STATE(unicode).ready = 1; + assert(_PyUnicode_CheckConsistency(unicode, 1)); + return 0; +} + +static void +unicode_dealloc(register PyObject *unicode) { switch (PyUnicode_CHECK_INTERNED(unicode)) { case SSTATE_NOT_INTERNED: @@ -393,7 +1503,7 @@ void unicode_dealloc(register PyUnicodeObject *unicode) case SSTATE_INTERNED_MORTAL: /* revive dead object temporarily for DelItem */ Py_REFCNT(unicode) = 3; - if (PyDict_DelItem(interned, (PyObject *)unicode) != 0) + if (PyDict_DelItem(interned, unicode) != 0) Py_FatalError( "deletion of interned string failed"); break; @@ -405,271 +1515,753 @@ void unicode_dealloc(register PyUnicodeObject *unicode) Py_FatalError("Inconsistent interned string state."); } - if (PyUnicode_CheckExact(unicode) && - numfree < PyUnicode_MAXFREELIST) { - /* Keep-Alive optimization */ - if (unicode->length >= KEEPALIVE_SIZE_LIMIT) { - PyObject_DEL(unicode->str); - unicode->str = NULL; - unicode->length = 0; - } - if (unicode->defenc) { - Py_CLEAR(unicode->defenc); - } - /* Add to free list */ - *(PyUnicodeObject **)unicode = free_list; - free_list = unicode; - numfree++; - } - else { - PyObject_DEL(unicode->str); - Py_XDECREF(unicode->defenc); - Py_TYPE(unicode)->tp_free((PyObject *)unicode); + if (_PyUnicode_HAS_WSTR_MEMORY(unicode)) + PyObject_DEL(_PyUnicode_WSTR(unicode)); + if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) + PyObject_DEL(_PyUnicode_UTF8(unicode)); + if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode)) + PyObject_DEL(_PyUnicode_DATA_ANY(unicode)); + + Py_TYPE(unicode)->tp_free(unicode); +} + +#ifdef Py_DEBUG +static int +unicode_is_singleton(PyObject *unicode) +{ + PyASCIIObject *ascii = (PyASCIIObject *)unicode; + if (unicode == unicode_empty) + return 1; + if (ascii->state.kind != PyUnicode_WCHAR_KIND && ascii->length == 1) + { + Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, 0); + if (ch < 256 && unicode_latin1[ch] == unicode) + return 1; } + return 0; } +#endif -static -int _PyUnicode_Resize(PyUnicodeObject **unicode, Py_ssize_t length) +static int +unicode_modifiable(PyObject *unicode) { - register PyUnicodeObject *v; + assert(_PyUnicode_CHECK(unicode)); + if (Py_REFCNT(unicode) != 1) + return 0; + if (_PyUnicode_HASH(unicode) != -1) + return 0; + if (PyUnicode_CHECK_INTERNED(unicode)) + return 0; + if (!PyUnicode_CheckExact(unicode)) + return 0; +#ifdef Py_DEBUG + /* singleton refcount is greater than 1 */ + assert(!unicode_is_singleton(unicode)); +#endif + return 1; +} - /* Argument checks */ - if (unicode == NULL) { +static int +unicode_resize(PyObject **p_unicode, Py_ssize_t length) +{ + PyObject *unicode; + Py_ssize_t old_length; + + assert(p_unicode != NULL); + unicode = *p_unicode; + + assert(unicode != NULL); + assert(PyUnicode_Check(unicode)); + assert(0 <= length); + + if (_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND) + old_length = PyUnicode_WSTR_LENGTH(unicode); + else + old_length = PyUnicode_GET_LENGTH(unicode); + if (old_length == length) + return 0; + + if (length == 0) { + Py_DECREF(*p_unicode); + *p_unicode = unicode_empty; + Py_INCREF(*p_unicode); + return 0; + } + + if (!unicode_modifiable(unicode)) { + PyObject *copy = resize_copy(unicode, length); + if (copy == NULL) + return -1; + Py_DECREF(*p_unicode); + *p_unicode = copy; + return 0; + } + + if (PyUnicode_IS_COMPACT(unicode)) { + PyObject *new_unicode = resize_compact(unicode, length); + if (new_unicode == NULL) + return -1; + *p_unicode = new_unicode; + return 0; + } + return resize_inplace(unicode, length); +} + +int +PyUnicode_Resize(PyObject **p_unicode, Py_ssize_t length) +{ + PyObject *unicode; + if (p_unicode == NULL) { PyErr_BadInternalCall(); return -1; } - v = *unicode; - if (v == NULL || !PyUnicode_Check(v) || Py_REFCNT(v) != 1 || length < 0) { + unicode = *p_unicode; + if (unicode == NULL || !PyUnicode_Check(unicode) || length < 0) + { PyErr_BadInternalCall(); return -1; } + return unicode_resize(p_unicode, length); +} - /* Resizing unicode_empty and single character objects is not - possible since these are being shared. We simply return a fresh - copy with the same Unicode content. */ - if (v->length != length && - (v == unicode_empty || v->length == 1)) { - PyUnicodeObject *w = _PyUnicode_New(length); - if (w == NULL) - return -1; - Py_UNICODE_COPY(w->str, v->str, - length < v->length ? length : v->length); - Py_DECREF(*unicode); - *unicode = w; +static int +unicode_widen(PyObject **p_unicode, Py_ssize_t length, + unsigned int maxchar) +{ + PyObject *result; + assert(PyUnicode_IS_READY(*p_unicode)); + assert(length <= PyUnicode_GET_LENGTH(*p_unicode)); + if (maxchar <= PyUnicode_MAX_CHAR_VALUE(*p_unicode)) return 0; + result = PyUnicode_New(PyUnicode_GET_LENGTH(*p_unicode), + maxchar); + if (result == NULL) + return -1; + _PyUnicode_FastCopyCharacters(result, 0, *p_unicode, 0, length); + Py_DECREF(*p_unicode); + *p_unicode = result; + return 0; +} + +static int +unicode_putchar(PyObject **p_unicode, Py_ssize_t *pos, + Py_UCS4 ch) +{ + assert(ch <= MAX_UNICODE); + if (unicode_widen(p_unicode, *pos, ch) < 0) + return -1; + PyUnicode_WRITE(PyUnicode_KIND(*p_unicode), + PyUnicode_DATA(*p_unicode), + (*pos)++, ch); + return 0; +} + +/* Copy a ASCII or latin1 char* string into a Python Unicode string. + + WARNING: The function doesn't copy the terminating null character and + doesn't check the maximum character (may write a latin1 character in an + ASCII string). */ +static void +unicode_write_cstr(PyObject *unicode, Py_ssize_t index, + const char *str, Py_ssize_t len) +{ + enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); + void *data = PyUnicode_DATA(unicode); + const char *end = str + len; + + switch (kind) { + case PyUnicode_1BYTE_KIND: { + assert(index + len <= PyUnicode_GET_LENGTH(unicode)); + memcpy((char *) data + index, str, len); + break; + } + case PyUnicode_2BYTE_KIND: { + Py_UCS2 *start = (Py_UCS2 *)data + index; + Py_UCS2 *ucs2 = start; + assert(index <= PyUnicode_GET_LENGTH(unicode)); + + for (; str < end; ++ucs2, ++str) + *ucs2 = (Py_UCS2)*str; + + assert((ucs2 - start) <= PyUnicode_GET_LENGTH(unicode)); + break; } + default: { + Py_UCS4 *start = (Py_UCS4 *)data + index; + Py_UCS4 *ucs4 = start; + assert(kind == PyUnicode_4BYTE_KIND); + assert(index <= PyUnicode_GET_LENGTH(unicode)); - /* Note that we don't have to modify *unicode for unshared Unicode - objects, since we can modify them in-place. */ - return unicode_resize(v, length); + for (; str < end; ++ucs4, ++str) + *ucs4 = (Py_UCS4)*str; + + assert((ucs4 - start) <= PyUnicode_GET_LENGTH(unicode)); + } + } } -int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length) + +static PyObject* +get_latin1_char(unsigned char ch) { - return _PyUnicode_Resize((PyUnicodeObject **)unicode, length); + PyObject *unicode = unicode_latin1[ch]; + if (!unicode) { + unicode = PyUnicode_New(1, ch); + if (!unicode) + return NULL; + PyUnicode_1BYTE_DATA(unicode)[0] = ch; + assert(_PyUnicode_CheckConsistency(unicode, 1)); + unicode_latin1[ch] = unicode; + } + Py_INCREF(unicode); + return unicode; } -PyObject *PyUnicode_FromUnicode(const Py_UNICODE *u, - Py_ssize_t size) +PyObject * +PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) { - PyUnicodeObject *unicode; + PyObject *unicode; + Py_UCS4 maxchar = 0; + Py_ssize_t num_surrogates; + + if (u == NULL) + return (PyObject*)_PyUnicode_New(size); /* If the Unicode data is known at construction time, we can apply some optimizations which share commonly used objects. */ - if (u != NULL) { - - /* Optimization for empty strings */ - if (size == 0 && unicode_empty != NULL) { - Py_INCREF(unicode_empty); - return (PyObject *)unicode_empty; - } - /* Single character Unicode objects in the Latin-1 range are - shared when using this constructor */ - if (size == 1 && *u < 256) { - unicode = unicode_latin1[*u]; - if (!unicode) { - unicode = _PyUnicode_New(1); - if (!unicode) - return NULL; - unicode->str[0] = *u; - unicode_latin1[*u] = unicode; - } - Py_INCREF(unicode); - return (PyObject *)unicode; - } + /* Optimization for empty strings */ + if (size == 0 && unicode_empty != NULL) { + Py_INCREF(unicode_empty); + return unicode_empty; } - unicode = _PyUnicode_New(size); + /* Single character Unicode objects in the Latin-1 range are + shared when using this constructor */ + if (size == 1 && *u < 256) + return get_latin1_char((unsigned char)*u); + + /* If not empty and not single character, copy the Unicode data + into the new object */ + if (find_maxchar_surrogates(u, u + size, + &maxchar, &num_surrogates) == -1) + return NULL; + + unicode = PyUnicode_New(size - num_surrogates, maxchar); if (!unicode) return NULL; - /* Copy the Unicode data into the new object */ - if (u != NULL) - Py_UNICODE_COPY(unicode->str, u, size); + switch (PyUnicode_KIND(unicode)) { + case PyUnicode_1BYTE_KIND: + _PyUnicode_CONVERT_BYTES(Py_UNICODE, unsigned char, + u, u + size, PyUnicode_1BYTE_DATA(unicode)); + break; + case PyUnicode_2BYTE_KIND: +#if Py_UNICODE_SIZE == 2 + Py_MEMCPY(PyUnicode_2BYTE_DATA(unicode), u, size * 2); +#else + _PyUnicode_CONVERT_BYTES(Py_UNICODE, Py_UCS2, + u, u + size, PyUnicode_2BYTE_DATA(unicode)); +#endif + break; + case PyUnicode_4BYTE_KIND: +#if SIZEOF_WCHAR_T == 2 + /* This is the only case which has to process surrogates, thus + a simple copy loop is not enough and we need a function. */ + unicode_convert_wchar_to_ucs4(u, u + size, unicode); +#else + assert(num_surrogates == 0); + Py_MEMCPY(PyUnicode_4BYTE_DATA(unicode), u, size * 4); +#endif + break; + default: + assert(0 && "Impossible state"); + } - return (PyObject *)unicode; + return unicode_result(unicode); } -PyObject *PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) +PyObject * +PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) { - PyUnicodeObject *unicode; - if (size < 0) { PyErr_SetString(PyExc_SystemError, "Negative size passed to PyUnicode_FromStringAndSize"); return NULL; } + if (u != NULL) + return PyUnicode_DecodeUTF8Stateful(u, size, NULL, NULL); + else + return (PyObject *)_PyUnicode_New(size); +} - /* If the Unicode data is known at construction time, we can apply - some optimizations which share commonly used objects. - Also, this means the input must be UTF-8, so fall back to the - UTF-8 decoder at the end. */ - if (u != NULL) { - - /* Optimization for empty strings */ - if (size == 0 && unicode_empty != NULL) { - Py_INCREF(unicode_empty); - return (PyObject *)unicode_empty; - } +PyObject * +PyUnicode_FromString(const char *u) +{ + size_t size = strlen(u); + if (size > PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, "input too long"); + return NULL; + } + return PyUnicode_DecodeUTF8Stateful(u, (Py_ssize_t)size, NULL, NULL); +} - /* Single characters are shared when using this constructor. - Restrict to ASCII, since the input must be UTF-8. */ - if (size == 1 && Py_CHARMASK(*u) < 128) { - unicode = unicode_latin1[Py_CHARMASK(*u)]; - if (!unicode) { - unicode = _PyUnicode_New(1); - if (!unicode) - return NULL; - unicode->str[0] = Py_CHARMASK(*u); - unicode_latin1[Py_CHARMASK(*u)] = unicode; - } - Py_INCREF(unicode); - return (PyObject *)unicode; - } +PyObject * +_PyUnicode_FromId(_Py_Identifier *id) +{ + if (!id->object) { + id->object = PyUnicode_DecodeUTF8Stateful(id->string, + strlen(id->string), + NULL, NULL); + if (!id->object) + return NULL; + PyUnicode_InternInPlace(&id->object); + assert(!id->next); + id->next = static_strings; + static_strings = id; + } + return id->object; +} - return PyUnicode_DecodeUTF8(u, size, NULL); +void +_PyUnicode_ClearStaticStrings() +{ + _Py_Identifier *i; + for (i = static_strings; i; i = i->next) { + Py_DECREF(i->object); + i->object = NULL; + i->next = NULL; } +} - unicode = _PyUnicode_New(size); +/* Internal function, doesn't check maximum character */ + +PyObject* +_PyUnicode_FromASCII(const char *buffer, Py_ssize_t size) +{ + const unsigned char *s = (const unsigned char *)buffer; + PyObject *unicode; + if (size == 1) { +#ifdef Py_DEBUG + assert(s[0] < 128); +#endif + return get_latin1_char(s[0]); + } + unicode = PyUnicode_New(size, 127); if (!unicode) return NULL; + memcpy(PyUnicode_1BYTE_DATA(unicode), s, size); + assert(_PyUnicode_CheckConsistency(unicode, 1)); + return unicode; +} + +static Py_UCS4 +kind_maxchar_limit(unsigned int kind) +{ + switch (kind) { + case PyUnicode_1BYTE_KIND: + return 0x80; + case PyUnicode_2BYTE_KIND: + return 0x100; + case PyUnicode_4BYTE_KIND: + return 0x10000; + default: + assert(0 && "invalid kind"); + return MAX_UNICODE; + } +} - return (PyObject *)unicode; +Py_LOCAL_INLINE(Py_UCS4) +align_maxchar(Py_UCS4 maxchar) +{ + if (maxchar <= 127) + return 127; + else if (maxchar <= 255) + return 255; + else if (maxchar <= 65535) + return 65535; + else + return MAX_UNICODE; } -PyObject *PyUnicode_FromString(const char *u) +static PyObject* +_PyUnicode_FromUCS1(const unsigned char* u, Py_ssize_t size) { - size_t size = strlen(u); - if (size > PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, "input too long"); - return NULL; + PyObject *res; + unsigned char max_char; + + if (size == 0) { + Py_INCREF(unicode_empty); + return unicode_empty; } + assert(size > 0); + if (size == 1) + return get_latin1_char(u[0]); - return PyUnicode_FromStringAndSize(u, size); + max_char = ucs1lib_find_max_char(u, u + size); + res = PyUnicode_New(size, max_char); + if (!res) + return NULL; + memcpy(PyUnicode_1BYTE_DATA(res), u, size); + assert(_PyUnicode_CheckConsistency(res, 1)); + return res; } -#ifdef HAVE_WCHAR_H +static PyObject* +_PyUnicode_FromUCS2(const Py_UCS2 *u, Py_ssize_t size) +{ + PyObject *res; + Py_UCS2 max_char; -#if (Py_UNICODE_SIZE == 2) && defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) -# define CONVERT_WCHAR_TO_SURROGATES -#endif + if (size == 0) { + Py_INCREF(unicode_empty); + return unicode_empty; + } + assert(size > 0); + if (size == 1) { + Py_UCS4 ch = u[0]; + if (ch < 256) + return get_latin1_char((unsigned char)ch); -#ifdef CONVERT_WCHAR_TO_SURROGATES + res = PyUnicode_New(1, ch); + if (res == NULL) + return NULL; + PyUnicode_WRITE(PyUnicode_KIND(res), PyUnicode_DATA(res), 0, ch); + assert(_PyUnicode_CheckConsistency(res, 1)); + return res; + } -/* Here sizeof(wchar_t) is 4 but Py_UNICODE_SIZE == 2, so we need - to convert from UTF32 to UTF16. */ + max_char = ucs2lib_find_max_char(u, u + size); + res = PyUnicode_New(size, max_char); + if (!res) + return NULL; + if (max_char >= 256) + memcpy(PyUnicode_2BYTE_DATA(res), u, sizeof(Py_UCS2)*size); + else { + _PyUnicode_CONVERT_BYTES( + Py_UCS2, Py_UCS1, u, u + size, PyUnicode_1BYTE_DATA(res)); + } + assert(_PyUnicode_CheckConsistency(res, 1)); + return res; +} -PyObject *PyUnicode_FromWideChar(register const wchar_t *w, - Py_ssize_t size) +static PyObject* +_PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size) { - PyUnicodeObject *unicode; - register Py_ssize_t i; - Py_ssize_t alloc; - const wchar_t *orig_w; + PyObject *res; + Py_UCS4 max_char; - if (w == NULL) { - if (size == 0) - return PyUnicode_FromStringAndSize(NULL, 0); - PyErr_BadInternalCall(); - return NULL; + if (size == 0) { + Py_INCREF(unicode_empty); + return unicode_empty; } + assert(size > 0); + if (size == 1) { + Py_UCS4 ch = u[0]; + if (ch < 256) + return get_latin1_char((unsigned char)ch); - if (size == -1) { - size = wcslen(w); + res = PyUnicode_New(1, ch); + if (res == NULL) + return NULL; + PyUnicode_WRITE(PyUnicode_KIND(res), PyUnicode_DATA(res), 0, ch); + assert(_PyUnicode_CheckConsistency(res, 1)); + return res; } - alloc = size; - orig_w = w; - for (i = size; i > 0; i--) { - if (*w > 0xFFFF) - alloc++; - w++; + max_char = ucs4lib_find_max_char(u, u + size); + res = PyUnicode_New(size, max_char); + if (!res) + return NULL; + if (max_char < 256) + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS1, u, u + size, + PyUnicode_1BYTE_DATA(res)); + else if (max_char < 0x10000) + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS2, u, u + size, + PyUnicode_2BYTE_DATA(res)); + else + memcpy(PyUnicode_4BYTE_DATA(res), u, sizeof(Py_UCS4)*size); + assert(_PyUnicode_CheckConsistency(res, 1)); + return res; +} + +PyObject* +PyUnicode_FromKindAndData(int kind, const void *buffer, Py_ssize_t size) +{ + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must be positive"); + return NULL; } - w = orig_w; - unicode = _PyUnicode_New(alloc); - if (!unicode) + switch (kind) { + case PyUnicode_1BYTE_KIND: + return _PyUnicode_FromUCS1(buffer, size); + case PyUnicode_2BYTE_KIND: + return _PyUnicode_FromUCS2(buffer, size); + case PyUnicode_4BYTE_KIND: + return _PyUnicode_FromUCS4(buffer, size); + default: + PyErr_SetString(PyExc_SystemError, "invalid kind"); return NULL; + } +} - /* Copy the wchar_t data into the new object */ - { - register Py_UNICODE *u; - u = PyUnicode_AS_UNICODE(unicode); - for (i = size; i > 0; i--) { - if (*w > 0xFFFF) { - wchar_t ordinal = *w++; - ordinal -= 0x10000; - *u++ = 0xD800 | (ordinal >> 10); - *u++ = 0xDC00 | (ordinal & 0x3FF); - } - else - *u++ = *w++; - } +Py_UCS4 +_PyUnicode_FindMaxChar(PyObject *unicode, Py_ssize_t start, Py_ssize_t end) +{ + enum PyUnicode_Kind kind; + void *startptr, *endptr; + + assert(PyUnicode_IS_READY(unicode)); + assert(0 <= start); + assert(end <= PyUnicode_GET_LENGTH(unicode)); + assert(start <= end); + + if (start == 0 && end == PyUnicode_GET_LENGTH(unicode)) + return PyUnicode_MAX_CHAR_VALUE(unicode); + + if (start == end) + return 127; + + if (PyUnicode_IS_ASCII(unicode)) + return 127; + + kind = PyUnicode_KIND(unicode); + startptr = PyUnicode_DATA(unicode); + endptr = (char *)startptr + end * kind; + startptr = (char *)startptr + start * kind; + switch(kind) { + case PyUnicode_1BYTE_KIND: + return ucs1lib_find_max_char(startptr, endptr); + case PyUnicode_2BYTE_KIND: + return ucs2lib_find_max_char(startptr, endptr); + case PyUnicode_4BYTE_KIND: + return ucs4lib_find_max_char(startptr, endptr); + default: + assert(0); + return 0; } - return (PyObject *)unicode; } -#else +/* Ensure that a string uses the most efficient storage, if it is not the + case: create a new string with of the right kind. Write NULL into *p_unicode + on error. */ +static void +unicode_adjust_maxchar(PyObject **p_unicode) +{ + PyObject *unicode, *copy; + Py_UCS4 max_char; + Py_ssize_t len; + unsigned int kind; -PyObject *PyUnicode_FromWideChar(register const wchar_t *w, - Py_ssize_t size) + assert(p_unicode != NULL); + unicode = *p_unicode; + assert(PyUnicode_IS_READY(unicode)); + if (PyUnicode_IS_ASCII(unicode)) + return; + + len = PyUnicode_GET_LENGTH(unicode); + kind = PyUnicode_KIND(unicode); + if (kind == PyUnicode_1BYTE_KIND) { + const Py_UCS1 *u = PyUnicode_1BYTE_DATA(unicode); + max_char = ucs1lib_find_max_char(u, u + len); + if (max_char >= 128) + return; + } + else if (kind == PyUnicode_2BYTE_KIND) { + const Py_UCS2 *u = PyUnicode_2BYTE_DATA(unicode); + max_char = ucs2lib_find_max_char(u, u + len); + if (max_char >= 256) + return; + } + else { + const Py_UCS4 *u = PyUnicode_4BYTE_DATA(unicode); + assert(kind == PyUnicode_4BYTE_KIND); + max_char = ucs4lib_find_max_char(u, u + len); + if (max_char >= 0x10000) + return; + } + copy = PyUnicode_New(len, max_char); + if (copy != NULL) + _PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, len); + Py_DECREF(unicode); + *p_unicode = copy; +} + +PyObject* +_PyUnicode_Copy(PyObject *unicode) { - PyUnicodeObject *unicode; + Py_ssize_t length; + PyObject *copy; - if (w == NULL) { - if (size == 0) - return PyUnicode_FromStringAndSize(NULL, 0); + if (!PyUnicode_Check(unicode)) { PyErr_BadInternalCall(); return NULL; } + if (PyUnicode_READY(unicode) == -1) + return NULL; - if (size == -1) { - size = wcslen(w); + length = PyUnicode_GET_LENGTH(unicode); + copy = PyUnicode_New(length, PyUnicode_MAX_CHAR_VALUE(unicode)); + if (!copy) + return NULL; + assert(PyUnicode_KIND(copy) == PyUnicode_KIND(unicode)); + + Py_MEMCPY(PyUnicode_DATA(copy), PyUnicode_DATA(unicode), + length * PyUnicode_KIND(unicode)); + assert(_PyUnicode_CheckConsistency(copy, 1)); + return copy; +} + + +/* Widen Unicode objects to larger buffers. Don't write terminating null + character. Return NULL on error. */ + +void* +_PyUnicode_AsKind(PyObject *s, unsigned int kind) +{ + Py_ssize_t len; + void *result; + unsigned int skind; + + if (PyUnicode_READY(s) == -1) + return NULL; + + len = PyUnicode_GET_LENGTH(s); + skind = PyUnicode_KIND(s); + if (skind >= kind) { + PyErr_SetString(PyExc_SystemError, "invalid widening attempt"); + return NULL; + } + switch (kind) { + case PyUnicode_2BYTE_KIND: + result = PyMem_Malloc(len * sizeof(Py_UCS2)); + if (!result) + return PyErr_NoMemory(); + assert(skind == PyUnicode_1BYTE_KIND); + _PyUnicode_CONVERT_BYTES( + Py_UCS1, Py_UCS2, + PyUnicode_1BYTE_DATA(s), + PyUnicode_1BYTE_DATA(s) + len, + result); + return result; + case PyUnicode_4BYTE_KIND: + result = PyMem_Malloc(len * sizeof(Py_UCS4)); + if (!result) + return PyErr_NoMemory(); + if (skind == PyUnicode_2BYTE_KIND) { + _PyUnicode_CONVERT_BYTES( + Py_UCS2, Py_UCS4, + PyUnicode_2BYTE_DATA(s), + PyUnicode_2BYTE_DATA(s) + len, + result); + } + else { + assert(skind == PyUnicode_1BYTE_KIND); + _PyUnicode_CONVERT_BYTES( + Py_UCS1, Py_UCS4, + PyUnicode_1BYTE_DATA(s), + PyUnicode_1BYTE_DATA(s) + len, + result); + } + return result; + default: + break; } + PyErr_SetString(PyExc_SystemError, "invalid kind"); + return NULL; +} - unicode = _PyUnicode_New(size); - if (!unicode) +static Py_UCS4* +as_ucs4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize, + int copy_null) +{ + int kind; + void *data; + Py_ssize_t len, targetlen; + if (PyUnicode_READY(string) == -1) return NULL; + kind = PyUnicode_KIND(string); + data = PyUnicode_DATA(string); + len = PyUnicode_GET_LENGTH(string); + targetlen = len; + if (copy_null) + targetlen++; + if (!target) { + if (PY_SSIZE_T_MAX / sizeof(Py_UCS4) < targetlen) { + PyErr_NoMemory(); + return NULL; + } + target = PyMem_Malloc(targetlen * sizeof(Py_UCS4)); + if (!target) { + PyErr_NoMemory(); + return NULL; + } + } + else { + if (targetsize < targetlen) { + PyErr_Format(PyExc_SystemError, + "string is longer than the buffer"); + if (copy_null && 0 < targetsize) + target[0] = 0; + return NULL; + } + } + if (kind == PyUnicode_1BYTE_KIND) { + Py_UCS1 *start = (Py_UCS1 *) data; + _PyUnicode_CONVERT_BYTES(Py_UCS1, Py_UCS4, start, start + len, target); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *start = (Py_UCS2 *) data; + _PyUnicode_CONVERT_BYTES(Py_UCS2, Py_UCS4, start, start + len, target); + } + else { + assert(kind == PyUnicode_4BYTE_KIND); + Py_MEMCPY(target, data, len * sizeof(Py_UCS4)); + } + if (copy_null) + target[len] = 0; + return target; +} - /* Copy the wchar_t data into the new object */ -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T - memcpy(unicode->str, w, size * sizeof(wchar_t)); -#else - { - register Py_UNICODE *u; - register Py_ssize_t i; - u = PyUnicode_AS_UNICODE(unicode); - for (i = size; i > 0; i--) - *u++ = *w++; +Py_UCS4* +PyUnicode_AsUCS4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize, + int copy_null) +{ + if (target == NULL || targetsize < 0) { + PyErr_BadInternalCall(); + return NULL; } -#endif + return as_ucs4(string, target, targetsize, copy_null); +} - return (PyObject *)unicode; +Py_UCS4* +PyUnicode_AsUCS4Copy(PyObject *string) +{ + return as_ucs4(string, NULL, 0, 1); } -#endif /* CONVERT_WCHAR_TO_SURROGATES */ +#ifdef HAVE_WCHAR_H + +PyObject * +PyUnicode_FromWideChar(register const wchar_t *w, Py_ssize_t size) +{ + if (w == NULL) { + if (size == 0) { + Py_INCREF(unicode_empty); + return unicode_empty; + } + PyErr_BadInternalCall(); + return NULL; + } + + if (size == -1) { + size = wcslen(w); + } + + return PyUnicode_FromUnicode(w, size); +} -#undef CONVERT_WCHAR_TO_SURROGATES +#endif /* HAVE_WCHAR_H */ static void makefmt(char *fmt, int longflag, int longlongflag, int size_tflag, @@ -707,10 +2299,71 @@ makefmt(char *fmt, int longflag, int longlongflag, int size_tflag, *fmt = '\0'; } -#define appendstring(string) {for (copy = string;*copy;) *s++ = *copy++;} +/* helper for PyUnicode_FromFormatV() */ + +static const char* +parse_format_flags(const char *f, + int *p_width, int *p_precision, + int *p_longflag, int *p_longlongflag, int *p_size_tflag) +{ + int width, precision, longflag, longlongflag, size_tflag; + + /* parse the width.precision part, e.g. "%2.5s" => width=2, precision=5 */ + f++; + width = 0; + while (Py_ISDIGIT((unsigned)*f)) + width = (width*10) + *f++ - '0'; + precision = 0; + if (*f == '.') { + f++; + while (Py_ISDIGIT((unsigned)*f)) + precision = (precision*10) + *f++ - '0'; + if (*f == '%') { + /* "%.3%s" => f points to "3" */ + f--; + } + } + if (*f == '\0') { + /* bogus format "%.1" => go backward, f points to "1" */ + f--; + } + if (p_width != NULL) + *p_width = width; + if (p_precision != NULL) + *p_precision = precision; + + /* Handle %ld, %lu, %lld and %llu. */ + longflag = 0; + longlongflag = 0; + size_tflag = 0; + + if (*f == 'l') { + if (f[1] == 'd' || f[1] == 'u' || f[1] == 'i') { + longflag = 1; + ++f; + } +#ifdef HAVE_LONG_LONG + else if (f[1] == 'l' && + (f[2] == 'd' || f[2] == 'u' || f[2] == 'i')) { + longlongflag = 1; + f += 2; + } +#endif + } + /* handle the size_t flag. */ + else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u' || f[1] == 'i')) { + size_tflag = 1; + ++f; + } + if (p_longflag != NULL) + *p_longflag = longflag; + if (p_longlongflag != NULL) + *p_longlongflag = longlongflag; + if (p_size_tflag != NULL) + *p_size_tflag = size_tflag; + return f; +} -/* size of fixed-size buffer for formatting single arguments */ -#define ITEM_BUFFER_LEN 21 /* maximum number of characters required for output of %ld. 21 characters allows for 64-bit integers (in decimal) and an optional sign. */ #define MAX_LONG_CHARS 21 @@ -731,132 +2384,197 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) int precision = 0; int zeropad; const char* f; - Py_UNICODE *s; PyObject *string; /* used by sprintf */ - char buffer[ITEM_BUFFER_LEN+1]; - /* use abuffer instead of buffer, if we need more space - * (which can happen if there's a format specifier with width). */ - char *abuffer = NULL; - char *realbuffer; - Py_ssize_t abuffersize = 0; char fmt[61]; /* should be enough for %0width.precisionlld */ - const char *copy; + Py_UCS4 maxchar = 127; /* result is ASCII by default */ + Py_UCS4 argmaxchar; + Py_ssize_t numbersize = 0; + char *numberresults = NULL; + char *numberresult = NULL; + Py_ssize_t i; + int kind; + void *data; Py_VA_COPY(count, vargs); /* step 1: count the number of %S/%R/%A/%s format specifications * (we call PyObject_Str()/PyObject_Repr()/PyObject_ASCII()/ * PyUnicode_DecodeUTF8() for these objects once during step 3 and put the - * result in an array) */ + * result in an array) + * also estimate a upper bound for all the number formats in the string, + * numbers will be formatted in step 3 and be kept in a '\0'-separated + * buffer before putting everything together. */ for (f = format; *f; f++) { - if (*f == '%') { - if (*(f+1)=='%') - continue; - if (*(f+1)=='S' || *(f+1)=='R' || *(f+1)=='A' || *(f+1) == 'V') - ++callcount; - while (Py_ISDIGIT((unsigned)*f)) - width = (width*10) + *f++ - '0'; - while (*++f && *f != '%' && !Py_ISALPHA((unsigned)*f)) - ; - if (*f == 's') - ++callcount; - } - else if (128 <= (unsigned char)*f) { - PyErr_Format(PyExc_ValueError, + if (*f == '%') { + int longlongflag; + /* skip width or width.precision (eg. "1.2" of "%1.2f") */ + f = parse_format_flags(f, &width, NULL, NULL, &longlongflag, NULL); + if (*f == 's' || *f=='S' || *f=='R' || *f=='A' || *f=='V') + ++callcount; + + else if (*f == 'd' || *f=='u' || *f=='i' || *f=='x' || *f=='p') { +#ifdef HAVE_LONG_LONG + if (longlongflag) { + if (width < MAX_LONG_LONG_CHARS) + width = MAX_LONG_LONG_CHARS; + } + else +#endif + /* MAX_LONG_CHARS is enough to hold a 64-bit integer, + including sign. Decimal takes the most space. This + isn't enough for octal. If a width is specified we + need more (which we allocate later). */ + if (width < MAX_LONG_CHARS) + width = MAX_LONG_CHARS; + + /* account for the size + '\0' to separate numbers + inside of the numberresults buffer */ + numbersize += (width + 1); + } + } + else if ((unsigned char)*f > 127) { + PyErr_Format(PyExc_ValueError, "PyUnicode_FromFormatV() expects an ASCII-encoded format " "string, got a non-ASCII byte: 0x%02x", (unsigned char)*f); - return NULL; - } + return NULL; + } } /* step 2: allocate memory for the results of * PyObject_Str()/PyObject_Repr()/PyUnicode_DecodeUTF8() calls */ if (callcount) { - callresults = PyObject_Malloc(sizeof(PyObject *)*callcount); + callresults = PyObject_Malloc(sizeof(PyObject *) * callcount); if (!callresults) { PyErr_NoMemory(); return NULL; } callresult = callresults; } - /* step 3: figure out how large a buffer we need */ + /* step 2.5: allocate memory for the results of formating numbers */ + if (numbersize) { + numberresults = PyObject_Malloc(numbersize); + if (!numberresults) { + PyErr_NoMemory(); + goto fail; + } + numberresult = numberresults; + } + + /* step 3: format numbers and figure out how large a buffer we need */ for (f = format; *f; f++) { if (*f == '%') { -#ifdef HAVE_LONG_LONG - int longlongflag = 0; -#endif - const char* p = f; - width = 0; - while (Py_ISDIGIT((unsigned)*f)) - width = (width*10) + *f++ - '0'; - while (*++f && *f != '%' && !Py_ISALPHA((unsigned)*f)) - ; - - /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since - * they don't affect the amount of space we reserve. - */ - if (*f == 'l') { - if (f[1] == 'd' || f[1] == 'u') { - ++f; - } -#ifdef HAVE_LONG_LONG - else if (f[1] == 'l' && - (f[2] == 'd' || f[2] == 'u')) { - longlongflag = 1; - f += 2; - } -#endif - } - else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { - ++f; - } - + const char* p; + int longflag; + int longlongflag; + int size_tflag; + int numprinted; + + p = f; + zeropad = (f[1] == '0'); + f = parse_format_flags(f, &width, &precision, + &longflag, &longlongflag, &size_tflag); switch (*f) { case 'c': { -#ifndef Py_UNICODE_WIDE - int ordinal = va_arg(count, int); - if (ordinal > 0xffff) - n += 2; - else - n++; -#else - (void)va_arg(count, int); + Py_UCS4 ordinal = va_arg(count, int); + maxchar = MAX_MAXCHAR(maxchar, ordinal); n++; -#endif break; } case '%': n++; break; - case 'd': case 'u': case 'i': case 'x': - (void) va_arg(count, int); + case 'i': + case 'd': + makefmt(fmt, longflag, longlongflag, size_tflag, zeropad, + width, precision, *f); + if (longflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, long)); #ifdef HAVE_LONG_LONG - if (longlongflag) { - if (width < MAX_LONG_LONG_CHARS) - width = MAX_LONG_LONG_CHARS; - } + else if (longlongflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, PY_LONG_LONG)); +#endif + else if (size_tflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, Py_ssize_t)); else + numprinted = sprintf(numberresult, fmt, + va_arg(count, int)); + n += numprinted; + /* advance by +1 to skip over the '\0' */ + numberresult += (numprinted + 1); + assert(*(numberresult - 1) == '\0'); + assert(*(numberresult - 2) != '\0'); + assert(numprinted >= 0); + assert(numberresult <= numberresults + numbersize); + break; + case 'u': + makefmt(fmt, longflag, longlongflag, size_tflag, zeropad, + width, precision, 'u'); + if (longflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, unsigned long)); +#ifdef HAVE_LONG_LONG + else if (longlongflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, unsigned PY_LONG_LONG)); #endif - /* MAX_LONG_CHARS is enough to hold a 64-bit integer, - including sign. Decimal takes the most space. This - isn't enough for octal. If a width is specified we - need more (which we allocate later). */ - if (width < MAX_LONG_CHARS) - width = MAX_LONG_CHARS; - n += width; - /* XXX should allow for large precision here too. */ - if (abuffersize < width) - abuffersize = width; + else if (size_tflag) + numprinted = sprintf(numberresult, fmt, + va_arg(count, size_t)); + else + numprinted = sprintf(numberresult, fmt, + va_arg(count, unsigned int)); + n += numprinted; + numberresult += (numprinted + 1); + assert(*(numberresult - 1) == '\0'); + assert(*(numberresult - 2) != '\0'); + assert(numprinted >= 0); + assert(numberresult <= numberresults + numbersize); + break; + case 'x': + makefmt(fmt, 0, 0, 0, zeropad, width, precision, 'x'); + numprinted = sprintf(numberresult, fmt, va_arg(count, int)); + n += numprinted; + numberresult += (numprinted + 1); + assert(*(numberresult - 1) == '\0'); + assert(*(numberresult - 2) != '\0'); + assert(numprinted >= 0); + assert(numberresult <= numberresults + numbersize); + break; + case 'p': + numprinted = sprintf(numberresult, "%p", va_arg(count, void*)); + /* %p is ill-defined: ensure leading 0x. */ + if (numberresult[1] == 'X') + numberresult[1] = 'x'; + else if (numberresult[1] != 'x') { + memmove(numberresult + 2, numberresult, + strlen(numberresult) + 1); + numberresult[0] = '0'; + numberresult[1] = 'x'; + numprinted += 2; + } + n += numprinted; + numberresult += (numprinted + 1); + assert(*(numberresult - 1) == '\0'); + assert(*(numberresult - 2) != '\0'); + assert(numprinted >= 0); + assert(numberresult <= numberresults + numbersize); break; case 's': { /* UTF-8 */ const char *s = va_arg(count, const char*); - PyObject *str = PyUnicode_DecodeUTF8(s, strlen(s), "replace"); + PyObject *str = PyUnicode_DecodeUTF8Stateful(s, strlen(s), "replace", NULL); if (!str) goto fail; - n += PyUnicode_GET_SIZE(str); + /* since PyUnicode_DecodeUTF8 returns already flexible + unicode objects, there is no need to call ready on them */ + argmaxchar = PyUnicode_MAX_CHAR_VALUE(str); + maxchar = MAX_MAXCHAR(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(str); /* Remember the str and switch to the next slot */ *callresult++ = str; break; @@ -864,8 +2582,12 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) case 'U': { PyObject *obj = va_arg(count, PyObject *); - assert(obj && PyUnicode_Check(obj)); - n += PyUnicode_GET_SIZE(obj); + assert(obj && _PyUnicode_CHECK(obj)); + if (PyUnicode_READY(obj) == -1) + goto fail; + argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj); + maxchar = MAX_MAXCHAR(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(obj); break; } case 'V': @@ -874,16 +2596,26 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) const char *str = va_arg(count, const char *); PyObject *str_obj; assert(obj || str); - assert(!obj || PyUnicode_Check(obj)); + assert(!obj || _PyUnicode_CHECK(obj)); if (obj) { - n += PyUnicode_GET_SIZE(obj); + if (PyUnicode_READY(obj) == -1) + goto fail; + argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj); + maxchar = MAX_MAXCHAR(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(obj); *callresult++ = NULL; } else { - str_obj = PyUnicode_DecodeUTF8(str, strlen(str), "replace"); + str_obj = PyUnicode_DecodeUTF8Stateful(str, strlen(str), "replace", NULL); if (!str_obj) goto fail; - n += PyUnicode_GET_SIZE(str_obj); + if (PyUnicode_READY(str_obj) == -1) { + Py_DECREF(str_obj); + goto fail; + } + argmaxchar = PyUnicode_MAX_CHAR_VALUE(str_obj); + maxchar = MAX_MAXCHAR(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(str_obj); *callresult++ = str_obj; } break; @@ -896,7 +2628,13 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) str = PyObject_Str(obj); if (!str) goto fail; - n += PyUnicode_GET_SIZE(str); + if (PyUnicode_READY(str) == -1) { + Py_DECREF(str); + goto fail; + } + argmaxchar = PyUnicode_MAX_CHAR_VALUE(str); + maxchar = MAX_MAXCHAR(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(str); /* Remember the str and switch to the next slot */ *callresult++ = str; break; @@ -909,7 +2647,13 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) repr = PyObject_Repr(obj); if (!repr) goto fail; - n += PyUnicode_GET_SIZE(repr); + if (PyUnicode_READY(repr) == -1) { + Py_DECREF(repr); + goto fail; + } + argmaxchar = PyUnicode_MAX_CHAR_VALUE(repr); + maxchar = MAX_MAXCHAR(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(repr); /* Remember the repr and switch to the next slot */ *callresult++ = repr; break; @@ -922,20 +2666,17 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) ascii = PyObject_ASCII(obj); if (!ascii) goto fail; - n += PyUnicode_GET_SIZE(ascii); + if (PyUnicode_READY(ascii) == -1) { + Py_DECREF(ascii); + goto fail; + } + argmaxchar = PyUnicode_MAX_CHAR_VALUE(ascii); + maxchar = MAX_MAXCHAR(maxchar, argmaxchar); + n += PyUnicode_GET_LENGTH(ascii); /* Remember the repr and switch to the next slot */ *callresult++ = ascii; break; } - case 'p': - (void) va_arg(count, int); - /* maximum 64-bit pointer representation: - * 0xffffffffffffffff - * so 19 characters is enough. - * XXX I count 18 -- what's the extra for? - */ - n += 19; - break; default: /* if we stumble upon an unknown formatting code, copy the rest of @@ -950,127 +2691,68 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) n++; } expand: - if (abuffersize > ITEM_BUFFER_LEN) { - /* add 1 for sprintf's trailing null byte */ - abuffer = PyObject_Malloc(abuffersize + 1); - if (!abuffer) { - PyErr_NoMemory(); - goto fail; - } - realbuffer = abuffer; - } - else - realbuffer = buffer; /* step 4: fill the buffer */ - /* Since we've analyzed how much space we need for the worst case, + /* Since we've analyzed how much space we need, we don't have to resize the string. There can be no errors beyond this point. */ - string = PyUnicode_FromUnicode(NULL, n); + string = PyUnicode_New(n, maxchar); if (!string) goto fail; - - s = PyUnicode_AS_UNICODE(string); + kind = PyUnicode_KIND(string); + data = PyUnicode_DATA(string); callresult = callresults; + numberresult = numberresults; - for (f = format; *f; f++) { + for (i = 0, f = format; *f; f++) { if (*f == '%') { - const char* p = f++; - int longflag = 0; - int longlongflag = 0; - int size_tflag = 0; - zeropad = (*f == '0'); - /* parse the width.precision part */ - width = 0; - while (Py_ISDIGIT((unsigned)*f)) - width = (width*10) + *f++ - '0'; - precision = 0; - if (*f == '.') { - f++; - while (Py_ISDIGIT((unsigned)*f)) - precision = (precision*10) + *f++ - '0'; - } - /* Handle %ld, %lu, %lld and %llu. */ - if (*f == 'l') { - if (f[1] == 'd' || f[1] == 'u') { - longflag = 1; - ++f; - } -#ifdef HAVE_LONG_LONG - else if (f[1] == 'l' && - (f[2] == 'd' || f[2] == 'u')) { - longlongflag = 1; - f += 2; - } -#endif - } - /* handle the size_t flag. */ - if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { - size_tflag = 1; - ++f; - } + const char* p; + + p = f; + f = parse_format_flags(f, NULL, NULL, NULL, NULL, NULL); + /* checking for == because the last argument could be a empty + string, which causes i to point to end, the assert at the end of + the loop */ + assert(i <= PyUnicode_GET_LENGTH(string)); switch (*f) { case 'c': { - int ordinal = va_arg(vargs, int); -#ifndef Py_UNICODE_WIDE - if (ordinal > 0xffff) { - ordinal -= 0x10000; - *s++ = 0xD800 | (ordinal >> 10); - *s++ = 0xDC00 | (ordinal & 0x3FF); - } else -#endif - *s++ = ordinal; + const int ordinal = va_arg(vargs, int); + PyUnicode_WRITE(kind, data, i++, ordinal); break; } + case 'i': case 'd': - makefmt(fmt, longflag, longlongflag, size_tflag, zeropad, - width, precision, 'd'); - if (longflag) - sprintf(realbuffer, fmt, va_arg(vargs, long)); -#ifdef HAVE_LONG_LONG - else if (longlongflag) - sprintf(realbuffer, fmt, va_arg(vargs, PY_LONG_LONG)); -#endif - else if (size_tflag) - sprintf(realbuffer, fmt, va_arg(vargs, Py_ssize_t)); - else - sprintf(realbuffer, fmt, va_arg(vargs, int)); - appendstring(realbuffer); - break; case 'u': - makefmt(fmt, longflag, longlongflag, size_tflag, zeropad, - width, precision, 'u'); - if (longflag) - sprintf(realbuffer, fmt, va_arg(vargs, unsigned long)); -#ifdef HAVE_LONG_LONG - else if (longlongflag) - sprintf(realbuffer, fmt, va_arg(vargs, - unsigned PY_LONG_LONG)); -#endif - else if (size_tflag) - sprintf(realbuffer, fmt, va_arg(vargs, size_t)); - else - sprintf(realbuffer, fmt, va_arg(vargs, unsigned int)); - appendstring(realbuffer); - break; - case 'i': - makefmt(fmt, 0, 0, 0, zeropad, width, precision, 'i'); - sprintf(realbuffer, fmt, va_arg(vargs, int)); - appendstring(realbuffer); - break; case 'x': - makefmt(fmt, 0, 0, 0, zeropad, width, precision, 'x'); - sprintf(realbuffer, fmt, va_arg(vargs, int)); - appendstring(realbuffer); + case 'p': + { + Py_ssize_t len; + /* unused, since we already have the result */ + if (*f == 'p') + (void) va_arg(vargs, void *); + else + (void) va_arg(vargs, int); + /* extract the result from numberresults and append. */ + len = strlen(numberresult); + unicode_write_cstr(string, i, numberresult, len); + /* skip over the separating '\0' */ + i += len; + numberresult += len; + assert(*numberresult == '\0'); + numberresult++; + assert(numberresult <= numberresults + numbersize); break; + } case 's': { /* unused, since we already have the result */ + Py_ssize_t size; (void) va_arg(vargs, char *); - Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(*callresult), - PyUnicode_GET_SIZE(*callresult)); - s += PyUnicode_GET_SIZE(*callresult); + size = PyUnicode_GET_LENGTH(*callresult); + assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); + _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size); + i += size; /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); /* switch to next unicode()/repr() result */ @@ -1080,23 +2762,29 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) case 'U': { PyObject *obj = va_arg(vargs, PyObject *); - Py_ssize_t size = PyUnicode_GET_SIZE(obj); - Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size); - s += size; + Py_ssize_t size; + assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); + size = PyUnicode_GET_LENGTH(obj); + _PyUnicode_FastCopyCharacters(string, i, obj, 0, size); + i += size; break; } case 'V': { + Py_ssize_t size; PyObject *obj = va_arg(vargs, PyObject *); va_arg(vargs, const char *); if (obj) { - Py_ssize_t size = PyUnicode_GET_SIZE(obj); - Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size); - s += size; + size = PyUnicode_GET_LENGTH(obj); + assert(PyUnicode_KIND(obj) <= PyUnicode_KIND(string)); + _PyUnicode_FastCopyCharacters(string, i, obj, 0, size); + i += size; } else { - Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(*callresult), - PyUnicode_GET_SIZE(*callresult)); - s += PyUnicode_GET_SIZE(*callresult); + size = PyUnicode_GET_LENGTH(*callresult); + assert(PyUnicode_KIND(*callresult) <= + PyUnicode_KIND(string)); + _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size); + i += size; Py_DECREF(*callresult); } ++callresult; @@ -1106,52 +2794,44 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) case 'R': case 'A': { - Py_UNICODE *ucopy; - Py_ssize_t usize; - Py_ssize_t upos; + Py_ssize_t size = PyUnicode_GET_LENGTH(*callresult); /* unused, since we already have the result */ (void) va_arg(vargs, PyObject *); - ucopy = PyUnicode_AS_UNICODE(*callresult); - usize = PyUnicode_GET_SIZE(*callresult); - for (upos = 0; upos<usize;) - *s++ = ucopy[upos++]; + assert(PyUnicode_KIND(*callresult) <= PyUnicode_KIND(string)); + _PyUnicode_FastCopyCharacters(string, i, *callresult, 0, size); + i += size; /* We're done with the unicode()/repr() => forget it */ Py_DECREF(*callresult); /* switch to next unicode()/repr() result */ ++callresult; break; } - case 'p': - sprintf(buffer, "%p", va_arg(vargs, void*)); - /* %p is ill-defined: ensure leading 0x. */ - if (buffer[1] == 'X') - buffer[1] = 'x'; - else if (buffer[1] != 'x') { - memmove(buffer+2, buffer, strlen(buffer)+1); - buffer[0] = '0'; - buffer[1] = 'x'; - } - appendstring(buffer); - break; case '%': - *s++ = '%'; + PyUnicode_WRITE(kind, data, i++, '%'); break; default: - appendstring(p); + { + Py_ssize_t len = strlen(p); + unicode_write_cstr(string, i, p, len); + i += len; + assert(i == PyUnicode_GET_LENGTH(string)); goto end; } + } + } + else { + assert(i < PyUnicode_GET_LENGTH(string)); + PyUnicode_WRITE(kind, data, i++, *f); } - else - *s++ = *f; } + assert(i == PyUnicode_GET_LENGTH(string)); end: if (callresults) PyObject_Free(callresults); - if (abuffer) - PyObject_Free(abuffer); - PyUnicode_Resize(&string, s - PyUnicode_AS_UNICODE(string)); - return string; + if (numberresults) + PyObject_Free(numberresults); + return unicode_result(string); fail: if (callresults) { PyObject **callresult2 = callresults; @@ -1161,13 +2841,11 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) } PyObject_Free(callresults); } - if (abuffer) - PyObject_Free(abuffer); + if (numberresults) + PyObject_Free(numberresults); return NULL; } -#undef appendstring - PyObject * PyUnicode_FromFormat(const char *format, ...) { @@ -1184,6 +2862,8 @@ PyUnicode_FromFormat(const char *format, ...) return ret; } +#ifdef HAVE_WCHAR_H + /* Helper function for PyUnicode_AsWideChar() and PyUnicode_AsWideCharString(): convert a Unicode object to a wide character string. @@ -1194,103 +2874,27 @@ PyUnicode_FromFormat(const char *format, ...) character) written into w. Write at most size wide characters (including the null character). */ static Py_ssize_t -unicode_aswidechar(PyUnicodeObject *unicode, +unicode_aswidechar(PyObject *unicode, wchar_t *w, Py_ssize_t size) { -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T Py_ssize_t res; + const wchar_t *wstr; + + wstr = PyUnicode_AsUnicodeAndSize(unicode, &res); + if (wstr == NULL) + return -1; + if (w != NULL) { - res = PyUnicode_GET_SIZE(unicode); if (size > res) size = res + 1; else res = size; - memcpy(w, unicode->str, size * sizeof(wchar_t)); + Py_MEMCPY(w, wstr, size * sizeof(wchar_t)); return res; } else - return PyUnicode_GET_SIZE(unicode) + 1; -#elif Py_UNICODE_SIZE == 2 && SIZEOF_WCHAR_T == 4 - register const Py_UNICODE *u; - const Py_UNICODE *uend; - const wchar_t *worig, *wend; - Py_ssize_t nchar; - - u = PyUnicode_AS_UNICODE(unicode); - uend = u + PyUnicode_GET_SIZE(unicode); - if (w != NULL) { - worig = w; - wend = w + size; - while (u != uend && w != wend) { - if (0xD800 <= u[0] && u[0] <= 0xDBFF - && 0xDC00 <= u[1] && u[1] <= 0xDFFF) - { - *w = (((u[0] & 0x3FF) << 10) | (u[1] & 0x3FF)) + 0x10000; - u += 2; - } - else { - *w = *u; - u++; - } - w++; - } - if (w != wend) - *w = L'\0'; - return w - worig; - } - else { - nchar = 1; /* null character at the end */ - while (u != uend) { - if (0xD800 <= u[0] && u[0] <= 0xDBFF - && 0xDC00 <= u[1] && u[1] <= 0xDFFF) - u += 2; - else - u++; - nchar++; - } - } - return nchar; -#elif Py_UNICODE_SIZE == 4 && SIZEOF_WCHAR_T == 2 - register Py_UNICODE *u, *uend, ordinal; - register Py_ssize_t i; - wchar_t *worig, *wend; - Py_ssize_t nchar; - - u = PyUnicode_AS_UNICODE(unicode); - uend = u + PyUnicode_GET_SIZE(u); - if (w != NULL) { - worig = w; - wend = w + size; - while (u != uend && w != wend) { - ordinal = *u; - if (ordinal > 0xffff) { - ordinal -= 0x10000; - *w++ = 0xD800 | (ordinal >> 10); - *w++ = 0xDC00 | (ordinal & 0x3FF); - } - else - *w++ = ordinal; - u++; - } - if (w != wend) - *w = 0; - return w - worig; - } - else { - nchar = 1; /* null character */ - while (u != uend) { - if (*u > 0xffff) - nchar += 2; - else - nchar++; - u++; - } - return nchar; - } -#else -# error "unsupported wchar_t and Py_UNICODE sizes, see issue #8670" -#endif + return res + 1; } Py_ssize_t @@ -1302,7 +2906,7 @@ PyUnicode_AsWideChar(PyObject *unicode, PyErr_BadInternalCall(); return -1; } - return unicode_aswidechar((PyUnicodeObject*)unicode, w, size); + return unicode_aswidechar(unicode, w, size); } wchar_t* @@ -1317,7 +2921,9 @@ PyUnicode_AsWideCharString(PyObject *unicode, return NULL; } - buflen = unicode_aswidechar((PyUnicodeObject *)unicode, NULL, 0); + buflen = unicode_aswidechar(unicode, NULL, 0); + if (buflen == -1) + return NULL; if (PY_SSIZE_T_MAX / sizeof(wchar_t) < buflen) { PyErr_NoMemory(); return NULL; @@ -1328,50 +2934,54 @@ PyUnicode_AsWideCharString(PyObject *unicode, PyErr_NoMemory(); return NULL; } - buflen = unicode_aswidechar((PyUnicodeObject *)unicode, buffer, buflen); + buflen = unicode_aswidechar(unicode, buffer, buflen); + if (buflen == -1) { + PyMem_FREE(buffer); + return NULL; + } if (size != NULL) *size = buflen; return buffer; } -#endif +#endif /* HAVE_WCHAR_H */ -PyObject *PyUnicode_FromOrdinal(int ordinal) +PyObject * +PyUnicode_FromOrdinal(int ordinal) { - Py_UNICODE s[2]; - - if (ordinal < 0 || ordinal > 0x10ffff) { + PyObject *v; + if (ordinal < 0 || ordinal > MAX_UNICODE) { PyErr_SetString(PyExc_ValueError, "chr() arg not in range(0x110000)"); return NULL; } -#ifndef Py_UNICODE_WIDE - if (ordinal > 0xffff) { - ordinal -= 0x10000; - s[0] = 0xD800 | (ordinal >> 10); - s[1] = 0xDC00 | (ordinal & 0x3FF); - return PyUnicode_FromUnicode(s, 2); - } -#endif + if (ordinal < 256) + return get_latin1_char(ordinal); - s[0] = (Py_UNICODE)ordinal; - return PyUnicode_FromUnicode(s, 1); + v = PyUnicode_New(1, ordinal); + if (v == NULL) + return NULL; + PyUnicode_WRITE(PyUnicode_KIND(v), PyUnicode_DATA(v), 0, ordinal); + assert(_PyUnicode_CheckConsistency(v, 1)); + return v; } -PyObject *PyUnicode_FromObject(register PyObject *obj) +PyObject * +PyUnicode_FromObject(register PyObject *obj) { /* XXX Perhaps we should make this API an alias of PyObject_Str() instead ?! */ if (PyUnicode_CheckExact(obj)) { + if (PyUnicode_READY(obj) == -1) + return NULL; Py_INCREF(obj); return obj; } if (PyUnicode_Check(obj)) { /* For a Unicode subtype that's not a Unicode object, return a true Unicode object with the same data. */ - return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(obj), - PyUnicode_GET_SIZE(obj)); + return _PyUnicode_Copy(obj); } PyErr_Format(PyExc_TypeError, "Can't convert '%.100s' object to str implicitly", @@ -1379,9 +2989,10 @@ PyObject *PyUnicode_FromObject(register PyObject *obj) return NULL; } -PyObject *PyUnicode_FromEncodedObject(register PyObject *obj, - const char *encoding, - const char *errors) +PyObject * +PyUnicode_FromEncodedObject(register PyObject *obj, + const char *encoding, + const char *errors) { Py_buffer buffer; PyObject *v; @@ -1395,7 +3006,7 @@ PyObject *PyUnicode_FromEncodedObject(register PyObject *obj, if (PyBytes_Check(obj)) { if (PyBytes_GET_SIZE(obj) == 0) { Py_INCREF(unicode_empty); - v = (PyObject *) unicode_empty; + v = unicode_empty; } else { v = PyUnicode_Decode( @@ -1422,7 +3033,7 @@ PyObject *PyUnicode_FromEncodedObject(register PyObject *obj, if (buffer.len == 0) { Py_INCREF(unicode_empty); - v = (PyObject *) unicode_empty; + v = unicode_empty; } else v = PyUnicode_Decode((char*) buffer.buf, buffer.len, encoding, errors); @@ -1443,6 +3054,10 @@ normalize_encoding(const char *encoding, char *l; char *l_end; + if (encoding == NULL) { + strcpy(lower, "utf-8"); + return 1; + } e = encoding; l = lower; l_end = &lower[lower_len - 1]; @@ -1464,26 +3079,26 @@ normalize_encoding(const char *encoding, return 1; } -PyObject *PyUnicode_Decode(const char *s, - Py_ssize_t size, - const char *encoding, - const char *errors) +PyObject * +PyUnicode_Decode(const char *s, + Py_ssize_t size, + const char *encoding, + const char *errors) { PyObject *buffer = NULL, *unicode; Py_buffer info; char lower[11]; /* Enough for any encoding shortcut */ - if (encoding == NULL) - encoding = PyUnicode_GetDefaultEncoding(); - /* Shortcuts for common default encodings */ if (normalize_encoding(encoding, lower, sizeof(lower))) { - if (strcmp(lower, "utf-8") == 0) - return PyUnicode_DecodeUTF8(s, size, errors); + if ((strcmp(lower, "utf-8") == 0) || + (strcmp(lower, "utf8") == 0)) + return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL); else if ((strcmp(lower, "latin-1") == 0) || + (strcmp(lower, "latin1") == 0) || (strcmp(lower, "iso-8859-1") == 0)) return PyUnicode_DecodeLatin1(s, size, errors); -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#ifdef HAVE_MBCS else if (strcmp(lower, "mbcs") == 0) return PyUnicode_DecodeMBCS(s, size, errors); #endif @@ -1513,16 +3128,17 @@ PyObject *PyUnicode_Decode(const char *s, goto onError; } Py_DECREF(buffer); - return unicode; + return unicode_result(unicode); onError: Py_XDECREF(buffer); return NULL; } -PyObject *PyUnicode_AsDecodedObject(PyObject *unicode, - const char *encoding, - const char *errors) +PyObject * +PyUnicode_AsDecodedObject(PyObject *unicode, + const char *encoding, + const char *errors) { PyObject *v; @@ -1538,15 +3154,16 @@ PyObject *PyUnicode_AsDecodedObject(PyObject *unicode, v = PyCodec_Decode(unicode, encoding, errors); if (v == NULL) goto onError; - return v; + return unicode_result(v); onError: return NULL; } -PyObject *PyUnicode_AsDecodedUnicode(PyObject *unicode, - const char *encoding, - const char *errors) +PyObject * +PyUnicode_AsDecodedUnicode(PyObject *unicode, + const char *encoding, + const char *errors) { PyObject *v; @@ -1569,16 +3186,17 @@ PyObject *PyUnicode_AsDecodedUnicode(PyObject *unicode, Py_DECREF(v); goto onError; } - return v; + return unicode_result(v); onError: return NULL; } -PyObject *PyUnicode_Encode(const Py_UNICODE *s, - Py_ssize_t size, - const char *encoding, - const char *errors) +PyObject * +PyUnicode_Encode(const Py_UNICODE *s, + Py_ssize_t size, + const char *encoding, + const char *errors) { PyObject *v, *unicode; @@ -1590,9 +3208,10 @@ PyObject *PyUnicode_Encode(const Py_UNICODE *s, return v; } -PyObject *PyUnicode_AsEncodedObject(PyObject *unicode, - const char *encoding, - const char *errors) +PyObject * +PyUnicode_AsEncodedObject(PyObject *unicode, + const char *encoding, + const char *errors) { PyObject *v; @@ -1614,17 +3233,192 @@ PyObject *PyUnicode_AsEncodedObject(PyObject *unicode, return NULL; } +static size_t +wcstombs_errorpos(const wchar_t *wstr) +{ + size_t len; +#if SIZEOF_WCHAR_T == 2 + wchar_t buf[3]; +#else + wchar_t buf[2]; +#endif + char outbuf[MB_LEN_MAX]; + const wchar_t *start, *previous; + +#if SIZEOF_WCHAR_T == 2 + buf[2] = 0; +#else + buf[1] = 0; +#endif + start = wstr; + while (*wstr != L'\0') + { + previous = wstr; +#if SIZEOF_WCHAR_T == 2 + if (Py_UNICODE_IS_HIGH_SURROGATE(wstr[0]) + && Py_UNICODE_IS_LOW_SURROGATE(wstr[1])) + { + buf[0] = wstr[0]; + buf[1] = wstr[1]; + wstr += 2; + } + else { + buf[0] = *wstr; + buf[1] = 0; + wstr++; + } +#else + buf[0] = *wstr; + wstr++; +#endif + len = wcstombs(outbuf, buf, sizeof(outbuf)); + if (len == (size_t)-1) + return previous - start; + } + + /* failed to find the unencodable character */ + return 0; +} + +static int +locale_error_handler(const char *errors, int *surrogateescape) +{ + if (errors == NULL) { + *surrogateescape = 0; + return 0; + } + + if (strcmp(errors, "strict") == 0) { + *surrogateescape = 0; + return 0; + } + if (strcmp(errors, "surrogateescape") == 0) { + *surrogateescape = 1; + return 0; + } + PyErr_Format(PyExc_ValueError, + "only 'strict' and 'surrogateescape' error handlers " + "are supported, not '%s'", + errors); + return -1; +} + +PyObject * +PyUnicode_EncodeLocale(PyObject *unicode, const char *errors) +{ + Py_ssize_t wlen, wlen2; + wchar_t *wstr; + PyObject *bytes = NULL; + char *errmsg; + PyObject *reason; + PyObject *exc; + size_t error_pos; + int surrogateescape; + + if (locale_error_handler(errors, &surrogateescape) < 0) + return NULL; + + wstr = PyUnicode_AsWideCharString(unicode, &wlen); + if (wstr == NULL) + return NULL; + + wlen2 = wcslen(wstr); + if (wlen2 != wlen) { + PyMem_Free(wstr); + PyErr_SetString(PyExc_TypeError, "embedded null character"); + return NULL; + } + + if (surrogateescape) { + /* locale encoding with surrogateescape */ + char *str; + + str = _Py_wchar2char(wstr, &error_pos); + if (str == NULL) { + if (error_pos == (size_t)-1) { + PyErr_NoMemory(); + PyMem_Free(wstr); + return NULL; + } + else { + goto encode_error; + } + } + PyMem_Free(wstr); + + bytes = PyBytes_FromString(str); + PyMem_Free(str); + } + else { + size_t len, len2; + + len = wcstombs(NULL, wstr, 0); + if (len == (size_t)-1) { + error_pos = (size_t)-1; + goto encode_error; + } + + bytes = PyBytes_FromStringAndSize(NULL, len); + if (bytes == NULL) { + PyMem_Free(wstr); + return NULL; + } + + len2 = wcstombs(PyBytes_AS_STRING(bytes), wstr, len+1); + if (len2 == (size_t)-1 || len2 > len) { + error_pos = (size_t)-1; + goto encode_error; + } + PyMem_Free(wstr); + } + return bytes; + +encode_error: + errmsg = strerror(errno); + assert(errmsg != NULL); + + if (error_pos == (size_t)-1) + error_pos = wcstombs_errorpos(wstr); + + PyMem_Free(wstr); + Py_XDECREF(bytes); + + if (errmsg != NULL) { + size_t errlen; + wstr = _Py_char2wchar(errmsg, &errlen); + if (wstr != NULL) { + reason = PyUnicode_FromWideChar(wstr, errlen); + PyMem_Free(wstr); + } else + errmsg = NULL; + } + if (errmsg == NULL) + reason = PyUnicode_FromString( + "wcstombs() encountered an unencodable " + "wide character"); + if (reason == NULL) + return NULL; + + exc = PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnnO", + "locale", unicode, + (Py_ssize_t)error_pos, + (Py_ssize_t)(error_pos+1), + reason); + Py_DECREF(reason); + if (exc != NULL) { + PyCodec_StrictErrors(exc); + Py_XDECREF(exc); + } + return NULL; +} + PyObject * PyUnicode_EncodeFSDefault(PyObject *unicode) { -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) - return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - NULL); +#ifdef HAVE_MBCS + return PyUnicode_EncodeCodePage(CP_ACP, unicode, NULL); #elif defined(__APPLE__) - return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - "surrogateescape"); + return _PyUnicode_AsUTF8String(unicode, "surrogateescape"); #else PyInterpreterState *interp = PyThreadState_GET()->interp; /* Bootstrap check: if the filesystem codec is implemented in Python, we @@ -1642,46 +3436,15 @@ PyUnicode_EncodeFSDefault(PyObject *unicode) "surrogateescape"); } else { - /* locale encoding with surrogateescape */ - wchar_t *wchar; - char *bytes; - PyObject *bytes_obj; - size_t error_pos; - - wchar = PyUnicode_AsWideCharString(unicode, NULL); - if (wchar == NULL) - return NULL; - bytes = _Py_wchar2char(wchar, &error_pos); - if (bytes == NULL) { - if (error_pos != (size_t)-1) { - char *errmsg = strerror(errno); - PyObject *exc = NULL; - if (errmsg == NULL) - errmsg = "Py_wchar2char() failed"; - raise_encode_exception(&exc, - "filesystemencoding", - PyUnicode_AS_UNICODE(unicode), PyUnicode_GET_SIZE(unicode), - error_pos, error_pos+1, - errmsg); - Py_XDECREF(exc); - } - else - PyErr_NoMemory(); - PyMem_Free(wchar); - return NULL; - } - PyMem_Free(wchar); - - bytes_obj = PyBytes_FromString(bytes); - PyMem_Free(bytes); - return bytes_obj; + return PyUnicode_EncodeLocale(unicode, "surrogateescape"); } #endif } -PyObject *PyUnicode_AsEncodedString(PyObject *unicode, - const char *encoding, - const char *errors) +PyObject * +PyUnicode_AsEncodedString(PyObject *unicode, + const char *encoding, + const char *errors) { PyObject *v; char lower[11]; /* Enough for any encoding shortcut */ @@ -1691,46 +3454,27 @@ PyObject *PyUnicode_AsEncodedString(PyObject *unicode, return NULL; } - if (encoding == NULL) - encoding = PyUnicode_GetDefaultEncoding(); - /* Shortcuts for common default encodings */ if (normalize_encoding(encoding, lower, sizeof(lower))) { - if (strcmp(lower, "utf-8") == 0) - return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); + if ((strcmp(lower, "utf-8") == 0) || + (strcmp(lower, "utf8") == 0)) + { + if (errors == NULL || strcmp(errors, "strict") == 0) + return _PyUnicode_AsUTF8String(unicode, NULL); + else + return _PyUnicode_AsUTF8String(unicode, errors); + } else if ((strcmp(lower, "latin-1") == 0) || + (strcmp(lower, "latin1") == 0) || (strcmp(lower, "iso-8859-1") == 0)) - return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) + return _PyUnicode_AsLatin1String(unicode, errors); +#ifdef HAVE_MBCS else if (strcmp(lower, "mbcs") == 0) - return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); + return PyUnicode_EncodeCodePage(CP_ACP, unicode, errors); #endif else if (strcmp(lower, "ascii") == 0) - return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); - } - /* During bootstrap, we may need to find the encodings - package, to load the file system encoding, and require the - file system encoding in order to load the encodings - package. - - Break out of this dependency by assuming that the path to - the encodings module is ASCII-only. XXX could try wcstombs - instead, if the file system encoding is the locale's - encoding. */ - if (Py_FileSystemDefaultEncoding && - strcmp(encoding, Py_FileSystemDefaultEncoding) == 0 && - !PyThreadState_GET()->interp->codecs_initialized) - return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); + return _PyUnicode_AsASCIIString(unicode, errors); + } /* Encode via the codec registry */ v = PyCodec_Encode(unicode, encoding, errors); @@ -1766,9 +3510,10 @@ PyObject *PyUnicode_AsEncodedString(PyObject *unicode, return NULL; } -PyObject *PyUnicode_AsEncodedUnicode(PyObject *unicode, - const char *encoding, - const char *errors) +PyObject * +PyUnicode_AsEncodedUnicode(PyObject *unicode, + const char *encoding, + const char *errors) { PyObject *v; @@ -1797,24 +3542,151 @@ PyObject *PyUnicode_AsEncodedUnicode(PyObject *unicode, return NULL; } -PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode, - const char *errors) +static size_t +mbstowcs_errorpos(const char *str, size_t len) { - PyObject *v = ((PyUnicodeObject *)unicode)->defenc; - if (v) - return v; - if (errors != NULL) - Py_FatalError("non-NULL encoding in _PyUnicode_AsDefaultEncodedString"); - v = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - NULL); - if (!v) - return NULL; - ((PyUnicodeObject *)unicode)->defenc = v; - return v; +#ifdef HAVE_MBRTOWC + const char *start = str; + mbstate_t mbs; + size_t converted; + wchar_t ch; + + memset(&mbs, 0, sizeof mbs); + while (len) + { + converted = mbrtowc(&ch, (char*)str, len, &mbs); + if (converted == 0) + /* Reached end of string */ + break; + if (converted == (size_t)-1 || converted == (size_t)-2) { + /* Conversion error or incomplete character */ + return str - start; + } + else { + str += converted; + len -= converted; + } + } + /* failed to find the undecodable byte sequence */ + return 0; +#endif + return 0; +} + +PyObject* +PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len, + const char *errors) +{ + wchar_t smallbuf[256]; + size_t smallbuf_len = Py_ARRAY_LENGTH(smallbuf); + wchar_t *wstr; + size_t wlen, wlen2; + PyObject *unicode; + int surrogateescape; + size_t error_pos; + char *errmsg; + PyObject *reason, *exc; + + if (locale_error_handler(errors, &surrogateescape) < 0) + return NULL; + + if (str[len] != '\0' || len != strlen(str)) { + PyErr_SetString(PyExc_TypeError, "embedded null character"); + return NULL; + } + + if (surrogateescape) + { + wstr = _Py_char2wchar(str, &wlen); + if (wstr == NULL) { + if (wlen == (size_t)-1) + PyErr_NoMemory(); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + unicode = PyUnicode_FromWideChar(wstr, wlen); + PyMem_Free(wstr); + } + else { +#ifndef HAVE_BROKEN_MBSTOWCS + wlen = mbstowcs(NULL, str, 0); +#else + wlen = len; +#endif + if (wlen == (size_t)-1) + goto decode_error; + if (wlen+1 <= smallbuf_len) { + wstr = smallbuf; + } + else { + if (wlen > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) + return PyErr_NoMemory(); + + wstr = PyMem_Malloc((wlen+1) * sizeof(wchar_t)); + if (!wstr) + return PyErr_NoMemory(); + } + + /* This shouldn't fail now */ + wlen2 = mbstowcs(wstr, str, wlen+1); + if (wlen2 == (size_t)-1) { + if (wstr != smallbuf) + PyMem_Free(wstr); + goto decode_error; + } +#ifdef HAVE_BROKEN_MBSTOWCS + assert(wlen2 == wlen); +#endif + unicode = PyUnicode_FromWideChar(wstr, wlen2); + if (wstr != smallbuf) + PyMem_Free(wstr); + } + return unicode; + +decode_error: + errmsg = strerror(errno); + assert(errmsg != NULL); + + error_pos = mbstowcs_errorpos(str, len); + if (errmsg != NULL) { + size_t errlen; + wstr = _Py_char2wchar(errmsg, &errlen); + if (wstr != NULL) { + reason = PyUnicode_FromWideChar(wstr, errlen); + PyMem_Free(wstr); + } else + errmsg = NULL; + } + if (errmsg == NULL) + reason = PyUnicode_FromString( + "mbstowcs() encountered an invalid multibyte sequence"); + if (reason == NULL) + return NULL; + + exc = PyObject_CallFunction(PyExc_UnicodeDecodeError, "sy#nnO", + "locale", str, len, + (Py_ssize_t)error_pos, + (Py_ssize_t)(error_pos+1), + reason); + Py_DECREF(reason); + if (exc != NULL) { + PyCodec_StrictErrors(exc); + Py_XDECREF(exc); + } + return NULL; } PyObject* +PyUnicode_DecodeLocale(const char *str, const char *errors) +{ + Py_ssize_t size = (Py_ssize_t)strlen(str); + return PyUnicode_DecodeLocaleAndSize(str, size, errors); +} + + +PyObject* PyUnicode_DecodeFSDefault(const char *s) { Py_ssize_t size = (Py_ssize_t)strlen(s); return PyUnicode_DecodeFSDefaultAndSize(s, size); @@ -1823,10 +3695,10 @@ PyUnicode_DecodeFSDefault(const char *s) { PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) { -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#ifdef HAVE_MBCS return PyUnicode_DecodeMBCS(s, size, NULL); #elif defined(__APPLE__) - return PyUnicode_DecodeUTF8(s, size, "surrogateescape"); + return PyUnicode_DecodeUTF8Stateful(s, size, "surrogateescape", NULL); #else PyInterpreterState *interp = PyThreadState_GET()->interp; /* Bootstrap check: if the filesystem codec is implemented in Python, we @@ -1844,23 +3716,7 @@ PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) "surrogateescape"); } else { - /* locale encoding with surrogateescape */ - wchar_t *wchar; - PyObject *unicode; - size_t len; - - if (s[size] != '\0' || size != strlen(s)) { - PyErr_SetString(PyExc_TypeError, "embedded NUL character"); - return NULL; - } - - wchar = _Py_char2wchar(s, &len); - if (wchar == NULL) - return PyErr_NoMemory(); - - unicode = PyUnicode_FromWideChar(wchar, len); - PyMem_Free(wchar); - return unicode; + return PyUnicode_DecodeLocaleAndSize(s, size, "surrogateescape"); } #endif } @@ -1923,13 +3779,13 @@ int PyUnicode_FSDecoder(PyObject* arg, void* addr) { PyObject *output = NULL; - Py_ssize_t size; - void *data; if (arg == NULL) { Py_DECREF(*(PyObject**)addr); return 1; } if (PyUnicode_Check(arg)) { + if (PyUnicode_READY(arg) == -1) + return 0; output = arg; Py_INCREF(output); } @@ -1948,9 +3804,12 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) return 0; } } - size = PyUnicode_GET_SIZE(output); - data = PyUnicode_AS_UNICODE(output); - if (size != Py_UNICODE_strlen(data)) { + if (PyUnicode_READY(output) == -1) { + Py_DECREF(output); + return 0; + } + if (findchar(PyUnicode_DATA(output), PyUnicode_KIND(output), + PyUnicode_GET_LENGTH(output), 0, 1) >= 0) { PyErr_SetString(PyExc_TypeError, "embedded NUL character"); Py_DECREF(output); return 0; @@ -1961,40 +3820,166 @@ PyUnicode_FSDecoder(PyObject* arg, void* addr) char* -_PyUnicode_AsStringAndSize(PyObject *unicode, Py_ssize_t *psize) +PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { PyObject *bytes; + if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - bytes = _PyUnicode_AsDefaultEncodedString(unicode, NULL); - if (bytes == NULL) + if (PyUnicode_READY(unicode) == -1) return NULL; - if (psize != NULL) - *psize = PyBytes_GET_SIZE(bytes); - return PyBytes_AS_STRING(bytes); + + if (PyUnicode_UTF8(unicode) == NULL) { + assert(!PyUnicode_IS_COMPACT_ASCII(unicode)); + bytes = _PyUnicode_AsUTF8String(unicode, "strict"); + if (bytes == NULL) + return NULL; + _PyUnicode_UTF8(unicode) = PyObject_MALLOC(PyBytes_GET_SIZE(bytes) + 1); + if (_PyUnicode_UTF8(unicode) == NULL) { + Py_DECREF(bytes); + return NULL; + } + _PyUnicode_UTF8_LENGTH(unicode) = PyBytes_GET_SIZE(bytes); + Py_MEMCPY(_PyUnicode_UTF8(unicode), + PyBytes_AS_STRING(bytes), + _PyUnicode_UTF8_LENGTH(unicode) + 1); + Py_DECREF(bytes); + } + + if (psize) + *psize = PyUnicode_UTF8_LENGTH(unicode); + return PyUnicode_UTF8(unicode); } char* -_PyUnicode_AsString(PyObject *unicode) +PyUnicode_AsUTF8(PyObject *unicode) { - return _PyUnicode_AsStringAndSize(unicode, NULL); + return PyUnicode_AsUTF8AndSize(unicode, NULL); } -Py_UNICODE *PyUnicode_AsUnicode(PyObject *unicode) +Py_UNICODE * +PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) { + const unsigned char *one_byte; +#if SIZEOF_WCHAR_T == 4 + const Py_UCS2 *two_bytes; +#else + const Py_UCS4 *four_bytes; + const Py_UCS4 *ucs4_end; + Py_ssize_t num_surrogates; +#endif + wchar_t *w; + wchar_t *wchar_end; + if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); - goto onError; + return NULL; } - return PyUnicode_AS_UNICODE(unicode); + if (_PyUnicode_WSTR(unicode) == NULL) { + /* Non-ASCII compact unicode object */ + assert(_PyUnicode_KIND(unicode) != 0); + assert(PyUnicode_IS_READY(unicode)); - onError: - return NULL; + if (PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND) { +#if SIZEOF_WCHAR_T == 2 + four_bytes = PyUnicode_4BYTE_DATA(unicode); + ucs4_end = four_bytes + _PyUnicode_LENGTH(unicode); + num_surrogates = 0; + + for (; four_bytes < ucs4_end; ++four_bytes) { + if (*four_bytes > 0xFFFF) + ++num_surrogates; + } + + _PyUnicode_WSTR(unicode) = (wchar_t *) PyObject_MALLOC( + sizeof(wchar_t) * (_PyUnicode_LENGTH(unicode) + 1 + num_surrogates)); + if (!_PyUnicode_WSTR(unicode)) { + PyErr_NoMemory(); + return NULL; + } + _PyUnicode_WSTR_LENGTH(unicode) = _PyUnicode_LENGTH(unicode) + num_surrogates; + + w = _PyUnicode_WSTR(unicode); + wchar_end = w + _PyUnicode_WSTR_LENGTH(unicode); + four_bytes = PyUnicode_4BYTE_DATA(unicode); + for (; four_bytes < ucs4_end; ++four_bytes, ++w) { + if (*four_bytes > 0xFFFF) { + assert(*four_bytes <= MAX_UNICODE); + /* encode surrogate pair in this case */ + *w++ = Py_UNICODE_HIGH_SURROGATE(*four_bytes); + *w = Py_UNICODE_LOW_SURROGATE(*four_bytes); + } + else + *w = *four_bytes; + + if (w > wchar_end) { + assert(0 && "Miscalculated string end"); + } + } + *w = 0; +#else + /* sizeof(wchar_t) == 4 */ + Py_FatalError("Impossible unicode object state, wstr and str " + "should share memory already."); + return NULL; +#endif + } + else { + _PyUnicode_WSTR(unicode) = (wchar_t *) PyObject_MALLOC(sizeof(wchar_t) * + (_PyUnicode_LENGTH(unicode) + 1)); + if (!_PyUnicode_WSTR(unicode)) { + PyErr_NoMemory(); + return NULL; + } + if (!PyUnicode_IS_COMPACT_ASCII(unicode)) + _PyUnicode_WSTR_LENGTH(unicode) = _PyUnicode_LENGTH(unicode); + w = _PyUnicode_WSTR(unicode); + wchar_end = w + _PyUnicode_LENGTH(unicode); + + if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND) { + one_byte = PyUnicode_1BYTE_DATA(unicode); + for (; w < wchar_end; ++one_byte, ++w) + *w = *one_byte; + /* null-terminate the wstr */ + *w = 0; + } + else if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) { +#if SIZEOF_WCHAR_T == 4 + two_bytes = PyUnicode_2BYTE_DATA(unicode); + for (; w < wchar_end; ++two_bytes, ++w) + *w = *two_bytes; + /* null-terminate the wstr */ + *w = 0; +#else + /* sizeof(wchar_t) == 2 */ + PyObject_FREE(_PyUnicode_WSTR(unicode)); + _PyUnicode_WSTR(unicode) = NULL; + Py_FatalError("Impossible unicode object state, wstr " + "and str should share memory already."); + return NULL; +#endif + } + else { + assert(0 && "This should never happen."); + } + } + } + if (size != NULL) + *size = PyUnicode_WSTR_LENGTH(unicode); + return _PyUnicode_WSTR(unicode); +} + +Py_UNICODE * +PyUnicode_AsUnicode(PyObject *unicode) +{ + return PyUnicode_AsUnicodeAndSize(unicode, NULL); } -Py_ssize_t PyUnicode_GetSize(PyObject *unicode) + +Py_ssize_t +PyUnicode_GetSize(PyObject *unicode) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); @@ -2006,7 +3991,57 @@ Py_ssize_t PyUnicode_GetSize(PyObject *unicode) return -1; } -const char *PyUnicode_GetDefaultEncoding(void) +Py_ssize_t +PyUnicode_GetLength(PyObject *unicode) +{ + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return -1; + } + if (PyUnicode_READY(unicode) == -1) + return -1; + return PyUnicode_GET_LENGTH(unicode); +} + +Py_UCS4 +PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index) +{ + if (!PyUnicode_Check(unicode) || PyUnicode_READY(unicode) == -1) { + PyErr_BadArgument(); + return (Py_UCS4)-1; + } + if (index < 0 || index >= PyUnicode_GET_LENGTH(unicode)) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return (Py_UCS4)-1; + } + return PyUnicode_READ_CHAR(unicode, index); +} + +int +PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, Py_UCS4 ch) +{ + if (!PyUnicode_Check(unicode) || !PyUnicode_IS_COMPACT(unicode)) { + PyErr_BadArgument(); + return -1; + } + assert(PyUnicode_IS_READY(unicode)); + if (index < 0 || index >= PyUnicode_GET_LENGTH(unicode)) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } + if (unicode_check_modifiable(unicode)) + return -1; + if (ch > PyUnicode_MAX_CHAR_VALUE(unicode)) { + PyErr_SetString(PyExc_ValueError, "character out of range"); + return -1; + } + PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode), + index, ch); + return 0; +} + +const char * +PyUnicode_GetDefaultEncoding(void) { return "utf-8"; } @@ -2045,26 +4080,29 @@ onError: return 0 on success, -1 on error */ -static -int unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler, - const char *encoding, const char *reason, - const char **input, const char **inend, Py_ssize_t *startinpos, - Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, - PyUnicodeObject **output, Py_ssize_t *outpos, Py_UNICODE **outptr) +static int +unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler, + const char *encoding, const char *reason, + const char **input, const char **inend, Py_ssize_t *startinpos, + Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, + PyObject **output, Py_ssize_t *outpos) { static char *argparse = "O!n;decoding error handler must return (str, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; - Py_ssize_t outsize = PyUnicode_GET_SIZE(*output); + Py_ssize_t outsize; Py_ssize_t insize; Py_ssize_t requiredsize; Py_ssize_t newpos; - Py_UNICODE *repptr; PyObject *inputobj = NULL; - Py_ssize_t repsize; int res = -1; + if (_PyUnicode_KIND(*output) != PyUnicode_WCHAR_KIND) + outsize = PyUnicode_GET_LENGTH(*output); + else + outsize = _PyUnicode_WSTR_LENGTH(*output); + if (*errorHandler == NULL) { *errorHandler = PyCodec_LookupError(errors); if (*errorHandler == NULL) @@ -2088,6 +4126,8 @@ int unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler } if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos)) goto onError; + if (PyUnicode_READY(repunicode) == -1) + goto onError; /* Copy back the bytes variables, which might have been modified by the callback */ @@ -2111,25 +4151,47 @@ int unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler goto onError; } - /* need more space? (at least enough for what we - have+the replacement+the rest of the string (starting - at the new input position), so we won't have to check space - when there are no errors in the rest of the string) */ - repptr = PyUnicode_AS_UNICODE(repunicode); - repsize = PyUnicode_GET_SIZE(repunicode); - requiredsize = *outpos + repsize + insize-newpos; - if (requiredsize > outsize) { - if (requiredsize<2*outsize) - requiredsize = 2*outsize; - if (_PyUnicode_Resize(output, requiredsize) < 0) + if (_PyUnicode_KIND(*output) != PyUnicode_WCHAR_KIND) { + /* need more space? (at least enough for what we + have+the replacement+the rest of the string (starting + at the new input position), so we won't have to check space + when there are no errors in the rest of the string) */ + Py_ssize_t replen = PyUnicode_GET_LENGTH(repunicode); + requiredsize = *outpos + replen + insize-newpos; + if (requiredsize > outsize) { + if (requiredsize<2*outsize) + requiredsize = 2*outsize; + if (unicode_resize(output, requiredsize) < 0) + goto onError; + } + if (unicode_widen(output, *outpos, + PyUnicode_MAX_CHAR_VALUE(repunicode)) < 0) goto onError; - *outptr = PyUnicode_AS_UNICODE(*output) + *outpos; + _PyUnicode_FastCopyCharacters(*output, *outpos, repunicode, 0, replen); + *outpos += replen; + } + else { + wchar_t *repwstr; + Py_ssize_t repwlen; + repwstr = PyUnicode_AsUnicodeAndSize(repunicode, &repwlen); + if (repwstr == NULL) + goto onError; + /* need more space? (at least enough for what we + have+the replacement+the rest of the string (starting + at the new input position), so we won't have to check space + when there are no errors in the rest of the string) */ + requiredsize = *outpos + repwlen + insize-newpos; + if (requiredsize > outsize) { + if (requiredsize < 2*outsize) + requiredsize = 2*outsize; + if (unicode_resize(output, requiredsize) < 0) + goto onError; + } + wcsncpy(_PyUnicode_WSTR(*output) + *outpos, repwstr, repwlen); + *outpos += repwlen; } *endinpos = newpos; *inptr = *input + newpos; - Py_UNICODE_COPY(*outptr, repptr, repsize); - *outptr += repsize; - *outpos += repsize; /* we made it! */ res = 0; @@ -2220,9 +4282,10 @@ char utf7_category[128] = { (directWS && (utf7_category[(c)] == 2)) || \ (directO && (utf7_category[(c)] == 1)))) -PyObject *PyUnicode_DecodeUTF7(const char *s, - Py_ssize_t size, - const char *errors) +PyObject * +PyUnicode_DecodeUTF7(const char *s, + Py_ssize_t size, + const char *errors) { return PyUnicode_DecodeUTF7Stateful(s, size, errors, NULL); } @@ -2234,42 +4297,42 @@ PyObject *PyUnicode_DecodeUTF7(const char *s, * all the shift state (seen bits, number of bits seen, high * surrogate). */ -PyObject *PyUnicode_DecodeUTF7Stateful(const char *s, - Py_ssize_t size, - const char *errors, - Py_ssize_t *consumed) +PyObject * +PyUnicode_DecodeUTF7Stateful(const char *s, + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) { const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; Py_ssize_t outpos; const char *e; - PyUnicodeObject *unicode; - Py_UNICODE *p; + PyObject *unicode; const char *errmsg = ""; int inShift = 0; - Py_UNICODE *shiftOutStart; + Py_ssize_t shiftOutStart; unsigned int base64bits = 0; unsigned long base64buffer = 0; - Py_UNICODE surrogate = 0; + Py_UCS4 surrogate = 0; PyObject *errorHandler = NULL; PyObject *exc = NULL; - unicode = _PyUnicode_New(size); + /* Start off assuming it's all ASCII. Widen later as necessary. */ + unicode = PyUnicode_New(size, 127); if (!unicode) return NULL; if (size == 0) { if (consumed) *consumed = 0; - return (PyObject *)unicode; + return unicode; } - p = unicode->str; - shiftOutStart = p; + shiftOutStart = outpos = 0; e = s + size; while (s < e) { - Py_UNICODE ch; + Py_UCS4 ch; restart: ch = (unsigned char) *s; @@ -2280,34 +4343,31 @@ PyObject *PyUnicode_DecodeUTF7Stateful(const char *s, s++; if (base64bits >= 16) { /* we have enough bits for a UTF-16 value */ - Py_UNICODE outCh = (Py_UNICODE) - (base64buffer >> (base64bits-16)); + Py_UCS4 outCh = (Py_UCS4)(base64buffer >> (base64bits-16)); base64bits -= 16; base64buffer &= (1 << base64bits) - 1; /* clear high bits */ if (surrogate) { /* expecting a second surrogate */ - if (outCh >= 0xDC00 && outCh <= 0xDFFF) { -#ifdef Py_UNICODE_WIDE - *p++ = (((surrogate & 0x3FF)<<10) - | (outCh & 0x3FF)) + 0x10000; -#else - *p++ = surrogate; - *p++ = outCh; -#endif + if (Py_UNICODE_IS_LOW_SURROGATE(outCh)) { + Py_UCS4 ch2 = Py_UNICODE_JOIN_SURROGATES(surrogate, outCh); + if (unicode_putchar(&unicode, &outpos, ch2) < 0) + goto onError; surrogate = 0; continue; } else { - *p++ = surrogate; + if (unicode_putchar(&unicode, &outpos, surrogate) < 0) + goto onError; surrogate = 0; } } - if (outCh >= 0xD800 && outCh <= 0xDBFF) { + if (Py_UNICODE_IS_HIGH_SURROGATE(outCh)) { /* first surrogate */ surrogate = outCh; } else { - *p++ = outCh; + if (unicode_putchar(&unicode, &outpos, outCh) < 0) + goto onError; } } } @@ -2315,7 +4375,8 @@ PyObject *PyUnicode_DecodeUTF7Stateful(const char *s, inShift = 0; s++; if (surrogate) { - *p++ = surrogate; + if (unicode_putchar(&unicode, &outpos, surrogate) < 0) + goto onError; surrogate = 0; } if (base64bits > 0) { /* left-over bits */ @@ -2335,7 +4396,8 @@ PyObject *PyUnicode_DecodeUTF7Stateful(const char *s, if (ch != '-') { /* '-' is absorbed; other terminating characters are preserved */ - *p++ = ch; + if (unicode_putchar(&unicode, &outpos, ch) < 0) + goto onError; } } } @@ -2344,16 +4406,18 @@ PyObject *PyUnicode_DecodeUTF7Stateful(const char *s, s++; /* consume '+' */ if (s < e && *s == '-') { /* '+-' encodes '+' */ s++; - *p++ = '+'; + if (unicode_putchar(&unicode, &outpos, '+') < 0) + goto onError; } else { /* begin base64-encoded section */ inShift = 1; - shiftOutStart = p; + shiftOutStart = outpos; base64bits = 0; } } else if (DECODE_DIRECT(ch)) { /* character decodes as itself */ - *p++ = ch; + if (unicode_putchar(&unicode, &outpos, ch) < 0) + goto onError; s++; } else { @@ -2364,13 +4428,12 @@ PyObject *PyUnicode_DecodeUTF7Stateful(const char *s, } continue; utf7Error: - outpos = p-PyUnicode_AS_UNICODE(unicode); endinpos = s-starts; if (unicode_decode_call_errorhandler( errors, &errorHandler, "utf7", errmsg, &starts, &e, &startinpos, &endinpos, &exc, &s, - &unicode, &outpos, &p)) + &unicode, &outpos)) goto onError; } @@ -2381,13 +4444,12 @@ utf7Error: if (surrogate || (base64bits >= 6) || (base64bits > 0 && base64buffer != 0)) { - outpos = p-PyUnicode_AS_UNICODE(unicode); endinpos = size; if (unicode_decode_call_errorhandler( errors, &errorHandler, "utf7", "unterminated shift sequence", &starts, &e, &startinpos, &endinpos, &exc, &s, - &unicode, &outpos, &p)) + &unicode, &outpos)) goto onError; if (s < e) goto restart; @@ -2397,7 +4459,7 @@ utf7Error: /* return state */ if (consumed) { if (inShift) { - p = shiftOutStart; /* back off output */ + outpos = shiftOutStart; /* back off output */ *consumed = startinpos; } else { @@ -2405,12 +4467,12 @@ utf7Error: } } - if (_PyUnicode_Resize(&unicode, p - PyUnicode_AS_UNICODE(unicode)) < 0) + if (unicode_resize(&unicode, outpos) < 0) goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); - return (PyObject *)unicode; + return unicode_result(unicode); onError: Py_XDECREF(errorHandler); @@ -2420,35 +4482,42 @@ utf7Error: } -PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s, - Py_ssize_t size, - int base64SetO, - int base64WhiteSpace, - const char *errors) +PyObject * +_PyUnicode_EncodeUTF7(PyObject *str, + int base64SetO, + int base64WhiteSpace, + const char *errors) { + int kind; + void *data; + Py_ssize_t len; PyObject *v; - /* It might be possible to tighten this worst case */ - Py_ssize_t allocated = 8 * size; int inShift = 0; - Py_ssize_t i = 0; + Py_ssize_t i; unsigned int base64bits = 0; unsigned long base64buffer = 0; char * out; char * start; - if (size == 0) + if (PyUnicode_READY(str) == -1) + return NULL; + kind = PyUnicode_KIND(str); + data = PyUnicode_DATA(str); + len = PyUnicode_GET_LENGTH(str); + + if (len == 0) return PyBytes_FromStringAndSize(NULL, 0); - if (allocated / 8 != size) + /* It might be possible to tighten this worst case */ + if (len > PY_SSIZE_T_MAX / 8) return PyErr_NoMemory(); - - v = PyBytes_FromStringAndSize(NULL, allocated); + v = PyBytes_FromStringAndSize(NULL, len * 8); if (v == NULL) return NULL; start = out = PyBytes_AS_STRING(v); - for (;i < size; ++i) { - Py_UNICODE ch = s[i]; + for (i = 0; i < len; ++i) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (inShift) { if (ENCODE_DIRECT(ch, !base64SetO, !base64WhiteSpace)) { @@ -2486,8 +4555,9 @@ PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s, } continue; encode_char: -#ifdef Py_UNICODE_WIDE if (ch >= 0x10000) { + assert(ch <= MAX_UNICODE); + /* code first surrogate */ base64bits += 16; base64buffer = (base64buffer << 16) | 0xd800 | ((ch-0x10000) >> 10); @@ -2496,9 +4566,8 @@ encode_char: base64bits -= 6; } /* prepare second surrogate */ - ch = 0xDC00 | ((ch-0x10000) & 0x3FF); + ch = Py_UNICODE_LOW_SURROGATE(ch); } -#endif base64bits += 16; base64buffer = (base64buffer << 16) | ch; while (base64bits >= 6) { @@ -2514,6 +4583,22 @@ encode_char: return NULL; return v; } +PyObject * +PyUnicode_EncodeUTF7(const Py_UNICODE *s, + Py_ssize_t size, + int base64SetO, + int base64WhiteSpace, + const char *errors) +{ + PyObject *result; + PyObject *tmp = PyUnicode_FromUnicode(s, size); + if (tmp == NULL) + return NULL; + result = _PyUnicode_EncodeUTF7(tmp, base64SetO, + base64WhiteSpace, errors); + Py_DECREF(tmp); + return result; +} #undef IS_BASE64 #undef FROM_BASE64 @@ -2523,272 +4608,206 @@ encode_char: /* --- UTF-8 Codec -------------------------------------------------------- */ -static -char utf8_code_length[256] = { - /* Map UTF-8 encoded prefix byte to sequence length. Zero means - illegal prefix. See RFC 3629 for details */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00-0F */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 70-7F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 80-8F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* B0-BF */ - 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* C0-C1 + C2-CF */ - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* D0-DF */ - 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* E0-EF */ - 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* F0-F4 + F5-FF */ -}; - -PyObject *PyUnicode_DecodeUTF8(const char *s, - Py_ssize_t size, - const char *errors) +PyObject * +PyUnicode_DecodeUTF8(const char *s, + Py_ssize_t size, + const char *errors) { return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL); } -/* Mask to check or force alignment of a pointer to C 'long' boundaries */ -#define LONG_PTR_MASK (size_t) (SIZEOF_LONG - 1) +#include "stringlib/asciilib.h" +#include "stringlib/codecs.h" +#include "stringlib/undef.h" + +#include "stringlib/ucs1lib.h" +#include "stringlib/codecs.h" +#include "stringlib/undef.h" + +#include "stringlib/ucs2lib.h" +#include "stringlib/codecs.h" +#include "stringlib/undef.h" + +#include "stringlib/ucs4lib.h" +#include "stringlib/codecs.h" +#include "stringlib/undef.h" /* Mask to quickly check whether a C 'long' contains a non-ASCII, UTF8-encoded char. */ #if (SIZEOF_LONG == 8) -# define ASCII_CHAR_MASK 0x8080808080808080L +# define ASCII_CHAR_MASK 0x8080808080808080UL #elif (SIZEOF_LONG == 4) -# define ASCII_CHAR_MASK 0x80808080L +# define ASCII_CHAR_MASK 0x80808080UL #else # error C 'long' size should be either 4 or 8! #endif -PyObject *PyUnicode_DecodeUTF8Stateful(const char *s, - Py_ssize_t size, - const char *errors, - Py_ssize_t *consumed) +static Py_ssize_t +ascii_decode(const char *start, const char *end, Py_UCS1 *dest) +{ + const char *p = start; + const char *aligned_end = (const char *) _Py_ALIGN_DOWN(end, SIZEOF_LONG); + +#if SIZEOF_LONG <= SIZEOF_VOID_P + assert(_Py_IS_ALIGNED(dest, SIZEOF_LONG)); + if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) { + /* Fast path, see in STRINGLIB(utf8_decode) for + an explanation. */ + /* Help register allocation */ + register const char *_p = p; + register Py_UCS1 * q = dest; + while (_p < aligned_end) { + unsigned long value = *(const unsigned long *) _p; + if (value & ASCII_CHAR_MASK) + break; + *((unsigned long *)q) = value; + _p += SIZEOF_LONG; + q += SIZEOF_LONG; + } + p = _p; + while (p < end) { + if ((unsigned char)*p & 0x80) + break; + *q++ = *p++; + } + return p - start; + } +#endif + while (p < end) { + /* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h + for an explanation. */ + if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) { + /* Help register allocation */ + register const char *_p = p; + while (_p < aligned_end) { + unsigned long value = *(unsigned long *) _p; + if (value & ASCII_CHAR_MASK) + break; + _p += SIZEOF_LONG; + } + p = _p; + if (_p == end) + break; + } + if ((unsigned char)*p & 0x80) + break; + ++p; + } + memcpy(dest, start, p - start); + return p - start; +} + +PyObject * +PyUnicode_DecodeUTF8Stateful(const char *s, + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) { + PyObject *unicode; const char *starts = s; - int n; - int k; + const char *end = s + size; + Py_ssize_t outpos; + Py_ssize_t startinpos; Py_ssize_t endinpos; - Py_ssize_t outpos; - const char *e, *aligned_end; - PyUnicodeObject *unicode; - Py_UNICODE *p; const char *errmsg = ""; PyObject *errorHandler = NULL; PyObject *exc = NULL; - /* Note: size will always be longer than the resulting Unicode - character count */ - unicode = _PyUnicode_New(size); - if (!unicode) - return NULL; if (size == 0) { if (consumed) *consumed = 0; - return (PyObject *)unicode; + Py_INCREF(unicode_empty); + return unicode_empty; } - /* Unpack UTF-8 encoded data */ - p = unicode->str; - e = s + size; - aligned_end = (const char *) ((size_t) e & ~LONG_PTR_MASK); - - while (s < e) { - Py_UCS4 ch = (unsigned char)*s; - - if (ch < 0x80) { - /* Fast path for runs of ASCII characters. Given that common UTF-8 - input will consist of an overwhelming majority of ASCII - characters, we try to optimize for this case by checking - as many characters as a C 'long' can contain. - First, check if we can do an aligned read, as most CPUs have - a penalty for unaligned reads. - */ - if (!((size_t) s & LONG_PTR_MASK)) { - /* Help register allocation */ - register const char *_s = s; - register Py_UNICODE *_p = p; - while (_s < aligned_end) { - /* Read a whole long at a time (either 4 or 8 bytes), - and do a fast unrolled copy if it only contains ASCII - characters. */ - unsigned long data = *(unsigned long *) _s; - if (data & ASCII_CHAR_MASK) - break; - _p[0] = (unsigned char) _s[0]; - _p[1] = (unsigned char) _s[1]; - _p[2] = (unsigned char) _s[2]; - _p[3] = (unsigned char) _s[3]; -#if (SIZEOF_LONG == 8) - _p[4] = (unsigned char) _s[4]; - _p[5] = (unsigned char) _s[5]; - _p[6] = (unsigned char) _s[6]; - _p[7] = (unsigned char) _s[7]; -#endif - _s += SIZEOF_LONG; - _p += SIZEOF_LONG; - } - s = _s; - p = _p; - if (s == e) - break; - ch = (unsigned char)*s; - } - } - - if (ch < 0x80) { - *p++ = (Py_UNICODE)ch; - s++; - continue; - } + /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if (size == 1 && (unsigned char)s[0] < 128) { + if (consumed) + *consumed = 1; + return get_latin1_char((unsigned char)s[0]); + } - n = utf8_code_length[ch]; + unicode = PyUnicode_New(size, 127); + if (!unicode) + return NULL; - if (s + n > e) { - if (consumed) - break; - else { - errmsg = "unexpected end of data"; - startinpos = s-starts; - endinpos = startinpos+1; - for (k=1; (k < size-startinpos) && ((s[k]&0xC0) == 0x80); k++) - endinpos++; - goto utf8Error; - } + outpos = ascii_decode(s, end, PyUnicode_1BYTE_DATA(unicode)); + s += outpos; + while (s < end) { + Py_UCS4 ch; + int kind = PyUnicode_KIND(unicode); + if (kind == PyUnicode_1BYTE_KIND) { + if (PyUnicode_IS_ASCII(unicode)) + ch = asciilib_utf8_decode(&s, end, + PyUnicode_1BYTE_DATA(unicode), &outpos); + else + ch = ucs1lib_utf8_decode(&s, end, + PyUnicode_1BYTE_DATA(unicode), &outpos); + } else if (kind == PyUnicode_2BYTE_KIND) { + ch = ucs2lib_utf8_decode(&s, end, + PyUnicode_2BYTE_DATA(unicode), &outpos); + } else { + assert(kind == PyUnicode_4BYTE_KIND); + ch = ucs4lib_utf8_decode(&s, end, + PyUnicode_4BYTE_DATA(unicode), &outpos); } - switch (n) { - + switch (ch) { case 0: - errmsg = "invalid start byte"; - startinpos = s-starts; - endinpos = startinpos+1; - goto utf8Error; - - case 1: - errmsg = "internal error"; - startinpos = s-starts; - endinpos = startinpos+1; - goto utf8Error; - - case 2: - if ((s[1] & 0xc0) != 0x80) { - errmsg = "invalid continuation byte"; - startinpos = s-starts; - endinpos = startinpos + 1; - goto utf8Error; - } - ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f); - assert ((ch > 0x007F) && (ch <= 0x07FF)); - *p++ = (Py_UNICODE)ch; + if (s == end || consumed) + goto End; + errmsg = "unexpected end of data"; + startinpos = s - starts; + endinpos = startinpos + 1; + while (endinpos < size && (starts[endinpos] & 0xC0) == 0x80) + endinpos++; break; - - case 3: - /* Decoding UTF-8 sequences in range \xed\xa0\x80-\xed\xbf\xbf - will result in surrogates in range d800-dfff. Surrogates are - not valid UTF-8 so they are rejected. - See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf - (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */ - if ((s[1] & 0xc0) != 0x80 || - (s[2] & 0xc0) != 0x80 || - ((unsigned char)s[0] == 0xE0 && - (unsigned char)s[1] < 0xA0) || - ((unsigned char)s[0] == 0xED && - (unsigned char)s[1] > 0x9F)) { - errmsg = "invalid continuation byte"; - startinpos = s-starts; - endinpos = startinpos + 1; - - /* if s[1] first two bits are 1 and 0, then the invalid - continuation byte is s[2], so increment endinpos by 1, - if not, s[1] is invalid and endinpos doesn't need to - be incremented. */ - if ((s[1] & 0xC0) == 0x80) - endinpos++; - goto utf8Error; - } - ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f); - assert ((ch > 0x07FF) && (ch <= 0xFFFF)); - *p++ = (Py_UNICODE)ch; + case 1: + errmsg = "invalid start byte"; + startinpos = s - starts; + endinpos = startinpos + 1; break; - - case 4: - if ((s[1] & 0xc0) != 0x80 || - (s[2] & 0xc0) != 0x80 || - (s[3] & 0xc0) != 0x80 || - ((unsigned char)s[0] == 0xF0 && - (unsigned char)s[1] < 0x90) || - ((unsigned char)s[0] == 0xF4 && - (unsigned char)s[1] > 0x8F)) { - errmsg = "invalid continuation byte"; - startinpos = s-starts; - endinpos = startinpos + 1; - if ((s[1] & 0xC0) == 0x80) { - endinpos++; - if ((s[2] & 0xC0) == 0x80) - endinpos++; - } - goto utf8Error; - } - ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) + - ((s[2] & 0x3f) << 6) + (s[3] & 0x3f); - assert ((ch > 0xFFFF) && (ch <= 0x10ffff)); - -#ifdef Py_UNICODE_WIDE - *p++ = (Py_UNICODE)ch; -#else - /* compute and append the two surrogates: */ - - /* translate from 10000..10FFFF to 0..FFFF */ - ch -= 0x10000; - - /* high surrogate = top 10 bits added to D800 */ - *p++ = (Py_UNICODE)(0xD800 + (ch >> 10)); - - /* low surrogate = bottom 10 bits added to DC00 */ - *p++ = (Py_UNICODE)(0xDC00 + (ch & 0x03FF)); -#endif + case 2: + errmsg = "invalid continuation byte"; + startinpos = s - starts; + endinpos = startinpos + 1; + while (endinpos < size && (starts[endinpos] & 0xC0) == 0x80) + endinpos++; break; + default: + if (unicode_putchar(&unicode, &outpos, ch) < 0) + goto onError; + continue; } - s += n; - continue; - utf8Error: - outpos = p-PyUnicode_AS_UNICODE(unicode); if (unicode_decode_call_errorhandler( errors, &errorHandler, "utf-8", errmsg, - &starts, &e, &startinpos, &endinpos, &exc, &s, - &unicode, &outpos, &p)) + &starts, &end, &startinpos, &endinpos, &exc, &s, + &unicode, &outpos)) goto onError; - aligned_end = (const char *) ((size_t) e & ~LONG_PTR_MASK); } - if (consumed) - *consumed = s-starts; - /* Adjust length */ - if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0) +End: + if (unicode_resize(&unicode, outpos) < 0) goto onError; + if (consumed) + *consumed = s - starts; + Py_XDECREF(errorHandler); Py_XDECREF(exc); - return (PyObject *)unicode; + assert(_PyUnicode_CheckConsistency(unicode, 1)); + return unicode; - onError: +onError: Py_XDECREF(errorHandler); Py_XDECREF(exc); - Py_DECREF(unicode); + Py_XDECREF(unicode); return NULL; } -#undef ASCII_CHAR_MASK - #ifdef __APPLE__ /* Simplified UTF-8 decoder using surrogateescape error handler, @@ -2797,9 +4816,9 @@ PyObject *PyUnicode_DecodeUTF8Stateful(const char *s, wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size) { - int n; const char *e; - wchar_t *unicode, *p; + wchar_t *unicode; + Py_ssize_t outpos; /* Note: size will always be longer than the resulting Unicode character count */ @@ -2812,277 +4831,101 @@ _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size) return NULL; /* Unpack UTF-8 encoded data */ - p = unicode; e = s + size; + outpos = 0; while (s < e) { - Py_UCS4 ch = (unsigned char)*s; - - if (ch < 0x80) { - *p++ = (wchar_t)ch; - s++; - continue; - } - - n = utf8_code_length[ch]; - if (s + n > e) { - goto surrogateescape; - } - - switch (n) { - case 0: - case 1: - goto surrogateescape; - - case 2: - if ((s[1] & 0xc0) != 0x80) - goto surrogateescape; - ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f); - assert ((ch > 0x007F) && (ch <= 0x07FF)); - *p++ = (wchar_t)ch; - break; - - case 3: - /* Decoding UTF-8 sequences in range \xed\xa0\x80-\xed\xbf\xbf - will result in surrogates in range d800-dfff. Surrogates are - not valid UTF-8 so they are rejected. - See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf - (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */ - if ((s[1] & 0xc0) != 0x80 || - (s[2] & 0xc0) != 0x80 || - ((unsigned char)s[0] == 0xE0 && - (unsigned char)s[1] < 0xA0) || - ((unsigned char)s[0] == 0xED && - (unsigned char)s[1] > 0x9F)) { - - goto surrogateescape; - } - ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f); - assert ((ch > 0x07FF) && (ch <= 0xFFFF)); - *p++ = (Py_UNICODE)ch; - break; - - case 4: - if ((s[1] & 0xc0) != 0x80 || - (s[2] & 0xc0) != 0x80 || - (s[3] & 0xc0) != 0x80 || - ((unsigned char)s[0] == 0xF0 && - (unsigned char)s[1] < 0x90) || - ((unsigned char)s[0] == 0xF4 && - (unsigned char)s[1] > 0x8F)) { - goto surrogateescape; - } - ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) + - ((s[2] & 0x3f) << 6) + (s[3] & 0x3f); - assert ((ch > 0xFFFF) && (ch <= 0x10ffff)); - + Py_UCS4 ch; +#if SIZEOF_WCHAR_T == 4 + ch = ucs4lib_utf8_decode(&s, e, (Py_UCS4 *)unicode, &outpos); +#else + ch = ucs2lib_utf8_decode(&s, e, (Py_UCS2 *)unicode, &outpos); +#endif + if (ch > 0xFF) { #if SIZEOF_WCHAR_T == 4 - *p++ = (wchar_t)ch; + assert(0); #else + assert(Py_UNICODE_IS_SURROGATE(ch)); /* compute and append the two surrogates: */ - - /* translate from 10000..10FFFF to 0..FFFF */ - ch -= 0x10000; - - /* high surrogate = top 10 bits added to D800 */ - *p++ = (wchar_t)(0xD800 + (ch >> 10)); - - /* low surrogate = bottom 10 bits added to DC00 */ - *p++ = (wchar_t)(0xDC00 + (ch & 0x03FF)); + unicode[outpos++] = (wchar_t)Py_UNICODE_HIGH_SURROGATE(ch); + unicode[outpos++] = (wchar_t)Py_UNICODE_LOW_SURROGATE(ch); #endif - break; } - s += n; - continue; - - surrogateescape: - *p++ = 0xDC00 + ch; - s++; + else { + if (!ch && s == e) + break; + /* surrogateescape */ + unicode[outpos++] = 0xDC00 + (unsigned char)*s++; + } } - *p = L'\0'; + unicode[outpos] = L'\0'; return unicode; } #endif /* __APPLE__ */ -/* Allocation strategy: if the string is short, convert into a stack buffer +/* Primary internal function which creates utf8 encoded bytes objects. + + Allocation strategy: if the string is short, convert into a stack buffer and allocate exactly as much space needed at the end. Else allocate the maximum possible needed (4 result bytes per Unicode character), and return the excess memory at the end. */ PyObject * -PyUnicode_EncodeUTF8(const Py_UNICODE *s, - Py_ssize_t size, - const char *errors) +_PyUnicode_AsUTF8String(PyObject *unicode, const char *errors) { -#define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */ - - Py_ssize_t i; /* index into s of next input byte */ - PyObject *result; /* result string object */ - char *p; /* next free byte in output buffer */ - Py_ssize_t nallocated; /* number of result bytes allocated */ - Py_ssize_t nneeded; /* number of result bytes needed */ - char stackbuf[MAX_SHORT_UNICHARS * 4]; - PyObject *errorHandler = NULL; - PyObject *exc = NULL; - - assert(s != NULL); - assert(size >= 0); + enum PyUnicode_Kind kind; + void *data; + Py_ssize_t size; - if (size <= MAX_SHORT_UNICHARS) { - /* Write into the stack buffer; nallocated can't overflow. - * At the end, we'll allocate exactly as much heap space as it - * turns out we need. - */ - nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int); - result = NULL; /* will allocate after we're done */ - p = stackbuf; - } - else { - /* Overallocate on the heap, and give the excess back at the end. */ - nallocated = size * 4; - if (nallocated / 4 != size) /* overflow! */ - return PyErr_NoMemory(); - result = PyBytes_FromStringAndSize(NULL, nallocated); - if (result == NULL) - return NULL; - p = PyBytes_AS_STRING(result); + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return NULL; } - for (i = 0; i < size;) { - Py_UCS4 ch = s[i++]; - - if (ch < 0x80) - /* Encode ASCII */ - *p++ = (char) ch; - - else if (ch < 0x0800) { - /* Encode Latin-1 */ - *p++ = (char)(0xc0 | (ch >> 6)); - *p++ = (char)(0x80 | (ch & 0x3f)); - } else if (0xD800 <= ch && ch <= 0xDFFF) { -#ifndef Py_UNICODE_WIDE - /* Special case: check for high and low surrogate */ - if (ch <= 0xDBFF && i != size && 0xDC00 <= s[i] && s[i] <= 0xDFFF) { - Py_UCS4 ch2 = s[i]; - /* Combine the two surrogates to form a UCS4 value */ - ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000; - i++; - - /* Encode UCS4 Unicode ordinals */ - *p++ = (char)(0xf0 | (ch >> 18)); - *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); - *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); - *p++ = (char)(0x80 | (ch & 0x3f)); - } else { -#endif - Py_ssize_t newpos; - PyObject *rep; - Py_ssize_t repsize, k; - rep = unicode_encode_call_errorhandler - (errors, &errorHandler, "utf-8", "surrogates not allowed", - s, size, &exc, i-1, i, &newpos); - if (!rep) - goto error; - - if (PyBytes_Check(rep)) - repsize = PyBytes_GET_SIZE(rep); - else - repsize = PyUnicode_GET_SIZE(rep); + if (PyUnicode_READY(unicode) == -1) + return NULL; - if (repsize > 4) { - Py_ssize_t offset; + if (PyUnicode_UTF8(unicode)) + return PyBytes_FromStringAndSize(PyUnicode_UTF8(unicode), + PyUnicode_UTF8_LENGTH(unicode)); - if (result == NULL) - offset = p - stackbuf; - else - offset = p - PyBytes_AS_STRING(result); + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + size = PyUnicode_GET_LENGTH(unicode); - if (nallocated > PY_SSIZE_T_MAX - repsize + 4) { - /* integer overflow */ - PyErr_NoMemory(); - goto error; - } - nallocated += repsize - 4; - if (result != NULL) { - if (_PyBytes_Resize(&result, nallocated) < 0) - goto error; - } else { - result = PyBytes_FromStringAndSize(NULL, nallocated); - if (result == NULL) - goto error; - Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); - } - p = PyBytes_AS_STRING(result) + offset; - } - - if (PyBytes_Check(rep)) { - char *prep = PyBytes_AS_STRING(rep); - for(k = repsize; k > 0; k--) - *p++ = *prep++; - } else /* rep is unicode */ { - Py_UNICODE *prep = PyUnicode_AS_UNICODE(rep); - Py_UNICODE c; - - for(k=0; k<repsize; k++) { - c = prep[k]; - if (0x80 <= c) { - raise_encode_exception(&exc, "utf-8", s, size, - i-1, i, "surrogates not allowed"); - goto error; - } - *p++ = (char)prep[k]; - } - } - Py_DECREF(rep); -#ifndef Py_UNICODE_WIDE - } -#endif - } else if (ch < 0x10000) { - *p++ = (char)(0xe0 | (ch >> 12)); - *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); - *p++ = (char)(0x80 | (ch & 0x3f)); - } else /* ch >= 0x10000 */ { - /* Encode UCS4 Unicode ordinals */ - *p++ = (char)(0xf0 | (ch >> 18)); - *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); - *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); - *p++ = (char)(0x80 | (ch & 0x3f)); - } + switch (kind) { + default: + assert(0); + case PyUnicode_1BYTE_KIND: + /* the string cannot be ASCII, or PyUnicode_UTF8() would be set */ + assert(!PyUnicode_IS_ASCII(unicode)); + return ucs1lib_utf8_encoder(unicode, data, size, errors); + case PyUnicode_2BYTE_KIND: + return ucs2lib_utf8_encoder(unicode, data, size, errors); + case PyUnicode_4BYTE_KIND: + return ucs4lib_utf8_encoder(unicode, data, size, errors); } +} - if (result == NULL) { - /* This was stack allocated. */ - nneeded = p - stackbuf; - assert(nneeded <= nallocated); - result = PyBytes_FromStringAndSize(stackbuf, nneeded); - } - else { - /* Cut back to size actually needed. */ - nneeded = p - PyBytes_AS_STRING(result); - assert(nneeded <= nallocated); - _PyBytes_Resize(&result, nneeded); - } - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - return result; - error: - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - Py_XDECREF(result); - return NULL; +PyObject * +PyUnicode_EncodeUTF8(const Py_UNICODE *s, + Py_ssize_t size, + const char *errors) +{ + PyObject *v, *unicode; -#undef MAX_SHORT_UNICHARS + unicode = PyUnicode_FromUnicode(s, size); + if (unicode == NULL) + return NULL; + v = _PyUnicode_AsUTF8String(unicode, errors); + Py_DECREF(unicode); + return v; } -PyObject *PyUnicode_AsUTF8String(PyObject *unicode) +PyObject * +PyUnicode_AsUTF8String(PyObject *unicode) { - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return NULL; - } - return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - NULL); + return _PyUnicode_AsUTF8String(unicode, NULL); } /* --- UTF-32 Codec ------------------------------------------------------- */ @@ -3107,14 +4950,7 @@ PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t startinpos; Py_ssize_t endinpos; Py_ssize_t outpos; - PyUnicodeObject *unicode; - Py_UNICODE *p; -#ifndef Py_UNICODE_WIDE - int pairs = 0; - const unsigned char *qq; -#else - const int pairs = 0; -#endif + PyObject *unicode; const unsigned char *q, *e; int bo = 0; /* assume native ordering by default */ const char *errmsg = ""; @@ -3178,23 +5014,13 @@ PyUnicode_DecodeUTF32Stateful(const char *s, iorder[3] = 0; } - /* On narrow builds we split characters outside the BMP into two - codepoints => count how much extra space we need. */ -#ifndef Py_UNICODE_WIDE - for (qq = q; qq < e; qq += 4) - if (qq[iorder[2]] != 0 || qq[iorder[3]] != 0) - pairs++; -#endif - /* This might be one to much, because of a BOM */ - unicode = _PyUnicode_New((size+3)/4+pairs); + unicode = PyUnicode_New((size+3)/4, 127); if (!unicode) return NULL; if (size == 0) - return (PyObject *)unicode; - - /* Unpack UTF-32 encoded data */ - p = unicode->str; + return unicode; + outpos = 0; while (q < e) { Py_UCS4 ch; @@ -3219,24 +5045,16 @@ PyUnicode_DecodeUTF32Stateful(const char *s, endinpos = startinpos+4; goto utf32Error; } -#ifndef Py_UNICODE_WIDE - if (ch >= 0x10000) - { - *p++ = 0xD800 | ((ch-0x10000) >> 10); - *p++ = 0xDC00 | ((ch-0x10000) & 0x3FF); - } - else -#endif - *p++ = ch; + if (unicode_putchar(&unicode, &outpos, ch) < 0) + goto onError; q += 4; continue; utf32Error: - outpos = p-PyUnicode_AS_UNICODE(unicode); if (unicode_decode_call_errorhandler( errors, &errorHandler, "utf32", errmsg, &starts, (const char **)&e, &startinpos, &endinpos, &exc, (const char **)&q, - &unicode, &outpos, &p)) + &unicode, &outpos)) goto onError; } @@ -3247,12 +5065,12 @@ PyUnicode_DecodeUTF32Stateful(const char *s, *consumed = (const char *)q-starts; /* Adjust length */ - if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0) + if (unicode_resize(&unicode, outpos) < 0) goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); - return (PyObject *)unicode; + return unicode_result(unicode); onError: Py_DECREF(unicode); @@ -3262,19 +5080,16 @@ PyUnicode_DecodeUTF32Stateful(const char *s, } PyObject * -PyUnicode_EncodeUTF32(const Py_UNICODE *s, - Py_ssize_t size, - const char *errors, - int byteorder) +_PyUnicode_EncodeUTF32(PyObject *str, + const char *errors, + int byteorder) { + int kind; + void *data; + Py_ssize_t len; PyObject *v; unsigned char *p; - Py_ssize_t nsize, bytesize; -#ifndef Py_UNICODE_WIDE - Py_ssize_t i, pairs; -#else - const int pairs = 0; -#endif + Py_ssize_t nsize, i; /* Offsets from p for storing byte pairs in the right order. */ #ifdef BYTEORDER_IS_LITTLE_ENDIAN int iorder[] = {0, 1, 2, 3}; @@ -3291,26 +5106,27 @@ PyUnicode_EncodeUTF32(const Py_UNICODE *s, p += 4; \ } while(0) - /* In narrow builds we can output surrogate pairs as one codepoint, - so we need less space. */ -#ifndef Py_UNICODE_WIDE - for (i = pairs = 0; i < size-1; i++) - if (0xD800 <= s[i] && s[i] <= 0xDBFF && - 0xDC00 <= s[i+1] && s[i+1] <= 0xDFFF) - pairs++; -#endif - nsize = (size - pairs + (byteorder == 0)); - bytesize = nsize * 4; - if (bytesize / 4 != nsize) + if (!PyUnicode_Check(str)) { + PyErr_BadArgument(); + return NULL; + } + if (PyUnicode_READY(str) == -1) + return NULL; + kind = PyUnicode_KIND(str); + data = PyUnicode_DATA(str); + len = PyUnicode_GET_LENGTH(str); + + nsize = len + (byteorder == 0); + if (nsize > PY_SSIZE_T_MAX / 4) return PyErr_NoMemory(); - v = PyBytes_FromStringAndSize(NULL, bytesize); + v = PyBytes_FromStringAndSize(NULL, nsize * 4); if (v == NULL) return NULL; p = (unsigned char *)PyBytes_AS_STRING(v); if (byteorder == 0) STORECHAR(0xFEFF); - if (size == 0) + if (len == 0) goto done; if (byteorder == -1) { @@ -3328,36 +5144,33 @@ PyUnicode_EncodeUTF32(const Py_UNICODE *s, iorder[3] = 0; } - while (size-- > 0) { - Py_UCS4 ch = *s++; -#ifndef Py_UNICODE_WIDE - if (0xD800 <= ch && ch <= 0xDBFF && size > 0) { - Py_UCS4 ch2 = *s; - if (0xDC00 <= ch2 && ch2 <= 0xDFFF) { - ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000; - s++; - size--; - } - } -#endif - STORECHAR(ch); - } + for (i = 0; i < len; i++) + STORECHAR(PyUnicode_READ(kind, data, i)); done: return v; #undef STORECHAR } -PyObject *PyUnicode_AsUTF32String(PyObject *unicode) +PyObject * +PyUnicode_EncodeUTF32(const Py_UNICODE *s, + Py_ssize_t size, + const char *errors, + int byteorder) { - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); + PyObject *result; + PyObject *tmp = PyUnicode_FromUnicode(s, size); + if (tmp == NULL) return NULL; - } - return PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - NULL, - 0); + result = _PyUnicode_EncodeUTF32(tmp, errors, byteorder); + Py_DECREF(tmp); + return result; +} + +PyObject * +PyUnicode_AsUTF32String(PyObject *unicode) +{ + return _PyUnicode_EncodeUTF32(unicode, NULL, 0); } /* --- UTF-16 Codec ------------------------------------------------------- */ @@ -3371,23 +5184,6 @@ PyUnicode_DecodeUTF16(const char *s, return PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder, NULL); } -/* Two masks for fast checking of whether a C 'long' may contain - UTF16-encoded surrogate characters. This is an efficient heuristic, - assuming that non-surrogate characters with a code point >= 0x8000 are - rare in most input. - FAST_CHAR_MASK is used when the input is in native byte ordering, - SWAPPED_FAST_CHAR_MASK when the input is in byteswapped ordering. -*/ -#if (SIZEOF_LONG == 8) -# define FAST_CHAR_MASK 0x8000800080008000L -# define SWAPPED_FAST_CHAR_MASK 0x0080008000800080L -#elif (SIZEOF_LONG == 4) -# define FAST_CHAR_MASK 0x80008000L -# define SWAPPED_FAST_CHAR_MASK 0x00800080L -#else -# error C 'long' size should be either 4 or 8! -#endif - PyObject * PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, @@ -3399,31 +5195,14 @@ PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t startinpos; Py_ssize_t endinpos; Py_ssize_t outpos; - PyUnicodeObject *unicode; - Py_UNICODE *p; - const unsigned char *q, *e, *aligned_end; + PyObject *unicode; + const unsigned char *q, *e; int bo = 0; /* assume native ordering by default */ - int native_ordering = 0; + int native_ordering; const char *errmsg = ""; - /* Offsets from q for retrieving byte pairs in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - int ihi = 1, ilo = 0; -#else - int ihi = 0, ilo = 1; -#endif PyObject *errorHandler = NULL; PyObject *exc = NULL; - /* Note: size will always be longer than the resulting Unicode - character count */ - unicode = _PyUnicode_New(size); - if (!unicode) - return NULL; - if (size == 0) - return (PyObject *)unicode; - - /* Unpack UTF-16 encoded data */ - p = unicode->str; q = (unsigned char *)s; e = q + size; @@ -3434,176 +5213,98 @@ PyUnicode_DecodeUTF16Stateful(const char *s, byte order setting accordingly. In native mode, the leading BOM mark is skipped, in all other modes, it is copied to the output stream as-is (giving a ZWNBSP character). */ - if (bo == 0) { - if (size >= 2) { - const Py_UNICODE bom = (q[ihi] << 8) | q[ilo]; -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - if (bom == 0xFEFF) { - q += 2; - bo = -1; - } - else if (bom == 0xFFFE) { - q += 2; - bo = 1; - } -#else - if (bom == 0xFEFF) { - q += 2; - bo = 1; - } - else if (bom == 0xFFFE) { - q += 2; - bo = -1; - } -#endif + if (bo == 0 && size >= 2) { + const Py_UCS4 bom = (q[1] << 8) | q[0]; + if (bom == 0xFEFF) { + q += 2; + bo = -1; } + else if (bom == 0xFFFE) { + q += 2; + bo = 1; + } + if (byteorder) + *byteorder = bo; } - if (bo == -1) { - /* force LE */ - ihi = 1; - ilo = 0; - } - else if (bo == 1) { - /* force BE */ - ihi = 0; - ilo = 1; + if (q == e) { + if (consumed) + *consumed = size; + Py_INCREF(unicode_empty); + return unicode_empty; } + #ifdef BYTEORDER_IS_LITTLE_ENDIAN - native_ordering = ilo < ihi; + native_ordering = bo <= 0; #else - native_ordering = ilo > ihi; + native_ordering = bo >= 0; #endif - aligned_end = (const unsigned char *) ((size_t) e & ~LONG_PTR_MASK); + /* Note: size will always be longer than the resulting Unicode + character count */ + unicode = PyUnicode_New((e - q + 1) / 2, 127); + if (!unicode) + return NULL; + + outpos = 0; while (1) { - Py_UNICODE ch; - if (e - q < 2) { + Py_UCS4 ch = 0; + if (e - q >= 2) { + int kind = PyUnicode_KIND(unicode); + if (kind == PyUnicode_1BYTE_KIND) { + if (PyUnicode_IS_ASCII(unicode)) + ch = asciilib_utf16_decode(&q, e, + PyUnicode_1BYTE_DATA(unicode), &outpos, + native_ordering); + else + ch = ucs1lib_utf16_decode(&q, e, + PyUnicode_1BYTE_DATA(unicode), &outpos, + native_ordering); + } else if (kind == PyUnicode_2BYTE_KIND) { + ch = ucs2lib_utf16_decode(&q, e, + PyUnicode_2BYTE_DATA(unicode), &outpos, + native_ordering); + } else { + assert(kind == PyUnicode_4BYTE_KIND); + ch = ucs4lib_utf16_decode(&q, e, + PyUnicode_4BYTE_DATA(unicode), &outpos, + native_ordering); + } + } + + switch (ch) + { + case 0: /* remaining byte at the end? (size should be even) */ if (q == e || consumed) - break; + goto End; errmsg = "truncated data"; startinpos = ((const char *)q) - starts; endinpos = ((const char *)e) - starts; - outpos = p - PyUnicode_AS_UNICODE(unicode); - goto utf16Error; + break; /* The remaining input chars are ignored if the callback chooses to skip the input */ - } - /* First check for possible aligned read of a C 'long'. Unaligned - reads are more expensive, better to defer to another iteration. */ - if (!((size_t) q & LONG_PTR_MASK)) { - /* Fast path for runs of non-surrogate chars. */ - register const unsigned char *_q = q; - Py_UNICODE *_p = p; - if (native_ordering) { - /* Native ordering is simple: as long as the input cannot - possibly contain a surrogate char, do an unrolled copy - of several 16-bit code points to the target object. - The non-surrogate check is done on several input bytes - at a time (as many as a C 'long' can contain). */ - while (_q < aligned_end) { - unsigned long data = * (unsigned long *) _q; - if (data & FAST_CHAR_MASK) - break; - _p[0] = ((unsigned short *) _q)[0]; - _p[1] = ((unsigned short *) _q)[1]; -#if (SIZEOF_LONG == 8) - _p[2] = ((unsigned short *) _q)[2]; - _p[3] = ((unsigned short *) _q)[3]; -#endif - _q += SIZEOF_LONG; - _p += SIZEOF_LONG / 2; - } - } - else { - /* Byteswapped ordering is similar, but we must decompose - the copy bytewise, and take care of zero'ing out the - upper bytes if the target object is in 32-bit units - (that is, in UCS-4 builds). */ - while (_q < aligned_end) { - unsigned long data = * (unsigned long *) _q; - if (data & SWAPPED_FAST_CHAR_MASK) - break; - /* Zero upper bytes in UCS-4 builds */ -#if (Py_UNICODE_SIZE > 2) - _p[0] = 0; - _p[1] = 0; -#if (SIZEOF_LONG == 8) - _p[2] = 0; - _p[3] = 0; -#endif -#endif - /* Issue #4916; UCS-4 builds on big endian machines must - fill the two last bytes of each 4-byte unit. */ -#if (!defined(BYTEORDER_IS_LITTLE_ENDIAN) && Py_UNICODE_SIZE > 2) -# define OFF 2 -#else -# define OFF 0 -#endif - ((unsigned char *) _p)[OFF + 1] = _q[0]; - ((unsigned char *) _p)[OFF + 0] = _q[1]; - ((unsigned char *) _p)[OFF + 1 + Py_UNICODE_SIZE] = _q[2]; - ((unsigned char *) _p)[OFF + 0 + Py_UNICODE_SIZE] = _q[3]; -#if (SIZEOF_LONG == 8) - ((unsigned char *) _p)[OFF + 1 + 2 * Py_UNICODE_SIZE] = _q[4]; - ((unsigned char *) _p)[OFF + 0 + 2 * Py_UNICODE_SIZE] = _q[5]; - ((unsigned char *) _p)[OFF + 1 + 3 * Py_UNICODE_SIZE] = _q[6]; - ((unsigned char *) _p)[OFF + 0 + 3 * Py_UNICODE_SIZE] = _q[7]; -#endif -#undef OFF - _q += SIZEOF_LONG; - _p += SIZEOF_LONG / 2; - } - } - p = _p; - q = _q; - if (e - q < 2) - continue; - } - ch = (q[ihi] << 8) | q[ilo]; - - q += 2; - - if (ch < 0xD800 || ch > 0xDFFF) { - *p++ = ch; - continue; - } - - /* UTF-16 code pair: */ - if (e - q < 2) { + case 1: errmsg = "unexpected end of data"; - startinpos = (((const char *)q) - 2) - starts; + startinpos = ((const char *)q) - 2 - starts; endinpos = ((const char *)e) - starts; - goto utf16Error; - } - if (0xD800 <= ch && ch <= 0xDBFF) { - Py_UNICODE ch2 = (q[ihi] << 8) | q[ilo]; - q += 2; - if (0xDC00 <= ch2 && ch2 <= 0xDFFF) { -#ifndef Py_UNICODE_WIDE - *p++ = ch; - *p++ = ch2; -#else - *p++ = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000; -#endif - continue; - } - else { - errmsg = "illegal UTF-16 surrogate"; - startinpos = (((const char *)q)-4)-starts; - endinpos = startinpos+2; - goto utf16Error; - } - + break; + case 2: + errmsg = "illegal encoding"; + startinpos = ((const char *)q) - 2 - starts; + endinpos = startinpos + 2; + break; + case 3: + errmsg = "illegal UTF-16 surrogate"; + startinpos = ((const char *)q) - 4 - starts; + endinpos = startinpos + 2; + break; + default: + if (unicode_putchar(&unicode, &outpos, ch) < 0) + goto onError; + continue; } - errmsg = "illegal encoding"; - startinpos = (((const char *)q)-2)-starts; - endinpos = startinpos+2; - /* Fall through to report the error */ - utf16Error: - outpos = p - PyUnicode_AS_UNICODE(unicode); if (unicode_decode_call_errorhandler( errors, &errorHandler, @@ -3615,27 +5316,21 @@ PyUnicode_DecodeUTF16Stateful(const char *s, &exc, (const char **)&q, &unicode, - &outpos, - &p)) + &outpos)) goto onError; - /* Update data because unicode_decode_call_errorhandler might have - changed the input object. */ - aligned_end = (const unsigned char *) ((size_t) e & ~LONG_PTR_MASK); } - if (byteorder) - *byteorder = bo; - +End: if (consumed) *consumed = (const char *)q-starts; /* Adjust length */ - if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0) + if (unicode_resize(&unicode, outpos) < 0) goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); - return (PyObject *)unicode; + return unicode_result(unicode); onError: Py_DECREF(unicode); @@ -3644,145 +5339,219 @@ PyUnicode_DecodeUTF16Stateful(const char *s, return NULL; } -#undef FAST_CHAR_MASK -#undef SWAPPED_FAST_CHAR_MASK - PyObject * -PyUnicode_EncodeUTF16(const Py_UNICODE *s, - Py_ssize_t size, - const char *errors, - int byteorder) +_PyUnicode_EncodeUTF16(PyObject *str, + const char *errors, + int byteorder) { + enum PyUnicode_Kind kind; + const void *data; + Py_ssize_t len; PyObject *v; - unsigned char *p; - Py_ssize_t nsize, bytesize; -#ifdef Py_UNICODE_WIDE - Py_ssize_t i, pairs; -#else - const int pairs = 0; -#endif - /* Offsets from p for storing byte pairs in the right order. */ -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - int ihi = 1, ilo = 0; + unsigned short *out; + Py_ssize_t bytesize; + Py_ssize_t pairs; +#ifdef WORDS_BIGENDIAN + int native_ordering = byteorder >= 0; #else - int ihi = 0, ilo = 1; + int native_ordering = byteorder <= 0; #endif -#define STORECHAR(CH) \ - do { \ - p[ihi] = ((CH) >> 8) & 0xff; \ - p[ilo] = (CH) & 0xff; \ - p += 2; \ - } while(0) - -#ifdef Py_UNICODE_WIDE - for (i = pairs = 0; i < size; i++) - if (s[i] >= 0x10000) - pairs++; -#endif - /* 2 * (size + pairs + (byteorder == 0)) */ - if (size > PY_SSIZE_T_MAX || - size > PY_SSIZE_T_MAX - pairs - (byteorder == 0)) - return PyErr_NoMemory(); - nsize = size + pairs + (byteorder == 0); - bytesize = nsize * 2; - if (bytesize / 2 != nsize) + if (!PyUnicode_Check(str)) { + PyErr_BadArgument(); + return NULL; + } + if (PyUnicode_READY(str) == -1) + return NULL; + kind = PyUnicode_KIND(str); + data = PyUnicode_DATA(str); + len = PyUnicode_GET_LENGTH(str); + + pairs = 0; + if (kind == PyUnicode_4BYTE_KIND) { + const Py_UCS4 *in = (const Py_UCS4 *)data; + const Py_UCS4 *end = in + len; + while (in < end) + if (*in++ >= 0x10000) + pairs++; + } + if (len > PY_SSIZE_T_MAX / 2 - pairs - (byteorder == 0)) return PyErr_NoMemory(); + bytesize = (len + pairs + (byteorder == 0)) * 2; v = PyBytes_FromStringAndSize(NULL, bytesize); if (v == NULL) return NULL; - p = (unsigned char *)PyBytes_AS_STRING(v); + /* output buffer is 2-bytes aligned */ + assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 2)); + out = (unsigned short *)PyBytes_AS_STRING(v); if (byteorder == 0) - STORECHAR(0xFEFF); - if (size == 0) + *out++ = 0xFEFF; + if (len == 0) goto done; - if (byteorder == -1) { - /* force LE */ - ihi = 1; - ilo = 0; + switch (kind) { + case PyUnicode_1BYTE_KIND: { + ucs1lib_utf16_encode(out, (const Py_UCS1 *)data, len, native_ordering); + break; } - else if (byteorder == 1) { - /* force BE */ - ihi = 0; - ilo = 1; + case PyUnicode_2BYTE_KIND: { + ucs2lib_utf16_encode(out, (const Py_UCS2 *)data, len, native_ordering); + break; } - - while (size-- > 0) { - Py_UNICODE ch = *s++; - Py_UNICODE ch2 = 0; -#ifdef Py_UNICODE_WIDE - if (ch >= 0x10000) { - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF); - ch = 0xD800 | ((ch-0x10000) >> 10); - } -#endif - STORECHAR(ch); - if (ch2) - STORECHAR(ch2); + case PyUnicode_4BYTE_KIND: { + ucs4lib_utf16_encode(out, (const Py_UCS4 *)data, len, native_ordering); + break; + } + default: + assert(0); } done: return v; -#undef STORECHAR } -PyObject *PyUnicode_AsUTF16String(PyObject *unicode) +PyObject * +PyUnicode_EncodeUTF16(const Py_UNICODE *s, + Py_ssize_t size, + const char *errors, + int byteorder) { - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); + PyObject *result; + PyObject *tmp = PyUnicode_FromUnicode(s, size); + if (tmp == NULL) return NULL; - } - return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - NULL, - 0); + result = _PyUnicode_EncodeUTF16(tmp, errors, byteorder); + Py_DECREF(tmp); + return result; +} + +PyObject * +PyUnicode_AsUTF16String(PyObject *unicode) +{ + return _PyUnicode_EncodeUTF16(unicode, NULL, 0); } /* --- Unicode Escape Codec ----------------------------------------------- */ +/* Helper function for PyUnicode_DecodeUnicodeEscape, determines + if all the escapes in the string make it still a valid ASCII string. + Returns -1 if any escapes were found which cause the string to + pop out of ASCII range. Otherwise returns the length of the + required buffer to hold the string. + */ +static Py_ssize_t +length_of_escaped_ascii_string(const char *s, Py_ssize_t size) +{ + const unsigned char *p = (const unsigned char *)s; + const unsigned char *end = p + size; + Py_ssize_t length = 0; + + if (size < 0) + return -1; + + for (; p < end; ++p) { + if (*p > 127) { + /* Non-ASCII */ + return -1; + } + else if (*p != '\\') { + /* Normal character */ + ++length; + } + else { + /* Backslash-escape, check next char */ + ++p; + /* Escape sequence reaches till end of string or + non-ASCII follow-up. */ + if (p >= end || *p > 127) + return -1; + switch (*p) { + case '\n': + /* backslash + \n result in zero characters */ + break; + case '\\': case '\'': case '\"': + case 'b': case 'f': case 't': + case 'n': case 'r': case 'v': case 'a': + ++length; + break; + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + case 'x': case 'u': case 'U': case 'N': + /* these do not guarantee ASCII characters */ + return -1; + default: + /* count the backslash + the other character */ + length += 2; + } + } + } + return length; +} + static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL; -PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, - Py_ssize_t size, - const char *errors) +PyObject * +PyUnicode_DecodeUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) { const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; - Py_ssize_t outpos; - int i; - PyUnicodeObject *v; - Py_UNICODE *p; + int j; + PyObject *v; const char *end; char* message; Py_UCS4 chr = 0xffffffff; /* in case 'getcode' messes up */ PyObject *errorHandler = NULL; PyObject *exc = NULL; + Py_ssize_t len; + Py_ssize_t i; - /* Escaped strings will always be longer than the resulting - Unicode string, so we start with size here and then reduce the - length after conversion to the true value. - (but if the error callback returns a long replacement string - we'll have to allocate more space) */ - v = _PyUnicode_New(size); - if (v == NULL) - goto onError; - if (size == 0) - return (PyObject *)v; + len = length_of_escaped_ascii_string(s, size); + + /* After length_of_escaped_ascii_string() there are two alternatives, + either the string is pure ASCII with named escapes like \n, etc. + and we determined it's exact size (common case) + or it contains \x, \u, ... escape sequences. then we create a + legacy wchar string and resize it at the end of this function. */ + if (len >= 0) { + v = PyUnicode_New(len, 127); + if (!v) + goto onError; + assert(PyUnicode_KIND(v) == PyUnicode_1BYTE_KIND); + } + else { + /* Escaped strings will always be longer than the resulting + Unicode string, so we start with size here and then reduce the + length after conversion to the true value. + (but if the error callback returns a long replacement string + we'll have to allocate more space) */ + v = PyUnicode_New(size, 127); + if (!v) + goto onError; + len = size; + } - p = PyUnicode_AS_UNICODE(v); + if (size == 0) + return v; + i = 0; end = s + size; while (s < end) { unsigned char c; - Py_UNICODE x; + Py_UCS4 x; int digits; + /* The only case in which i == ascii_length is a backslash + followed by a newline. */ + assert(i <= len); + /* Non-escape characters are interpreted as Unicode ordinals */ if (*s != '\\') { - *p++ = (unsigned char) *s++; + if (unicode_putchar(&v, &i, (unsigned char) *s++) < 0) + goto onError; continue; } @@ -3792,20 +5561,34 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, c = *s++; if (s > end) c = '\0'; /* Invalid after \ */ + + /* The only case in which i == ascii_length is a backslash + followed by a newline. */ + assert(i < len || (i == len && c == '\n')); + switch (c) { /* \x escapes */ +#define WRITECHAR(ch) \ + do { \ + if (unicode_putchar(&v, &i, ch) < 0) \ + goto onError; \ + }while(0) + case '\n': break; - case '\\': *p++ = '\\'; break; - case '\'': *p++ = '\''; break; - case '\"': *p++ = '\"'; break; - case 'b': *p++ = '\b'; break; - case 'f': *p++ = '\014'; break; /* FF */ - case 't': *p++ = '\t'; break; - case 'n': *p++ = '\n'; break; - case 'r': *p++ = '\r'; break; - case 'v': *p++ = '\013'; break; /* VT */ - case 'a': *p++ = '\007'; break; /* BEL, not classic C */ + case '\\': WRITECHAR('\\'); break; + case '\'': WRITECHAR('\''); break; + case '\"': WRITECHAR('\"'); break; + case 'b': WRITECHAR('\b'); break; + /* FF */ + case 'f': WRITECHAR('\014'); break; + case 't': WRITECHAR('\t'); break; + case 'n': WRITECHAR('\n'); break; + case 'r': WRITECHAR('\r'); break; + /* VT */ + case 'v': WRITECHAR('\013'); break; + /* BEL, not classic C */ + case 'a': WRITECHAR('\007'); break; /* \OOO (octal) escapes */ case '0': case '1': case '2': case '3': @@ -3816,7 +5599,7 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, if (s < end && '0' <= *s && *s <= '7') x = (x<<3) + *s++ - '0'; } - *p++ = x; + WRITECHAR(x); break; /* hex escapes */ @@ -3838,27 +5621,27 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, message = "truncated \\UXXXXXXXX escape"; hexescape: chr = 0; - outpos = p-PyUnicode_AS_UNICODE(v); if (s+digits>end) { endinpos = size; if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", "end of string in escape sequence", &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i)) goto onError; goto nextByte; } - for (i = 0; i < digits; ++i) { - c = (unsigned char) s[i]; + for (j = 0; j < digits; ++j) { + c = (unsigned char) s[j]; if (!Py_ISXDIGIT(c)) { - endinpos = (s+i+1)-starts; + endinpos = (s+j+1)-starts; if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", message, &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i)) goto onError; + len = PyUnicode_GET_LENGTH(v); goto nextByte; } chr = (chr<<4) & ~0xF; @@ -3869,34 +5652,22 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, else chr += 10 + c - 'A'; } - s += i; + s += j; if (chr == 0xffffffff && PyErr_Occurred()) /* _decoding_error will have already written into the target buffer. */ break; store: /* when we get here, chr is a 32-bit unicode character */ - if (chr <= 0xffff) - /* UCS-2 character */ - *p++ = (Py_UNICODE) chr; - else if (chr <= 0x10ffff) { - /* UCS-4 character. Either store directly, or as - surrogate pair. */ -#ifdef Py_UNICODE_WIDE - *p++ = chr; -#else - chr -= 0x10000L; - *p++ = 0xD800 + (Py_UNICODE) (chr >> 10); - *p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF); -#endif + if (chr <= MAX_UNICODE) { + WRITECHAR(chr); } else { endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", "illegal Unicode character", &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i)) goto onError; } break; @@ -3906,7 +5677,8 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, message = "malformed \\N character escape"; if (ucnhash_CAPI == NULL) { /* load the unicode data module */ - ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import(PyUnicodeData_CAPSULE_NAME, 1); + ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import( + PyUnicodeData_CAPSULE_NAME, 1); if (ucnhash_CAPI == NULL) goto ucnhashError; } @@ -3919,17 +5691,17 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, /* found a name. look it up in the unicode database */ message = "unknown Unicode character name"; s++; - if (ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), &chr)) + if (ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), + &chr, 0)) goto store; } } endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", message, &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i)) goto onError; break; @@ -3938,28 +5710,29 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, message = "\\ at end of string"; s--; endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", message, &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &i)) goto onError; } else { - *p++ = '\\'; - *p++ = (unsigned char)s[-1]; + WRITECHAR('\\'); + WRITECHAR(s[-1]); } break; } nextByte: ; } - if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) +#undef WRITECHAR + + if (unicode_resize(&v, i) < 0) goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); - return (PyObject *)v; + return unicode_result(v); ucnhashError: PyErr_SetString( @@ -3985,70 +5758,56 @@ PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, */ -Py_LOCAL_INLINE(const Py_UNICODE *) findchar(const Py_UNICODE *s, - Py_ssize_t size, - Py_UNICODE ch) -{ - /* like wcschr, but doesn't stop at NULL characters */ - - while (size-- > 0) { - if (*s == ch) - return s; - s++; - } - - return NULL; -} - -static const char *hexdigits = "0123456789abcdef"; - -PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, - Py_ssize_t size) +PyObject * +PyUnicode_AsUnicodeEscapeString(PyObject *unicode) { + Py_ssize_t i, len; PyObject *repr; char *p; + int kind; + void *data; + Py_ssize_t expandsize = 0; -#ifdef Py_UNICODE_WIDE - const Py_ssize_t expandsize = 10; -#else - const Py_ssize_t expandsize = 6; -#endif - - /* XXX(nnorwitz): rather than over-allocating, it would be - better to choose a different scheme. Perhaps scan the - first N-chars of the string and allocate based on that size. - */ - /* Initial allocation is based on the longest-possible unichr + /* Initial allocation is based on the longest-possible character escape. - In wide (UTF-32) builds '\U00xxxxxx' is 10 chars per source - unichr, so in this case it's the longest unichr escape. In - narrow (UTF-16) builds this is five chars per source unichr - since there are two unichrs in the surrogate pair, so in narrow - (UTF-16) builds it's not the longest unichr escape. - - In wide or narrow builds '\uxxxx' is 6 chars per source unichr, - so in the narrow (UTF-16) build case it's the longest unichr - escape. + For UCS1 strings it's '\xxx', 4 bytes per source character. + For UCS2 strings it's '\uxxxx', 6 bytes per source character. + For UCS4 strings it's '\U00xxxxxx', 10 bytes per source character. */ - if (size == 0) + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return NULL; + } + if (PyUnicode_READY(unicode) == -1) + return NULL; + len = PyUnicode_GET_LENGTH(unicode); + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + switch (kind) { + case PyUnicode_1BYTE_KIND: expandsize = 4; break; + case PyUnicode_2BYTE_KIND: expandsize = 6; break; + case PyUnicode_4BYTE_KIND: expandsize = 10; break; + } + + if (len == 0) return PyBytes_FromStringAndSize(NULL, 0); - if (size > (PY_SSIZE_T_MAX - 2 - 1) / expandsize) + if (len > (PY_SSIZE_T_MAX - 2 - 1) / expandsize) return PyErr_NoMemory(); repr = PyBytes_FromStringAndSize(NULL, 2 - + expandsize*size + + expandsize*len + 1); if (repr == NULL) return NULL; p = PyBytes_AS_STRING(repr); - while (size-- > 0) { - Py_UNICODE ch = *s++; + for (i = 0; i < len; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Escape backslashes */ if (ch == '\\') { @@ -4057,57 +5816,30 @@ PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, continue; } -#ifdef Py_UNICODE_WIDE /* Map 21-bit characters to '\U00xxxxxx' */ else if (ch >= 0x10000) { + assert(ch <= MAX_UNICODE); *p++ = '\\'; *p++ = 'U'; - *p++ = hexdigits[(ch >> 28) & 0x0000000F]; - *p++ = hexdigits[(ch >> 24) & 0x0000000F]; - *p++ = hexdigits[(ch >> 20) & 0x0000000F]; - *p++ = hexdigits[(ch >> 16) & 0x0000000F]; - *p++ = hexdigits[(ch >> 12) & 0x0000000F]; - *p++ = hexdigits[(ch >> 8) & 0x0000000F]; - *p++ = hexdigits[(ch >> 4) & 0x0000000F]; - *p++ = hexdigits[ch & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 28) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 24) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 20) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 16) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 12) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 8) & 0x0000000F]; + *p++ = Py_hexdigits[(ch >> 4) & 0x0000000F]; + *p++ = Py_hexdigits[ch & 0x0000000F]; continue; } -#else - /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */ - else if (ch >= 0xD800 && ch < 0xDC00) { - Py_UNICODE ch2; - Py_UCS4 ucs; - - ch2 = *s++; - size--; - if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) { - ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000; - *p++ = '\\'; - *p++ = 'U'; - *p++ = hexdigits[(ucs >> 28) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 24) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 20) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 16) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 12) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 8) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 4) & 0x0000000F]; - *p++ = hexdigits[ucs & 0x0000000F]; - continue; - } - /* Fall through: isolated surrogates are copied as-is */ - s--; - size++; - } -#endif /* Map 16-bit characters to '\uxxxx' */ if (ch >= 256) { *p++ = '\\'; *p++ = 'u'; - *p++ = hexdigits[(ch >> 12) & 0x000F]; - *p++ = hexdigits[(ch >> 8) & 0x000F]; - *p++ = hexdigits[(ch >> 4) & 0x000F]; - *p++ = hexdigits[ch & 0x000F]; + *p++ = Py_hexdigits[(ch >> 12) & 0x000F]; + *p++ = Py_hexdigits[(ch >> 8) & 0x000F]; + *p++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *p++ = Py_hexdigits[ch & 0x000F]; } /* Map special whitespace to '\t', \n', '\r' */ @@ -4128,8 +5860,8 @@ PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, else if (ch < ' ' || ch >= 0x7F) { *p++ = '\\'; *p++ = 'x'; - *p++ = hexdigits[(ch >> 4) & 0x000F]; - *p++ = hexdigits[ch & 0x000F]; + *p++ = Py_hexdigits[(ch >> 4) & 0x000F]; + *p++ = Py_hexdigits[ch & 0x000F]; } /* Copy everything else as-is */ @@ -4143,30 +5875,31 @@ PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, return repr; } -PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode) +PyObject * +PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, + Py_ssize_t size) { - PyObject *s; - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); + PyObject *result; + PyObject *tmp = PyUnicode_FromUnicode(s, size); + if (tmp == NULL) return NULL; - } - s = PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode)); - return s; + result = PyUnicode_AsUnicodeEscapeString(tmp); + Py_DECREF(tmp); + return result; } /* --- Raw Unicode Escape Codec ------------------------------------------- */ -PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s, - Py_ssize_t size, - const char *errors) +PyObject * +PyUnicode_DecodeRawUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) { const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; Py_ssize_t outpos; - PyUnicodeObject *v; - Py_UNICODE *p; + PyObject *v; const char *end; const char *bs; PyObject *errorHandler = NULL; @@ -4176,12 +5909,12 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s, Unicode string, so we start with size here and then reduce the length after conversion to the true value. (But decoding error handler might have to resize the string) */ - v = _PyUnicode_New(size); + v = PyUnicode_New(size, 127); if (v == NULL) goto onError; if (size == 0) - return (PyObject *)v; - p = PyUnicode_AS_UNICODE(v); + return v; + outpos = 0; end = s + size; while (s < end) { unsigned char c; @@ -4191,7 +5924,8 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s, /* Non-escape characters are interpreted as Unicode ordinals */ if (*s != '\\') { - *p++ = (unsigned char)*s++; + if (unicode_putchar(&v, &outpos, (unsigned char)*s++) < 0) + goto onError; continue; } startinpos = s-starts; @@ -4202,19 +5936,19 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s, for (;s < end;) { if (*s != '\\') break; - *p++ = (unsigned char)*s++; + if (unicode_putchar(&v, &outpos, (unsigned char)*s++) < 0) + goto onError; } if (((s - bs) & 1) == 0 || s >= end || (*s != 'u' && *s != 'U')) { continue; } - p--; + outpos--; count = *s=='u' ? 4 : 8; s++; /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ - outpos = p-PyUnicode_AS_UNICODE(v); for (x = 0, i = 0; i < count; ++i, ++s) { c = (unsigned char)*s; if (!Py_ISXDIGIT(c)) { @@ -4223,7 +5957,7 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s, errors, &errorHandler, "rawunicodeescape", "truncated \\uXXXX", &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &outpos)) goto onError; goto nextByte; } @@ -4235,37 +5969,26 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s, else x += 10 + c - 'A'; } - if (x <= 0xffff) - /* UCS-2 character */ - *p++ = (Py_UNICODE) x; - else if (x <= 0x10ffff) { - /* UCS-4 character. Either store directly, or as - surrogate pair. */ -#ifdef Py_UNICODE_WIDE - *p++ = (Py_UNICODE) x; -#else - x -= 0x10000L; - *p++ = 0xD800 + (Py_UNICODE) (x >> 10); - *p++ = 0xDC00 + (Py_UNICODE) (x & 0x03FF); -#endif + if (x <= MAX_UNICODE) { + if (unicode_putchar(&v, &outpos, x) < 0) + goto onError; } else { endinpos = s-starts; - outpos = p-PyUnicode_AS_UNICODE(v); if (unicode_decode_call_errorhandler( errors, &errorHandler, "rawunicodeescape", "\\Uxxxxxxxx out of range", &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &v, &outpos)) goto onError; } nextByte: ; } - if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) + if (unicode_resize(&v, outpos) < 0) goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); - return (PyObject *)v; + return unicode_result(v); onError: Py_XDECREF(v); @@ -4274,144 +5997,139 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s, return NULL; } -PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, - Py_ssize_t size) + +PyObject * +PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) { PyObject *repr; char *p; char *q; + Py_ssize_t expandsize, pos; + int kind; + void *data; + Py_ssize_t len; -#ifdef Py_UNICODE_WIDE - const Py_ssize_t expandsize = 10; -#else - const Py_ssize_t expandsize = 6; -#endif - - if (size > PY_SSIZE_T_MAX / expandsize) + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return NULL; + } + if (PyUnicode_READY(unicode) == -1) + return NULL; + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + len = PyUnicode_GET_LENGTH(unicode); + /* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6 + bytes, and 1 byte characters 4. */ + expandsize = kind * 2 + 2; + + if (len > PY_SSIZE_T_MAX / expandsize) return PyErr_NoMemory(); - repr = PyBytes_FromStringAndSize(NULL, expandsize * size); + repr = PyBytes_FromStringAndSize(NULL, expandsize * len); if (repr == NULL) return NULL; - if (size == 0) + if (len == 0) return repr; p = q = PyBytes_AS_STRING(repr); - while (size-- > 0) { - Py_UNICODE ch = *s++; -#ifdef Py_UNICODE_WIDE + for (pos = 0; pos < len; pos++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, pos); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { + assert(ch <= MAX_UNICODE); *p++ = '\\'; *p++ = 'U'; - *p++ = hexdigits[(ch >> 28) & 0xf]; - *p++ = hexdigits[(ch >> 24) & 0xf]; - *p++ = hexdigits[(ch >> 20) & 0xf]; - *p++ = hexdigits[(ch >> 16) & 0xf]; - *p++ = hexdigits[(ch >> 12) & 0xf]; - *p++ = hexdigits[(ch >> 8) & 0xf]; - *p++ = hexdigits[(ch >> 4) & 0xf]; - *p++ = hexdigits[ch & 15]; + *p++ = Py_hexdigits[(ch >> 28) & 0xf]; + *p++ = Py_hexdigits[(ch >> 24) & 0xf]; + *p++ = Py_hexdigits[(ch >> 20) & 0xf]; + *p++ = Py_hexdigits[(ch >> 16) & 0xf]; + *p++ = Py_hexdigits[(ch >> 12) & 0xf]; + *p++ = Py_hexdigits[(ch >> 8) & 0xf]; + *p++ = Py_hexdigits[(ch >> 4) & 0xf]; + *p++ = Py_hexdigits[ch & 15]; } - else -#else - /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */ - if (ch >= 0xD800 && ch < 0xDC00) { - Py_UNICODE ch2; - Py_UCS4 ucs; - - ch2 = *s++; - size--; - if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) { - ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000; - *p++ = '\\'; - *p++ = 'U'; - *p++ = hexdigits[(ucs >> 28) & 0xf]; - *p++ = hexdigits[(ucs >> 24) & 0xf]; - *p++ = hexdigits[(ucs >> 20) & 0xf]; - *p++ = hexdigits[(ucs >> 16) & 0xf]; - *p++ = hexdigits[(ucs >> 12) & 0xf]; - *p++ = hexdigits[(ucs >> 8) & 0xf]; - *p++ = hexdigits[(ucs >> 4) & 0xf]; - *p++ = hexdigits[ucs & 0xf]; - continue; - } - /* Fall through: isolated surrogates are copied as-is */ - s--; - size++; - } -#endif /* Map 16-bit characters to '\uxxxx' */ - if (ch >= 256) { + else if (ch >= 256) { *p++ = '\\'; *p++ = 'u'; - *p++ = hexdigits[(ch >> 12) & 0xf]; - *p++ = hexdigits[(ch >> 8) & 0xf]; - *p++ = hexdigits[(ch >> 4) & 0xf]; - *p++ = hexdigits[ch & 15]; + *p++ = Py_hexdigits[(ch >> 12) & 0xf]; + *p++ = Py_hexdigits[(ch >> 8) & 0xf]; + *p++ = Py_hexdigits[(ch >> 4) & 0xf]; + *p++ = Py_hexdigits[ch & 15]; } /* Copy everything else as-is */ else *p++ = (char) ch; } - size = p - q; - assert(size > 0); - if (_PyBytes_Resize(&repr, size) < 0) + assert(p > q); + if (_PyBytes_Resize(&repr, p - q) < 0) return NULL; return repr; } -PyObject *PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) +PyObject * +PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, + Py_ssize_t size) { - PyObject *s; - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); + PyObject *result; + PyObject *tmp = PyUnicode_FromUnicode(s, size); + if (tmp == NULL) return NULL; - } - s = PyUnicode_EncodeRawUnicodeEscape(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode)); - - return s; + result = PyUnicode_AsRawUnicodeEscapeString(tmp); + Py_DECREF(tmp); + return result; } /* --- Unicode Internal Codec ------------------------------------------- */ -PyObject *_PyUnicode_DecodeUnicodeInternal(const char *s, - Py_ssize_t size, - const char *errors) +PyObject * +_PyUnicode_DecodeUnicodeInternal(const char *s, + Py_ssize_t size, + const char *errors) { const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; Py_ssize_t outpos; - PyUnicodeObject *v; - Py_UNICODE *p; + PyObject *v; const char *end; const char *reason; PyObject *errorHandler = NULL; PyObject *exc = NULL; -#ifdef Py_UNICODE_WIDE - Py_UNICODE unimax = PyUnicode_GetMax(); -#endif + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "unicode_internal codec has been deprecated", + 1)) + return NULL; /* XXX overflow detection missing */ - v = _PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE); + v = PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE, 127); if (v == NULL) goto onError; - if (PyUnicode_GetSize((PyObject *)v) == 0) - return (PyObject *)v; - p = PyUnicode_AS_UNICODE(v); + if (PyUnicode_GET_LENGTH(v) == 0) + return v; + outpos = 0; end = s + size; while (s < end) { - memcpy(p, s, sizeof(Py_UNICODE)); + Py_UNICODE uch; + Py_UCS4 ch; + /* We copy the raw representation one byte at a time because the + pointer may be unaligned (see test_codeccallbacks). */ + ((char *) &uch)[0] = s[0]; + ((char *) &uch)[1] = s[1]; +#ifdef Py_UNICODE_WIDE + ((char *) &uch)[2] = s[2]; + ((char *) &uch)[3] = s[3]; +#endif + ch = uch; + /* We have to sanity check the raw data, otherwise doom looms for some malformed UCS-4 data. */ if ( #ifdef Py_UNICODE_WIDE - *p > unimax || *p < 0 || + ch > 0x10ffff || #endif end-s < Py_UNICODE_SIZE ) @@ -4425,26 +6143,39 @@ PyObject *_PyUnicode_DecodeUnicodeInternal(const char *s, endinpos = s - starts + Py_UNICODE_SIZE; reason = "illegal code point (> 0x10FFFF)"; } - outpos = p - PyUnicode_AS_UNICODE(v); if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicode_internal", reason, &starts, &end, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) { + &v, &outpos)) goto onError; - } + continue; } - else { - p++; - s += Py_UNICODE_SIZE; + + s += Py_UNICODE_SIZE; +#ifndef Py_UNICODE_WIDE + if (Py_UNICODE_IS_HIGH_SURROGATE(ch) && s < end) + { + Py_UNICODE uch2; + ((char *) &uch2)[0] = s[0]; + ((char *) &uch2)[1] = s[1]; + if (Py_UNICODE_IS_LOW_SURROGATE(uch2)) + { + ch = Py_UNICODE_JOIN_SURROGATES(uch, uch2); + s += Py_UNICODE_SIZE; + } } +#endif + + if (unicode_putchar(&v, &outpos, ch) < 0) + goto onError; } - if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) + if (unicode_resize(&v, outpos) < 0) goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); - return (PyObject *)v; + return unicode_result(v); onError: Py_XDECREF(v); @@ -4455,57 +6186,27 @@ PyObject *_PyUnicode_DecodeUnicodeInternal(const char *s, /* --- Latin-1 Codec ------------------------------------------------------ */ -PyObject *PyUnicode_DecodeLatin1(const char *s, - Py_ssize_t size, - const char *errors) +PyObject * +PyUnicode_DecodeLatin1(const char *s, + Py_ssize_t size, + const char *errors) { - PyUnicodeObject *v; - Py_UNICODE *p; - const char *e, *unrolled_end; - /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ - if (size == 1) { - Py_UNICODE r = *(unsigned char*)s; - return PyUnicode_FromUnicode(&r, 1); - } - - v = _PyUnicode_New(size); - if (v == NULL) - goto onError; - if (size == 0) - return (PyObject *)v; - p = PyUnicode_AS_UNICODE(v); - e = s + size; - /* Unrolling the copy makes it much faster by reducing the looping - overhead. This is similar to what many memcpy() implementations do. */ - unrolled_end = e - 4; - while (s < unrolled_end) { - p[0] = (unsigned char) s[0]; - p[1] = (unsigned char) s[1]; - p[2] = (unsigned char) s[2]; - p[3] = (unsigned char) s[3]; - s += 4; - p += 4; - } - while (s < e) - *p++ = (unsigned char) *s++; - return (PyObject *)v; - - onError: - Py_XDECREF(v); - return NULL; + return _PyUnicode_FromUCS1((unsigned char*)s, size); } /* create or adjust a UnicodeEncodeError */ -static void make_encode_exception(PyObject **exceptionObject, - const char *encoding, - const Py_UNICODE *unicode, Py_ssize_t size, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason) +static void +make_encode_exception(PyObject **exceptionObject, + const char *encoding, + PyObject *unicode, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason) { if (*exceptionObject == NULL) { - *exceptionObject = PyUnicodeEncodeError_Create( - encoding, unicode, size, startpos, endpos, reason); + *exceptionObject = PyObject_CallFunction( + PyExc_UnicodeEncodeError, "sOnns", + encoding, unicode, startpos, endpos, reason); } else { if (PyUnicodeEncodeError_SetStart(*exceptionObject, startpos)) @@ -4522,14 +6223,15 @@ static void make_encode_exception(PyObject **exceptionObject, } /* raises a UnicodeEncodeError */ -static void raise_encode_exception(PyObject **exceptionObject, - const char *encoding, - const Py_UNICODE *unicode, Py_ssize_t size, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason) +static void +raise_encode_exception(PyObject **exceptionObject, + const char *encoding, + PyObject *unicode, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason) { make_encode_exception(exceptionObject, - encoding, unicode, size, startpos, endpos, reason); + encoding, unicode, startpos, endpos, reason); if (*exceptionObject != NULL) PyCodec_StrictErrors(*exceptionObject); } @@ -4538,15 +6240,16 @@ static void raise_encode_exception(PyObject **exceptionObject, build arguments, call the callback and check the arguments, put the result into newpos and return the replacement string, which has to be freed by the caller */ -static PyObject *unicode_encode_call_errorhandler(const char *errors, - PyObject **errorHandler, - const char *encoding, const char *reason, - const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, - Py_ssize_t startpos, Py_ssize_t endpos, - Py_ssize_t *newpos) +static PyObject * +unicode_encode_call_errorhandler(const char *errors, + PyObject **errorHandler, + const char *encoding, const char *reason, + PyObject *unicode, PyObject **exceptionObject, + Py_ssize_t startpos, Py_ssize_t endpos, + Py_ssize_t *newpos) { static char *argparse = "On;encoding error handler must return (str/bytes, int) tuple"; - + Py_ssize_t len; PyObject *restuple; PyObject *resunicode; @@ -4556,8 +6259,12 @@ static PyObject *unicode_encode_call_errorhandler(const char *errors, return NULL; } + if (PyUnicode_READY(unicode) == -1) + return NULL; + len = PyUnicode_GET_LENGTH(unicode); + make_encode_exception(exceptionObject, - encoding, unicode, size, startpos, endpos, reason); + encoding, unicode, startpos, endpos, reason); if (*exceptionObject == NULL) return NULL; @@ -4581,8 +6288,8 @@ static PyObject *unicode_encode_call_errorhandler(const char *errors, return NULL; } if (*newpos<0) - *newpos = size+*newpos; - if (*newpos<0 || *newpos>size) { + *newpos = len + *newpos; + if (*newpos<0 || *newpos>len) { PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos); Py_DECREF(restuple); return NULL; @@ -4592,18 +6299,17 @@ static PyObject *unicode_encode_call_errorhandler(const char *errors, return resunicode; } -static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, - Py_ssize_t size, - const char *errors, - int limit) +static PyObject * +unicode_encode_ucs1(PyObject *unicode, + const char *errors, + unsigned int limit) { + /* input state */ + Py_ssize_t pos=0, size; + int kind; + void *data; /* output object */ PyObject *res; - /* pointers to the beginning and end+1 of input */ - const Py_UNICODE *startp = p; - const Py_UNICODE *endp = p + size; - /* pointer to the beginning of the unencodable characters */ - /* const Py_UNICODE *badp = NULL; */ /* pointer into the output */ char *str; /* current output position */ @@ -4616,6 +6322,11 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */ int known_errorHandler = -1; + if (PyUnicode_READY(unicode) == -1) + return NULL; + size = PyUnicode_GET_LENGTH(unicode); + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); /* allocate enough for a simple encoding without replacements, if we need more, we'll resize */ if (size == 0) @@ -4626,28 +6337,24 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, str = PyBytes_AS_STRING(res); ressize = size; - while (p<endp) { - Py_UNICODE c = *p; + while (pos < size) { + Py_UCS4 c = PyUnicode_READ(kind, data, pos); /* can we encode this? */ if (c<limit) { /* no overflow check, because we know that the space is enough */ *str++ = (char)c; - ++p; + ++pos; } else { - Py_ssize_t unicodepos = p-startp; Py_ssize_t requiredsize; PyObject *repunicode; - Py_ssize_t repsize; - Py_ssize_t newpos; - Py_ssize_t respos; - Py_UNICODE *uni2; + Py_ssize_t repsize, newpos, respos, i; /* startpos for collecting unencodable chars */ - const Py_UNICODE *collstart = p; - const Py_UNICODE *collend = p; + Py_ssize_t collstart = pos; + Py_ssize_t collend = pos; /* find all unecodable characters */ - while ((collend < endp) && ((*collend)>=limit)) + while ((collend < size) && (PyUnicode_READ(kind, data, collend)>=limit)) ++collend; /* cache callback name lookup (if not done yet, i.e. it's the first error) */ if (known_errorHandler==-1) { @@ -4664,39 +6371,37 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, } switch (known_errorHandler) { case 1: /* strict */ - raise_encode_exception(&exc, encoding, startp, size, collstart-startp, collend-startp, reason); + raise_encode_exception(&exc, encoding, unicode, collstart, collend, reason); goto onError; case 2: /* replace */ while (collstart++<collend) *str++ = '?'; /* fall through */ case 3: /* ignore */ - p = collend; + pos = collend; break; case 4: /* xmlcharrefreplace */ respos = str - PyBytes_AS_STRING(res); - /* determine replacement size (temporarily (mis)uses p) */ - for (p = collstart, repsize = 0; p < collend; ++p) { - if (*p<10) + /* determine replacement size */ + for (i = collstart, repsize = 0; i < collend; ++i) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch < 10) repsize += 2+1+1; - else if (*p<100) + else if (ch < 100) repsize += 2+2+1; - else if (*p<1000) + else if (ch < 1000) repsize += 2+3+1; - else if (*p<10000) + else if (ch < 10000) repsize += 2+4+1; -#ifndef Py_UNICODE_WIDE - else + else if (ch < 100000) repsize += 2+5+1; -#else - else if (*p<100000) - repsize += 2+5+1; - else if (*p<1000000) + else if (ch < 1000000) repsize += 2+6+1; - else + else { + assert(ch <= MAX_UNICODE); repsize += 2+7+1; -#endif + } } - requiredsize = respos+repsize+(endp-collend); + requiredsize = respos+repsize+(size-collend); if (requiredsize > ressize) { if (requiredsize<2*ressize) requiredsize = 2*ressize; @@ -4705,17 +6410,18 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, str = PyBytes_AS_STRING(res) + respos; ressize = requiredsize; } - /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) { - str += sprintf(str, "&#%d;", (int)*p); + /* generate replacement */ + for (i = collstart; i < collend; ++i) { + str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); } - p = collend; + pos = collend; break; default: repunicode = unicode_encode_call_errorhandler(errors, &errorHandler, - encoding, reason, startp, size, &exc, - collstart-startp, collend-startp, &newpos); - if (repunicode == NULL) + encoding, reason, unicode, &exc, + collstart, collend, &newpos); + if (repunicode == NULL || (PyUnicode_Check(repunicode) && + PyUnicode_READY(repunicode) == -1)) goto onError; if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ @@ -4732,7 +6438,7 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, } memcpy(str, PyBytes_AsString(repunicode), repsize); str += repsize; - p = startp + newpos; + pos = newpos; Py_DECREF(repunicode); break; } @@ -4740,8 +6446,8 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, have+the replacement+the rest of the string, so we won't have to check space for encodable characters) */ respos = str - PyBytes_AS_STRING(res); - repsize = PyUnicode_GET_SIZE(repunicode); - requiredsize = respos+repsize+(endp-collend); + repsize = PyUnicode_GET_LENGTH(repunicode); + requiredsize = respos+repsize+(size-collend); if (requiredsize > ressize) { if (requiredsize<2*ressize) requiredsize = 2*ressize; @@ -4754,17 +6460,17 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, } /* check if there is anything unencodable in the replacement and copy it to the output */ - for (uni2 = PyUnicode_AS_UNICODE(repunicode);repsize-->0; ++uni2, ++str) { - c = *uni2; + for (i = 0; repsize-->0; ++i, ++str) { + c = PyUnicode_READ_CHAR(repunicode, i); if (c >= limit) { - raise_encode_exception(&exc, encoding, startp, size, - unicodepos, unicodepos+1, reason); + raise_encode_exception(&exc, encoding, unicode, + pos, pos+1, reason); Py_DECREF(repunicode); goto onError; } *str = (char)c; } - p = startp + newpos; + pos = newpos; Py_DECREF(repunicode); } } @@ -4788,33 +6494,57 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, return NULL; } -PyObject *PyUnicode_EncodeLatin1(const Py_UNICODE *p, - Py_ssize_t size, - const char *errors) +/* Deprecated */ +PyObject * +PyUnicode_EncodeLatin1(const Py_UNICODE *p, + Py_ssize_t size, + const char *errors) { - return unicode_encode_ucs1(p, size, errors, 256); + PyObject *result; + PyObject *unicode = PyUnicode_FromUnicode(p, size); + if (unicode == NULL) + return NULL; + result = unicode_encode_ucs1(unicode, errors, 256); + Py_DECREF(unicode); + return result; } -PyObject *PyUnicode_AsLatin1String(PyObject *unicode) +PyObject * +_PyUnicode_AsLatin1String(PyObject *unicode, const char *errors) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - NULL); + if (PyUnicode_READY(unicode) == -1) + return NULL; + /* Fast path: if it is a one-byte string, construct + bytes object directly. */ + if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND) + return PyBytes_FromStringAndSize(PyUnicode_DATA(unicode), + PyUnicode_GET_LENGTH(unicode)); + /* Non-Latin-1 characters present. Defer to above function to + raise the exception. */ + return unicode_encode_ucs1(unicode, errors, 256); +} + +PyObject* +PyUnicode_AsLatin1String(PyObject *unicode) +{ + return _PyUnicode_AsLatin1String(unicode, NULL); } /* --- 7-bit ASCII Codec -------------------------------------------------- */ -PyObject *PyUnicode_DecodeASCII(const char *s, - Py_ssize_t size, - const char *errors) +PyObject * +PyUnicode_DecodeASCII(const char *s, + Py_ssize_t size, + const char *errors) { const char *starts = s; - PyUnicodeObject *v; - Py_UNICODE *p; + PyObject *unicode; + int kind; + void *data; Py_ssize_t startinpos; Py_ssize_t endinpos; Py_ssize_t outpos; @@ -4822,70 +6552,99 @@ PyObject *PyUnicode_DecodeASCII(const char *s, PyObject *errorHandler = NULL; PyObject *exc = NULL; - /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 && *(unsigned char*)s < 128) { - Py_UNICODE r = *(unsigned char*)s; - return PyUnicode_FromUnicode(&r, 1); + if (size == 0) { + Py_INCREF(unicode_empty); + return unicode_empty; } - v = _PyUnicode_New(size); - if (v == NULL) + /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if (size == 1 && (unsigned char)s[0] < 128) + return get_latin1_char((unsigned char)s[0]); + + unicode = PyUnicode_New(size, 127); + if (unicode == NULL) goto onError; - if (size == 0) - return (PyObject *)v; - p = PyUnicode_AS_UNICODE(v); + e = s + size; + data = PyUnicode_1BYTE_DATA(unicode); + outpos = ascii_decode(s, e, (Py_UCS1 *)data); + if (outpos == size) + return unicode; + + s += outpos; + kind = PyUnicode_1BYTE_KIND; while (s < e) { register unsigned char c = (unsigned char)*s; if (c < 128) { - *p++ = c; + PyUnicode_WRITE(kind, data, outpos++, c); ++s; } else { startinpos = s-starts; endinpos = startinpos + 1; - outpos = p - (Py_UNICODE *)PyUnicode_AS_UNICODE(v); if (unicode_decode_call_errorhandler( errors, &errorHandler, "ascii", "ordinal not in range(128)", &starts, &e, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) + &unicode, &outpos)) goto onError; + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); } } - if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v)) - if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) - goto onError; + if (unicode_resize(&unicode, outpos) < 0) + goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); - return (PyObject *)v; + assert(_PyUnicode_CheckConsistency(unicode, 1)); + return unicode; onError: - Py_XDECREF(v); + Py_XDECREF(unicode); Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; } -PyObject *PyUnicode_EncodeASCII(const Py_UNICODE *p, - Py_ssize_t size, - const char *errors) +/* Deprecated */ +PyObject * +PyUnicode_EncodeASCII(const Py_UNICODE *p, + Py_ssize_t size, + const char *errors) { - return unicode_encode_ucs1(p, size, errors, 128); + PyObject *result; + PyObject *unicode = PyUnicode_FromUnicode(p, size); + if (unicode == NULL) + return NULL; + result = unicode_encode_ucs1(unicode, errors, 128); + Py_DECREF(unicode); + return result; } -PyObject *PyUnicode_AsASCIIString(PyObject *unicode) +PyObject * +_PyUnicode_AsASCIIString(PyObject *unicode, const char *errors) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - NULL); + if (PyUnicode_READY(unicode) == -1) + return NULL; + /* Fast path: if it is an ASCII-only string, construct bytes object + directly. Else defer to above function to raise the exception. */ + if (PyUnicode_MAX_CHAR_VALUE(unicode) < 128) + return PyBytes_FromStringAndSize(PyUnicode_DATA(unicode), + PyUnicode_GET_LENGTH(unicode)); + return unicode_encode_ucs1(unicode, errors, 128); } -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +PyObject * +PyUnicode_AsASCIIString(PyObject *unicode) +{ + return _PyUnicode_AsASCIIString(unicode, NULL); +} + +#ifdef HAVE_MBCS /* --- MBCS codecs for Windows -------------------------------------------- */ @@ -4893,336 +6652,842 @@ PyObject *PyUnicode_AsASCIIString(PyObject *unicode) #define NEED_RETRY #endif -/* XXX This code is limited to "true" double-byte encodings, as - a) it assumes an incomplete character consists of a single byte, and - b) IsDBCSLeadByte (probably) does not work for non-DBCS multi-byte - encodings, see IsDBCSLeadByteEx documentation. */ +#ifndef WC_ERR_INVALID_CHARS +# define WC_ERR_INVALID_CHARS 0x0080 +#endif -static int is_dbcs_lead_byte(const char *s, int offset) +static char* +code_page_name(UINT code_page, PyObject **obj) { - const char *curr = s + offset; + *obj = NULL; + if (code_page == CP_ACP) + return "mbcs"; + if (code_page == CP_UTF7) + return "CP_UTF7"; + if (code_page == CP_UTF8) + return "CP_UTF8"; - if (IsDBCSLeadByte(*curr)) { - const char *prev = CharPrev(s, curr); - return (prev == curr) || !IsDBCSLeadByte(*prev) || (curr - prev == 2); - } - return 0; + *obj = PyBytes_FromFormat("cp%u", code_page); + if (*obj == NULL) + return NULL; + return PyBytes_AS_STRING(*obj); } -/* - * Decode MBCS string into unicode object. If 'final' is set, converts - * trailing lead-byte too. Returns consumed size if succeed, -1 otherwise. - */ -static int decode_mbcs(PyUnicodeObject **v, - const char *s, /* MBCS string */ - int size, /* sizeof MBCS string */ - int final, - const char *errors) +static int +is_dbcs_lead_byte(UINT code_page, const char *s, int offset) { - Py_UNICODE *p; - Py_ssize_t n; - DWORD usize; - DWORD flags; + const char *curr = s + offset; + const char *prev; - assert(size >= 0); + if (!IsDBCSLeadByteEx(code_page, *curr)) + return 0; - /* check and handle 'errors' arg */ - if (errors==NULL || strcmp(errors, "strict")==0) - flags = MB_ERR_INVALID_CHARS; - else if (strcmp(errors, "ignore")==0) - flags = 0; - else { - PyErr_Format(PyExc_ValueError, - "mbcs encoding does not support errors='%s'", - errors); - return -1; + prev = CharPrevExA(code_page, s, curr, 0); + if (prev == curr) + return 1; + /* FIXME: This code is limited to "true" double-byte encodings, + as it assumes an incomplete character consists of a single + byte. */ + if (curr - prev == 2) + return 1; + if (!IsDBCSLeadByteEx(code_page, *prev)) + return 1; + return 0; +} + +static DWORD +decode_code_page_flags(UINT code_page) +{ + if (code_page == CP_UTF7) { + /* The CP_UTF7 decoder only supports flags=0 */ + return 0; } + else + return MB_ERR_INVALID_CHARS; +} - /* Skip trailing lead-byte unless 'final' is set */ - if (!final && size >= 1 && is_dbcs_lead_byte(s, size - 1)) - --size; +/* + * Decode a byte string from a Windows code page into unicode object in strict + * mode. + * + * Returns consumed size if succeed, returns -2 on decode error, or raise a + * WindowsError and returns -1 on other error. + */ +static int +decode_code_page_strict(UINT code_page, + PyObject **v, + const char *in, + int insize) +{ + const DWORD flags = decode_code_page_flags(code_page); + wchar_t *out; + DWORD outsize; /* First get the size of the result */ - if (size > 0) { - usize = MultiByteToWideChar(CP_ACP, flags, s, size, NULL, 0); - if (usize==0) - goto mbcs_decode_error; - } else - usize = 0; + assert(insize > 0); + outsize = MultiByteToWideChar(code_page, flags, in, insize, NULL, 0); + if (outsize <= 0) + goto error; if (*v == NULL) { /* Create unicode object */ - *v = _PyUnicode_New(usize); + /* FIXME: don't use _PyUnicode_New(), but allocate a wchar_t* buffer */ + *v = (PyObject*)_PyUnicode_New(outsize); if (*v == NULL) return -1; - n = 0; + out = PyUnicode_AS_UNICODE(*v); } else { /* Extend unicode object */ - n = PyUnicode_GET_SIZE(*v); - if (_PyUnicode_Resize(v, n + usize) < 0) + Py_ssize_t n = PyUnicode_GET_SIZE(*v); + if (unicode_resize(v, n + outsize) < 0) return -1; + out = PyUnicode_AS_UNICODE(*v) + n; } /* Do the conversion */ - if (usize > 0) { - p = PyUnicode_AS_UNICODE(*v) + n; - if (0 == MultiByteToWideChar(CP_ACP, flags, s, size, p, usize)) { - goto mbcs_decode_error; - } - } - return size; + outsize = MultiByteToWideChar(code_page, flags, in, insize, out, outsize); + if (outsize <= 0) + goto error; + return insize; -mbcs_decode_error: - /* If the last error was ERROR_NO_UNICODE_TRANSLATION, then - we raise a UnicodeDecodeError - else it is a 'generic' - windows error - */ - if (GetLastError()==ERROR_NO_UNICODE_TRANSLATION) { - /* Ideally, we should get reason from FormatMessage - this - is the Windows 2000 English version of the message - */ - PyObject *exc = NULL; - const char *reason = "No mapping for the Unicode character exists " - "in the target multi-byte code page."; - make_decode_exception(&exc, "mbcs", s, size, 0, 0, reason); +error: + if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION) + return -2; + PyErr_SetFromWindowsErr(0); + return -1; +} + +/* + * Decode a byte string from a code page into unicode object with an error + * handler. + * + * Returns consumed size if succeed, or raise a WindowsError or + * UnicodeDecodeError exception and returns -1 on error. + */ +static int +decode_code_page_errors(UINT code_page, + PyObject **v, + const char *in, const int size, + const char *errors) +{ + const char *startin = in; + const char *endin = in + size; + const DWORD flags = decode_code_page_flags(code_page); + /* Ideally, we should get reason from FormatMessage. This is the Windows + 2000 English version of the message. */ + const char *reason = "No mapping for the Unicode character exists " + "in the target code page."; + /* each step cannot decode more than 1 character, but a character can be + represented as a surrogate pair */ + wchar_t buffer[2], *startout, *out; + int insize, outsize; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + PyObject *encoding_obj = NULL; + char *encoding; + DWORD err; + int ret = -1; + + assert(size > 0); + + encoding = code_page_name(code_page, &encoding_obj); + if (encoding == NULL) + return -1; + + if (errors == NULL || strcmp(errors, "strict") == 0) { + /* The last error was ERROR_NO_UNICODE_TRANSLATION, then we raise a + UnicodeDecodeError. */ + make_decode_exception(&exc, encoding, in, size, 0, 0, reason); if (exc != NULL) { PyCodec_StrictErrors(exc); - Py_DECREF(exc); + Py_CLEAR(exc); } - } else { - PyErr_SetFromWindowsErrWithFilename(0, NULL); + goto error; } - return -1; + + if (*v == NULL) { + /* Create unicode object */ + if (size > PY_SSIZE_T_MAX / (Py_ssize_t)Py_ARRAY_LENGTH(buffer)) { + PyErr_NoMemory(); + goto error; + } + /* FIXME: don't use _PyUnicode_New(), but allocate a wchar_t* buffer */ + *v = (PyObject*)_PyUnicode_New(size * Py_ARRAY_LENGTH(buffer)); + if (*v == NULL) + goto error; + startout = PyUnicode_AS_UNICODE(*v); + } + else { + /* Extend unicode object */ + Py_ssize_t n = PyUnicode_GET_SIZE(*v); + if (size > (PY_SSIZE_T_MAX - n) / (Py_ssize_t)Py_ARRAY_LENGTH(buffer)) { + PyErr_NoMemory(); + goto error; + } + if (unicode_resize(v, n + size * Py_ARRAY_LENGTH(buffer)) < 0) + goto error; + startout = PyUnicode_AS_UNICODE(*v) + n; + } + + /* Decode the byte string character per character */ + out = startout; + while (in < endin) + { + /* Decode a character */ + insize = 1; + do + { + outsize = MultiByteToWideChar(code_page, flags, + in, insize, + buffer, Py_ARRAY_LENGTH(buffer)); + if (outsize > 0) + break; + err = GetLastError(); + if (err != ERROR_NO_UNICODE_TRANSLATION + && err != ERROR_INSUFFICIENT_BUFFER) + { + PyErr_SetFromWindowsErr(0); + goto error; + } + insize++; + } + /* 4=maximum length of a UTF-8 sequence */ + while (insize <= 4 && (in + insize) <= endin); + + if (outsize <= 0) { + Py_ssize_t startinpos, endinpos, outpos; + + startinpos = in - startin; + endinpos = startinpos + 1; + outpos = out - PyUnicode_AS_UNICODE(*v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + encoding, reason, + &startin, &endin, &startinpos, &endinpos, &exc, &in, + v, &outpos)) + { + goto error; + } + out = PyUnicode_AS_UNICODE(*v) + outpos; + } + else { + in += insize; + memcpy(out, buffer, outsize * sizeof(wchar_t)); + out += outsize; + } + } + + /* write a NUL character at the end */ + *out = 0; + + /* Extend unicode object */ + outsize = out - startout; + assert(outsize <= PyUnicode_WSTR_LENGTH(*v)); + if (unicode_resize(v, outsize) < 0) + goto error; + ret = size; + +error: + Py_XDECREF(encoding_obj); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return ret; } -PyObject *PyUnicode_DecodeMBCSStateful(const char *s, - Py_ssize_t size, - const char *errors, - Py_ssize_t *consumed) +static PyObject * +decode_code_page_stateful(int code_page, + const char *s, Py_ssize_t size, + const char *errors, Py_ssize_t *consumed) { - PyUnicodeObject *v = NULL; - int done; + PyObject *v = NULL; + int chunk_size, final, converted, done; + + if (code_page < 0) { + PyErr_SetString(PyExc_ValueError, "invalid code page number"); + return NULL; + } if (consumed) *consumed = 0; + do + { #ifdef NEED_RETRY - retry: - if (size > INT_MAX) - done = decode_mbcs(&v, s, INT_MAX, 0, errors); - else + if (size > INT_MAX) { + chunk_size = INT_MAX; + final = 0; + done = 0; + } + else #endif - done = decode_mbcs(&v, s, (int)size, !consumed, errors); + { + chunk_size = (int)size; + final = (consumed == NULL); + done = 1; + } - if (done < 0) { - Py_XDECREF(v); - return NULL; - } + /* Skip trailing lead-byte unless 'final' is set */ + if (!final && is_dbcs_lead_byte(code_page, s, chunk_size - 1)) + --chunk_size; - if (consumed) - *consumed += done; + if (chunk_size == 0 && done) { + if (v != NULL) + break; + Py_INCREF(unicode_empty); + return unicode_empty; + } -#ifdef NEED_RETRY - if (size > INT_MAX) { - s += done; - size -= done; - goto retry; - } -#endif - return (PyObject *)v; + converted = decode_code_page_strict(code_page, &v, + s, chunk_size); + if (converted == -2) + converted = decode_code_page_errors(code_page, &v, + s, chunk_size, + errors); + assert(converted != 0); + + if (converted < 0) { + Py_XDECREF(v); + return NULL; + } + + if (consumed) + *consumed += converted; + + s += converted; + size -= converted; + } while (!done); + + return unicode_result(v); +} + +PyObject * +PyUnicode_DecodeCodePageStateful(int code_page, + const char *s, + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) +{ + return decode_code_page_stateful(code_page, s, size, errors, consumed); +} + +PyObject * +PyUnicode_DecodeMBCSStateful(const char *s, + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) +{ + return decode_code_page_stateful(CP_ACP, s, size, errors, consumed); } -PyObject *PyUnicode_DecodeMBCS(const char *s, - Py_ssize_t size, - const char *errors) +PyObject * +PyUnicode_DecodeMBCS(const char *s, + Py_ssize_t size, + const char *errors) { return PyUnicode_DecodeMBCSStateful(s, size, errors, NULL); } +static DWORD +encode_code_page_flags(UINT code_page, const char *errors) +{ + if (code_page == CP_UTF8) { + if (winver.dwMajorVersion >= 6) + /* CP_UTF8 supports WC_ERR_INVALID_CHARS on Windows Vista + and later */ + return WC_ERR_INVALID_CHARS; + else + /* CP_UTF8 only supports flags=0 on Windows older than Vista */ + return 0; + } + else if (code_page == CP_UTF7) { + /* CP_UTF7 only supports flags=0 */ + return 0; + } + else { + if (errors != NULL && strcmp(errors, "replace") == 0) + return 0; + else + return WC_NO_BEST_FIT_CHARS; + } +} + /* - * Convert unicode into string object (MBCS). - * Returns 0 if succeed, -1 otherwise. + * Encode a Unicode string to a Windows code page into a byte string in strict + * mode. + * + * Returns consumed characters if succeed, returns -2 on encode error, or raise + * a WindowsError and returns -1 on other error. */ -static int encode_mbcs(PyObject **repr, - const Py_UNICODE *p, /* unicode */ - int size, /* size of unicode */ - const char* errors) +static int +encode_code_page_strict(UINT code_page, PyObject **outbytes, + PyObject *unicode, Py_ssize_t offset, int len, + const char* errors) { BOOL usedDefaultChar = FALSE; - BOOL *pusedDefaultChar; - int mbcssize; - Py_ssize_t n; + BOOL *pusedDefaultChar = &usedDefaultChar; + int outsize; PyObject *exc = NULL; - DWORD flags; + wchar_t *p; + Py_ssize_t size; + const DWORD flags = encode_code_page_flags(code_page, NULL); + char *out; + /* Create a substring so that we can get the UTF-16 representation + of just the slice under consideration. */ + PyObject *substring; - assert(size >= 0); + assert(len > 0); - /* check and handle 'errors' arg */ - if (errors==NULL || strcmp(errors, "strict")==0) { - flags = WC_NO_BEST_FIT_CHARS; + if (code_page != CP_UTF8 && code_page != CP_UTF7) pusedDefaultChar = &usedDefaultChar; - } else if (strcmp(errors, "replace")==0) { - flags = 0; + else pusedDefaultChar = NULL; - } else { - PyErr_Format(PyExc_ValueError, - "mbcs encoding does not support errors='%s'", - errors); - return -1; + + substring = PyUnicode_Substring(unicode, offset, offset+len); + if (substring == NULL) + return -1; + p = PyUnicode_AsUnicodeAndSize(substring, &size); + if (p == NULL) { + Py_DECREF(substring); + return -1; } /* First get the size of the result */ - if (size > 0) { - mbcssize = WideCharToMultiByte(CP_ACP, flags, p, size, NULL, 0, - NULL, pusedDefaultChar); - if (mbcssize == 0) { - PyErr_SetFromWindowsErrWithFilename(0, NULL); - return -1; - } - /* If we used a default char, then we failed! */ - if (pusedDefaultChar && *pusedDefaultChar) - goto mbcs_encode_error; - } else { - mbcssize = 0; + outsize = WideCharToMultiByte(code_page, flags, + p, size, + NULL, 0, + NULL, pusedDefaultChar); + if (outsize <= 0) + goto error; + /* If we used a default char, then we failed! */ + if (pusedDefaultChar && *pusedDefaultChar) { + Py_DECREF(substring); + return -2; } - if (*repr == NULL) { + if (*outbytes == NULL) { /* Create string object */ - *repr = PyBytes_FromStringAndSize(NULL, mbcssize); - if (*repr == NULL) + *outbytes = PyBytes_FromStringAndSize(NULL, outsize); + if (*outbytes == NULL) { + Py_DECREF(substring); return -1; - n = 0; + } + out = PyBytes_AS_STRING(*outbytes); } else { /* Extend string object */ - n = PyBytes_Size(*repr); - if (_PyBytes_Resize(repr, n + mbcssize) < 0) + const Py_ssize_t n = PyBytes_Size(*outbytes); + if (outsize > PY_SSIZE_T_MAX - n) { + PyErr_NoMemory(); + Py_DECREF(substring); + return -1; + } + if (_PyBytes_Resize(outbytes, n + outsize) < 0) { + Py_DECREF(substring); return -1; + } + out = PyBytes_AS_STRING(*outbytes) + n; } /* Do the conversion */ - if (size > 0) { - char *s = PyBytes_AS_STRING(*repr) + n; - if (0 == WideCharToMultiByte(CP_ACP, flags, p, size, s, mbcssize, - NULL, pusedDefaultChar)) { - PyErr_SetFromWindowsErrWithFilename(0, NULL); - return -1; + outsize = WideCharToMultiByte(code_page, flags, + p, size, + out, outsize, + NULL, pusedDefaultChar); + Py_CLEAR(substring); + if (outsize <= 0) + goto error; + if (pusedDefaultChar && *pusedDefaultChar) + return -2; + return 0; + +error: + Py_XDECREF(substring); + if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION) + return -2; + PyErr_SetFromWindowsErr(0); + return -1; +} + +/* + * Encode a Unicode string to a Windows code page into a byte string using a + * error handler. + * + * Returns consumed characters if succeed, or raise a WindowsError and returns + * -1 on other error. + */ +static int +encode_code_page_errors(UINT code_page, PyObject **outbytes, + PyObject *unicode, Py_ssize_t unicode_offset, + Py_ssize_t insize, const char* errors) +{ + const DWORD flags = encode_code_page_flags(code_page, errors); + Py_ssize_t pos = unicode_offset; + Py_ssize_t endin = unicode_offset + insize; + /* Ideally, we should get reason from FormatMessage. This is the Windows + 2000 English version of the message. */ + const char *reason = "invalid character"; + /* 4=maximum length of a UTF-8 sequence */ + char buffer[4]; + BOOL usedDefaultChar = FALSE, *pusedDefaultChar; + Py_ssize_t outsize; + char *out; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + PyObject *encoding_obj = NULL; + char *encoding; + Py_ssize_t newpos, newoutsize; + PyObject *rep; + int ret = -1; + + assert(insize > 0); + + encoding = code_page_name(code_page, &encoding_obj); + if (encoding == NULL) + return -1; + + if (errors == NULL || strcmp(errors, "strict") == 0) { + /* The last error was ERROR_NO_UNICODE_TRANSLATION, + then we raise a UnicodeEncodeError. */ + make_encode_exception(&exc, encoding, unicode, 0, 0, reason); + if (exc != NULL) { + PyCodec_StrictErrors(exc); + Py_DECREF(exc); } - if (pusedDefaultChar && *pusedDefaultChar) - goto mbcs_encode_error; + Py_XDECREF(encoding_obj); + return -1; } - return 0; -mbcs_encode_error: - raise_encode_exception(&exc, "mbcs", p, size, 0, 0, "invalid character"); + if (code_page != CP_UTF8 && code_page != CP_UTF7) + pusedDefaultChar = &usedDefaultChar; + else + pusedDefaultChar = NULL; + + if (Py_ARRAY_LENGTH(buffer) > PY_SSIZE_T_MAX / insize) { + PyErr_NoMemory(); + goto error; + } + outsize = insize * Py_ARRAY_LENGTH(buffer); + + if (*outbytes == NULL) { + /* Create string object */ + *outbytes = PyBytes_FromStringAndSize(NULL, outsize); + if (*outbytes == NULL) + goto error; + out = PyBytes_AS_STRING(*outbytes); + } + else { + /* Extend string object */ + Py_ssize_t n = PyBytes_Size(*outbytes); + if (n > PY_SSIZE_T_MAX - outsize) { + PyErr_NoMemory(); + goto error; + } + if (_PyBytes_Resize(outbytes, n + outsize) < 0) + goto error; + out = PyBytes_AS_STRING(*outbytes) + n; + } + + /* Encode the string character per character */ + while (pos < endin) + { + Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, pos); + wchar_t chars[2]; + int charsize; + if (ch < 0x10000) { + chars[0] = (wchar_t)ch; + charsize = 1; + } + else { + ch -= 0x10000; + chars[0] = 0xd800 + (ch >> 10); + chars[1] = 0xdc00 + (ch & 0x3ff); + charsize = 2; + } + + outsize = WideCharToMultiByte(code_page, flags, + chars, charsize, + buffer, Py_ARRAY_LENGTH(buffer), + NULL, pusedDefaultChar); + if (outsize > 0) { + if (pusedDefaultChar == NULL || !(*pusedDefaultChar)) + { + pos++; + memcpy(out, buffer, outsize); + out += outsize; + continue; + } + } + else if (GetLastError() != ERROR_NO_UNICODE_TRANSLATION) { + PyErr_SetFromWindowsErr(0); + goto error; + } + + rep = unicode_encode_call_errorhandler( + errors, &errorHandler, encoding, reason, + unicode, &exc, + pos, pos + 1, &newpos); + if (rep == NULL) + goto error; + pos = newpos; + + if (PyBytes_Check(rep)) { + outsize = PyBytes_GET_SIZE(rep); + if (outsize != 1) { + Py_ssize_t offset = out - PyBytes_AS_STRING(*outbytes); + newoutsize = PyBytes_GET_SIZE(*outbytes) + (outsize - 1); + if (_PyBytes_Resize(outbytes, newoutsize) < 0) { + Py_DECREF(rep); + goto error; + } + out = PyBytes_AS_STRING(*outbytes) + offset; + } + memcpy(out, PyBytes_AS_STRING(rep), outsize); + out += outsize; + } + else { + Py_ssize_t i; + enum PyUnicode_Kind kind; + void *data; + + if (PyUnicode_READY(rep) == -1) { + Py_DECREF(rep); + goto error; + } + + outsize = PyUnicode_GET_LENGTH(rep); + if (outsize != 1) { + Py_ssize_t offset = out - PyBytes_AS_STRING(*outbytes); + newoutsize = PyBytes_GET_SIZE(*outbytes) + (outsize - 1); + if (_PyBytes_Resize(outbytes, newoutsize) < 0) { + Py_DECREF(rep); + goto error; + } + out = PyBytes_AS_STRING(*outbytes) + offset; + } + kind = PyUnicode_KIND(rep); + data = PyUnicode_DATA(rep); + for (i=0; i < outsize; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch > 127) { + raise_encode_exception(&exc, + encoding, unicode, + pos, pos + 1, + "unable to encode error handler result to ASCII"); + Py_DECREF(rep); + goto error; + } + *out = (unsigned char)ch; + out++; + } + } + Py_DECREF(rep); + } + /* write a NUL byte */ + *out = 0; + outsize = out - PyBytes_AS_STRING(*outbytes); + assert(outsize <= PyBytes_GET_SIZE(*outbytes)); + if (_PyBytes_Resize(outbytes, outsize) < 0) + goto error; + ret = 0; + +error: + Py_XDECREF(encoding_obj); + Py_XDECREF(errorHandler); Py_XDECREF(exc); - return -1; + return ret; } -PyObject *PyUnicode_EncodeMBCS(const Py_UNICODE *p, - Py_ssize_t size, - const char *errors) +static PyObject * +encode_code_page(int code_page, + PyObject *unicode, + const char *errors) { - PyObject *repr = NULL; - int ret; + Py_ssize_t len; + PyObject *outbytes = NULL; + Py_ssize_t offset; + int chunk_len, ret, done; -#ifdef NEED_RETRY - retry: - if (size > INT_MAX) - ret = encode_mbcs(&repr, p, INT_MAX, errors); - else -#endif - ret = encode_mbcs(&repr, p, (int)size, errors); + if (PyUnicode_READY(unicode) == -1) + return NULL; + len = PyUnicode_GET_LENGTH(unicode); - if (ret < 0) { - Py_XDECREF(repr); + if (code_page < 0) { + PyErr_SetString(PyExc_ValueError, "invalid code page number"); return NULL; } + if (len == 0) + return PyBytes_FromStringAndSize(NULL, 0); + + offset = 0; + do + { #ifdef NEED_RETRY - if (size > INT_MAX) { - p += INT_MAX; - size -= INT_MAX; - goto retry; - } + /* UTF-16 encoding may double the size, so use only INT_MAX/2 + chunks. */ + if (len > INT_MAX/2) { + chunk_len = INT_MAX/2; + done = 0; + } + else #endif + { + chunk_len = (int)len; + done = 1; + } + + ret = encode_code_page_strict(code_page, &outbytes, + unicode, offset, chunk_len, + errors); + if (ret == -2) + ret = encode_code_page_errors(code_page, &outbytes, + unicode, offset, + chunk_len, errors); + if (ret < 0) { + Py_XDECREF(outbytes); + return NULL; + } - return repr; + offset += chunk_len; + len -= chunk_len; + } while (!done); + + return outbytes; } -PyObject *PyUnicode_AsMBCSString(PyObject *unicode) +PyObject * +PyUnicode_EncodeMBCS(const Py_UNICODE *p, + Py_ssize_t size, + const char *errors) +{ + PyObject *unicode, *res; + unicode = PyUnicode_FromUnicode(p, size); + if (unicode == NULL) + return NULL; + res = encode_code_page(CP_ACP, unicode, errors); + Py_DECREF(unicode); + return res; +} + +PyObject * +PyUnicode_EncodeCodePage(int code_page, + PyObject *unicode, + const char *errors) +{ + return encode_code_page(code_page, unicode, errors); +} + +PyObject * +PyUnicode_AsMBCSString(PyObject *unicode) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - NULL); + return PyUnicode_EncodeCodePage(CP_ACP, unicode, NULL); } #undef NEED_RETRY -#endif /* MS_WINDOWS */ +#endif /* HAVE_MBCS */ /* --- Character Mapping Codec -------------------------------------------- */ -PyObject *PyUnicode_DecodeCharmap(const char *s, - Py_ssize_t size, - PyObject *mapping, - const char *errors) +PyObject * +PyUnicode_DecodeCharmap(const char *s, + Py_ssize_t size, + PyObject *mapping, + const char *errors) { const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; Py_ssize_t outpos; const char *e; - PyUnicodeObject *v; - Py_UNICODE *p; + PyObject *v; Py_ssize_t extrachars = 0; PyObject *errorHandler = NULL; PyObject *exc = NULL; - Py_UNICODE *mapstring = NULL; - Py_ssize_t maplen = 0; /* Default to Latin-1 */ if (mapping == NULL) return PyUnicode_DecodeLatin1(s, size, errors); - v = _PyUnicode_New(size); + v = PyUnicode_New(size, 127); if (v == NULL) goto onError; if (size == 0) - return (PyObject *)v; - p = PyUnicode_AS_UNICODE(v); + return v; + outpos = 0; e = s + size; if (PyUnicode_CheckExact(mapping)) { - mapstring = PyUnicode_AS_UNICODE(mapping); - maplen = PyUnicode_GET_SIZE(mapping); + Py_ssize_t maplen; + enum PyUnicode_Kind mapkind; + void *mapdata; + Py_UCS4 x; + + if (PyUnicode_READY(mapping) == -1) + return NULL; + + maplen = PyUnicode_GET_LENGTH(mapping); + mapdata = PyUnicode_DATA(mapping); + mapkind = PyUnicode_KIND(mapping); while (s < e) { - unsigned char ch = *s; - Py_UNICODE x = 0xfffe; /* illegal value */ + unsigned char ch; + if (mapkind == PyUnicode_2BYTE_KIND && maplen >= 256) { + enum PyUnicode_Kind outkind = PyUnicode_KIND(v); + if (outkind == PyUnicode_1BYTE_KIND) { + void *outdata = PyUnicode_DATA(v); + Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(v); + while (s < e) { + unsigned char ch = *s; + x = PyUnicode_READ(PyUnicode_2BYTE_KIND, mapdata, ch); + if (x > maxchar) + goto Error; + PyUnicode_WRITE(PyUnicode_1BYTE_KIND, outdata, outpos++, x); + ++s; + } + break; + } + else if (outkind == PyUnicode_2BYTE_KIND) { + void *outdata = PyUnicode_DATA(v); + while (s < e) { + unsigned char ch = *s; + x = PyUnicode_READ(PyUnicode_2BYTE_KIND, mapdata, ch); + if (x == 0xFFFE) + goto Error; + PyUnicode_WRITE(PyUnicode_2BYTE_KIND, outdata, outpos++, x); + ++s; + } + break; + } + } + ch = *s; if (ch < maplen) - x = mapstring[ch]; - - if (x == 0xfffe) { + x = PyUnicode_READ(mapkind, mapdata, ch); + else + x = 0xfffe; /* invalid value */ +Error: + if (x == 0xfffe) + { /* undefined mapping */ - outpos = p-PyUnicode_AS_UNICODE(v); startinpos = s-starts; endinpos = startinpos+1; if (unicode_decode_call_errorhandler( errors, &errorHandler, "charmap", "character maps to <undefined>", &starts, &e, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) { + &v, &outpos)) { goto onError; } continue; } - *p++ = x; + + if (unicode_putchar(&v, &outpos, x) < 0) + goto onError; ++s; } } @@ -5250,48 +7515,25 @@ PyObject *PyUnicode_DecodeCharmap(const char *s, /* Apply mapping */ if (PyLong_Check(x)) { long value = PyLong_AS_LONG(x); - if (value < 0 || value > 0x10FFFF) { - PyErr_SetString(PyExc_TypeError, - "character mapping must be in range(0x110000)"); + if (value < 0 || value > MAX_UNICODE) { + PyErr_Format(PyExc_TypeError, + "character mapping must be in range(0x%lx)", + (unsigned long)MAX_UNICODE + 1); Py_DECREF(x); goto onError; } - -#ifndef Py_UNICODE_WIDE - if (value > 0xFFFF) { - /* see the code for 1-n mapping below */ - if (extrachars < 2) { - /* resize first */ - Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v); - Py_ssize_t needed = 10 - extrachars; - extrachars += needed; - /* XXX overflow detection missing */ - if (_PyUnicode_Resize(&v, - PyUnicode_GET_SIZE(v) + needed) < 0) { - Py_DECREF(x); - goto onError; - } - p = PyUnicode_AS_UNICODE(v) + oldpos; - } - value -= 0x10000; - *p++ = 0xD800 | (value >> 10); - *p++ = 0xDC00 | (value & 0x3FF); - extrachars -= 2; - } - else -#endif - *p++ = (Py_UNICODE)value; + if (unicode_putchar(&v, &outpos, value) < 0) + goto onError; } else if (x == Py_None) { /* undefined mapping */ - outpos = p-PyUnicode_AS_UNICODE(v); startinpos = s-starts; endinpos = startinpos+1; if (unicode_decode_call_errorhandler( errors, &errorHandler, "charmap", "character maps to <undefined>", &starts, &e, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) { + &v, &outpos)) { Py_DECREF(x); goto onError; } @@ -5299,32 +7541,37 @@ PyObject *PyUnicode_DecodeCharmap(const char *s, continue; } else if (PyUnicode_Check(x)) { - Py_ssize_t targetsize = PyUnicode_GET_SIZE(x); + Py_ssize_t targetsize; - if (targetsize == 1) - /* 1-1 mapping */ - *p++ = *PyUnicode_AS_UNICODE(x); + if (PyUnicode_READY(x) == -1) + goto onError; + targetsize = PyUnicode_GET_LENGTH(x); + if (targetsize == 1) { + /* 1-1 mapping */ + if (unicode_putchar(&v, &outpos, + PyUnicode_READ_CHAR(x, 0)) < 0) + goto onError; + } else if (targetsize > 1) { /* 1-n mapping */ if (targetsize > extrachars) { /* resize first */ - Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v); Py_ssize_t needed = (targetsize - extrachars) + \ (targetsize << 2); extrachars += needed; /* XXX overflow detection missing */ - if (_PyUnicode_Resize(&v, - PyUnicode_GET_SIZE(v) + needed) < 0) { + if (unicode_resize(&v, + PyUnicode_GET_LENGTH(v) + needed) < 0) + { Py_DECREF(x); goto onError; } - p = PyUnicode_AS_UNICODE(v) + oldpos; } - Py_UNICODE_COPY(p, - PyUnicode_AS_UNICODE(x), - targetsize); - p += targetsize; + if (unicode_widen(&v, outpos, PyUnicode_MAX_CHAR_VALUE(x)) < 0) + goto onError; + PyUnicode_CopyCharacters(v, outpos, x, 0, targetsize); + outpos += targetsize; extrachars -= targetsize; } /* 1-0 mapping: skip the character */ @@ -5340,12 +7587,11 @@ PyObject *PyUnicode_DecodeCharmap(const char *s, ++s; } } - if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v)) - if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) - goto onError; + if (unicode_resize(&v, outpos) < 0) + goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); - return (PyObject *)v; + return unicode_result(v); onError: Py_XDECREF(errorHandler); @@ -5356,7 +7602,7 @@ PyObject *PyUnicode_DecodeCharmap(const char *s, /* Charmap encoding: the lookup table */ -struct encoding_map{ +struct encoding_map { PyObject_HEAD unsigned char level1[32]; int count2, count3; @@ -5430,7 +7676,6 @@ static PyTypeObject EncodingMapType = { PyObject* PyUnicode_BuildEncodingMap(PyObject* string) { - Py_UNICODE *decode; PyObject *result; struct encoding_map *mresult; int i; @@ -5439,35 +7684,39 @@ PyUnicode_BuildEncodingMap(PyObject* string) unsigned char level2[512]; unsigned char *mlevel1, *mlevel2, *mlevel3; int count2 = 0, count3 = 0; + int kind; + void *data; + Py_ssize_t length; + Py_UCS4 ch; - if (!PyUnicode_Check(string) || PyUnicode_GetSize(string) != 256) { + if (!PyUnicode_Check(string) || !PyUnicode_GET_LENGTH(string)) { PyErr_BadArgument(); return NULL; } - decode = PyUnicode_AS_UNICODE(string); + kind = PyUnicode_KIND(string); + data = PyUnicode_DATA(string); + length = PyUnicode_GET_LENGTH(string); + length = Py_MIN(length, 256); memset(level1, 0xFF, sizeof level1); memset(level2, 0xFF, sizeof level2); /* If there isn't a one-to-one mapping of NULL to \0, or if there are non-BMP characters, we need to use a mapping dictionary. */ - if (decode[0] != 0) + if (PyUnicode_READ(kind, data, 0) != 0) need_dict = 1; - for (i = 1; i < 256; i++) { + for (i = 1; i < length; i++) { int l1, l2; - if (decode[i] == 0 -#ifdef Py_UNICODE_WIDE - || decode[i] > 0xFFFF -#endif - ) { + ch = PyUnicode_READ(kind, data, i); + if (ch == 0 || ch > 0xFFFF) { need_dict = 1; break; } - if (decode[i] == 0xFFFE) + if (ch == 0xFFFE) /* unmapped character */ continue; - l1 = decode[i] >> 11; - l2 = decode[i] >> 7; + l1 = ch >> 11; + l2 = ch >> 7; if (level1[l1] == 0xFF) level1[l1] = count2++; if (level2[l2] == 0xFF) @@ -5482,9 +7731,8 @@ PyUnicode_BuildEncodingMap(PyObject* string) PyObject *key, *value; if (!result) return NULL; - for (i = 0; i < 256; i++) { - key = value = NULL; - key = PyLong_FromLong(decode[i]); + for (i = 0; i < length; i++) { + key = PyLong_FromLong(PyUnicode_READ(kind, data, i)); value = PyLong_FromLong(i); if (!key || !value) goto failed1; @@ -5517,17 +7765,18 @@ PyUnicode_BuildEncodingMap(PyObject* string) memset(mlevel2, 0xFF, 16*count2); memset(mlevel3, 0, 128*count3); count3 = 0; - for (i = 1; i < 256; i++) { + for (i = 1; i < length; i++) { int o1, o2, o3, i2, i3; - if (decode[i] == 0xFFFE) + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch == 0xFFFE) /* unmapped character */ continue; - o1 = decode[i]>>11; - o2 = (decode[i]>>7) & 0xF; + o1 = ch>>11; + o2 = (ch>>7) & 0xF; i2 = 16*mlevel1[o1] + o2; if (mlevel2[i2] == 0xFF) mlevel2[i2] = count3++; - o3 = decode[i] & 0x7F; + o3 = ch & 0x7F; i3 = 128*mlevel2[i2] + o3; mlevel3[i3] = i; } @@ -5535,7 +7784,7 @@ PyUnicode_BuildEncodingMap(PyObject* string) } static int -encoding_map_lookup(Py_UNICODE c, PyObject *mapping) +encoding_map_lookup(Py_UCS4 c, PyObject *mapping) { struct encoding_map *map = (struct encoding_map*)mapping; int l1 = c>>11; @@ -5543,11 +7792,8 @@ encoding_map_lookup(Py_UNICODE c, PyObject *mapping) int l3 = c & 0x7F; int i; -#ifdef Py_UNICODE_WIDE - if (c > 0xFFFF) { + if (c > 0xFFFF) return -1; - } -#endif if (c == 0) return 0; /* level 1*/ @@ -5571,7 +7817,8 @@ encoding_map_lookup(Py_UNICODE c, PyObject *mapping) /* Lookup the character ch in the mapping. If the character can't be found, Py_None is returned (or NULL, if another error occurred). */ -static PyObject *charmapencode_lookup(Py_UNICODE c, PyObject *mapping) +static PyObject * +charmapencode_lookup(Py_UCS4 c, PyObject *mapping) { PyObject *w = PyLong_FromLong((long)c); PyObject *x; @@ -5628,16 +7875,16 @@ charmapencode_resize(PyObject **outobj, Py_ssize_t *outpos, Py_ssize_t requireds typedef enum charmapencode_result { enc_SUCCESS, enc_FAILED, enc_EXCEPTION -}charmapencode_result; +} charmapencode_result; /* lookup the character, put the result in the output string and adjust various state variables. Resize the output bytes object if not enough space is available. Return a new reference to the object that was put in the output buffer, or Py_None, if the mapping was undefined (in which case no character was written) or NULL, if a reallocation error occurred. The caller must decref the result */ -static -charmapencode_result charmapencode_output(Py_UNICODE c, PyObject *mapping, - PyObject **outobj, Py_ssize_t *outpos) +static charmapencode_result +charmapencode_output(Py_UCS4 c, PyObject *mapping, + PyObject **outobj, Py_ssize_t *outpos) { PyObject *rep; char *outstart; @@ -5693,17 +7940,19 @@ charmapencode_result charmapencode_output(Py_UNICODE c, PyObject *mapping, /* handle an error in PyUnicode_EncodeCharmap Return 0 on success, -1 on error */ -static -int charmap_encoding_error( - const Py_UNICODE *p, Py_ssize_t size, Py_ssize_t *inpos, PyObject *mapping, +static int +charmap_encoding_error( + PyObject *unicode, Py_ssize_t *inpos, PyObject *mapping, PyObject **exceptionObject, int *known_errorHandler, PyObject **errorHandler, const char *errors, PyObject **res, Py_ssize_t *respos) { PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ - Py_ssize_t repsize; + Py_ssize_t size, repsize; Py_ssize_t newpos; - Py_UNICODE *uni2; + enum PyUnicode_Kind kind; + void *data; + Py_ssize_t index; /* startpos for collecting unencodable chars */ Py_ssize_t collstartpos = *inpos; Py_ssize_t collendpos = *inpos+1; @@ -5711,19 +7960,26 @@ int charmap_encoding_error( char *encoding = "charmap"; char *reason = "character maps to <undefined>"; charmapencode_result x; + Py_UCS4 ch; + int val; + if (PyUnicode_READY(unicode) == -1) + return -1; + size = PyUnicode_GET_LENGTH(unicode); /* find all unencodable characters */ while (collendpos < size) { PyObject *rep; if (Py_TYPE(mapping) == &EncodingMapType) { - int res = encoding_map_lookup(p[collendpos], mapping); - if (res != -1) + ch = PyUnicode_READ_CHAR(unicode, collendpos); + val = encoding_map_lookup(ch, mapping); + if (val != -1) break; ++collendpos; continue; } - rep = charmapencode_lookup(p[collendpos], mapping); + ch = PyUnicode_READ_CHAR(unicode, collendpos); + rep = charmapencode_lookup(ch, mapping); if (rep==NULL) return -1; else if (rep!=Py_None) { @@ -5749,7 +8005,7 @@ int charmap_encoding_error( } switch (*known_errorHandler) { case 1: /* strict */ - raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); + raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; case 2: /* replace */ for (collpos = collstartpos; collpos<collendpos; ++collpos) { @@ -5758,7 +8014,7 @@ int charmap_encoding_error( return -1; } else if (x==enc_FAILED) { - raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); + raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; } } @@ -5771,13 +8027,13 @@ int charmap_encoding_error( for (collpos = collstartpos; collpos < collendpos; ++collpos) { char buffer[2+29+1+1]; char *cp; - sprintf(buffer, "&#%d;", (int)p[collpos]); + sprintf(buffer, "&#%d;", (int)PyUnicode_READ_CHAR(unicode, collpos)); for (cp = buffer; *cp; ++cp) { x = charmapencode_output(*cp, mapping, res, respos); if (x==enc_EXCEPTION) return -1; else if (x==enc_FAILED) { - raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); + raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; } } @@ -5786,7 +8042,7 @@ int charmap_encoding_error( break; default: repunicode = unicode_encode_call_errorhandler(errors, errorHandler, - encoding, reason, p, size, exceptionObject, + encoding, reason, unicode, exceptionObject, collstartpos, collendpos, &newpos); if (repunicode == NULL) return -1; @@ -5810,15 +8066,23 @@ int charmap_encoding_error( break; } /* generate replacement */ - repsize = PyUnicode_GET_SIZE(repunicode); - for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) { - x = charmapencode_output(*uni2, mapping, res, respos); + if (PyUnicode_READY(repunicode) == -1) { + Py_DECREF(repunicode); + return -1; + } + repsize = PyUnicode_GET_LENGTH(repunicode); + data = PyUnicode_DATA(repunicode); + kind = PyUnicode_KIND(repunicode); + for (index = 0; index < repsize; index++) { + Py_UCS4 repch = PyUnicode_READ(kind, data, index); + x = charmapencode_output(repch, mapping, res, respos); if (x==enc_EXCEPTION) { + Py_DECREF(repunicode); return -1; } else if (x==enc_FAILED) { Py_DECREF(repunicode); - raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); + raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; } } @@ -5828,15 +8092,16 @@ int charmap_encoding_error( return 0; } -PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p, - Py_ssize_t size, - PyObject *mapping, - const char *errors) +PyObject * +_PyUnicode_EncodeCharmap(PyObject *unicode, + PyObject *mapping, + const char *errors) { /* output object */ PyObject *res = NULL; /* current input position */ Py_ssize_t inpos = 0; + Py_ssize_t size; /* current output position */ Py_ssize_t respos = 0; PyObject *errorHandler = NULL; @@ -5846,9 +8111,13 @@ PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p, * 3=ignore, 4=xmlcharrefreplace */ int known_errorHandler = -1; + if (PyUnicode_READY(unicode) == -1) + return NULL; + size = PyUnicode_GET_LENGTH(unicode); + /* Default to Latin-1 */ if (mapping == NULL) - return PyUnicode_EncodeLatin1(p, size, errors); + return unicode_encode_ucs1(unicode, errors, 256); /* allocate enough for a simple encoding without replacements, if we need more, we'll resize */ @@ -5859,12 +8128,13 @@ PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p, return res; while (inpos<size) { + Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, inpos); /* try to encode it */ - charmapencode_result x = charmapencode_output(p[inpos], mapping, &res, &respos); + charmapencode_result x = charmapencode_output(ch, mapping, &res, &respos); if (x==enc_EXCEPTION) /* error */ goto onError; if (x==enc_FAILED) { /* unencodable character */ - if (charmap_encoding_error(p, size, &inpos, mapping, + if (charmap_encoding_error(unicode, &inpos, mapping, &exc, &known_errorHandler, &errorHandler, errors, &res, &respos)) { @@ -5892,28 +8162,43 @@ PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p, return NULL; } -PyObject *PyUnicode_AsCharmapString(PyObject *unicode, - PyObject *mapping) +/* Deprecated */ +PyObject * +PyUnicode_EncodeCharmap(const Py_UNICODE *p, + Py_ssize_t size, + PyObject *mapping, + const char *errors) +{ + PyObject *result; + PyObject *unicode = PyUnicode_FromUnicode(p, size); + if (unicode == NULL) + return NULL; + result = _PyUnicode_EncodeCharmap(unicode, mapping, errors); + Py_DECREF(unicode); + return result; +} + +PyObject * +PyUnicode_AsCharmapString(PyObject *unicode, + PyObject *mapping) { if (!PyUnicode_Check(unicode) || mapping == NULL) { PyErr_BadArgument(); return NULL; } - return PyUnicode_EncodeCharmap(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - mapping, - NULL); + return _PyUnicode_EncodeCharmap(unicode, mapping, NULL); } /* create or adjust a UnicodeTranslateError */ -static void make_translate_exception(PyObject **exceptionObject, - const Py_UNICODE *unicode, Py_ssize_t size, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason) +static void +make_translate_exception(PyObject **exceptionObject, + PyObject *unicode, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason) { if (*exceptionObject == NULL) { - *exceptionObject = PyUnicodeTranslateError_Create( - unicode, size, startpos, endpos, reason); + *exceptionObject = _PyUnicodeTranslateError_Create( + unicode, startpos, endpos, reason); } else { if (PyUnicodeTranslateError_SetStart(*exceptionObject, startpos)) @@ -5930,13 +8215,14 @@ static void make_translate_exception(PyObject **exceptionObject, } /* raises a UnicodeTranslateError */ -static void raise_translate_exception(PyObject **exceptionObject, - const Py_UNICODE *unicode, Py_ssize_t size, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason) +static void +raise_translate_exception(PyObject **exceptionObject, + PyObject *unicode, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason) { make_translate_exception(exceptionObject, - unicode, size, startpos, endpos, reason); + unicode, startpos, endpos, reason); if (*exceptionObject != NULL) PyCodec_StrictErrors(*exceptionObject); } @@ -5945,12 +8231,13 @@ static void raise_translate_exception(PyObject **exceptionObject, build arguments, call the callback and check the arguments, put the result into newpos and return the replacement string, which has to be freed by the caller */ -static PyObject *unicode_translate_call_errorhandler(const char *errors, - PyObject **errorHandler, - const char *reason, - const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, - Py_ssize_t startpos, Py_ssize_t endpos, - Py_ssize_t *newpos) +static PyObject * +unicode_translate_call_errorhandler(const char *errors, + PyObject **errorHandler, + const char *reason, + PyObject *unicode, PyObject **exceptionObject, + Py_ssize_t startpos, Py_ssize_t endpos, + Py_ssize_t *newpos) { static char *argparse = "O!n;translating error handler must return (str, int) tuple"; @@ -5965,7 +8252,7 @@ static PyObject *unicode_translate_call_errorhandler(const char *errors, } make_translate_exception(exceptionObject, - unicode, size, startpos, endpos, reason); + unicode, startpos, endpos, reason); if (*exceptionObject == NULL) return NULL; @@ -5984,10 +8271,10 @@ static PyObject *unicode_translate_call_errorhandler(const char *errors, return NULL; } if (i_newpos<0) - *newpos = size+i_newpos; + *newpos = PyUnicode_GET_LENGTH(unicode)+i_newpos; else *newpos = i_newpos; - if (*newpos<0 || *newpos>size) { + if (*newpos<0 || *newpos>PyUnicode_GET_LENGTH(unicode)) { PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos); Py_DECREF(restuple); return NULL; @@ -6000,8 +8287,8 @@ static PyObject *unicode_translate_call_errorhandler(const char *errors, /* Lookup the character ch in the mapping and put the result in result, which must be decrefed by the caller. Return 0 on success, -1 on error */ -static -int charmaptranslate_lookup(Py_UNICODE c, PyObject *mapping, PyObject **result) +static int +charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) { PyObject *w = PyLong_FromLong((long)c); PyObject *x; @@ -6050,20 +8337,21 @@ int charmaptranslate_lookup(Py_UNICODE c, PyObject *mapping, PyObject **result) /* ensure that *outobj is at least requiredsize characters long, if not reallocate and adjust various state variables. Return 0 on success, -1 on error */ -static -int charmaptranslate_makespace(PyObject **outobj, Py_UNICODE **outp, +static int +charmaptranslate_makespace(Py_UCS4 **outobj, Py_ssize_t *psize, Py_ssize_t requiredsize) { - Py_ssize_t oldsize = PyUnicode_GET_SIZE(*outobj); + Py_ssize_t oldsize = *psize; + Py_UCS4 *new_outobj; if (requiredsize > oldsize) { - /* remember old output position */ - Py_ssize_t outpos = *outp-PyUnicode_AS_UNICODE(*outobj); /* exponentially overallocate to minimize reallocations */ if (requiredsize < 2 * oldsize) requiredsize = 2 * oldsize; - if (PyUnicode_Resize(outobj, requiredsize) < 0) + new_outobj = PyMem_Realloc(*outobj, requiredsize * sizeof(Py_UCS4)); + if (new_outobj == 0) return -1; - *outp = PyUnicode_AS_UNICODE(*outobj) + outpos; + *outobj = new_outobj; + *psize = requiredsize; } return 0; } @@ -6073,38 +8361,44 @@ int charmaptranslate_makespace(PyObject **outobj, Py_UNICODE **outp, undefined (in which case no character was written). The called must decref result. Return 0 on success, -1 on error. */ -static -int charmaptranslate_output(const Py_UNICODE *startinp, const Py_UNICODE *curinp, - Py_ssize_t insize, PyObject *mapping, PyObject **outobj, Py_UNICODE **outp, - PyObject **res) +static int +charmaptranslate_output(PyObject *input, Py_ssize_t ipos, + PyObject *mapping, Py_UCS4 **output, + Py_ssize_t *osize, Py_ssize_t *opos, + PyObject **res) { - if (charmaptranslate_lookup(*curinp, mapping, res)) + Py_UCS4 curinp = PyUnicode_READ_CHAR(input, ipos); + if (charmaptranslate_lookup(curinp, mapping, res)) return -1; if (*res==NULL) { /* not found => default to 1:1 mapping */ - *(*outp)++ = *curinp; + (*output)[(*opos)++] = curinp; } else if (*res==Py_None) ; else if (PyLong_Check(*res)) { /* no overflow check, because we know that the space is enough */ - *(*outp)++ = (Py_UNICODE)PyLong_AS_LONG(*res); + (*output)[(*opos)++] = (Py_UCS4)PyLong_AS_LONG(*res); } else if (PyUnicode_Check(*res)) { - Py_ssize_t repsize = PyUnicode_GET_SIZE(*res); + Py_ssize_t repsize; + if (PyUnicode_READY(*res) == -1) + return -1; + repsize = PyUnicode_GET_LENGTH(*res); if (repsize==1) { /* no overflow check, because we know that the space is enough */ - *(*outp)++ = *PyUnicode_AS_UNICODE(*res); + (*output)[(*opos)++] = PyUnicode_READ_CHAR(*res, 0); } else if (repsize!=0) { /* more than one character */ - Py_ssize_t requiredsize = (*outp-PyUnicode_AS_UNICODE(*outobj)) + - (insize - (curinp-startinp)) + + Py_ssize_t requiredsize = *opos + + (PyUnicode_GET_LENGTH(input) - ipos) + repsize - 1; - if (charmaptranslate_makespace(outobj, outp, requiredsize)) + Py_ssize_t i; + if (charmaptranslate_makespace(output, osize, requiredsize)) return -1; - memcpy(*outp, PyUnicode_AS_UNICODE(*res), sizeof(Py_UNICODE)*repsize); - *outp += repsize; + for(i = 0; i < repsize; i++) + (*output)[(*opos)++] = PyUnicode_READ_CHAR(*res, i); } } else @@ -6112,20 +8406,21 @@ int charmaptranslate_output(const Py_UNICODE *startinp, const Py_UNICODE *curinp return 0; } -PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p, - Py_ssize_t size, - PyObject *mapping, - const char *errors) +PyObject * +_PyUnicode_TranslateCharmap(PyObject *input, + PyObject *mapping, + const char *errors) { - /* output object */ - PyObject *res = NULL; - /* pointers to the beginning and end+1 of input */ - const Py_UNICODE *startp = p; - const Py_UNICODE *endp = p + size; - /* pointer into the output */ - Py_UNICODE *str; + /* input object */ + char *idata; + Py_ssize_t size, i; + int kind; + /* output buffer */ + Py_UCS4 *output = NULL; + Py_ssize_t osize; + PyObject *res; /* current output position */ - Py_ssize_t respos = 0; + Py_ssize_t opos; char *reason = "character maps to <undefined>"; PyObject *errorHandler = NULL; PyObject *exc = NULL; @@ -6139,38 +8434,52 @@ PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p, return NULL; } + if (PyUnicode_READY(input) == -1) + return NULL; + idata = (char*)PyUnicode_DATA(input); + kind = PyUnicode_KIND(input); + size = PyUnicode_GET_LENGTH(input); + i = 0; + + if (size == 0) { + Py_INCREF(input); + return input; + } + /* allocate enough for a simple 1:1 translation without replacements, if we need more, we'll resize */ - res = PyUnicode_FromUnicode(NULL, size); - if (res == NULL) + osize = size; + output = PyMem_Malloc(osize * sizeof(Py_UCS4)); + opos = 0; + if (output == NULL) { + PyErr_NoMemory(); goto onError; - if (size == 0) - return res; - str = PyUnicode_AS_UNICODE(res); + } - while (p<endp) { + while (i<size) { /* try to encode it */ PyObject *x = NULL; - if (charmaptranslate_output(startp, p, size, mapping, &res, &str, &x)) { + if (charmaptranslate_output(input, i, mapping, + &output, &osize, &opos, &x)) { Py_XDECREF(x); goto onError; } Py_XDECREF(x); if (x!=Py_None) /* it worked => adjust input pointer */ - ++p; + ++i; else { /* untranslatable character */ PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ Py_ssize_t repsize; Py_ssize_t newpos; - Py_UNICODE *uni2; + Py_ssize_t uni2; /* startpos for collecting untranslatable chars */ - const Py_UNICODE *collstart = p; - const Py_UNICODE *collend = p+1; - const Py_UNICODE *coll; + Py_ssize_t collstart = i; + Py_ssize_t collend = i+1; + Py_ssize_t coll; /* find all untranslatable characters */ - while (collend < endp) { - if (charmaptranslate_lookup(*collend, mapping, &x)) + while (collend < size) { + if (charmaptranslate_lookup(PyUnicode_READ(kind,idata, collend), mapping, &x)) goto onError; Py_XDECREF(x); if (x!=Py_None) @@ -6193,261 +8502,419 @@ PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p, } switch (known_errorHandler) { case 1: /* strict */ - raise_translate_exception(&exc, startp, size, collstart-startp, collend-startp, reason); + raise_translate_exception(&exc, input, collstart, + collend, reason); goto onError; case 2: /* replace */ /* No need to check for space, this is a 1:1 replacement */ - for (coll = collstart; coll<collend; ++coll) - *str++ = '?'; + for (coll = collstart; coll<collend; coll++) + output[opos++] = '?'; /* fall through */ case 3: /* ignore */ - p = collend; + i = collend; break; case 4: /* xmlcharrefreplace */ - /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) { + /* generate replacement (temporarily (mis)uses i) */ + for (i = collstart; i < collend; ++i) { char buffer[2+29+1+1]; char *cp; - sprintf(buffer, "&#%d;", (int)*p); - if (charmaptranslate_makespace(&res, &str, - (str-PyUnicode_AS_UNICODE(res))+strlen(buffer)+(endp-collend))) + sprintf(buffer, "&#%d;", PyUnicode_READ(kind, idata, i)); + if (charmaptranslate_makespace(&output, &osize, + opos+strlen(buffer)+(size-collend))) goto onError; for (cp = buffer; *cp; ++cp) - *str++ = *cp; + output[opos++] = *cp; } - p = collend; + i = collend; break; default: repunicode = unicode_translate_call_errorhandler(errors, &errorHandler, - reason, startp, size, &exc, - collstart-startp, collend-startp, &newpos); + reason, input, &exc, + collstart, collend, &newpos); if (repunicode == NULL) goto onError; + if (PyUnicode_READY(repunicode) == -1) { + Py_DECREF(repunicode); + goto onError; + } /* generate replacement */ - repsize = PyUnicode_GET_SIZE(repunicode); - if (charmaptranslate_makespace(&res, &str, - (str-PyUnicode_AS_UNICODE(res))+repsize+(endp-collend))) { + repsize = PyUnicode_GET_LENGTH(repunicode); + if (charmaptranslate_makespace(&output, &osize, + opos+repsize+(size-collend))) { Py_DECREF(repunicode); goto onError; } - for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) - *str++ = *uni2; - p = startp + newpos; + for (uni2 = 0; repsize-->0; ++uni2) + output[opos++] = PyUnicode_READ_CHAR(repunicode, uni2); + i = newpos; Py_DECREF(repunicode); } } } - /* Resize if we allocated to much */ - respos = str-PyUnicode_AS_UNICODE(res); - if (respos<PyUnicode_GET_SIZE(res)) { - if (PyUnicode_Resize(&res, respos) < 0) - goto onError; - } + res = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, output, opos); + if (!res) + goto onError; + PyMem_Free(output); Py_XDECREF(exc); Py_XDECREF(errorHandler); return res; onError: - Py_XDECREF(res); + PyMem_Free(output); Py_XDECREF(exc); Py_XDECREF(errorHandler); return NULL; } -PyObject *PyUnicode_Translate(PyObject *str, - PyObject *mapping, - const char *errors) +/* Deprecated. Use PyUnicode_Translate instead. */ +PyObject * +PyUnicode_TranslateCharmap(const Py_UNICODE *p, + Py_ssize_t size, + PyObject *mapping, + const char *errors) +{ + PyObject *result; + PyObject *unicode = PyUnicode_FromUnicode(p, size); + if (!unicode) + return NULL; + result = _PyUnicode_TranslateCharmap(unicode, mapping, errors); + Py_DECREF(unicode); + return result; +} + +PyObject * +PyUnicode_Translate(PyObject *str, + PyObject *mapping, + const char *errors) { PyObject *result; str = PyUnicode_FromObject(str); if (str == NULL) - goto onError; - result = PyUnicode_TranslateCharmap(PyUnicode_AS_UNICODE(str), - PyUnicode_GET_SIZE(str), - mapping, - errors); + return NULL; + result = _PyUnicode_TranslateCharmap(str, mapping, errors); Py_DECREF(str); return result; +} - onError: - Py_XDECREF(str); - return NULL; +static Py_UCS4 +fix_decimal_and_space_to_ascii(PyObject *self) +{ + /* No need to call PyUnicode_READY(self) because this function is only + called as a callback from fixup() which does it already. */ + const Py_ssize_t len = PyUnicode_GET_LENGTH(self); + const int kind = PyUnicode_KIND(self); + void *data = PyUnicode_DATA(self); + Py_UCS4 maxchar = 127, ch, fixed; + int modified = 0; + Py_ssize_t i; + + for (i = 0; i < len; ++i) { + ch = PyUnicode_READ(kind, data, i); + fixed = 0; + if (ch > 127) { + if (Py_UNICODE_ISSPACE(ch)) + fixed = ' '; + else { + const int decimal = Py_UNICODE_TODECIMAL(ch); + if (decimal >= 0) + fixed = '0' + decimal; + } + if (fixed != 0) { + modified = 1; + maxchar = MAX_MAXCHAR(maxchar, fixed); + PyUnicode_WRITE(kind, data, i, fixed); + } + else + maxchar = MAX_MAXCHAR(maxchar, ch); + } + } + + return (modified) ? maxchar : 0; +} + +PyObject * +_PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode) +{ + if (!PyUnicode_Check(unicode)) { + PyErr_BadInternalCall(); + return NULL; + } + if (PyUnicode_READY(unicode) == -1) + return NULL; + if (PyUnicode_MAX_CHAR_VALUE(unicode) <= 127) { + /* If the string is already ASCII, just return the same string */ + Py_INCREF(unicode); + return unicode; + } + return fixup(unicode, fix_decimal_and_space_to_ascii); } PyObject * PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, Py_ssize_t length) { - PyObject *result; - Py_UNICODE *p; /* write pointer into result */ + PyObject *decimal; Py_ssize_t i; + Py_UCS4 maxchar; + enum PyUnicode_Kind kind; + void *data; + + maxchar = 127; + for (i = 0; i < length; i++) { + Py_UNICODE ch = s[i]; + if (ch > 127) { + int decimal = Py_UNICODE_TODECIMAL(ch); + if (decimal >= 0) + ch = '0' + decimal; + maxchar = MAX_MAXCHAR(maxchar, ch); + } + } + /* Copy to a new string */ - result = (PyObject *)_PyUnicode_New(length); - Py_UNICODE_COPY(PyUnicode_AS_UNICODE(result), s, length); - if (result == NULL) - return result; - p = PyUnicode_AS_UNICODE(result); + decimal = PyUnicode_New(length, maxchar); + if (decimal == NULL) + return decimal; + kind = PyUnicode_KIND(decimal); + data = PyUnicode_DATA(decimal); /* Iterate over code points */ for (i = 0; i < length; i++) { - Py_UNICODE ch =s[i]; + Py_UNICODE ch = s[i]; if (ch > 127) { int decimal = Py_UNICODE_TODECIMAL(ch); if (decimal >= 0) - p[i] = '0' + decimal; + ch = '0' + decimal; } + PyUnicode_WRITE(kind, data, i, ch); } - return result; + return unicode_result(decimal); } /* --- Decimal Encoder ---------------------------------------------------- */ -int PyUnicode_EncodeDecimal(Py_UNICODE *s, - Py_ssize_t length, - char *output, - const char *errors) +int +PyUnicode_EncodeDecimal(Py_UNICODE *s, + Py_ssize_t length, + char *output, + const char *errors) { - Py_UNICODE *p, *end; - PyObject *errorHandler = NULL; - PyObject *exc = NULL; - const char *encoding = "decimal"; - const char *reason = "invalid decimal Unicode string"; - /* the following variable is used for caching string comparisons - * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */ - int known_errorHandler = -1; + PyObject *unicode; + Py_ssize_t i; + enum PyUnicode_Kind kind; + void *data; if (output == NULL) { PyErr_BadArgument(); return -1; } - p = s; - end = s + length; - while (p < end) { - register Py_UNICODE ch = *p; + unicode = PyUnicode_FromUnicode(s, length); + if (unicode == NULL) + return -1; + + if (PyUnicode_READY(unicode) == -1) { + Py_DECREF(unicode); + return -1; + } + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + + for (i=0; i < length; ) { + PyObject *exc; + Py_UCS4 ch; int decimal; - PyObject *repunicode; - Py_ssize_t repsize; - Py_ssize_t newpos; - Py_UNICODE *uni2; - Py_UNICODE *collstart; - Py_UNICODE *collend; + Py_ssize_t startpos; + + ch = PyUnicode_READ(kind, data, i); if (Py_UNICODE_ISSPACE(ch)) { *output++ = ' '; - ++p; + i++; continue; } decimal = Py_UNICODE_TODECIMAL(ch); if (decimal >= 0) { *output++ = '0' + decimal; - ++p; + i++; continue; } if (0 < ch && ch < 256) { *output++ = (char)ch; - ++p; + i++; continue; } - /* All other characters are considered unencodable */ - collstart = p; - for (collend = p+1; collend < end; collend++) { - if ((0 < *collend && *collend < 256) || - Py_UNICODE_ISSPACE(*collend) || - 0 <= Py_UNICODE_TODECIMAL(*collend)) - break; - } - /* cache callback name lookup - * (if not done yet, i.e. it's the first error) */ - if (known_errorHandler==-1) { - if ((errors==NULL) || (!strcmp(errors, "strict"))) - known_errorHandler = 1; - else if (!strcmp(errors, "replace")) - known_errorHandler = 2; - else if (!strcmp(errors, "ignore")) - known_errorHandler = 3; - else if (!strcmp(errors, "xmlcharrefreplace")) - known_errorHandler = 4; + + startpos = i; + exc = NULL; + raise_encode_exception(&exc, "decimal", unicode, + startpos, startpos+1, + "invalid decimal Unicode string"); + Py_XDECREF(exc); + Py_DECREF(unicode); + return -1; + } + /* 0-terminate the output string */ + *output++ = '\0'; + Py_DECREF(unicode); + return 0; +} + +/* --- Helpers ------------------------------------------------------------ */ + +static Py_ssize_t +any_find_slice(int direction, PyObject* s1, PyObject* s2, + Py_ssize_t start, + Py_ssize_t end) +{ + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2, result; + + kind1 = PyUnicode_KIND(s1); + kind2 = PyUnicode_KIND(s2); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(s1); + buf2 = PyUnicode_DATA(s2); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind(s1, kind); + if (!buf1) + return -2; + if (kind2 != kind) + buf2 = _PyUnicode_AsKind(s2, kind); + if (!buf2) { + if (kind1 != kind) PyMem_Free(buf1); + return -2; + } + len1 = PyUnicode_GET_LENGTH(s1); + len2 = PyUnicode_GET_LENGTH(s2); + + if (direction > 0) { + switch (kind) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(s1) && PyUnicode_IS_ASCII(s2)) + result = asciilib_find_slice(buf1, len1, buf2, len2, start, end); else - known_errorHandler = 0; + result = ucs1lib_find_slice(buf1, len1, buf2, len2, start, end); + break; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_find_slice(buf1, len1, buf2, len2, start, end); + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_find_slice(buf1, len1, buf2, len2, start, end); + break; + default: + assert(0); result = -2; } - switch (known_errorHandler) { - case 1: /* strict */ - raise_encode_exception(&exc, encoding, s, length, collstart-s, collend-s, reason); - goto onError; - case 2: /* replace */ - for (p = collstart; p < collend; ++p) - *output++ = '?'; - /* fall through */ - case 3: /* ignore */ - p = collend; + } + else { + switch (kind) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(s1) && PyUnicode_IS_ASCII(s2)) + result = asciilib_rfind_slice(buf1, len1, buf2, len2, start, end); + else + result = ucs1lib_rfind_slice(buf1, len1, buf2, len2, start, end); break; - case 4: /* xmlcharrefreplace */ - /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) - output += sprintf(output, "&#%d;", (int)*p); - p = collend; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_rfind_slice(buf1, len1, buf2, len2, start, end); + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_rfind_slice(buf1, len1, buf2, len2, start, end); break; default: - repunicode = unicode_encode_call_errorhandler(errors, &errorHandler, - encoding, reason, s, length, &exc, - collstart-s, collend-s, &newpos); - if (repunicode == NULL) - goto onError; - if (!PyUnicode_Check(repunicode)) { - /* Byte results not supported, since they have no decimal property. */ - PyErr_SetString(PyExc_TypeError, "error handler should return unicode"); - Py_DECREF(repunicode); - goto onError; - } - /* generate replacement */ - repsize = PyUnicode_GET_SIZE(repunicode); - for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) { - Py_UNICODE ch = *uni2; - if (Py_UNICODE_ISSPACE(ch)) - *output++ = ' '; - else { - decimal = Py_UNICODE_TODECIMAL(ch); - if (decimal >= 0) - *output++ = '0' + decimal; - else if (0 < ch && ch < 256) - *output++ = (char)ch; - else { - Py_DECREF(repunicode); - raise_encode_exception(&exc, encoding, - s, length, collstart-s, collend-s, reason); - goto onError; - } - } - } - p = s + newpos; - Py_DECREF(repunicode); + assert(0); result = -2; } } - /* 0-terminate the output string */ - *output++ = '\0'; - Py_XDECREF(exc); - Py_XDECREF(errorHandler); - return 0; - onError: - Py_XDECREF(exc); - Py_XDECREF(errorHandler); - return -1; + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); + + return result; } -/* --- Helpers ------------------------------------------------------------ */ +Py_ssize_t +_PyUnicode_InsertThousandsGrouping( + PyObject *unicode, Py_ssize_t index, + Py_ssize_t n_buffer, + void *digits, Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, PyObject *thousands_sep, + Py_UCS4 *maxchar) +{ + unsigned int kind, thousands_sep_kind; + char *data, *thousands_sep_data; + Py_ssize_t thousands_sep_len; + Py_ssize_t len; -#include "stringlib/unicodedefs.h" -#include "stringlib/fastsearch.h" + if (unicode != NULL) { + kind = PyUnicode_KIND(unicode); + data = (char *) PyUnicode_DATA(unicode) + index * kind; + } + else { + kind = PyUnicode_1BYTE_KIND; + data = NULL; + } + thousands_sep_kind = PyUnicode_KIND(thousands_sep); + thousands_sep_data = PyUnicode_DATA(thousands_sep); + thousands_sep_len = PyUnicode_GET_LENGTH(thousands_sep); + if (unicode != NULL && thousands_sep_kind != kind) { + if (thousands_sep_kind < kind) { + thousands_sep_data = _PyUnicode_AsKind(thousands_sep, kind); + if (!thousands_sep_data) + return -1; + } + else { + data = _PyUnicode_AsKind(unicode, thousands_sep_kind); + if (!data) + return -1; + } + } -#include "stringlib/count.h" -#include "stringlib/find.h" -#include "stringlib/partition.h" -#include "stringlib/split.h" + switch (kind) { + case PyUnicode_1BYTE_KIND: + if (unicode != NULL && PyUnicode_IS_ASCII(unicode)) + len = asciilib_InsertThousandsGrouping( + (Py_UCS1 *) data, n_buffer, (Py_UCS1 *) digits, n_digits, + min_width, grouping, + (Py_UCS1 *) thousands_sep_data, thousands_sep_len); + else + len = ucs1lib_InsertThousandsGrouping( + (Py_UCS1*)data, n_buffer, (Py_UCS1*)digits, n_digits, + min_width, grouping, + (Py_UCS1 *) thousands_sep_data, thousands_sep_len); + break; + case PyUnicode_2BYTE_KIND: + len = ucs2lib_InsertThousandsGrouping( + (Py_UCS2 *) data, n_buffer, (Py_UCS2 *) digits, n_digits, + min_width, grouping, + (Py_UCS2 *) thousands_sep_data, thousands_sep_len); + break; + case PyUnicode_4BYTE_KIND: + len = ucs4lib_InsertThousandsGrouping( + (Py_UCS4 *) data, n_buffer, (Py_UCS4 *) digits, n_digits, + min_width, grouping, + (Py_UCS4 *) thousands_sep_data, thousands_sep_len); + break; + default: + assert(0); + return -1; + } + if (unicode != NULL && thousands_sep_kind != kind) { + if (thousands_sep_kind < kind) + PyMem_Free(thousands_sep_data); + else + PyMem_Free(data); + } + if (unicode == NULL) { + *maxchar = 127; + if (len != n_digits) { + *maxchar = MAX_MAXCHAR(*maxchar, + PyUnicode_MAX_CHAR_VALUE(thousands_sep)); + } + } + return len; +} -#define _Py_InsertThousandsGrouping _PyUnicode_InsertThousandsGrouping -#define _Py_InsertThousandsGroupingLocale _PyUnicode_InsertThousandsGroupingLocale -#include "stringlib/localeutil.h" /* helper macro to fixup start/end slice values */ #define ADJUST_INDICES(start, end, len) \ @@ -6464,72 +8931,102 @@ int PyUnicode_EncodeDecimal(Py_UNICODE *s, start = 0; \ } -/* _Py_UNICODE_NEXT is a private macro used to retrieve the character pointed - * by 'ptr', possibly combining surrogate pairs on narrow builds. - * 'ptr' and 'end' must be Py_UNICODE*, with 'ptr' pointing at the character - * that should be returned and 'end' pointing to the end of the buffer. - * ('end' is used on narrow builds to detect a lone surrogate at the - * end of the buffer that should be returned unchanged.) - * The ptr and end arguments should be side-effect free and ptr must an lvalue. - * The type of the returned char is always Py_UCS4. - * - * Note: the macro advances ptr to next char, so it might have side-effects - * (especially if used with other macros). - */ - -/* helper macros used by _Py_UNICODE_NEXT */ -#define _Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF) -#define _Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF) -/* Join two surrogate characters and return a single Py_UCS4 value. */ -#define _Py_UNICODE_JOIN_SURROGATES(high, low) \ - (((((Py_UCS4)(high) & 0x03FF) << 10) | \ - ((Py_UCS4)(low) & 0x03FF)) + 0x10000) - -#ifdef Py_UNICODE_WIDE -#define _Py_UNICODE_NEXT(ptr, end) *(ptr)++ -#else -#define _Py_UNICODE_NEXT(ptr, end) \ - (((_Py_UNICODE_IS_HIGH_SURROGATE(*(ptr)) && (ptr) < (end)) && \ - _Py_UNICODE_IS_LOW_SURROGATE((ptr)[1])) ? \ - ((ptr) += 2,_Py_UNICODE_JOIN_SURROGATES((ptr)[-2], (ptr)[-1])) : \ - (Py_UCS4)*(ptr)++) -#endif - -Py_ssize_t PyUnicode_Count(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end) +Py_ssize_t +PyUnicode_Count(PyObject *str, + PyObject *substr, + Py_ssize_t start, + Py_ssize_t end) { Py_ssize_t result; - PyUnicodeObject* str_obj; - PyUnicodeObject* sub_obj; + PyObject* str_obj; + PyObject* sub_obj; + int kind1, kind2, kind; + void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; - str_obj = (PyUnicodeObject*) PyUnicode_FromObject(str); + str_obj = PyUnicode_FromObject(str); if (!str_obj) return -1; - sub_obj = (PyUnicodeObject*) PyUnicode_FromObject(substr); + sub_obj = PyUnicode_FromObject(substr); if (!sub_obj) { Py_DECREF(str_obj); return -1; } + if (PyUnicode_READY(sub_obj) == -1 || PyUnicode_READY(str_obj) == -1) { + Py_DECREF(sub_obj); + Py_DECREF(str_obj); + return -1; + } - ADJUST_INDICES(start, end, str_obj->length); - result = stringlib_count( - str_obj->str + start, end - start, sub_obj->str, sub_obj->length, - PY_SSIZE_T_MAX - ); + kind1 = PyUnicode_KIND(str_obj); + kind2 = PyUnicode_KIND(sub_obj); + kind = kind1; + buf1 = PyUnicode_DATA(str_obj); + buf2 = PyUnicode_DATA(sub_obj); + if (kind2 != kind) { + if (kind2 > kind) { + Py_DECREF(sub_obj); + Py_DECREF(str_obj); + return 0; + } + buf2 = _PyUnicode_AsKind(sub_obj, kind); + } + if (!buf2) + goto onError; + len1 = PyUnicode_GET_LENGTH(str_obj); + len2 = PyUnicode_GET_LENGTH(sub_obj); + + ADJUST_INDICES(start, end, len1); + switch (kind) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sub_obj)) + result = asciilib_count( + ((Py_UCS1*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + else + result = ucs1lib_count( + ((Py_UCS1*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_count( + ((Py_UCS2*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_count( + ((Py_UCS4*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + default: + assert(0); result = 0; + } Py_DECREF(sub_obj); Py_DECREF(str_obj); + if (kind2 != kind) + PyMem_Free(buf2); + return result; + onError: + Py_DECREF(sub_obj); + Py_DECREF(str_obj); + if (kind2 != kind && buf2) + PyMem_Free(buf2); + return -1; } -Py_ssize_t PyUnicode_Find(PyObject *str, - PyObject *sub, - Py_ssize_t start, - Py_ssize_t end, - int direction) +Py_ssize_t +PyUnicode_Find(PyObject *str, + PyObject *sub, + Py_ssize_t start, + Py_ssize_t end, + int direction) { Py_ssize_t result; @@ -6541,19 +9038,15 @@ Py_ssize_t PyUnicode_Find(PyObject *str, Py_DECREF(str); return -2; } + if (PyUnicode_READY(sub) == -1 || PyUnicode_READY(str) == -1) { + Py_DECREF(sub); + Py_DECREF(str); + return -2; + } - if (direction > 0) - result = stringlib_find_slice( - PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str), - PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub), - start, end - ); - else - result = stringlib_rfind_slice( - PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str), - PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub), - start, end - ); + result = any_find_slice(direction, + str, sub, start, end + ); Py_DECREF(str); Py_DECREF(sub); @@ -6561,37 +9054,104 @@ Py_ssize_t PyUnicode_Find(PyObject *str, return result; } -static -int tailmatch(PyUnicodeObject *self, - PyUnicodeObject *substring, - Py_ssize_t start, - Py_ssize_t end, - int direction) +Py_ssize_t +PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, + Py_ssize_t start, Py_ssize_t end, + int direction) { - if (substring->length == 0) + int kind; + Py_ssize_t result; + if (PyUnicode_READY(str) == -1) + return -2; + if (start < 0 || end < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -2; + } + if (end > PyUnicode_GET_LENGTH(str)) + end = PyUnicode_GET_LENGTH(str); + kind = PyUnicode_KIND(str); + result = findchar(PyUnicode_1BYTE_DATA(str) + kind*start, + kind, end-start, ch, direction); + if (result == -1) + return -1; + else + return start + result; +} + +static int +tailmatch(PyObject *self, + PyObject *substring, + Py_ssize_t start, + Py_ssize_t end, + int direction) +{ + int kind_self; + int kind_sub; + void *data_self; + void *data_sub; + Py_ssize_t offset; + Py_ssize_t i; + Py_ssize_t end_sub; + + if (PyUnicode_READY(self) == -1 || + PyUnicode_READY(substring) == -1) + return 0; + + if (PyUnicode_GET_LENGTH(substring) == 0) return 1; - ADJUST_INDICES(start, end, self->length); - end -= substring->length; + ADJUST_INDICES(start, end, PyUnicode_GET_LENGTH(self)); + end -= PyUnicode_GET_LENGTH(substring); if (end < start) return 0; - if (direction > 0) { - if (Py_UNICODE_MATCH(self, end, substring)) - return 1; - } else { - if (Py_UNICODE_MATCH(self, start, substring)) + kind_self = PyUnicode_KIND(self); + data_self = PyUnicode_DATA(self); + kind_sub = PyUnicode_KIND(substring); + data_sub = PyUnicode_DATA(substring); + end_sub = PyUnicode_GET_LENGTH(substring) - 1; + + if (direction > 0) + offset = end; + else + offset = start; + + if (PyUnicode_READ(kind_self, data_self, offset) == + PyUnicode_READ(kind_sub, data_sub, 0) && + PyUnicode_READ(kind_self, data_self, offset + end_sub) == + PyUnicode_READ(kind_sub, data_sub, end_sub)) { + /* If both are of the same kind, memcmp is sufficient */ + if (kind_self == kind_sub) { + return ! memcmp((char *)data_self + + (offset * PyUnicode_KIND(substring)), + data_sub, + PyUnicode_GET_LENGTH(substring) * + PyUnicode_KIND(substring)); + } + /* otherwise we have to compare each character by first accesing it */ + else { + /* We do not need to compare 0 and len(substring)-1 because + the if statement above ensured already that they are equal + when we end up here. */ + /* TODO: honor direction and do a forward or backwards search */ + for (i = 1; i < end_sub; ++i) { + if (PyUnicode_READ(kind_self, data_self, offset + i) != + PyUnicode_READ(kind_sub, data_sub, i)) + return 0; + } return 1; + } } return 0; } -Py_ssize_t PyUnicode_Tailmatch(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end, - int direction) +Py_ssize_t +PyUnicode_Tailmatch(PyObject *str, + PyObject *substr, + Py_ssize_t start, + Py_ssize_t end, + int direction) { Py_ssize_t result; @@ -6604,8 +9164,7 @@ Py_ssize_t PyUnicode_Tailmatch(PyObject *str, return -1; } - result = tailmatch((PyUnicodeObject *)str, - (PyUnicodeObject *)substr, + result = tailmatch(str, substr, start, end, direction); Py_DECREF(str); Py_DECREF(substr); @@ -6615,168 +9174,312 @@ Py_ssize_t PyUnicode_Tailmatch(PyObject *str, /* Apply fixfct filter to the Unicode object self and return a reference to the modified object */ -static -PyObject *fixup(PyUnicodeObject *self, - int (*fixfct)(PyUnicodeObject *s)) +static PyObject * +fixup(PyObject *self, + Py_UCS4 (*fixfct)(PyObject *s)) { + PyObject *u; + Py_UCS4 maxchar_old, maxchar_new = 0; + PyObject *v; - PyUnicodeObject *u; - - u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length); + u = _PyUnicode_Copy(self); if (u == NULL) return NULL; + maxchar_old = PyUnicode_MAX_CHAR_VALUE(u); + + /* fix functions return the new maximum character in a string, + if the kind of the resulting unicode object does not change, + everything is fine. Otherwise we need to change the string kind + and re-run the fix function. */ + maxchar_new = fixfct(u); + + if (maxchar_new == 0) { + /* no changes */; + if (PyUnicode_CheckExact(self)) { + Py_DECREF(u); + Py_INCREF(self); + return self; + } + else + return u; + } + + maxchar_new = align_maxchar(maxchar_new); - Py_UNICODE_COPY(u->str, self->str, self->length); + if (maxchar_new == maxchar_old) + return u; - if (!fixfct(u) && PyUnicode_CheckExact(self)) { - /* fixfct should return TRUE if it modified the buffer. If - FALSE, return a reference to the original buffer instead - (to save space, not time) */ - Py_INCREF(self); + /* In case the maximum character changed, we need to + convert the string to the new category. */ + v = PyUnicode_New(PyUnicode_GET_LENGTH(self), maxchar_new); + if (v == NULL) { Py_DECREF(u); - return (PyObject*) self; + return NULL; + } + if (maxchar_new > maxchar_old) { + /* If the maxchar increased so that the kind changed, not all + characters are representable anymore and we need to fix the + string again. This only happens in very few cases. */ + _PyUnicode_FastCopyCharacters(v, 0, + self, 0, PyUnicode_GET_LENGTH(self)); + maxchar_old = fixfct(v); + assert(maxchar_old > 0 && maxchar_old <= maxchar_new); + } + else { + _PyUnicode_FastCopyCharacters(v, 0, + u, 0, PyUnicode_GET_LENGTH(self)); } - return (PyObject*) u; + Py_DECREF(u); + assert(_PyUnicode_CheckConsistency(v, 1)); + return v; } -static -int fixupper(PyUnicodeObject *self) +static PyObject * +ascii_upper_or_lower(PyObject *self, int lower) +{ + Py_ssize_t len = PyUnicode_GET_LENGTH(self); + char *resdata, *data = PyUnicode_DATA(self); + PyObject *res; + + res = PyUnicode_New(len, 127); + if (res == NULL) + return NULL; + resdata = PyUnicode_DATA(res); + if (lower) + _Py_bytes_lower(resdata, data, len); + else + _Py_bytes_upper(resdata, data, len); + return res; +} + +static Py_UCS4 +handle_capital_sigma(int kind, void *data, Py_ssize_t length, Py_ssize_t i) { - Py_ssize_t len = self->length; - Py_UNICODE *s = self->str; - int status = 0; + Py_ssize_t j; + int final_sigma; + Py_UCS4 c; + /* U+03A3 is in the Final_Sigma context when, it is found like this: - while (len-- > 0) { - register Py_UNICODE ch; + \p{cased}\p{case-ignorable}*U+03A3!(\p{case-ignorable}*\p{cased}) - ch = Py_UNICODE_TOUPPER(*s); - if (ch != *s) { - status = 1; - *s = ch; + where ! is a negation and \p{xxx} is a character with property xxx. + */ + for (j = i - 1; j >= 0; j--) { + c = PyUnicode_READ(kind, data, j); + if (!_PyUnicode_IsCaseIgnorable(c)) + break; + } + final_sigma = j >= 0 && _PyUnicode_IsCased(c); + if (final_sigma) { + for (j = i + 1; j < length; j++) { + c = PyUnicode_READ(kind, data, j); + if (!_PyUnicode_IsCaseIgnorable(c)) + break; } - s++; + final_sigma = j == length || !_PyUnicode_IsCased(c); } - - return status; + return (final_sigma) ? 0x3C2 : 0x3C3; } -static -int fixlower(PyUnicodeObject *self) +static int +lower_ucs4(int kind, void *data, Py_ssize_t length, Py_ssize_t i, + Py_UCS4 c, Py_UCS4 *mapped) { - Py_ssize_t len = self->length; - Py_UNICODE *s = self->str; - int status = 0; + /* Obscure special case. */ + if (c == 0x3A3) { + mapped[0] = handle_capital_sigma(kind, data, length, i); + return 1; + } + return _PyUnicode_ToLowerFull(c, mapped); +} - while (len-- > 0) { - register Py_UNICODE ch; +static Py_ssize_t +do_capitalize(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) +{ + Py_ssize_t i, k = 0; + int n_res, j; + Py_UCS4 c, mapped[3]; - ch = Py_UNICODE_TOLOWER(*s); - if (ch != *s) { - status = 1; - *s = ch; + c = PyUnicode_READ(kind, data, 0); + n_res = _PyUnicode_ToUpperFull(c, mapped); + for (j = 0; j < n_res; j++) { + *maxchar = MAX_MAXCHAR(*maxchar, mapped[j]); + res[k++] = mapped[j]; + } + for (i = 1; i < length; i++) { + c = PyUnicode_READ(kind, data, i); + n_res = lower_ucs4(kind, data, length, i, c, mapped); + for (j = 0; j < n_res; j++) { + *maxchar = MAX_MAXCHAR(*maxchar, mapped[j]); + res[k++] = mapped[j]; } - s++; } + return k; +} + +static Py_ssize_t +do_swapcase(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) { + Py_ssize_t i, k = 0; - return status; + for (i = 0; i < length; i++) { + Py_UCS4 c = PyUnicode_READ(kind, data, i), mapped[3]; + int n_res, j; + if (Py_UNICODE_ISUPPER(c)) { + n_res = lower_ucs4(kind, data, length, i, c, mapped); + } + else if (Py_UNICODE_ISLOWER(c)) { + n_res = _PyUnicode_ToUpperFull(c, mapped); + } + else { + n_res = 1; + mapped[0] = c; + } + for (j = 0; j < n_res; j++) { + *maxchar = MAX_MAXCHAR(*maxchar, mapped[j]); + res[k++] = mapped[j]; + } + } + return k; } -static -int fixswapcase(PyUnicodeObject *self) +static Py_ssize_t +do_upper_or_lower(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, + Py_UCS4 *maxchar, int lower) { - Py_ssize_t len = self->length; - Py_UNICODE *s = self->str; - int status = 0; + Py_ssize_t i, k = 0; - while (len-- > 0) { - if (Py_UNICODE_ISUPPER(*s)) { - *s = Py_UNICODE_TOLOWER(*s); - status = 1; - } else if (Py_UNICODE_ISLOWER(*s)) { - *s = Py_UNICODE_TOUPPER(*s); - status = 1; + for (i = 0; i < length; i++) { + Py_UCS4 c = PyUnicode_READ(kind, data, i), mapped[3]; + int n_res, j; + if (lower) + n_res = lower_ucs4(kind, data, length, i, c, mapped); + else + n_res = _PyUnicode_ToUpperFull(c, mapped); + for (j = 0; j < n_res; j++) { + *maxchar = MAX_MAXCHAR(*maxchar, mapped[j]); + res[k++] = mapped[j]; } - s++; } + return k; +} - return status; +static Py_ssize_t +do_upper(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) +{ + return do_upper_or_lower(kind, data, length, res, maxchar, 0); } -static -int fixcapitalize(PyUnicodeObject *self) +static Py_ssize_t +do_lower(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) { - Py_ssize_t len = self->length; - Py_UNICODE *s = self->str; - int status = 0; + return do_upper_or_lower(kind, data, length, res, maxchar, 1); +} - if (len == 0) - return 0; - if (!Py_UNICODE_ISUPPER(*s)) { - *s = Py_UNICODE_TOUPPER(*s); - status = 1; - } - s++; - while (--len > 0) { - if (!Py_UNICODE_ISLOWER(*s)) { - *s = Py_UNICODE_TOLOWER(*s); - status = 1; +static Py_ssize_t +do_casefold(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) +{ + Py_ssize_t i, k = 0; + + for (i = 0; i < length; i++) { + Py_UCS4 c = PyUnicode_READ(kind, data, i); + Py_UCS4 mapped[3]; + int j, n_res = _PyUnicode_ToFoldedFull(c, mapped); + for (j = 0; j < n_res; j++) { + *maxchar = MAX_MAXCHAR(*maxchar, mapped[j]); + res[k++] = mapped[j]; } - s++; } - return status; + return k; } -static -int fixtitle(PyUnicodeObject *self) +static Py_ssize_t +do_title(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) { - register Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register Py_UNICODE *e; + Py_ssize_t i, k = 0; int previous_is_cased; - /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1) { - Py_UNICODE ch = Py_UNICODE_TOTITLE(*p); - if (*p != ch) { - *p = ch; - return 1; - } - else - return 0; - } - - e = p + PyUnicode_GET_SIZE(self); previous_is_cased = 0; - for (; p < e; p++) { - register const Py_UNICODE ch = *p; + for (i = 0; i < length; i++) { + const Py_UCS4 c = PyUnicode_READ(kind, data, i); + Py_UCS4 mapped[3]; + int n_res, j; if (previous_is_cased) - *p = Py_UNICODE_TOLOWER(ch); + n_res = lower_ucs4(kind, data, length, i, c, mapped); else - *p = Py_UNICODE_TOTITLE(ch); + n_res = _PyUnicode_ToTitleFull(c, mapped); - if (Py_UNICODE_ISLOWER(ch) || - Py_UNICODE_ISUPPER(ch) || - Py_UNICODE_ISTITLE(ch)) - previous_is_cased = 1; - else - previous_is_cased = 0; + for (j = 0; j < n_res; j++) { + *maxchar = MAX_MAXCHAR(*maxchar, mapped[j]); + res[k++] = mapped[j]; + } + + previous_is_cased = _PyUnicode_IsCased(c); } - return 1; + return k; +} + +static PyObject * +case_operation(PyObject *self, + Py_ssize_t (*perform)(int, void *, Py_ssize_t, Py_UCS4 *, Py_UCS4 *)) +{ + PyObject *res = NULL; + Py_ssize_t length, newlength = 0; + int kind, outkind; + void *data, *outdata; + Py_UCS4 maxchar = 0, *tmp, *tmpend; + + assert(PyUnicode_IS_READY(self)); + + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + length = PyUnicode_GET_LENGTH(self); + tmp = PyMem_MALLOC(sizeof(Py_UCS4) * 3 * length); + if (tmp == NULL) + return PyErr_NoMemory(); + newlength = perform(kind, data, length, tmp, &maxchar); + res = PyUnicode_New(newlength, maxchar); + if (res == NULL) + goto leave; + tmpend = tmp + newlength; + outdata = PyUnicode_DATA(res); + outkind = PyUnicode_KIND(res); + switch (outkind) { + case PyUnicode_1BYTE_KIND: + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS1, tmp, tmpend, outdata); + break; + case PyUnicode_2BYTE_KIND: + _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS2, tmp, tmpend, outdata); + break; + case PyUnicode_4BYTE_KIND: + memcpy(outdata, tmp, sizeof(Py_UCS4) * newlength); + break; + default: + assert(0); + break; + } + leave: + PyMem_FREE(tmp); + return res; } PyObject * PyUnicode_Join(PyObject *separator, PyObject *seq) { - const Py_UNICODE blank = ' '; - const Py_UNICODE *sep = ␣ - Py_ssize_t seplen = 1; - PyUnicodeObject *res = NULL; /* the result */ - Py_UNICODE *res_p; /* pointer to free byte in res's string area */ + PyObject *sep = NULL; + Py_ssize_t seplen; + PyObject *res = NULL; /* the result */ PyObject *fseq; /* PySequence_Fast(seq) */ Py_ssize_t seqlen; /* len(fseq) -- number of items in sequence */ PyObject **items; PyObject *item; - Py_ssize_t sz, i; + Py_ssize_t sz, i, res_offset; + Py_UCS4 maxchar; + Py_UCS4 item_maxchar; + int use_memcpy; + unsigned char *res_data = NULL, *sep_data = NULL; + PyObject *last_obj; + unsigned int kind = 0; fseq = PySequence_Fast(seq, ""); if (fseq == NULL) { @@ -6790,24 +9493,34 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) seqlen = PySequence_Fast_GET_SIZE(fseq); /* If empty sequence, return u"". */ if (seqlen == 0) { - res = _PyUnicode_New(0); /* empty sequence; return u"" */ - goto Done; + Py_DECREF(fseq); + Py_INCREF(unicode_empty); + res = unicode_empty; + return res; } - items = PySequence_Fast_ITEMS(fseq); + /* If singleton sequence with an exact Unicode, return that. */ + last_obj = NULL; + items = PySequence_Fast_ITEMS(fseq); if (seqlen == 1) { - item = items[0]; - if (PyUnicode_CheckExact(item)) { - Py_INCREF(item); - res = (PyUnicodeObject *)item; - goto Done; + if (PyUnicode_CheckExact(items[0])) { + res = items[0]; + Py_INCREF(res); + Py_DECREF(fseq); + return res; } + seplen = 0; + maxchar = 0; } else { /* Set up sep and seplen */ if (separator == NULL) { - sep = ␣ + /* fall back to a blank space separator */ + sep = PyUnicode_FromOrdinal(' '); + if (!sep) + goto onError; seplen = 1; + maxchar = 32; } else { if (!PyUnicode_Check(separator)) { @@ -6817,9 +9530,16 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) Py_TYPE(separator)->tp_name); goto onError; } - sep = PyUnicode_AS_UNICODE(separator); - seplen = PyUnicode_GET_SIZE(separator); + if (PyUnicode_READY(separator)) + goto onError; + sep = separator; + seplen = PyUnicode_GET_LENGTH(separator); + maxchar = PyUnicode_MAX_CHAR_VALUE(separator); + /* inc refcount to keep this code path symmetric with the + above case of a blank separator */ + Py_INCREF(sep); } + last_obj = sep; } /* There are at least two things to join, or else we have a subclass @@ -6828,6 +9548,11 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) * need (sz), and see whether all argument are strings. */ sz = 0; +#ifdef Py_DEBUG + use_memcpy = 0; +#else + use_memcpy = 1; +#endif for (i = 0; i < seqlen; i++) { const Py_ssize_t old_sz = sz; item = items[i]; @@ -6838,7 +9563,11 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) i, Py_TYPE(item)->tp_name); goto onError; } - sz += PyUnicode_GET_SIZE(item); + if (PyUnicode_READY(item) == -1) + goto onError; + sz += PyUnicode_GET_LENGTH(item); + item_maxchar = PyUnicode_MAX_CHAR_VALUE(item); + maxchar = MAX_MAXCHAR(maxchar, item_maxchar); if (i != 0) sz += seplen; if (sz < old_sz || sz > PY_SSIZE_T_MAX) { @@ -6846,266 +9575,725 @@ PyUnicode_Join(PyObject *separator, PyObject *seq) "join() result is too long for a Python string"); goto onError; } + if (use_memcpy && last_obj != NULL) { + if (PyUnicode_KIND(last_obj) != PyUnicode_KIND(item)) + use_memcpy = 0; + } + last_obj = item; } - res = _PyUnicode_New(sz); + res = PyUnicode_New(sz, maxchar); if (res == NULL) goto onError; /* Catenate everything. */ - res_p = PyUnicode_AS_UNICODE(res); - for (i = 0; i < seqlen; ++i) { +#ifdef Py_DEBUG + use_memcpy = 0; +#else + if (use_memcpy) { + res_data = PyUnicode_1BYTE_DATA(res); + kind = PyUnicode_KIND(res); + if (seplen != 0) + sep_data = PyUnicode_1BYTE_DATA(sep); + } +#endif + for (i = 0, res_offset = 0; i < seqlen; ++i) { Py_ssize_t itemlen; item = items[i]; - itemlen = PyUnicode_GET_SIZE(item); /* Copy item, and maybe the separator. */ - if (i) { - Py_UNICODE_COPY(res_p, sep, seplen); - res_p += seplen; + if (i && seplen != 0) { + if (use_memcpy) { + Py_MEMCPY(res_data, + sep_data, + kind * seplen); + res_data += kind * seplen; + } + else { + _PyUnicode_FastCopyCharacters(res, res_offset, sep, 0, seplen); + res_offset += seplen; + } + } + itemlen = PyUnicode_GET_LENGTH(item); + if (itemlen != 0) { + if (use_memcpy) { + Py_MEMCPY(res_data, + PyUnicode_DATA(item), + kind * itemlen); + res_data += kind * itemlen; + } + else { + _PyUnicode_FastCopyCharacters(res, res_offset, item, 0, itemlen); + res_offset += itemlen; + } } - Py_UNICODE_COPY(res_p, PyUnicode_AS_UNICODE(item), itemlen); - res_p += itemlen; } + if (use_memcpy) + assert(res_data == PyUnicode_1BYTE_DATA(res) + + kind * PyUnicode_GET_LENGTH(res)); + else + assert(res_offset == PyUnicode_GET_LENGTH(res)); - Done: Py_DECREF(fseq); - return (PyObject *)res; + Py_XDECREF(sep); + assert(_PyUnicode_CheckConsistency(res, 1)); + return res; onError: Py_DECREF(fseq); + Py_XDECREF(sep); Py_XDECREF(res); return NULL; } -static -PyUnicodeObject *pad(PyUnicodeObject *self, - Py_ssize_t left, - Py_ssize_t right, - Py_UNICODE fill) +#define FILL(kind, data, value, start, length) \ + do { \ + Py_ssize_t i_ = 0; \ + assert(kind != PyUnicode_WCHAR_KIND); \ + switch ((kind)) { \ + case PyUnicode_1BYTE_KIND: { \ + unsigned char * to_ = (unsigned char *)((data)) + (start); \ + memset(to_, (unsigned char)value, (length)); \ + break; \ + } \ + case PyUnicode_2BYTE_KIND: { \ + Py_UCS2 * to_ = (Py_UCS2 *)((data)) + (start); \ + for (; i_ < (length); ++i_, ++to_) *to_ = (value); \ + break; \ + } \ + case PyUnicode_4BYTE_KIND: { \ + Py_UCS4 * to_ = (Py_UCS4 *)((data)) + (start); \ + for (; i_ < (length); ++i_, ++to_) *to_ = (value); \ + break; \ + default: assert(0); \ + } \ + } \ + } while (0) + +void +_PyUnicode_FastFill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, + Py_UCS4 fill_char) +{ + const enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); + const void *data = PyUnicode_DATA(unicode); + assert(PyUnicode_IS_READY(unicode)); + assert(unicode_modifiable(unicode)); + assert(fill_char <= PyUnicode_MAX_CHAR_VALUE(unicode)); + assert(start >= 0); + assert(start + length <= PyUnicode_GET_LENGTH(unicode)); + FILL(kind, data, fill_char, start, length); +} + +Py_ssize_t +PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, + Py_UCS4 fill_char) { - PyUnicodeObject *u; + Py_ssize_t maxlen; + + if (!PyUnicode_Check(unicode)) { + PyErr_BadInternalCall(); + return -1; + } + if (PyUnicode_READY(unicode) == -1) + return -1; + if (unicode_check_modifiable(unicode)) + return -1; + + if (start < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return -1; + } + if (fill_char > PyUnicode_MAX_CHAR_VALUE(unicode)) { + PyErr_SetString(PyExc_ValueError, + "fill character is bigger than " + "the string maximum character"); + return -1; + } + + maxlen = PyUnicode_GET_LENGTH(unicode) - start; + length = Py_MIN(maxlen, length); + if (length <= 0) + return 0; + + _PyUnicode_FastFill(unicode, start, length, fill_char); + return length; +} + +static PyObject * +pad(PyObject *self, + Py_ssize_t left, + Py_ssize_t right, + Py_UCS4 fill) +{ + PyObject *u; + Py_UCS4 maxchar; + int kind; + void *data; if (left < 0) left = 0; if (right < 0) right = 0; - if (left == 0 && right == 0 && PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return self; - } + if (left == 0 && right == 0) + return unicode_result_unchanged(self); - if (left > PY_SSIZE_T_MAX - self->length || - right > PY_SSIZE_T_MAX - (left + self->length)) { + if (left > PY_SSIZE_T_MAX - _PyUnicode_LENGTH(self) || + right > PY_SSIZE_T_MAX - (left + _PyUnicode_LENGTH(self))) { PyErr_SetString(PyExc_OverflowError, "padded string is too long"); return NULL; } - u = _PyUnicode_New(left + self->length + right); - if (u) { - if (left) - Py_UNICODE_FILL(u->str, fill, left); - Py_UNICODE_COPY(u->str + left, self->str, self->length); - if (right) - Py_UNICODE_FILL(u->str + left + self->length, fill, right); - } + maxchar = PyUnicode_MAX_CHAR_VALUE(self); + maxchar = MAX_MAXCHAR(maxchar, fill); + u = PyUnicode_New(left + _PyUnicode_LENGTH(self) + right, maxchar); + if (!u) + return NULL; + kind = PyUnicode_KIND(u); + data = PyUnicode_DATA(u); + if (left) + FILL(kind, data, fill, 0, left); + if (right) + FILL(kind, data, fill, left + _PyUnicode_LENGTH(self), right); + _PyUnicode_FastCopyCharacters(u, left, self, 0, _PyUnicode_LENGTH(self)); + assert(_PyUnicode_CheckConsistency(u, 1)); return u; } -PyObject *PyUnicode_Splitlines(PyObject *string, int keepends) +PyObject * +PyUnicode_Splitlines(PyObject *string, int keepends) { PyObject *list; string = PyUnicode_FromObject(string); if (string == NULL) return NULL; + if (PyUnicode_READY(string) == -1) { + Py_DECREF(string); + return NULL; + } - list = stringlib_splitlines( - (PyObject*) string, PyUnicode_AS_UNICODE(string), - PyUnicode_GET_SIZE(string), keepends); - + switch (PyUnicode_KIND(string)) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(string)) + list = asciilib_splitlines( + string, PyUnicode_1BYTE_DATA(string), + PyUnicode_GET_LENGTH(string), keepends); + else + list = ucs1lib_splitlines( + string, PyUnicode_1BYTE_DATA(string), + PyUnicode_GET_LENGTH(string), keepends); + break; + case PyUnicode_2BYTE_KIND: + list = ucs2lib_splitlines( + string, PyUnicode_2BYTE_DATA(string), + PyUnicode_GET_LENGTH(string), keepends); + break; + case PyUnicode_4BYTE_KIND: + list = ucs4lib_splitlines( + string, PyUnicode_4BYTE_DATA(string), + PyUnicode_GET_LENGTH(string), keepends); + break; + default: + assert(0); + list = 0; + } Py_DECREF(string); return list; } -static -PyObject *split(PyUnicodeObject *self, - PyUnicodeObject *substring, - Py_ssize_t maxcount) +static PyObject * +split(PyObject *self, + PyObject *substring, + Py_ssize_t maxcount) { + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2; + PyObject* out; + if (maxcount < 0) maxcount = PY_SSIZE_T_MAX; + if (PyUnicode_READY(self) == -1) + return NULL; + if (substring == NULL) - return stringlib_split_whitespace( - (PyObject*) self, self->str, self->length, maxcount - ); + switch (PyUnicode_KIND(self)) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(self)) + return asciilib_split_whitespace( + self, PyUnicode_1BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + else + return ucs1lib_split_whitespace( + self, PyUnicode_1BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + case PyUnicode_2BYTE_KIND: + return ucs2lib_split_whitespace( + self, PyUnicode_2BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + case PyUnicode_4BYTE_KIND: + return ucs4lib_split_whitespace( + self, PyUnicode_4BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + default: + assert(0); + return NULL; + } - return stringlib_split( - (PyObject*) self, self->str, self->length, - substring->str, substring->length, - maxcount - ); + if (PyUnicode_READY(substring) == -1) + return NULL; + + kind1 = PyUnicode_KIND(self); + kind2 = PyUnicode_KIND(substring); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(self); + buf2 = PyUnicode_DATA(substring); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind(self, kind); + if (!buf1) + return NULL; + if (kind2 != kind) + buf2 = _PyUnicode_AsKind(substring, kind); + if (!buf2) { + if (kind1 != kind) PyMem_Free(buf1); + return NULL; + } + len1 = PyUnicode_GET_LENGTH(self); + len2 = PyUnicode_GET_LENGTH(substring); + + switch (kind) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(self) && PyUnicode_IS_ASCII(substring)) + out = asciilib_split( + self, buf1, len1, buf2, len2, maxcount); + else + out = ucs1lib_split( + self, buf1, len1, buf2, len2, maxcount); + break; + case PyUnicode_2BYTE_KIND: + out = ucs2lib_split( + self, buf1, len1, buf2, len2, maxcount); + break; + case PyUnicode_4BYTE_KIND: + out = ucs4lib_split( + self, buf1, len1, buf2, len2, maxcount); + break; + default: + out = NULL; + } + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); + return out; } -static -PyObject *rsplit(PyUnicodeObject *self, - PyUnicodeObject *substring, - Py_ssize_t maxcount) +static PyObject * +rsplit(PyObject *self, + PyObject *substring, + Py_ssize_t maxcount) { + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2; + PyObject* out; + if (maxcount < 0) maxcount = PY_SSIZE_T_MAX; + if (PyUnicode_READY(self) == -1) + return NULL; + if (substring == NULL) - return stringlib_rsplit_whitespace( - (PyObject*) self, self->str, self->length, maxcount - ); + switch (PyUnicode_KIND(self)) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(self)) + return asciilib_rsplit_whitespace( + self, PyUnicode_1BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + else + return ucs1lib_rsplit_whitespace( + self, PyUnicode_1BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + case PyUnicode_2BYTE_KIND: + return ucs2lib_rsplit_whitespace( + self, PyUnicode_2BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + case PyUnicode_4BYTE_KIND: + return ucs4lib_rsplit_whitespace( + self, PyUnicode_4BYTE_DATA(self), + PyUnicode_GET_LENGTH(self), maxcount + ); + default: + assert(0); + return NULL; + } - return stringlib_rsplit( - (PyObject*) self, self->str, self->length, - substring->str, substring->length, - maxcount - ); + if (PyUnicode_READY(substring) == -1) + return NULL; + + kind1 = PyUnicode_KIND(self); + kind2 = PyUnicode_KIND(substring); + kind = kind1 > kind2 ? kind1 : kind2; + buf1 = PyUnicode_DATA(self); + buf2 = PyUnicode_DATA(substring); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind(self, kind); + if (!buf1) + return NULL; + if (kind2 != kind) + buf2 = _PyUnicode_AsKind(substring, kind); + if (!buf2) { + if (kind1 != kind) PyMem_Free(buf1); + return NULL; + } + len1 = PyUnicode_GET_LENGTH(self); + len2 = PyUnicode_GET_LENGTH(substring); + + switch (kind) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(self) && PyUnicode_IS_ASCII(substring)) + out = asciilib_rsplit( + self, buf1, len1, buf2, len2, maxcount); + else + out = ucs1lib_rsplit( + self, buf1, len1, buf2, len2, maxcount); + break; + case PyUnicode_2BYTE_KIND: + out = ucs2lib_rsplit( + self, buf1, len1, buf2, len2, maxcount); + break; + case PyUnicode_4BYTE_KIND: + out = ucs4lib_rsplit( + self, buf1, len1, buf2, len2, maxcount); + break; + default: + out = NULL; + } + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); + return out; } -static -PyObject *replace(PyUnicodeObject *self, - PyUnicodeObject *str1, - PyUnicodeObject *str2, - Py_ssize_t maxcount) +static Py_ssize_t +anylib_find(int kind, PyObject *str1, void *buf1, Py_ssize_t len1, + PyObject *str2, void *buf2, Py_ssize_t len2, Py_ssize_t offset) { - PyUnicodeObject *u; + switch (kind) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(str1) && PyUnicode_IS_ASCII(str2)) + return asciilib_find(buf1, len1, buf2, len2, offset); + else + return ucs1lib_find(buf1, len1, buf2, len2, offset); + case PyUnicode_2BYTE_KIND: + return ucs2lib_find(buf1, len1, buf2, len2, offset); + case PyUnicode_4BYTE_KIND: + return ucs4lib_find(buf1, len1, buf2, len2, offset); + } + assert(0); + return -1; +} + +static Py_ssize_t +anylib_count(int kind, PyObject *sstr, void* sbuf, Py_ssize_t slen, + PyObject *str1, void *buf1, Py_ssize_t len1, Py_ssize_t maxcount) +{ + switch (kind) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(sstr) && PyUnicode_IS_ASCII(str1)) + return asciilib_count(sbuf, slen, buf1, len1, maxcount); + else + return ucs1lib_count(sbuf, slen, buf1, len1, maxcount); + case PyUnicode_2BYTE_KIND: + return ucs2lib_count(sbuf, slen, buf1, len1, maxcount); + case PyUnicode_4BYTE_KIND: + return ucs4lib_count(sbuf, slen, buf1, len1, maxcount); + } + assert(0); + return 0; +} + +static PyObject * +replace(PyObject *self, PyObject *str1, + PyObject *str2, Py_ssize_t maxcount) +{ + PyObject *u; + char *sbuf = PyUnicode_DATA(self); + char *buf1 = PyUnicode_DATA(str1); + char *buf2 = PyUnicode_DATA(str2); + int srelease = 0, release1 = 0, release2 = 0; + int skind = PyUnicode_KIND(self); + int kind1 = PyUnicode_KIND(str1); + int kind2 = PyUnicode_KIND(str2); + Py_ssize_t slen = PyUnicode_GET_LENGTH(self); + Py_ssize_t len1 = PyUnicode_GET_LENGTH(str1); + Py_ssize_t len2 = PyUnicode_GET_LENGTH(str2); + int mayshrink; + Py_UCS4 maxchar, maxchar_str2; if (maxcount < 0) maxcount = PY_SSIZE_T_MAX; - else if (maxcount == 0 || self->length == 0) + else if (maxcount == 0 || slen == 0) goto nothing; - if (str1->length == str2->length) { - Py_ssize_t i; + if (str1 == str2) + goto nothing; + if (skind < kind1) + /* substring too wide to be present */ + goto nothing; + + maxchar = PyUnicode_MAX_CHAR_VALUE(self); + maxchar_str2 = PyUnicode_MAX_CHAR_VALUE(str2); + /* Replacing str1 with str2 may cause a maxchar reduction in the + result string. */ + mayshrink = (maxchar_str2 < maxchar); + maxchar = MAX_MAXCHAR(maxchar, maxchar_str2); + + if (len1 == len2) { /* same length */ - if (str1->length == 0) + if (len1 == 0) goto nothing; - if (str1->length == 1) { + if (len1 == 1) { /* replace characters */ - Py_UNICODE u1, u2; - if (!findchar(self->str, self->length, str1->str[0])) + Py_UCS4 u1, u2; + int rkind; + Py_ssize_t index, pos; + char *src; + + u1 = PyUnicode_READ_CHAR(str1, 0); + pos = findchar(sbuf, PyUnicode_KIND(self), slen, u1, 1); + if (pos < 0) goto nothing; - u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length); + u2 = PyUnicode_READ_CHAR(str2, 0); + u = PyUnicode_New(slen, maxchar); if (!u) - return NULL; - Py_UNICODE_COPY(u->str, self->str, self->length); - u1 = str1->str[0]; - u2 = str2->str[0]; - for (i = 0; i < u->length; i++) - if (u->str[i] == u1) { - if (--maxcount < 0) - break; - u->str[i] = u2; - } - } else { - i = stringlib_find( - self->str, self->length, str1->str, str1->length, 0 - ); + goto error; + _PyUnicode_FastCopyCharacters(u, 0, self, 0, slen); + rkind = PyUnicode_KIND(u); + + PyUnicode_WRITE(rkind, PyUnicode_DATA(u), pos, u2); + index = 0; + src = sbuf; + while (--maxcount) + { + pos++; + src += pos * PyUnicode_KIND(self); + slen -= pos; + index += pos; + pos = findchar(src, PyUnicode_KIND(self), slen, u1, 1); + if (pos < 0) + break; + PyUnicode_WRITE(rkind, PyUnicode_DATA(u), index + pos, u2); + } + } + else { + int rkind = skind; + char *res; + Py_ssize_t i; + + if (kind1 < rkind) { + /* widen substring */ + buf1 = _PyUnicode_AsKind(str1, rkind); + if (!buf1) goto error; + release1 = 1; + } + i = anylib_find(rkind, self, sbuf, slen, str1, buf1, len1, 0); if (i < 0) goto nothing; - u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length); + if (rkind > kind2) { + /* widen replacement */ + buf2 = _PyUnicode_AsKind(str2, rkind); + if (!buf2) goto error; + release2 = 1; + } + else if (rkind < kind2) { + /* widen self and buf1 */ + rkind = kind2; + if (release1) PyMem_Free(buf1); + sbuf = _PyUnicode_AsKind(self, rkind); + if (!sbuf) goto error; + srelease = 1; + buf1 = _PyUnicode_AsKind(str1, rkind); + if (!buf1) goto error; + release1 = 1; + } + u = PyUnicode_New(slen, maxchar); if (!u) - return NULL; - Py_UNICODE_COPY(u->str, self->str, self->length); + goto error; + assert(PyUnicode_KIND(u) == rkind); + res = PyUnicode_DATA(u); + memcpy(res, sbuf, rkind * slen); /* change everything in-place, starting with this one */ - Py_UNICODE_COPY(u->str+i, str2->str, str2->length); - i += str1->length; + memcpy(res + rkind * i, + buf2, + rkind * len2); + i += len1; while ( --maxcount > 0) { - i = stringlib_find(self->str+i, self->length-i, - str1->str, str1->length, - i); + i = anylib_find(rkind, self, + sbuf+rkind*i, slen-i, + str1, buf1, len1, i); if (i == -1) break; - Py_UNICODE_COPY(u->str+i, str2->str, str2->length); - i += str1->length; + memcpy(res + rkind * i, + buf2, + rkind * len2); + i += len1; } } - } else { - - Py_ssize_t n, i, j; - Py_ssize_t product, new_size, delta; - Py_UNICODE *p; - - /* replace strings */ - n = stringlib_count(self->str, self->length, str1->str, str1->length, - maxcount); + } + else { + Py_ssize_t n, i, j, ires; + Py_ssize_t new_size; + int rkind = skind; + char *res; + + if (kind1 < rkind) { + /* widen substring */ + buf1 = _PyUnicode_AsKind(str1, rkind); + if (!buf1) goto error; + release1 = 1; + } + n = anylib_count(rkind, self, sbuf, slen, str1, buf1, len1, maxcount); if (n == 0) goto nothing; - /* new_size = self->length + n * (str2->length - str1->length)); */ - delta = (str2->length - str1->length); - if (delta == 0) { - new_size = self->length; - } else { - product = n * (str2->length - str1->length); - if ((product / (str2->length - str1->length)) != n) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - new_size = self->length + product; - if (new_size < 0) { + if (kind2 < rkind) { + /* widen replacement */ + buf2 = _PyUnicode_AsKind(str2, rkind); + if (!buf2) goto error; + release2 = 1; + } + else if (kind2 > rkind) { + /* widen self and buf1 */ + rkind = kind2; + sbuf = _PyUnicode_AsKind(self, rkind); + if (!sbuf) goto error; + srelease = 1; + if (release1) PyMem_Free(buf1); + buf1 = _PyUnicode_AsKind(str1, rkind); + if (!buf1) goto error; + release1 = 1; + } + /* new_size = PyUnicode_GET_LENGTH(self) + n * (PyUnicode_GET_LENGTH(str2) - + PyUnicode_GET_LENGTH(str1))); */ + if (len2 > len1 && len2 - len1 > (PY_SSIZE_T_MAX - slen) / n) { PyErr_SetString(PyExc_OverflowError, "replace string is too long"); - return NULL; - } + goto error; + } + new_size = slen + n * (len2 - len1); + if (new_size == 0) { + Py_INCREF(unicode_empty); + u = unicode_empty; + goto done; + } + if (new_size > (PY_SSIZE_T_MAX >> (rkind-1))) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + goto error; } - u = _PyUnicode_New(new_size); + u = PyUnicode_New(new_size, maxchar); if (!u) - return NULL; - i = 0; - p = u->str; - if (str1->length > 0) { + goto error; + assert(PyUnicode_KIND(u) == rkind); + res = PyUnicode_DATA(u); + ires = i = 0; + if (len1 > 0) { while (n-- > 0) { /* look for next match */ - j = stringlib_find(self->str+i, self->length-i, - str1->str, str1->length, - i); + j = anylib_find(rkind, self, + sbuf + rkind * i, slen-i, + str1, buf1, len1, i); if (j == -1) break; else if (j > i) { /* copy unchanged part [i:j] */ - Py_UNICODE_COPY(p, self->str+i, j-i); - p += j - i; + memcpy(res + rkind * ires, + sbuf + rkind * i, + rkind * (j-i)); + ires += j - i; } /* copy substitution string */ - if (str2->length > 0) { - Py_UNICODE_COPY(p, str2->str, str2->length); - p += str2->length; + if (len2 > 0) { + memcpy(res + rkind * ires, + buf2, + rkind * len2); + ires += len2; } - i = j + str1->length; + i = j + len1; } - if (i < self->length) + if (i < slen) /* copy tail [i:] */ - Py_UNICODE_COPY(p, self->str+i, self->length-i); - } else { + memcpy(res + rkind * ires, + sbuf + rkind * i, + rkind * (slen-i)); + } + else { /* interleave */ while (n > 0) { - Py_UNICODE_COPY(p, str2->str, str2->length); - p += str2->length; + memcpy(res + rkind * ires, + buf2, + rkind * len2); + ires += len2; if (--n <= 0) break; - *p++ = self->str[i++]; + memcpy(res + rkind * ires, + sbuf + rkind * i, + rkind); + ires++; + i++; } - Py_UNICODE_COPY(p, self->str+i, self->length-i); + memcpy(res + rkind * ires, + sbuf + rkind * i, + rkind * (slen-i)); } } - return (PyObject *) u; + + if (mayshrink) { + unicode_adjust_maxchar(&u); + if (u == NULL) + goto error; + } + + done: + if (srelease) + PyMem_FREE(sbuf); + if (release1) + PyMem_FREE(buf1); + if (release2) + PyMem_FREE(buf2); + assert(_PyUnicode_CheckConsistency(u, 1)); + return u; nothing: /* nothing to replace; return original string (when possible) */ - if (PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return (PyObject *) self; - } - return PyUnicode_FromUnicode(self->str, self->length); + if (srelease) + PyMem_FREE(sbuf); + if (release1) + PyMem_FREE(buf1); + if (release2) + PyMem_FREE(buf2); + return unicode_result_unchanged(self); + + error: + if (srelease && sbuf) + PyMem_FREE(sbuf); + if (release1 && buf1) + PyMem_FREE(buf1); + if (release2 && buf2) + PyMem_FREE(buf2); + return NULL; } /* --- Unicode Object Methods --------------------------------------------- */ @@ -7117,9 +10305,11 @@ Return a titlecased version of S, i.e. words start with title case\n\ characters, all remaining cased characters have lower case."); static PyObject* -unicode_title(PyUnicodeObject *self) +unicode_title(PyObject *self) { - return fixup(self, fixtitle); + if (PyUnicode_READY(self) == -1) + return NULL; + return case_operation(self, do_title); } PyDoc_STRVAR(capitalize__doc__, @@ -7129,57 +10319,38 @@ Return a capitalized version of S, i.e. make the first character\n\ have upper case and the rest lower case."); static PyObject* -unicode_capitalize(PyUnicodeObject *self) +unicode_capitalize(PyObject *self) { - return fixup(self, fixcapitalize); + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_GET_LENGTH(self) == 0) + return unicode_result_unchanged(self); + return case_operation(self, do_capitalize); } -#if 0 -PyDoc_STRVAR(capwords__doc__, - "S.capwords() -> str\n\ +PyDoc_STRVAR(casefold__doc__, + "S.casefold() -> str\n\ \n\ -Apply .capitalize() to all words in S and return the result with\n\ -normalized whitespace (all whitespace strings are replaced by ' ')."); +Return a version of S suitable for caseless comparisons."); -static PyObject* -unicode_capwords(PyUnicodeObject *self) +static PyObject * +unicode_casefold(PyObject *self) { - PyObject *list; - PyObject *item; - Py_ssize_t i; - - /* Split into words */ - list = split(self, NULL, -1); - if (!list) + if (PyUnicode_READY(self) == -1) return NULL; - - /* Capitalize each word */ - for (i = 0; i < PyList_GET_SIZE(list); i++) { - item = fixup((PyUnicodeObject *)PyList_GET_ITEM(list, i), - fixcapitalize); - if (item == NULL) - goto onError; - Py_DECREF(PyList_GET_ITEM(list, i)); - PyList_SET_ITEM(list, i, item); - } - - /* Join the words to form a new string */ - item = PyUnicode_Join(NULL, list); - - onError: - Py_DECREF(list); - return (PyObject *)item; + if (PyUnicode_IS_ASCII(self)) + return ascii_upper_or_lower(self, 1); + return case_operation(self, do_casefold); } -#endif + /* Argument converter. Coerces to a single unicode character */ static int convert_uc(PyObject *obj, void *addr) { - Py_UNICODE *fillcharloc = (Py_UNICODE *)addr; + Py_UCS4 *fillcharloc = (Py_UCS4 *)addr; PyObject *uniobj; - Py_UNICODE *unistr; uniobj = PyUnicode_FromObject(obj); if (uniobj == NULL) { @@ -7187,14 +10358,13 @@ convert_uc(PyObject *obj, void *addr) "The fill character cannot be converted to Unicode"); return 0; } - if (PyUnicode_GET_SIZE(uniobj) != 1) { + if (PyUnicode_GET_LENGTH(uniobj) != 1) { PyErr_SetString(PyExc_TypeError, "The fill character must be exactly one character long"); Py_DECREF(uniobj); return 0; } - unistr = PyUnicode_AS_UNICODE(uniobj); - *fillcharloc = unistr[0]; + *fillcharloc = PyUnicode_READ_CHAR(uniobj, 0); Py_DECREF(uniobj); return 1; } @@ -7206,112 +10376,64 @@ Return S centered in a string of length width. Padding is\n\ done using the specified fill character (default is a space)"); static PyObject * -unicode_center(PyUnicodeObject *self, PyObject *args) +unicode_center(PyObject *self, PyObject *args) { Py_ssize_t marg, left; Py_ssize_t width; - Py_UNICODE fillchar = ' '; + Py_UCS4 fillchar = ' '; if (!PyArg_ParseTuple(args, "n|O&:center", &width, convert_uc, &fillchar)) return NULL; - if (self->length >= width && PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return (PyObject*) self; - } + if (PyUnicode_READY(self) == -1) + return NULL; - marg = width - self->length; + if (PyUnicode_GET_LENGTH(self) >= width) + return unicode_result_unchanged(self); + + marg = width - PyUnicode_GET_LENGTH(self); left = marg / 2 + (marg & width & 1); - return (PyObject*) pad(self, left, marg - left, fillchar); + return pad(self, left, marg - left, fillchar); } -#if 0 - -/* This code should go into some future Unicode collation support - module. The basic comparison should compare ordinals on a naive - basis (this is what Java does and thus Jython too). */ - -/* speedy UTF-16 code point order comparison */ -/* gleaned from: */ -/* http://www-4.ibm.com/software/developer/library/utf16.html?dwzone=unicode */ - -static short utf16Fixup[32] = -{ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0x2000, -0x800, -0x800, -0x800, -0x800 -}; +/* This function assumes that str1 and str2 are readied by the caller. */ static int -unicode_compare(PyUnicodeObject *str1, PyUnicodeObject *str2) +unicode_compare(PyObject *str1, PyObject *str2) { - Py_ssize_t len1, len2; - - Py_UNICODE *s1 = str1->str; - Py_UNICODE *s2 = str2->str; - - len1 = str1->length; - len2 = str2->length; - - while (len1 > 0 && len2 > 0) { - Py_UNICODE c1, c2; + int kind1, kind2; + void *data1, *data2; + Py_ssize_t len1, len2, i; - c1 = *s1++; - c2 = *s2++; + kind1 = PyUnicode_KIND(str1); + kind2 = PyUnicode_KIND(str2); + data1 = PyUnicode_DATA(str1); + data2 = PyUnicode_DATA(str2); + len1 = PyUnicode_GET_LENGTH(str1); + len2 = PyUnicode_GET_LENGTH(str2); - if (c1 > (1<<11) * 26) - c1 += utf16Fixup[c1>>11]; - if (c2 > (1<<11) * 26) - c2 += utf16Fixup[c2>>11]; - /* now c1 and c2 are in UTF-32-compatible order */ + for (i = 0; i < len1 && i < len2; ++i) { + Py_UCS4 c1, c2; + c1 = PyUnicode_READ(kind1, data1, i); + c2 = PyUnicode_READ(kind2, data2, i); if (c1 != c2) return (c1 < c2) ? -1 : 1; - - len1--; len2--; } return (len1 < len2) ? -1 : (len1 != len2); } -#else - -static int -unicode_compare(PyUnicodeObject *str1, PyUnicodeObject *str2) +int +PyUnicode_Compare(PyObject *left, PyObject *right) { - register Py_ssize_t len1, len2; - - Py_UNICODE *s1 = str1->str; - Py_UNICODE *s2 = str2->str; - - len1 = str1->length; - len2 = str2->length; - - while (len1 > 0 && len2 > 0) { - Py_UNICODE c1, c2; - - c1 = *s1++; - c2 = *s2++; - - if (c1 != c2) - return (c1 < c2) ? -1 : 1; - - len1--; len2--; + if (PyUnicode_Check(left) && PyUnicode_Check(right)) { + if (PyUnicode_READY(left) == -1 || + PyUnicode_READY(right) == -1) + return -1; + return unicode_compare(left, right); } - - return (len1 < len2) ? -1 : (len1 != len2); -} - -#endif - -int PyUnicode_Compare(PyObject *left, - PyObject *right) -{ - if (PyUnicode_Check(left) && PyUnicode_Check(right)) - return unicode_compare((PyUnicodeObject *)left, - (PyUnicodeObject *)right); PyErr_Format(PyExc_TypeError, "Can't compare %.100s and %.100s", left->ob_type->tp_name, @@ -7322,17 +10444,23 @@ int PyUnicode_Compare(PyObject *left, int PyUnicode_CompareWithASCIIString(PyObject* uni, const char* str) { - int i; - Py_UNICODE *id; - assert(PyUnicode_Check(uni)); - id = PyUnicode_AS_UNICODE(uni); + Py_ssize_t i; + int kind; + void *data; + Py_UCS4 chr; + + assert(_PyUnicode_CHECK(uni)); + if (PyUnicode_READY(uni) == -1) + return -1; + kind = PyUnicode_KIND(uni); + data = PyUnicode_DATA(uni); /* Compare Unicode string and source character set string */ - for (i = 0; id[i] && str[i]; i++) - if (id[i] != str[i]) - return ((int)id[i] < (int)str[i]) ? -1 : 1; + for (i = 0; (chr = PyUnicode_READ(kind, data, i)) && str[i]; i++) + if (chr != str[i]) + return (chr < (unsigned char)(str[i])) ? -1 : 1; /* This check keeps Python strings that end in '\0' from comparing equal to C strings identical up to that point. */ - if (PyUnicode_GET_SIZE(uni) != i || id[i]) + if (PyUnicode_GET_LENGTH(uni) != i || chr) return 1; /* uni is longer */ if (str[i]) return -1; /* str is longer */ @@ -7343,16 +10471,18 @@ PyUnicode_CompareWithASCIIString(PyObject* uni, const char* str) #define TEST_COND(cond) \ ((cond) ? Py_True : Py_False) -PyObject *PyUnicode_RichCompare(PyObject *left, - PyObject *right, - int op) +PyObject * +PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) { int result; if (PyUnicode_Check(left) && PyUnicode_Check(right)) { PyObject *v; - if (((PyUnicodeObject *) left)->length != - ((PyUnicodeObject *) right)->length) { + if (PyUnicode_READY(left) == -1 || + PyUnicode_READY(right) == -1) + return NULL; + if (PyUnicode_GET_LENGTH(left) != PyUnicode_GET_LENGTH(right) || + PyUnicode_KIND(left) != PyUnicode_KIND(right)) { if (op == Py_EQ) { Py_INCREF(Py_False); return Py_False; @@ -7365,8 +10495,7 @@ PyObject *PyUnicode_RichCompare(PyObject *left, if (left == right) result = 0; else - result = unicode_compare((PyUnicodeObject *)left, - (PyUnicodeObject *)right); + result = unicode_compare(left, right); /* Convert the return value to a Boolean */ switch (op) { @@ -7396,14 +10525,16 @@ PyObject *PyUnicode_RichCompare(PyObject *left, return v; } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } -int PyUnicode_Contains(PyObject *container, - PyObject *element) +int +PyUnicode_Contains(PyObject *container, PyObject *element) { PyObject *str, *sub; + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2; int result; /* Coerce the two arguments */ @@ -7420,50 +10551,106 @@ int PyUnicode_Contains(PyObject *container, Py_DECREF(sub); return -1; } + if (PyUnicode_READY(sub) == -1 || PyUnicode_READY(str) == -1) { + Py_DECREF(sub); + Py_DECREF(str); + } + + kind1 = PyUnicode_KIND(str); + kind2 = PyUnicode_KIND(sub); + kind = kind1; + buf1 = PyUnicode_DATA(str); + buf2 = PyUnicode_DATA(sub); + if (kind2 != kind) { + if (kind2 > kind) { + Py_DECREF(sub); + Py_DECREF(str); + return 0; + } + buf2 = _PyUnicode_AsKind(sub, kind); + } + if (!buf2) { + Py_DECREF(sub); + Py_DECREF(str); + return -1; + } + len1 = PyUnicode_GET_LENGTH(str); + len2 = PyUnicode_GET_LENGTH(sub); - result = stringlib_contains_obj(str, sub); + switch (kind) { + case PyUnicode_1BYTE_KIND: + result = ucs1lib_find(buf1, len1, buf2, len2, 0) != -1; + break; + case PyUnicode_2BYTE_KIND: + result = ucs2lib_find(buf1, len1, buf2, len2, 0) != -1; + break; + case PyUnicode_4BYTE_KIND: + result = ucs4lib_find(buf1, len1, buf2, len2, 0) != -1; + break; + default: + result = -1; + assert(0); + } Py_DECREF(str); Py_DECREF(sub); + if (kind2 != kind) + PyMem_Free(buf2); + return result; } /* Concat to string or Unicode object giving a new Unicode object. */ -PyObject *PyUnicode_Concat(PyObject *left, - PyObject *right) +PyObject * +PyUnicode_Concat(PyObject *left, PyObject *right) { - PyUnicodeObject *u = NULL, *v = NULL, *w; + PyObject *u = NULL, *v = NULL, *w; + Py_UCS4 maxchar, maxchar2; + Py_ssize_t u_len, v_len, new_len; /* Coerce the two arguments */ - u = (PyUnicodeObject *)PyUnicode_FromObject(left); + u = PyUnicode_FromObject(left); if (u == NULL) goto onError; - v = (PyUnicodeObject *)PyUnicode_FromObject(right); + v = PyUnicode_FromObject(right); if (v == NULL) goto onError; /* Shortcuts */ if (v == unicode_empty) { Py_DECREF(v); - return (PyObject *)u; + return u; } if (u == unicode_empty) { Py_DECREF(u); - return (PyObject *)v; + return v; } + u_len = PyUnicode_GET_LENGTH(u); + v_len = PyUnicode_GET_LENGTH(v); + if (u_len > PY_SSIZE_T_MAX - v_len) { + PyErr_SetString(PyExc_OverflowError, + "strings are too large to concat"); + goto onError; + } + new_len = u_len + v_len; + + maxchar = PyUnicode_MAX_CHAR_VALUE(u); + maxchar2 = PyUnicode_MAX_CHAR_VALUE(v); + maxchar = MAX_MAXCHAR(maxchar, maxchar2); + /* Concat the two Unicode strings */ - w = _PyUnicode_New(u->length + v->length); + w = PyUnicode_New(new_len, maxchar); if (w == NULL) goto onError; - Py_UNICODE_COPY(w->str, u->str, u->length); - Py_UNICODE_COPY(w->str + u->length, v->str, v->length); - + _PyUnicode_FastCopyCharacters(w, 0, u, 0, u_len); + _PyUnicode_FastCopyCharacters(w, u_len, v, 0, v_len); Py_DECREF(u); Py_DECREF(v); - return (PyObject *)w; + assert(_PyUnicode_CheckConsistency(w, 1)); + return w; onError: Py_XDECREF(u); @@ -7472,19 +10659,89 @@ PyObject *PyUnicode_Concat(PyObject *left, } void -PyUnicode_Append(PyObject **pleft, PyObject *right) +PyUnicode_Append(PyObject **p_left, PyObject *right) { - PyObject *new; - if (*pleft == NULL) + PyObject *left, *res; + Py_UCS4 maxchar, maxchar2; + Py_ssize_t left_len, right_len, new_len; + + if (p_left == NULL) { + if (!PyErr_Occurred()) + PyErr_BadInternalCall(); + return; + } + left = *p_left; + if (right == NULL || !PyUnicode_Check(left)) { + if (!PyErr_Occurred()) + PyErr_BadInternalCall(); + goto error; + } + + if (PyUnicode_READY(left) == -1) + goto error; + if (PyUnicode_READY(right) == -1) + goto error; + + /* Shortcuts */ + if (left == unicode_empty) { + Py_DECREF(left); + Py_INCREF(right); + *p_left = right; return; - if (right == NULL || !PyUnicode_Check(*pleft)) { - Py_DECREF(*pleft); - *pleft = NULL; + } + if (right == unicode_empty) return; + + left_len = PyUnicode_GET_LENGTH(left); + right_len = PyUnicode_GET_LENGTH(right); + if (left_len > PY_SSIZE_T_MAX - right_len) { + PyErr_SetString(PyExc_OverflowError, + "strings are too large to concat"); + goto error; + } + new_len = left_len + right_len; + + if (unicode_modifiable(left) + && PyUnicode_CheckExact(right) + && PyUnicode_KIND(right) <= PyUnicode_KIND(left) + /* Don't resize for ascii += latin1. Convert ascii to latin1 requires + to change the structure size, but characters are stored just after + the structure, and so it requires to move all characters which is + not so different than duplicating the string. */ + && !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) + { + /* append inplace */ + if (unicode_resize(p_left, new_len) != 0) { + /* XXX if _PyUnicode_Resize() fails, 'left' has been + * deallocated so it cannot be put back into + * 'variable'. The MemoryError is raised when there + * is no value in 'variable', which might (very + * remotely) be a cause of incompatibilities. + */ + goto error; + } + /* copy 'right' into the newly allocated area of 'left' */ + _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len); } - new = PyUnicode_Concat(*pleft, right); - Py_DECREF(*pleft); - *pleft = new; + else { + maxchar = PyUnicode_MAX_CHAR_VALUE(left); + maxchar2 = PyUnicode_MAX_CHAR_VALUE(right); + maxchar = MAX_MAXCHAR(maxchar, maxchar2); + + /* Concat the two Unicode strings */ + res = PyUnicode_New(new_len, maxchar); + if (res == NULL) + goto error; + _PyUnicode_FastCopyCharacters(res, 0, left, 0, left_len); + _PyUnicode_FastCopyCharacters(res, left_len, right, 0, right_len); + Py_DECREF(left); + *p_left = res; + } + assert(_PyUnicode_CheckConsistency(*p_left, 1)); + return; + +error: + Py_CLEAR(*p_left); } void @@ -7502,23 +10759,64 @@ string S[start:end]. Optional arguments start and end are\n\ interpreted as in slice notation."); static PyObject * -unicode_count(PyUnicodeObject *self, PyObject *args) +unicode_count(PyObject *self, PyObject *args) { - PyUnicodeObject *substring; + PyObject *substring; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; PyObject *result; + int kind1, kind2, kind; + void *buf1, *buf2; + Py_ssize_t len1, len2, iresult; if (!stringlib_parse_args_finds_unicode("count", args, &substring, &start, &end)) return NULL; - ADJUST_INDICES(start, end, self->length); - result = PyLong_FromSsize_t( - stringlib_count(self->str + start, end - start, - substring->str, substring->length, - PY_SSIZE_T_MAX) - ); + kind1 = PyUnicode_KIND(self); + kind2 = PyUnicode_KIND(substring); + if (kind2 > kind1) + return PyLong_FromLong(0); + kind = kind1; + buf1 = PyUnicode_DATA(self); + buf2 = PyUnicode_DATA(substring); + if (kind2 != kind) + buf2 = _PyUnicode_AsKind(substring, kind); + if (!buf2) { + Py_DECREF(substring); + return NULL; + } + len1 = PyUnicode_GET_LENGTH(self); + len2 = PyUnicode_GET_LENGTH(substring); + + ADJUST_INDICES(start, end, len1); + switch (kind) { + case PyUnicode_1BYTE_KIND: + iresult = ucs1lib_count( + ((Py_UCS1*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_2BYTE_KIND: + iresult = ucs2lib_count( + ((Py_UCS2*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + case PyUnicode_4BYTE_KIND: + iresult = ucs4lib_count( + ((Py_UCS4*)buf1) + start, end - start, + buf2, len2, PY_SSIZE_T_MAX + ); + break; + default: + assert(0); iresult = 0; + } + + result = PyLong_FromSsize_t(iresult); + + if (kind2 != kind) + PyMem_Free(buf2); Py_DECREF(substring); @@ -7536,7 +10834,7 @@ a UnicodeEncodeError. Other possible values are 'ignore', 'replace' and\n\ codecs.register_error that can handle UnicodeEncodeErrors."); static PyObject * -unicode_encode(PyUnicodeObject *self, PyObject *args, PyObject *kwargs) +unicode_encode(PyObject *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = {"encoding", "errors", 0}; char *encoding = NULL; @@ -7545,7 +10843,7 @@ unicode_encode(PyUnicodeObject *self, PyObject *args, PyObject *kwargs) if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ss:encode", kwlist, &encoding, &errors)) return NULL; - return PyUnicode_AsEncodedString((PyObject *)self, encoding, errors); + return PyUnicode_AsEncodedString(self, encoding, errors); } PyDoc_STRVAR(expandtabs__doc__, @@ -7555,82 +10853,82 @@ Return a copy of S where all tab characters are expanded using spaces.\n\ If tabsize is not given, a tab size of 8 characters is assumed."); static PyObject* -unicode_expandtabs(PyUnicodeObject *self, PyObject *args) -{ - Py_UNICODE *e; - Py_UNICODE *p; - Py_UNICODE *q; - Py_UNICODE *qe; - Py_ssize_t i, j, incr; - PyUnicodeObject *u; +unicode_expandtabs(PyObject *self, PyObject *args) +{ + Py_ssize_t i, j, line_pos, src_len, incr; + Py_UCS4 ch; + PyObject *u; + void *src_data, *dest_data; int tabsize = 8; + int kind; + int found; if (!PyArg_ParseTuple(args, "|i:expandtabs", &tabsize)) return NULL; + if (PyUnicode_READY(self) == -1) + return NULL; + /* First pass: determine size of output string */ - i = 0; /* chars up to and including most recent \n or \r */ - j = 0; /* chars since most recent \n or \r (use in tab calculations) */ - e = self->str + self->length; /* end of input */ - for (p = self->str; p < e; p++) - if (*p == '\t') { + src_len = PyUnicode_GET_LENGTH(self); + i = j = line_pos = 0; + kind = PyUnicode_KIND(self); + src_data = PyUnicode_DATA(self); + found = 0; + for (; i < src_len; i++) { + ch = PyUnicode_READ(kind, src_data, i); + if (ch == '\t') { + found = 1; if (tabsize > 0) { - incr = tabsize - (j % tabsize); /* cannot overflow */ + incr = tabsize - (line_pos % tabsize); /* cannot overflow */ if (j > PY_SSIZE_T_MAX - incr) - goto overflow1; + goto overflow; + line_pos += incr; j += incr; } } else { if (j > PY_SSIZE_T_MAX - 1) - goto overflow1; + goto overflow; + line_pos++; j++; - if (*p == '\n' || *p == '\r') { - if (i > PY_SSIZE_T_MAX - j) - goto overflow1; - i += j; - j = 0; - } + if (ch == '\n' || ch == '\r') + line_pos = 0; } - - if (i > PY_SSIZE_T_MAX - j) - goto overflow1; + } + if (!found) + return unicode_result_unchanged(self); /* Second pass: create output string and fill it */ - u = _PyUnicode_New(i + j); + u = PyUnicode_New(j, PyUnicode_MAX_CHAR_VALUE(self)); if (!u) return NULL; + dest_data = PyUnicode_DATA(u); - j = 0; /* same as in first pass */ - q = u->str; /* next output char */ - qe = u->str + u->length; /* end of output */ + i = j = line_pos = 0; - for (p = self->str; p < e; p++) - if (*p == '\t') { + for (; i < src_len; i++) { + ch = PyUnicode_READ(kind, src_data, i); + if (ch == '\t') { if (tabsize > 0) { - i = tabsize - (j % tabsize); - j += i; - while (i--) { - if (q >= qe) - goto overflow2; - *q++ = ' '; - } + incr = tabsize - (line_pos % tabsize); + line_pos += incr; + FILL(kind, dest_data, ' ', j, incr); + j += incr; } } else { - if (q >= qe) - goto overflow2; - *q++ = *p; + line_pos++; + PyUnicode_WRITE(kind, dest_data, j, ch); j++; - if (*p == '\n' || *p == '\r') - j = 0; + if (ch == '\n' || ch == '\r') + line_pos = 0; } + } + assert (j == PyUnicode_GET_LENGTH(u)); + return unicode_result(u); - return (PyObject*) u; - - overflow2: - Py_DECREF(u); - overflow1: + overflow: PyErr_SetString(PyExc_OverflowError, "new string is too long"); return NULL; } @@ -7645,9 +10943,9 @@ arguments start and end are interpreted as in slice notation.\n\ Return -1 on failure."); static PyObject * -unicode_find(PyUnicodeObject *self, PyObject *args) +unicode_find(PyObject *self, PyObject *args) { - PyUnicodeObject *substring; + PyObject *substring; Py_ssize_t start; Py_ssize_t end; Py_ssize_t result; @@ -7656,63 +10954,114 @@ unicode_find(PyUnicodeObject *self, PyObject *args) &start, &end)) return NULL; - result = stringlib_find_slice( - PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), - PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), - start, end - ); + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_READY(substring) == -1) + return NULL; + + result = any_find_slice(1, self, substring, start, end); Py_DECREF(substring); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); } static PyObject * -unicode_getitem(PyUnicodeObject *self, Py_ssize_t index) +unicode_getitem(PyObject *self, Py_ssize_t index) { - if (index < 0 || index >= self->length) { + void *data; + enum PyUnicode_Kind kind; + Py_UCS4 ch; + PyObject *res; + + if (!PyUnicode_Check(self) || PyUnicode_READY(self) == -1) { + PyErr_BadArgument(); + return NULL; + } + if (index < 0 || index >= PyUnicode_GET_LENGTH(self)) { PyErr_SetString(PyExc_IndexError, "string index out of range"); return NULL; } + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + ch = PyUnicode_READ(kind, data, index); + if (ch < 256) + return get_latin1_char(ch); - return (PyObject*) PyUnicode_FromUnicode(&self->str[index], 1); + res = PyUnicode_New(1, ch); + if (res == NULL) + return NULL; + kind = PyUnicode_KIND(res); + data = PyUnicode_DATA(res); + PyUnicode_WRITE(kind, data, 0, ch); + assert(_PyUnicode_CheckConsistency(res, 1)); + return res; } /* Believe it or not, this produces the same value for ASCII strings - as string_hash(). */ + as bytes_hash(). */ static Py_hash_t -unicode_hash(PyUnicodeObject *self) +unicode_hash(PyObject *self) { Py_ssize_t len; - Py_UNICODE *p; - Py_hash_t x; + Py_uhash_t x; #ifdef Py_DEBUG assert(_Py_HashSecret_Initialized); #endif - if (self->hash != -1) - return self->hash; - len = Py_SIZE(self); + if (_PyUnicode_HASH(self) != -1) + return _PyUnicode_HASH(self); + if (PyUnicode_READY(self) == -1) + return -1; + len = PyUnicode_GET_LENGTH(self); /* We make the hash of the empty string be 0, rather than using (prefix ^ suffix), since this slightly obfuscates the hash secret */ if (len == 0) { - self->hash = 0; + _PyUnicode_HASH(self) = 0; return 0; } - p = self->str; - x = _Py_HashSecret.prefix; - x ^= *p << 7; - while (--len >= 0) - x = (_PyHASH_MULTIPLIER*x) ^ *p++; - x ^= Py_SIZE(self); - x ^= _Py_HashSecret.suffix; + + /* The hash function as a macro, gets expanded three times below. */ +#define HASH(P) \ + x ^= (Py_uhash_t) *P << 7; \ + while (--len >= 0) \ + x = (_PyHASH_MULTIPLIER * x) ^ (Py_uhash_t) *P++; \ + + x = (Py_uhash_t) _Py_HashSecret.prefix; + switch (PyUnicode_KIND(self)) { + case PyUnicode_1BYTE_KIND: { + const unsigned char *c = PyUnicode_1BYTE_DATA(self); + HASH(c); + break; + } + case PyUnicode_2BYTE_KIND: { + const Py_UCS2 *s = PyUnicode_2BYTE_DATA(self); + HASH(s); + break; + } + default: { + Py_UCS4 *l; + assert(PyUnicode_KIND(self) == PyUnicode_4BYTE_KIND && + "Impossible switch case in unicode_hash"); + l = PyUnicode_4BYTE_DATA(self); + HASH(l); + break; + } + } + x ^= (Py_uhash_t) PyUnicode_GET_LENGTH(self); + x ^= (Py_uhash_t) _Py_HashSecret.suffix; + if (x == -1) x = -2; - self->hash = x; + _PyUnicode_HASH(self) = x; return x; } +#undef HASH PyDoc_STRVAR(index__doc__, "S.index(sub[, start[, end]]) -> int\n\ @@ -7720,10 +11069,10 @@ PyDoc_STRVAR(index__doc__, Like S.find() but raise ValueError when the substring is not found."); static PyObject * -unicode_index(PyUnicodeObject *self, PyObject *args) +unicode_index(PyObject *self, PyObject *args) { Py_ssize_t result; - PyUnicodeObject *substring; + PyObject *substring; Py_ssize_t start; Py_ssize_t end; @@ -7731,14 +11080,18 @@ unicode_index(PyUnicodeObject *self, PyObject *args) &start, &end)) return NULL; - result = stringlib_find_slice( - PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), - PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), - start, end - ); + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_READY(substring) == -1) + return NULL; + + result = any_find_slice(1, self, substring, start, end); Py_DECREF(substring); + if (result == -2) + return NULL; + if (result < 0) { PyErr_SetString(PyExc_ValueError, "substring not found"); return NULL; @@ -7754,24 +11107,31 @@ Return True if all cased characters in S are lowercase and there is\n\ at least one cased character in S, False otherwise."); static PyObject* -unicode_islower(PyUnicodeObject *self) +unicode_islower(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; int cased; + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1) - return PyBool_FromLong(Py_UNICODE_ISLOWER(*p)); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISLOWER(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); cased = 0; - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < length; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch)) return PyBool_FromLong(0); @@ -7788,24 +11148,31 @@ Return True if all cased characters in S are uppercase and there is\n\ at least one cased character in S, False otherwise."); static PyObject* -unicode_isupper(PyUnicodeObject *self) +unicode_isupper(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; int cased; + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1) - return PyBool_FromLong(Py_UNICODE_ISUPPER(*p) != 0); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISUPPER(PyUnicode_READ(kind, data, 0)) != 0); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); cased = 0; - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < length; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (Py_UNICODE_ISLOWER(ch) || Py_UNICODE_ISTITLE(ch)) return PyBool_FromLong(0); @@ -7824,26 +11191,34 @@ follow uncased characters and lowercase characters only cased ones.\n\ Return False otherwise."); static PyObject* -unicode_istitle(PyUnicodeObject *self) +unicode_istitle(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; int cased, previous_is_cased; + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1) - return PyBool_FromLong((Py_UNICODE_ISTITLE(*p) != 0) || - (Py_UNICODE_ISUPPER(*p) != 0)); + if (length == 1) { + Py_UCS4 ch = PyUnicode_READ(kind, data, 0); + return PyBool_FromLong((Py_UNICODE_ISTITLE(ch) != 0) || + (Py_UNICODE_ISUPPER(ch) != 0)); + } /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); cased = 0; previous_is_cased = 0; - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < length; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch)) { if (previous_is_cased) @@ -7870,23 +11245,29 @@ Return True if all characters in S are whitespace\n\ and there is at least one character in S, False otherwise."); static PyObject* -unicode_isspace(PyUnicodeObject *self) +unicode_isspace(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISSPACE(*p)) - return PyBool_FromLong(1); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < length; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (!Py_UNICODE_ISSPACE(ch)) return PyBool_FromLong(0); } @@ -7900,23 +11281,29 @@ Return True if all characters in S are alphabetic\n\ and there is at least one character in S, False otherwise."); static PyObject* -unicode_isalpha(PyUnicodeObject *self) +unicode_isalpha(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISALPHA(*p)) - return PyBool_FromLong(1); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISALPHA(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISALPHA(_Py_UNICODE_NEXT(p, e))) + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISALPHA(PyUnicode_READ(kind, data, i))) return PyBool_FromLong(0); } return PyBool_FromLong(1); @@ -7929,23 +11316,31 @@ Return True if all characters in S are alphanumeric\n\ and there is at least one character in S, False otherwise."); static PyObject* -unicode_isalnum(PyUnicodeObject *self) +unicode_isalnum(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + int kind; + void *data; + Py_ssize_t len, i; + + if (PyUnicode_READY(self) == -1) + return NULL; + + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + len = PyUnicode_GET_LENGTH(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISALNUM(*p)) - return PyBool_FromLong(1); + if (len == 1) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, 0); + return PyBool_FromLong(Py_UNICODE_ISALNUM(ch)); + } /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (len == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - const Py_UCS4 ch = _Py_UNICODE_NEXT(p, e); + for (i = 0; i < len; i++) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, i); if (!Py_UNICODE_ISALNUM(ch)) return PyBool_FromLong(0); } @@ -7959,23 +11354,29 @@ Return True if there are only decimal characters in S,\n\ False otherwise."); static PyObject* -unicode_isdecimal(PyUnicodeObject *self) +unicode_isdecimal(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISDECIMAL(*p)) - return PyBool_FromLong(1); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISDECIMAL(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISDECIMAL(_Py_UNICODE_NEXT(p, e))) + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISDECIMAL(PyUnicode_READ(kind, data, i))) return PyBool_FromLong(0); } return PyBool_FromLong(1); @@ -7988,23 +11389,30 @@ Return True if all characters in S are digits\n\ and there is at least one character in S, False otherwise."); static PyObject* -unicode_isdigit(PyUnicodeObject *self) +unicode_isdigit(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISDIGIT(*p)) - return PyBool_FromLong(1); + if (length == 1) { + const Py_UCS4 ch = PyUnicode_READ(kind, data, 0); + return PyBool_FromLong(Py_UNICODE_ISDIGIT(ch)); + } /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISDIGIT(_Py_UNICODE_NEXT(p, e))) + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISDIGIT(PyUnicode_READ(kind, data, i))) return PyBool_FromLong(0); } return PyBool_FromLong(1); @@ -8017,23 +11425,29 @@ Return True if there are only numeric characters in S,\n\ False otherwise."); static PyObject* -unicode_isnumeric(PyUnicodeObject *self) +unicode_isnumeric(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && - Py_UNICODE_ISNUMERIC(*p)) - return PyBool_FromLong(1); + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISNUMERIC(PyUnicode_READ(kind, data, 0))); /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (length == 0) return PyBool_FromLong(0); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISNUMERIC(_Py_UNICODE_NEXT(p, e))) + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISNUMERIC(PyUnicode_READ(kind, data, i))) return PyBool_FromLong(0); } return PyBool_FromLong(1); @@ -8042,13 +11456,21 @@ unicode_isnumeric(PyUnicodeObject *self) int PyUnicode_IsIdentifier(PyObject *self) { - const Py_UNICODE *p = PyUnicode_AS_UNICODE((PyUnicodeObject*)self); - const Py_UNICODE *e; + int kind; + void *data; + Py_ssize_t i; Py_UCS4 first; + if (PyUnicode_READY(self) == -1) { + Py_FatalError("identifier not ready"); + return 0; + } + /* Special case for empty strings */ - if (PyUnicode_GET_SIZE(self) == 0) + if (PyUnicode_GET_LENGTH(self) == 0) return 0; + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* PEP 3131 says that the first character must be in XID_Start and subsequent characters in XID_Continue, @@ -8058,13 +11480,12 @@ PyUnicode_IsIdentifier(PyObject *self) definition of XID_Start and XID_Continue, it is sufficient to check just for these, except that _ must be allowed as starting an identifier. */ - e = p + PyUnicode_GET_SIZE(self); - first = _Py_UNICODE_NEXT(p, e); + first = PyUnicode_READ(kind, data, 0); if (!_PyUnicode_IsXidStart(first) && first != 0x5F /* LOW LINE */) return 0; - while (p < e) - if (!_PyUnicode_IsXidContinue(_Py_UNICODE_NEXT(p, e))) + for (i = 1; i < PyUnicode_GET_LENGTH(self); i++) + if (!_PyUnicode_IsXidContinue(PyUnicode_READ(kind, data, i))) return 0; return 1; } @@ -8090,17 +11511,23 @@ printable in repr() or S is empty, False otherwise."); static PyObject* unicode_isprintable(PyObject *self) { - register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); - register const Py_UNICODE *e; + Py_ssize_t i, length; + int kind; + void *data; + + if (PyUnicode_READY(self) == -1) + return NULL; + length = PyUnicode_GET_LENGTH(self); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); /* Shortcut for single character strings */ - if (PyUnicode_GET_SIZE(self) == 1 && Py_UNICODE_ISPRINTABLE(*p)) { - Py_RETURN_TRUE; - } + if (length == 1) + return PyBool_FromLong( + Py_UNICODE_ISPRINTABLE(PyUnicode_READ(kind, data, 0))); - e = p + PyUnicode_GET_SIZE(self); - while (p < e) { - if (!Py_UNICODE_ISPRINTABLE(_Py_UNICODE_NEXT(p, e))) { + for (i = 0; i < length; i++) { + if (!Py_UNICODE_ISPRINTABLE(PyUnicode_READ(kind, data, i))) { Py_RETURN_FALSE; } } @@ -8120,9 +11547,11 @@ unicode_join(PyObject *self, PyObject *data) } static Py_ssize_t -unicode_length(PyUnicodeObject *self) +unicode_length(PyObject *self) { - return self->length; + if (PyUnicode_READY(self) == -1) + return -1; + return PyUnicode_GET_LENGTH(self); } PyDoc_STRVAR(ljust__doc__, @@ -8132,20 +11561,21 @@ Return S left-justified in a Unicode string of length width. Padding is\n\ done using the specified fill character (default is a space)."); static PyObject * -unicode_ljust(PyUnicodeObject *self, PyObject *args) +unicode_ljust(PyObject *self, PyObject *args) { Py_ssize_t width; - Py_UNICODE fillchar = ' '; + Py_UCS4 fillchar = ' '; if (!PyArg_ParseTuple(args, "n|O&:ljust", &width, convert_uc, &fillchar)) return NULL; - if (self->length >= width && PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return (PyObject*) self; - } + if (PyUnicode_READY(self) == -1) + return NULL; + + if (PyUnicode_GET_LENGTH(self) >= width) + return unicode_result_unchanged(self); - return (PyObject*) pad(self, 0, width - self->length, fillchar); + return pad(self, 0, width - PyUnicode_GET_LENGTH(self), fillchar); } PyDoc_STRVAR(lower__doc__, @@ -8154,9 +11584,13 @@ PyDoc_STRVAR(lower__doc__, Return a copy of the string S converted to lowercase."); static PyObject* -unicode_lower(PyUnicodeObject *self) +unicode_lower(PyObject *self) { - return fixup(self, fixlower); + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_IS_ASCII(self)) + return ascii_upper_or_lower(self, 1); + return case_operation(self, do_lower); } #define LEFTSTRIP 0 @@ -8170,19 +11604,27 @@ static const char *stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; /* externally visible for str.strip(unicode) */ PyObject * -_PyUnicode_XStrip(PyUnicodeObject *self, int striptype, PyObject *sepobj) +_PyUnicode_XStrip(PyObject *self, int striptype, PyObject *sepobj) { - Py_UNICODE *s = PyUnicode_AS_UNICODE(self); - Py_ssize_t len = PyUnicode_GET_SIZE(self); - Py_UNICODE *sep = PyUnicode_AS_UNICODE(sepobj); - Py_ssize_t seplen = PyUnicode_GET_SIZE(sepobj); - Py_ssize_t i, j; + void *data; + int kind; + Py_ssize_t i, j, len; + BLOOM_MASK sepmask; + + if (PyUnicode_READY(self) == -1 || PyUnicode_READY(sepobj) == -1) + return NULL; - BLOOM_MASK sepmask = make_bloom_mask(sep, seplen); + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + len = PyUnicode_GET_LENGTH(self); + sepmask = make_bloom_mask(PyUnicode_KIND(sepobj), + PyUnicode_DATA(sepobj), + PyUnicode_GET_LENGTH(sepobj)); i = 0; if (striptype != RIGHTSTRIP) { - while (i < len && BLOOM_MEMBER(sepmask, s[i], sep, seplen)) { + while (i < len && + BLOOM_MEMBER(sepmask, PyUnicode_READ(kind, data, i), sepobj)) { i++; } } @@ -8191,28 +11633,70 @@ _PyUnicode_XStrip(PyUnicodeObject *self, int striptype, PyObject *sepobj) if (striptype != LEFTSTRIP) { do { j--; - } while (j >= i && BLOOM_MEMBER(sepmask, s[j], sep, seplen)); + } while (j >= i && + BLOOM_MEMBER(sepmask, PyUnicode_READ(kind, data, j), sepobj)); j++; } - if (i == 0 && j == len && PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return (PyObject*)self; - } - else - return PyUnicode_FromUnicode(s+i, j-i); + return PyUnicode_Substring(self, i, j); } +PyObject* +PyUnicode_Substring(PyObject *self, Py_ssize_t start, Py_ssize_t end) +{ + unsigned char *data; + int kind; + Py_ssize_t length; + + if (PyUnicode_READY(self) == -1) + return NULL; + + length = PyUnicode_GET_LENGTH(self); + end = Py_MIN(end, length); + + if (start == 0 && end == length) + return unicode_result_unchanged(self); + + if (start < 0 || end < 0) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return NULL; + } + if (start >= length || end < start) { + Py_INCREF(unicode_empty); + return unicode_empty; + } + + length = end - start; + if (PyUnicode_IS_ASCII(self)) { + data = PyUnicode_1BYTE_DATA(self); + return _PyUnicode_FromASCII((char*)(data + start), length); + } + else { + kind = PyUnicode_KIND(self); + data = PyUnicode_1BYTE_DATA(self); + return PyUnicode_FromKindAndData(kind, + data + kind * start, + length); + } +} static PyObject * -do_strip(PyUnicodeObject *self, int striptype) +do_strip(PyObject *self, int striptype) { - Py_UNICODE *s = PyUnicode_AS_UNICODE(self); - Py_ssize_t len = PyUnicode_GET_SIZE(self), i, j; + int kind; + void *data; + Py_ssize_t len, i, j; + + if (PyUnicode_READY(self) == -1) + return NULL; + + kind = PyUnicode_KIND(self); + data = PyUnicode_DATA(self); + len = PyUnicode_GET_LENGTH(self); i = 0; if (striptype != RIGHTSTRIP) { - while (i < len && Py_UNICODE_ISSPACE(s[i])) { + while (i < len && Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, i))) { i++; } } @@ -8221,21 +11705,16 @@ do_strip(PyUnicodeObject *self, int striptype) if (striptype != LEFTSTRIP) { do { j--; - } while (j >= i && Py_UNICODE_ISSPACE(s[j])); + } while (j >= i && Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, j))); j++; } - if (i == 0 && j == len && PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return (PyObject*)self; - } - else - return PyUnicode_FromUnicode(s+i, j-i); + return PyUnicode_Substring(self, i, j); } static PyObject * -do_argstrip(PyUnicodeObject *self, int striptype, PyObject *args) +do_argstrip(PyObject *self, int striptype, PyObject *args) { PyObject *sep = NULL; @@ -8265,7 +11744,7 @@ whitespace removed.\n\ If chars is given and not None, remove characters in chars instead."); static PyObject * -unicode_strip(PyUnicodeObject *self, PyObject *args) +unicode_strip(PyObject *self, PyObject *args) { if (PyTuple_GET_SIZE(args) == 0) return do_strip(self, BOTHSTRIP); /* Common case */ @@ -8281,7 +11760,7 @@ Return a copy of the string S with leading whitespace removed.\n\ If chars is given and not None, remove characters in chars instead."); static PyObject * -unicode_lstrip(PyUnicodeObject *self, PyObject *args) +unicode_lstrip(PyObject *self, PyObject *args) { if (PyTuple_GET_SIZE(args) == 0) return do_strip(self, LEFTSTRIP); /* Common case */ @@ -8297,7 +11776,7 @@ Return a copy of the string S with trailing whitespace removed.\n\ If chars is given and not None, remove characters in chars instead."); static PyObject * -unicode_rstrip(PyUnicodeObject *self, PyObject *args) +unicode_rstrip(PyObject *self, PyObject *args) { if (PyTuple_GET_SIZE(args) == 0) return do_strip(self, RIGHTSTRIP); /* Common case */ @@ -8307,64 +11786,76 @@ unicode_rstrip(PyUnicodeObject *self, PyObject *args) static PyObject* -unicode_repeat(PyUnicodeObject *str, Py_ssize_t len) +unicode_repeat(PyObject *str, Py_ssize_t len) { - PyUnicodeObject *u; - Py_UNICODE *p; - Py_ssize_t nchars; - size_t nbytes; + PyObject *u; + Py_ssize_t nchars, n; if (len < 1) { Py_INCREF(unicode_empty); - return (PyObject *)unicode_empty; + return unicode_empty; } - if (len == 1 && PyUnicode_CheckExact(str)) { - /* no repeat, return original string */ - Py_INCREF(str); - return (PyObject*) str; - } + /* no repeat, return original string */ + if (len == 1) + return unicode_result_unchanged(str); - /* ensure # of chars needed doesn't overflow int and # of bytes - * needed doesn't overflow size_t - */ - nchars = len * str->length; - if (nchars / len != str->length) { - PyErr_SetString(PyExc_OverflowError, - "repeated string is too long"); + if (PyUnicode_READY(str) == -1) return NULL; - } - nbytes = (nchars + 1) * sizeof(Py_UNICODE); - if (nbytes / sizeof(Py_UNICODE) != (size_t)(nchars + 1)) { + + if (PyUnicode_GET_LENGTH(str) > PY_SSIZE_T_MAX / len) { PyErr_SetString(PyExc_OverflowError, "repeated string is too long"); return NULL; } - u = _PyUnicode_New(nchars); + nchars = len * PyUnicode_GET_LENGTH(str); + + u = PyUnicode_New(nchars, PyUnicode_MAX_CHAR_VALUE(str)); if (!u) return NULL; - - p = u->str; - - if (str->length == 1) { - Py_UNICODE_FILL(p, str->str[0], len); - } else { - Py_ssize_t done = str->length; /* number of characters copied this far */ - Py_UNICODE_COPY(p, str->str, str->length); + assert(PyUnicode_KIND(u) == PyUnicode_KIND(str)); + + if (PyUnicode_GET_LENGTH(str) == 1) { + const int kind = PyUnicode_KIND(str); + const Py_UCS4 fill_char = PyUnicode_READ(kind, PyUnicode_DATA(str), 0); + if (kind == PyUnicode_1BYTE_KIND) { + void *to = PyUnicode_DATA(u); + memset(to, (unsigned char)fill_char, len); + } + else if (kind == PyUnicode_2BYTE_KIND) { + Py_UCS2 *ucs2 = PyUnicode_2BYTE_DATA(u); + for (n = 0; n < len; ++n) + ucs2[n] = fill_char; + } else { + Py_UCS4 *ucs4 = PyUnicode_4BYTE_DATA(u); + assert(kind == PyUnicode_4BYTE_KIND); + for (n = 0; n < len; ++n) + ucs4[n] = fill_char; + } + } + else { + /* number of characters copied this far */ + Py_ssize_t done = PyUnicode_GET_LENGTH(str); + const Py_ssize_t char_size = PyUnicode_KIND(str); + char *to = (char *) PyUnicode_DATA(u); + Py_MEMCPY(to, PyUnicode_DATA(str), + PyUnicode_GET_LENGTH(str) * char_size); while (done < nchars) { - Py_ssize_t n = (done <= nchars-done) ? done : nchars-done; - Py_UNICODE_COPY(p+done, p, n); + n = (done <= nchars-done) ? done : nchars-done; + Py_MEMCPY(to + (done * char_size), to, n * char_size); done += n; } } - return (PyObject*) u; + assert(_PyUnicode_CheckConsistency(u, 1)); + return u; } -PyObject *PyUnicode_Replace(PyObject *obj, - PyObject *subobj, - PyObject *replobj, - Py_ssize_t maxcount) +PyObject * +PyUnicode_Replace(PyObject *obj, + PyObject *subobj, + PyObject *replobj, + Py_ssize_t maxcount) { PyObject *self; PyObject *str1; @@ -8385,10 +11876,12 @@ PyObject *PyUnicode_Replace(PyObject *obj, Py_DECREF(str1); return NULL; } - result = replace((PyUnicodeObject *)self, - (PyUnicodeObject *)str1, - (PyUnicodeObject *)str2, - maxcount); + if (PyUnicode_READY(self) == -1 || + PyUnicode_READY(str1) == -1 || + PyUnicode_READY(str2) == -1) + result = NULL; + else + result = replace(self, str1, str2, maxcount); Py_DECREF(self); Py_DECREF(str1); Py_DECREF(str2); @@ -8403,177 +11896,181 @@ old replaced by new. If the optional argument count is\n\ given, only the first count occurrences are replaced."); static PyObject* -unicode_replace(PyUnicodeObject *self, PyObject *args) +unicode_replace(PyObject *self, PyObject *args) { - PyUnicodeObject *str1; - PyUnicodeObject *str2; + PyObject *str1; + PyObject *str2; Py_ssize_t maxcount = -1; PyObject *result; if (!PyArg_ParseTuple(args, "OO|n:replace", &str1, &str2, &maxcount)) return NULL; - str1 = (PyUnicodeObject *)PyUnicode_FromObject((PyObject *)str1); + if (PyUnicode_READY(self) == -1) + return NULL; + str1 = PyUnicode_FromObject(str1); if (str1 == NULL) return NULL; - str2 = (PyUnicodeObject *)PyUnicode_FromObject((PyObject *)str2); + str2 = PyUnicode_FromObject(str2); if (str2 == NULL) { Py_DECREF(str1); return NULL; } - - result = replace(self, str1, str2, maxcount); + if (PyUnicode_READY(str1) == -1 || PyUnicode_READY(str2) == -1) + result = NULL; + else + result = replace(self, str1, str2, maxcount); Py_DECREF(str1); Py_DECREF(str2); return result; } -static -PyObject *unicode_repr(PyObject *unicode) +static PyObject * +unicode_repr(PyObject *unicode) { PyObject *repr; - Py_UNICODE *p; - Py_UNICODE *s = PyUnicode_AS_UNICODE(unicode); - Py_ssize_t size = PyUnicode_GET_SIZE(unicode); + Py_ssize_t isize; + Py_ssize_t osize, squote, dquote, i, o; + Py_UCS4 max, quote; + int ikind, okind; + void *idata, *odata; - /* XXX(nnorwitz): rather than over-allocating, it would be - better to choose a different scheme. Perhaps scan the - first N-chars of the string and allocate based on that size. - */ - /* Initial allocation is based on the longest-possible unichr - escape. + if (PyUnicode_READY(unicode) == -1) + return NULL; - In wide (UTF-32) builds '\U00xxxxxx' is 10 chars per source - unichr, so in this case it's the longest unichr escape. In - narrow (UTF-16) builds this is five chars per source unichr - since there are two unichrs in the surrogate pair, so in narrow - (UTF-16) builds it's not the longest unichr escape. + isize = PyUnicode_GET_LENGTH(unicode); + idata = PyUnicode_DATA(unicode); + + /* Compute length of output, quote characters, and + maximum character */ + osize = 2; /* quotes */ + max = 127; + squote = dquote = 0; + ikind = PyUnicode_KIND(unicode); + for (i = 0; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); + switch (ch) { + case '\'': squote++; osize++; break; + case '"': dquote++; osize++; break; + case '\\': case '\t': case '\r': case '\n': + osize += 2; break; + default: + /* Fast-path ASCII */ + if (ch < ' ' || ch == 0x7f) + osize += 4; /* \xHH */ + else if (ch < 0x7f) + osize++; + else if (Py_UNICODE_ISPRINTABLE(ch)) { + osize++; + max = ch > max ? ch : max; + } + else if (ch < 0x100) + osize += 4; /* \xHH */ + else if (ch < 0x10000) + osize += 6; /* \uHHHH */ + else + osize += 10; /* \uHHHHHHHH */ + } + } - In wide or narrow builds '\uxxxx' is 6 chars per source unichr, - so in the narrow (UTF-16) build case it's the longest unichr - escape. - */ + quote = '\''; + if (squote) { + if (dquote) + /* Both squote and dquote present. Use squote, + and escape them */ + osize += squote; + else + quote = '"'; + } - repr = PyUnicode_FromUnicode(NULL, - 2 /* quotes */ -#ifdef Py_UNICODE_WIDE - + 10*size -#else - + 6*size -#endif - + 1); + repr = PyUnicode_New(osize, max); if (repr == NULL) return NULL; + okind = PyUnicode_KIND(repr); + odata = PyUnicode_DATA(repr); - p = PyUnicode_AS_UNICODE(repr); + PyUnicode_WRITE(okind, odata, 0, quote); + PyUnicode_WRITE(okind, odata, osize-1, quote); - /* Add quote */ - *p++ = (findchar(s, size, '\'') && - !findchar(s, size, '"')) ? '"' : '\''; - while (size-- > 0) { - Py_UNICODE ch = *s++; + for (i = 0, o = 1; i < isize; i++) { + Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); /* Escape quotes and backslashes */ - if ((ch == PyUnicode_AS_UNICODE(repr)[0]) || (ch == '\\')) { - *p++ = '\\'; - *p++ = ch; + if ((ch == quote) || (ch == '\\')) { + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, ch); continue; } /* Map special whitespace to '\t', \n', '\r' */ if (ch == '\t') { - *p++ = '\\'; - *p++ = 't'; + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 't'); } else if (ch == '\n') { - *p++ = '\\'; - *p++ = 'n'; + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 'n'); } else if (ch == '\r') { - *p++ = '\\'; - *p++ = 'r'; + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 'r'); } /* Map non-printable US ASCII to '\xhh' */ else if (ch < ' ' || ch == 0x7F) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(ch >> 4) & 0x000F]; - *p++ = hexdigits[ch & 0x000F]; + PyUnicode_WRITE(okind, odata, o++, '\\'); + PyUnicode_WRITE(okind, odata, o++, 'x'); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); } /* Copy ASCII characters as-is */ else if (ch < 0x7F) { - *p++ = ch; + PyUnicode_WRITE(okind, odata, o++, ch); } /* Non-ASCII characters */ else { - Py_UCS4 ucs = ch; - -#ifndef Py_UNICODE_WIDE - Py_UNICODE ch2 = 0; - /* Get code point from surrogate pair */ - if (size > 0) { - ch2 = *s; - if (ch >= 0xD800 && ch < 0xDC00 && ch2 >= 0xDC00 - && ch2 <= 0xDFFF) { - ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) - + 0x00010000; - s++; - size--; - } - } -#endif /* Map Unicode whitespace and control characters (categories Z* and C* except ASCII space) */ - if (!Py_UNICODE_ISPRINTABLE(ucs)) { + if (!Py_UNICODE_ISPRINTABLE(ch)) { + PyUnicode_WRITE(okind, odata, o++, '\\'); /* Map 8-bit characters to '\xhh' */ - if (ucs <= 0xff) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(ch >> 4) & 0x000F]; - *p++ = hexdigits[ch & 0x000F]; - } - /* Map 21-bit characters to '\U00xxxxxx' */ - else if (ucs >= 0x10000) { - *p++ = '\\'; - *p++ = 'U'; - *p++ = hexdigits[(ucs >> 28) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 24) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 20) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 16) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 12) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 8) & 0x0000000F]; - *p++ = hexdigits[(ucs >> 4) & 0x0000000F]; - *p++ = hexdigits[ucs & 0x0000000F]; + if (ch <= 0xff) { + PyUnicode_WRITE(okind, odata, o++, 'x'); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); } /* Map 16-bit characters to '\uxxxx' */ + else if (ch <= 0xffff) { + PyUnicode_WRITE(okind, odata, o++, 'u'); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); + } + /* Map 21-bit characters to '\U00xxxxxx' */ else { - *p++ = '\\'; - *p++ = 'u'; - *p++ = hexdigits[(ucs >> 12) & 0x000F]; - *p++ = hexdigits[(ucs >> 8) & 0x000F]; - *p++ = hexdigits[(ucs >> 4) & 0x000F]; - *p++ = hexdigits[ucs & 0x000F]; + PyUnicode_WRITE(okind, odata, o++, 'U'); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 28) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 24) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 20) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 16) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); + PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); } } /* Copy characters as-is */ else { - *p++ = ch; -#ifndef Py_UNICODE_WIDE - if (ucs >= 0x10000) - *p++ = ch2; -#endif + PyUnicode_WRITE(okind, odata, o++, ch); } } } - /* Add quote */ - *p++ = PyUnicode_AS_UNICODE(repr)[0]; - - *p = '\0'; - PyUnicode_Resize(&repr, p - PyUnicode_AS_UNICODE(repr)); + /* Closing quote already added at the beginning */ + assert(_PyUnicode_CheckConsistency(repr, 1)); return repr; } @@ -8587,9 +12084,9 @@ arguments start and end are interpreted as in slice notation.\n\ Return -1 on failure."); static PyObject * -unicode_rfind(PyUnicodeObject *self, PyObject *args) +unicode_rfind(PyObject *self, PyObject *args) { - PyUnicodeObject *substring; + PyObject *substring; Py_ssize_t start; Py_ssize_t end; Py_ssize_t result; @@ -8598,14 +12095,18 @@ unicode_rfind(PyUnicodeObject *self, PyObject *args) &start, &end)) return NULL; - result = stringlib_rfind_slice( - PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), - PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), - start, end - ); + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_READY(substring) == -1) + return NULL; + + result = any_find_slice(-1, self, substring, start, end); Py_DECREF(substring); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); } @@ -8615,9 +12116,9 @@ PyDoc_STRVAR(rindex__doc__, Like S.rfind() but raise ValueError when the substring is not found."); static PyObject * -unicode_rindex(PyUnicodeObject *self, PyObject *args) +unicode_rindex(PyObject *self, PyObject *args) { - PyUnicodeObject *substring; + PyObject *substring; Py_ssize_t start; Py_ssize_t end; Py_ssize_t result; @@ -8626,18 +12127,23 @@ unicode_rindex(PyUnicodeObject *self, PyObject *args) &start, &end)) return NULL; - result = stringlib_rfind_slice( - PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), - PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), - start, end - ); + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_READY(substring) == -1) + return NULL; + + result = any_find_slice(-1, self, substring, start, end); Py_DECREF(substring); + if (result == -2) + return NULL; + if (result < 0) { PyErr_SetString(PyExc_ValueError, "substring not found"); return NULL; } + return PyLong_FromSsize_t(result); } @@ -8648,25 +12154,25 @@ Return S right-justified in a string of length width. Padding is\n\ done using the specified fill character (default is a space)."); static PyObject * -unicode_rjust(PyUnicodeObject *self, PyObject *args) +unicode_rjust(PyObject *self, PyObject *args) { Py_ssize_t width; - Py_UNICODE fillchar = ' '; + Py_UCS4 fillchar = ' '; if (!PyArg_ParseTuple(args, "n|O&:rjust", &width, convert_uc, &fillchar)) return NULL; - if (self->length >= width && PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return (PyObject*) self; - } + if (PyUnicode_READY(self) == -1) + return NULL; + + if (PyUnicode_GET_LENGTH(self) >= width) + return unicode_result_unchanged(self); - return (PyObject*) pad(self, width - self->length, 0, fillchar); + return pad(self, width - PyUnicode_GET_LENGTH(self), 0, fillchar); } -PyObject *PyUnicode_Split(PyObject *s, - PyObject *sep, - Py_ssize_t maxsplit) +PyObject * +PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) { PyObject *result; @@ -8681,7 +12187,7 @@ PyObject *PyUnicode_Split(PyObject *s, } } - result = split((PyUnicodeObject *)s, (PyUnicodeObject *)sep, maxsplit); + result = split(s, sep, maxsplit); Py_DECREF(s); Py_XDECREF(sep); @@ -8689,7 +12195,7 @@ PyObject *PyUnicode_Split(PyObject *s, } PyDoc_STRVAR(split__doc__, - "S.split([sep[, maxsplit]]) -> list of strings\n\ + "S.split(sep=None, maxsplit=-1) -> list of strings\n\ \n\ Return a list of the words in S, using sep as the\n\ delimiter string. If maxsplit is given, at most maxsplit\n\ @@ -8698,20 +12204,22 @@ whitespace string is a separator and empty strings are\n\ removed from the result."); static PyObject* -unicode_split(PyUnicodeObject *self, PyObject *args) +unicode_split(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sep", "maxsplit", 0}; PyObject *substring = Py_None; Py_ssize_t maxcount = -1; - if (!PyArg_ParseTuple(args, "|On:split", &substring, &maxcount)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|On:split", + kwlist, &substring, &maxcount)) return NULL; if (substring == Py_None) return split(self, NULL, maxcount); else if (PyUnicode_Check(substring)) - return split(self, (PyUnicodeObject *)substring, maxcount); + return split(self, substring, maxcount); else - return PyUnicode_Split((PyObject *)self, substring, maxcount); + return PyUnicode_Split(self, substring, maxcount); } PyObject * @@ -8720,6 +12228,9 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) PyObject* str_obj; PyObject* sep_obj; PyObject* out; + int kind1, kind2, kind; + void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; str_obj = PyUnicode_FromObject(str_in); if (!str_obj) @@ -8729,16 +12240,62 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) Py_DECREF(str_obj); return NULL; } + if (PyUnicode_READY(sep_obj) == -1 || PyUnicode_READY(str_obj) == -1) { + Py_DECREF(sep_obj); + Py_DECREF(str_obj); + return NULL; + } - out = stringlib_partition( - str_obj, PyUnicode_AS_UNICODE(str_obj), PyUnicode_GET_SIZE(str_obj), - sep_obj, PyUnicode_AS_UNICODE(sep_obj), PyUnicode_GET_SIZE(sep_obj) - ); + kind1 = PyUnicode_KIND(str_obj); + kind2 = PyUnicode_KIND(sep_obj); + kind = Py_MAX(kind1, kind2); + buf1 = PyUnicode_DATA(str_obj); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind(str_obj, kind); + if (!buf1) + goto onError; + buf2 = PyUnicode_DATA(sep_obj); + if (kind2 != kind) + buf2 = _PyUnicode_AsKind(sep_obj, kind); + if (!buf2) + goto onError; + len1 = PyUnicode_GET_LENGTH(str_obj); + len2 = PyUnicode_GET_LENGTH(sep_obj); + + switch (PyUnicode_KIND(str_obj)) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sep_obj)) + out = asciilib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); + else + out = ucs1lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + case PyUnicode_2BYTE_KIND: + out = ucs2lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + case PyUnicode_4BYTE_KIND: + out = ucs4lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + default: + assert(0); + out = 0; + } Py_DECREF(sep_obj); Py_DECREF(str_obj); + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); return out; + onError: + Py_DECREF(sep_obj); + Py_DECREF(str_obj); + if (kind1 != kind && buf1) + PyMem_Free(buf1); + if (kind2 != kind && buf2) + PyMem_Free(buf2); + return NULL; } @@ -8748,6 +12305,9 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) PyObject* str_obj; PyObject* sep_obj; PyObject* out; + int kind1, kind2, kind; + void *buf1 = NULL, *buf2 = NULL; + Py_ssize_t len1, len2; str_obj = PyUnicode_FromObject(str_in); if (!str_obj) @@ -8758,15 +12318,56 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) return NULL; } - out = stringlib_rpartition( - str_obj, PyUnicode_AS_UNICODE(str_obj), PyUnicode_GET_SIZE(str_obj), - sep_obj, PyUnicode_AS_UNICODE(sep_obj), PyUnicode_GET_SIZE(sep_obj) - ); + kind1 = PyUnicode_KIND(str_in); + kind2 = PyUnicode_KIND(sep_obj); + kind = Py_MAX(kind1, kind2); + buf1 = PyUnicode_DATA(str_in); + if (kind1 != kind) + buf1 = _PyUnicode_AsKind(str_in, kind); + if (!buf1) + goto onError; + buf2 = PyUnicode_DATA(sep_obj); + if (kind2 != kind) + buf2 = _PyUnicode_AsKind(sep_obj, kind); + if (!buf2) + goto onError; + len1 = PyUnicode_GET_LENGTH(str_obj); + len2 = PyUnicode_GET_LENGTH(sep_obj); + + switch (PyUnicode_KIND(str_in)) { + case PyUnicode_1BYTE_KIND: + if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sep_obj)) + out = asciilib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); + else + out = ucs1lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + case PyUnicode_2BYTE_KIND: + out = ucs2lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + case PyUnicode_4BYTE_KIND: + out = ucs4lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); + break; + default: + assert(0); + out = 0; + } Py_DECREF(sep_obj); Py_DECREF(str_obj); + if (kind1 != kind) + PyMem_Free(buf1); + if (kind2 != kind) + PyMem_Free(buf2); return out; + onError: + Py_DECREF(sep_obj); + Py_DECREF(str_obj); + if (kind1 != kind && buf1) + PyMem_Free(buf1); + if (kind2 != kind && buf2) + PyMem_Free(buf2); + return NULL; } PyDoc_STRVAR(partition__doc__, @@ -8777,9 +12378,9 @@ the separator itself, and the part after it. If the separator is not\n\ found, return S and two empty strings."); static PyObject* -unicode_partition(PyUnicodeObject *self, PyObject *separator) +unicode_partition(PyObject *self, PyObject *separator) { - return PyUnicode_Partition((PyObject *)self, separator); + return PyUnicode_Partition(self, separator); } PyDoc_STRVAR(rpartition__doc__, @@ -8790,14 +12391,13 @@ the part before it, the separator itself, and the part after it. If the\n\ separator is not found, return two empty strings and S."); static PyObject* -unicode_rpartition(PyUnicodeObject *self, PyObject *separator) +unicode_rpartition(PyObject *self, PyObject *separator) { - return PyUnicode_RPartition((PyObject *)self, separator); + return PyUnicode_RPartition(self, separator); } -PyObject *PyUnicode_RSplit(PyObject *s, - PyObject *sep, - Py_ssize_t maxsplit) +PyObject * +PyUnicode_RSplit(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) { PyObject *result; @@ -8812,7 +12412,7 @@ PyObject *PyUnicode_RSplit(PyObject *s, } } - result = rsplit((PyUnicodeObject *)s, (PyUnicodeObject *)sep, maxsplit); + result = rsplit(s, sep, maxsplit); Py_DECREF(s); Py_XDECREF(sep); @@ -8820,7 +12420,7 @@ PyObject *PyUnicode_RSplit(PyObject *s, } PyDoc_STRVAR(rsplit__doc__, - "S.rsplit([sep[, maxsplit]]) -> list of strings\n\ + "S.rsplit(sep=None, maxsplit=-1) -> list of strings\n\ \n\ Return a list of the words in S, using sep as the\n\ delimiter string, starting at the end of the string and\n\ @@ -8829,20 +12429,22 @@ splits are done. If sep is not specified, any whitespace string\n\ is a separator."); static PyObject* -unicode_rsplit(PyUnicodeObject *self, PyObject *args) +unicode_rsplit(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"sep", "maxsplit", 0}; PyObject *substring = Py_None; Py_ssize_t maxcount = -1; - if (!PyArg_ParseTuple(args, "|On:rsplit", &substring, &maxcount)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|On:rsplit", + kwlist, &substring, &maxcount)) return NULL; if (substring == Py_None) return rsplit(self, NULL, maxcount); else if (PyUnicode_Check(substring)) - return rsplit(self, (PyUnicodeObject *)substring, maxcount); + return rsplit(self, substring, maxcount); else - return PyUnicode_RSplit((PyObject *)self, substring, maxcount); + return PyUnicode_RSplit(self, substring, maxcount); } PyDoc_STRVAR(splitlines__doc__, @@ -8853,26 +12455,22 @@ Line breaks are not included in the resulting list unless keepends\n\ is given and true."); static PyObject* -unicode_splitlines(PyUnicodeObject *self, PyObject *args) +unicode_splitlines(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"keepends", 0}; int keepends = 0; - if (!PyArg_ParseTuple(args, "|i:splitlines", &keepends)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i:splitlines", + kwlist, &keepends)) return NULL; - return PyUnicode_Splitlines((PyObject *)self, keepends); + return PyUnicode_Splitlines(self, keepends); } static PyObject *unicode_str(PyObject *self) { - if (PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return self; - } else - /* Subtype -- return genuine unicode string with the same value. */ - return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(self), - PyUnicode_GET_SIZE(self)); + return unicode_result_unchanged(self); } PyDoc_STRVAR(swapcase__doc__, @@ -8882,9 +12480,11 @@ Return a copy of S with uppercase characters converted to lowercase\n\ and vice versa."); static PyObject* -unicode_swapcase(PyUnicodeObject *self) +unicode_swapcase(PyObject *self) { - return fixup(self, fixswapcase); + if (PyUnicode_READY(self) == -1) + return NULL; + return case_operation(self, do_swapcase); } PyDoc_STRVAR(maketrans__doc__, @@ -8900,7 +12500,7 @@ character at the same position in y. If there is a third argument, it\n\ must be a string, whose characters will be mapped to None in the result."); static PyObject* -unicode_maketrans(PyUnicodeObject *null, PyObject *args) +unicode_maketrans(PyObject *null, PyObject *args) { PyObject *x, *y = NULL, *z = NULL; PyObject *new = NULL, *key, *value; @@ -8913,24 +12513,30 @@ unicode_maketrans(PyUnicodeObject *null, PyObject *args) if (!new) return NULL; if (y != NULL) { + int x_kind, y_kind, z_kind; + void *x_data, *y_data, *z_data; + /* x must be a string too, of equal length */ - Py_ssize_t ylen = PyUnicode_GET_SIZE(y); if (!PyUnicode_Check(x)) { PyErr_SetString(PyExc_TypeError, "first maketrans argument must " "be a string if there is a second argument"); goto err; } - if (PyUnicode_GET_SIZE(x) != ylen) { + if (PyUnicode_GET_LENGTH(x) != PyUnicode_GET_LENGTH(y)) { PyErr_SetString(PyExc_ValueError, "the first two maketrans " "arguments must have equal length"); goto err; } /* create entries for translating chars in x to those in y */ - for (i = 0; i < PyUnicode_GET_SIZE(x); i++) { - key = PyLong_FromLong(PyUnicode_AS_UNICODE(x)[i]); + x_kind = PyUnicode_KIND(x); + y_kind = PyUnicode_KIND(y); + x_data = PyUnicode_DATA(x); + y_data = PyUnicode_DATA(y); + for (i = 0; i < PyUnicode_GET_LENGTH(x); i++) { + key = PyLong_FromLong(PyUnicode_READ(x_kind, x_data, i)); if (!key) goto err; - value = PyLong_FromLong(PyUnicode_AS_UNICODE(y)[i]); + value = PyLong_FromLong(PyUnicode_READ(y_kind, y_data, i)); if (!value) { Py_DECREF(key); goto err; @@ -8943,8 +12549,10 @@ unicode_maketrans(PyUnicodeObject *null, PyObject *args) } /* create entries for deleting chars in z */ if (z != NULL) { - for (i = 0; i < PyUnicode_GET_SIZE(z); i++) { - key = PyLong_FromLong(PyUnicode_AS_UNICODE(z)[i]); + z_kind = PyUnicode_KIND(z); + z_data = PyUnicode_DATA(z); + for (i = 0; i < PyUnicode_GET_LENGTH(z); i++) { + key = PyLong_FromLong(PyUnicode_READ(z_kind, z_data, i)); if (!key) goto err; res = PyDict_SetItem(new, key, Py_None); @@ -8954,6 +12562,9 @@ unicode_maketrans(PyUnicodeObject *null, PyObject *args) } } } else { + int kind; + void *data; + /* x must be a dict */ if (!PyDict_CheckExact(x)) { PyErr_SetString(PyExc_TypeError, "if you give only one argument " @@ -8965,12 +12576,14 @@ unicode_maketrans(PyUnicodeObject *null, PyObject *args) if (PyUnicode_Check(key)) { /* convert string keys to integer keys */ PyObject *newkey; - if (PyUnicode_GET_SIZE(key) != 1) { + if (PyUnicode_GET_LENGTH(key) != 1) { PyErr_SetString(PyExc_ValueError, "string keys in translate " "table must be of length 1"); goto err; } - newkey = PyLong_FromLong(PyUnicode_AS_UNICODE(key)[0]); + kind = PyUnicode_KIND(key); + data = PyUnicode_DATA(key); + newkey = PyLong_FromLong(PyUnicode_READ(kind, data, 0)); if (!newkey) goto err; res = PyDict_SetItem(new, newkey, value); @@ -9004,9 +12617,9 @@ Unmapped characters are left untouched. Characters mapped to None\n\ are deleted."); static PyObject* -unicode_translate(PyUnicodeObject *self, PyObject *table) +unicode_translate(PyObject *self, PyObject *table) { - return PyUnicode_TranslateCharmap(self->str, self->length, table, "ignore"); + return _PyUnicode_TranslateCharmap(self, table, "ignore"); } PyDoc_STRVAR(upper__doc__, @@ -9015,9 +12628,13 @@ PyDoc_STRVAR(upper__doc__, Return a copy of S converted to uppercase."); static PyObject* -unicode_upper(PyUnicodeObject *self) +unicode_upper(PyObject *self) { - return fixup(self, fixupper); + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyUnicode_IS_ASCII(self)) + return ascii_upper_or_lower(self, 0); + return case_operation(self, do_upper); } PyDoc_STRVAR(zfill__doc__, @@ -9027,55 +12644,50 @@ Pad a numeric string S with zeros on the left, to fill a field\n\ of the specified width. The string S is never truncated."); static PyObject * -unicode_zfill(PyUnicodeObject *self, PyObject *args) +unicode_zfill(PyObject *self, PyObject *args) { Py_ssize_t fill; - PyUnicodeObject *u; - + PyObject *u; Py_ssize_t width; + int kind; + void *data; + Py_UCS4 chr; + if (!PyArg_ParseTuple(args, "n:zfill", &width)) return NULL; - if (self->length >= width) { - if (PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return (PyObject*) self; - } - else - return PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(self), - PyUnicode_GET_SIZE(self) - ); - } + if (PyUnicode_READY(self) == -1) + return NULL; - fill = width - self->length; + if (PyUnicode_GET_LENGTH(self) >= width) + return unicode_result_unchanged(self); + + fill = width - PyUnicode_GET_LENGTH(self); u = pad(self, fill, 0, '0'); if (u == NULL) return NULL; - if (u->str[fill] == '+' || u->str[fill] == '-') { + kind = PyUnicode_KIND(u); + data = PyUnicode_DATA(u); + chr = PyUnicode_READ(kind, data, fill); + + if (chr == '+' || chr == '-') { /* move sign to beginning of string */ - u->str[0] = u->str[fill]; - u->str[fill] = '0'; + PyUnicode_WRITE(kind, data, 0, chr); + PyUnicode_WRITE(kind, data, fill, '0'); } - return (PyObject*) u; + assert(_PyUnicode_CheckConsistency(u, 1)); + return u; } #if 0 -static PyObject* -unicode_freelistsize(PyUnicodeObject *self) -{ - return PyLong_FromLong(numfree); -} - static PyObject * unicode__decimal2ascii(PyObject *self) { - return PyUnicode_TransformDecimalToASCII(PyUnicode_AS_UNICODE(self), - PyUnicode_GET_SIZE(self)); + return PyUnicode_TransformDecimalAndSpaceToASCII(self); } #endif @@ -9088,11 +12700,11 @@ With optional end, stop comparing S at that position.\n\ prefix can also be a tuple of strings to try."); static PyObject * -unicode_startswith(PyUnicodeObject *self, +unicode_startswith(PyObject *self, PyObject *args) { PyObject *subobj; - PyUnicodeObject *substring; + PyObject *substring; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; int result; @@ -9102,8 +12714,7 @@ unicode_startswith(PyUnicodeObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = (PyUnicodeObject *)PyUnicode_FromObject( - PyTuple_GET_ITEM(subobj, i)); + substring = PyUnicode_FromObject(PyTuple_GET_ITEM(subobj, i)); if (substring == NULL) return NULL; result = tailmatch(self, substring, start, end, -1); @@ -9115,7 +12726,7 @@ unicode_startswith(PyUnicodeObject *self, /* nothing matched */ Py_RETURN_FALSE; } - substring = (PyUnicodeObject *)PyUnicode_FromObject(subobj); + substring = PyUnicode_FromObject(subobj); if (substring == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) PyErr_Format(PyExc_TypeError, "startswith first arg must be str or " @@ -9137,11 +12748,11 @@ With optional end, stop comparing S at that position.\n\ suffix can also be a tuple of strings to try."); static PyObject * -unicode_endswith(PyUnicodeObject *self, +unicode_endswith(PyObject *self, PyObject *args) { PyObject *subobj; - PyUnicodeObject *substring; + PyObject *substring; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; int result; @@ -9151,7 +12762,7 @@ unicode_endswith(PyUnicodeObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = (PyUnicodeObject *)PyUnicode_FromObject( + substring = PyUnicode_FromObject( PyTuple_GET_ITEM(subobj, i)); if (substring == NULL) return NULL; @@ -9163,7 +12774,7 @@ unicode_endswith(PyUnicodeObject *self, } Py_RETURN_FALSE; } - substring = (PyUnicodeObject *)PyUnicode_FromObject(subobj); + substring = PyUnicode_FromObject(subobj); if (substring == NULL) { if (PyErr_ExceptionMatches(PyExc_TypeError)) PyErr_Format(PyExc_TypeError, "endswith first arg must be str or " @@ -9175,7 +12786,160 @@ unicode_endswith(PyUnicodeObject *self, return PyBool_FromLong(result); } -#include "stringlib/string_format.h" +Py_LOCAL_INLINE(void) +_PyUnicodeWriter_Update(_PyUnicodeWriter *writer) +{ + writer->size = PyUnicode_GET_LENGTH(writer->buffer); + writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); + writer->data = PyUnicode_DATA(writer->buffer); + writer->kind = PyUnicode_KIND(writer->buffer); +} + +void +_PyUnicodeWriter_Init(_PyUnicodeWriter *writer, Py_ssize_t min_length) +{ + memset(writer, 0, sizeof(*writer)); +#ifdef Py_DEBUG + writer->kind = 5; /* invalid kind */ +#endif + writer->min_length = Py_MAX(min_length, 100); + writer->overallocate = (min_length > 0); +} + +int +_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, + Py_ssize_t length, Py_UCS4 maxchar) +{ + Py_ssize_t newlen; + PyObject *newbuffer; + + assert(length > 0); + + if (length > PY_SSIZE_T_MAX - writer->pos) { + PyErr_NoMemory(); + return -1; + } + newlen = writer->pos + length; + + if (writer->buffer == NULL) { + if (writer->overallocate) { + /* overallocate 25% to limit the number of resize */ + if (newlen <= (PY_SSIZE_T_MAX - newlen / 4)) + newlen += newlen / 4; + if (newlen < writer->min_length) + newlen = writer->min_length; + } + writer->buffer = PyUnicode_New(newlen, maxchar); + if (writer->buffer == NULL) + return -1; + _PyUnicodeWriter_Update(writer); + return 0; + } + + if (newlen > writer->size) { + if (writer->overallocate) { + /* overallocate 25% to limit the number of resize */ + if (newlen <= (PY_SSIZE_T_MAX - newlen / 4)) + newlen += newlen / 4; + if (newlen < writer->min_length) + newlen = writer->min_length; + } + + if (maxchar > writer->maxchar || writer->readonly) { + /* resize + widen */ + newbuffer = PyUnicode_New(newlen, maxchar); + if (newbuffer == NULL) + return -1; + _PyUnicode_FastCopyCharacters(newbuffer, 0, + writer->buffer, 0, writer->pos); + Py_DECREF(writer->buffer); + writer->readonly = 0; + } + else { + newbuffer = resize_compact(writer->buffer, newlen); + if (newbuffer == NULL) + return -1; + } + writer->buffer = newbuffer; + _PyUnicodeWriter_Update(writer); + } + else if (maxchar > writer->maxchar) { + assert(!writer->readonly); + newbuffer = PyUnicode_New(writer->size, maxchar); + if (newbuffer == NULL) + return -1; + _PyUnicode_FastCopyCharacters(newbuffer, 0, + writer->buffer, 0, writer->pos); + Py_DECREF(writer->buffer); + writer->buffer = newbuffer; + _PyUnicodeWriter_Update(writer); + } + return 0; +} + +int +_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) +{ + Py_UCS4 maxchar; + Py_ssize_t len; + + if (PyUnicode_READY(str) == -1) + return -1; + len = PyUnicode_GET_LENGTH(str); + if (len == 0) + return 0; + maxchar = PyUnicode_MAX_CHAR_VALUE(str); + if (maxchar > writer->maxchar || len > writer->size - writer->pos) { + if (writer->buffer == NULL && !writer->overallocate) { + Py_INCREF(str); + writer->buffer = str; + _PyUnicodeWriter_Update(writer); + writer->readonly = 1; + writer->size = 0; + writer->pos += len; + return 0; + } + if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1) + return -1; + } + _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, + str, 0, len); + writer->pos += len; + return 0; +} + +PyObject * +_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) +{ + if (writer->pos == 0) { + Py_XDECREF(writer->buffer); + Py_INCREF(unicode_empty); + return unicode_empty; + } + if (writer->readonly) { + assert(PyUnicode_GET_LENGTH(writer->buffer) == writer->pos); + return writer->buffer; + } + if (PyUnicode_GET_LENGTH(writer->buffer) != writer->pos) { + PyObject *newbuffer; + newbuffer = resize_compact(writer->buffer, writer->pos); + if (newbuffer == NULL) { + Py_DECREF(writer->buffer); + return NULL; + } + writer->buffer = newbuffer; + } + assert(_PyUnicode_CheckConsistency(writer->buffer, 1)); + return writer->buffer; +} + +void +_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) +{ + Py_CLEAR(writer->buffer); +} + +#include "stringlib/unicode_format.h" PyDoc_STRVAR(format__doc__, "S.format(*args, **kwargs) -> str\n\ @@ -9193,13 +12957,23 @@ static PyObject * unicode__format__(PyObject* self, PyObject* args) { PyObject *format_spec; + _PyUnicodeWriter writer; + int ret; if (!PyArg_ParseTuple(args, "U:__format__", &format_spec)) return NULL; - return _PyUnicode_FormatAdvanced(self, - PyUnicode_AS_UNICODE(format_spec), - PyUnicode_GET_SIZE(format_spec)); + if (PyUnicode_READY(self) == -1) + return NULL; + _PyUnicodeWriter_Init(&writer, 0); + ret = _PyUnicode_FormatAdvancedWriter(&writer, + self, format_spec, 0, + PyUnicode_GET_LENGTH(format_spec)); + if (ret == -1) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + return _PyUnicodeWriter_Finish(&writer); } PyDoc_STRVAR(p_format__doc__, @@ -9208,28 +12982,55 @@ PyDoc_STRVAR(p_format__doc__, Return a formatted version of S as described by format_spec."); static PyObject * -unicode__sizeof__(PyUnicodeObject *v) +unicode__sizeof__(PyObject *v) { - return PyLong_FromSsize_t(sizeof(PyUnicodeObject) + - sizeof(Py_UNICODE) * (v->length + 1)); + Py_ssize_t size; + + /* If it's a compact object, account for base structure + + character data. */ + if (PyUnicode_IS_COMPACT_ASCII(v)) + size = sizeof(PyASCIIObject) + PyUnicode_GET_LENGTH(v) + 1; + else if (PyUnicode_IS_COMPACT(v)) + size = sizeof(PyCompactUnicodeObject) + + (PyUnicode_GET_LENGTH(v) + 1) * PyUnicode_KIND(v); + else { + /* If it is a two-block object, account for base object, and + for character block if present. */ + size = sizeof(PyUnicodeObject); + if (_PyUnicode_DATA_ANY(v)) + size += (PyUnicode_GET_LENGTH(v) + 1) * + PyUnicode_KIND(v); + } + /* If the wstr pointer is present, account for it unless it is shared + with the data pointer. Check if the data is not shared. */ + if (_PyUnicode_HAS_WSTR_MEMORY(v)) + size += (PyUnicode_WSTR_LENGTH(v) + 1) * sizeof(wchar_t); + if (_PyUnicode_HAS_UTF8_MEMORY(v)) + size += PyUnicode_UTF8_LENGTH(v) + 1; + + return PyLong_FromSsize_t(size); } PyDoc_STRVAR(sizeof__doc__, "S.__sizeof__() -> size of S in memory, in bytes"); static PyObject * -unicode_getnewargs(PyUnicodeObject *v) +unicode_getnewargs(PyObject *v) { - return Py_BuildValue("(u#)", v->str, v->length); + PyObject *copy = _PyUnicode_Copy(v); + if (!copy) + return NULL; + return Py_BuildValue("(N)", copy); } static PyMethodDef unicode_methods[] = { {"encode", (PyCFunction) unicode_encode, METH_VARARGS | METH_KEYWORDS, encode__doc__}, {"replace", (PyCFunction) unicode_replace, METH_VARARGS, replace__doc__}, - {"split", (PyCFunction) unicode_split, METH_VARARGS, split__doc__}, - {"rsplit", (PyCFunction) unicode_rsplit, METH_VARARGS, rsplit__doc__}, + {"split", (PyCFunction) unicode_split, METH_VARARGS | METH_KEYWORDS, split__doc__}, + {"rsplit", (PyCFunction) unicode_rsplit, METH_VARARGS | METH_KEYWORDS, rsplit__doc__}, {"join", (PyCFunction) unicode_join, METH_O, join__doc__}, {"capitalize", (PyCFunction) unicode_capitalize, METH_NOARGS, capitalize__doc__}, + {"casefold", (PyCFunction) unicode_casefold, METH_NOARGS, casefold__doc__}, {"title", (PyCFunction) unicode_title, METH_NOARGS, title__doc__}, {"center", (PyCFunction) unicode_center, METH_VARARGS, center__doc__}, {"count", (PyCFunction) unicode_count, METH_VARARGS, count__doc__}, @@ -9245,7 +13046,7 @@ static PyMethodDef unicode_methods[] = { {"rjust", (PyCFunction) unicode_rjust, METH_VARARGS, rjust__doc__}, {"rstrip", (PyCFunction) unicode_rstrip, METH_VARARGS, rstrip__doc__}, {"rpartition", (PyCFunction) unicode_rpartition, METH_O, rpartition__doc__}, - {"splitlines", (PyCFunction) unicode_splitlines, METH_VARARGS, splitlines__doc__}, + {"splitlines", (PyCFunction) unicode_splitlines, METH_VARARGS | METH_KEYWORDS, splitlines__doc__}, {"strip", (PyCFunction) unicode_strip, METH_VARARGS, strip__doc__}, {"swapcase", (PyCFunction) unicode_swapcase, METH_NOARGS, swapcase__doc__}, {"translate", (PyCFunction) unicode_translate, METH_O, translate__doc__}, @@ -9271,12 +13072,7 @@ static PyMethodDef unicode_methods[] = { METH_VARARGS | METH_STATIC, maketrans__doc__}, {"__sizeof__", (PyCFunction) unicode__sizeof__, METH_NOARGS, sizeof__doc__}, #if 0 - {"capwords", (PyCFunction) unicode_capwords, METH_NOARGS, capwords__doc__}, -#endif - -#if 0 /* These methods are just used for debugging the implementation. */ - {"freelistsize", (PyCFunction) unicode_freelistsize, METH_NOARGS}, {"_decimal2ascii", (PyCFunction) unicode__decimal2ascii, METH_NOARGS}, #endif @@ -9287,10 +13083,8 @@ static PyMethodDef unicode_methods[] = { static PyObject * unicode_mod(PyObject *v, PyObject *w) { - if (!PyUnicode_Check(v)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyUnicode_Check(v)) + Py_RETURN_NOTIMPLEMENTED; return PyUnicode_Format(v, w); } @@ -9313,50 +13107,69 @@ static PySequenceMethods unicode_as_sequence = { }; static PyObject* -unicode_subscript(PyUnicodeObject* self, PyObject* item) +unicode_subscript(PyObject* self, PyObject* item) { + if (PyUnicode_READY(self) == -1) + return NULL; + if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) - i += PyUnicode_GET_SIZE(self); + i += PyUnicode_GET_LENGTH(self); return unicode_getitem(self, i); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, cur, i; - Py_UNICODE* source_buf; - Py_UNICODE* result_buf; - PyObject* result; + PyObject *result; + void *src_data, *dest_data; + int src_kind, dest_kind; + Py_UCS4 ch, max_char, kind_limit; - if (PySlice_GetIndicesEx(item, PyUnicode_GET_SIZE(self), + if (PySlice_GetIndicesEx(item, PyUnicode_GET_LENGTH(self), &start, &stop, &step, &slicelength) < 0) { return NULL; } if (slicelength <= 0) { - return PyUnicode_FromUnicode(NULL, 0); - } else if (start == 0 && step == 1 && slicelength == self->length && - PyUnicode_CheckExact(self)) { - Py_INCREF(self); - return (PyObject *)self; + Py_INCREF(unicode_empty); + return unicode_empty; + } else if (start == 0 && step == 1 && + slicelength == PyUnicode_GET_LENGTH(self)) { + return unicode_result_unchanged(self); } else if (step == 1) { - return PyUnicode_FromUnicode(self->str + start, slicelength); - } else { - source_buf = PyUnicode_AS_UNICODE((PyObject*)self); - result_buf = (Py_UNICODE *)PyObject_MALLOC(slicelength* - sizeof(Py_UNICODE)); - - if (result_buf == NULL) - return PyErr_NoMemory(); - + return PyUnicode_Substring(self, + start, start + slicelength); + } + /* General case */ + src_kind = PyUnicode_KIND(self); + src_data = PyUnicode_DATA(self); + if (!PyUnicode_IS_ASCII(self)) { + kind_limit = kind_maxchar_limit(src_kind); + max_char = 0; for (cur = start, i = 0; i < slicelength; cur += step, i++) { - result_buf[i] = source_buf[cur]; + ch = PyUnicode_READ(src_kind, src_data, cur); + if (ch > max_char) { + max_char = ch; + if (max_char >= kind_limit) + break; + } } + } + else + max_char = 127; + result = PyUnicode_New(slicelength, max_char); + if (result == NULL) + return NULL; + dest_kind = PyUnicode_KIND(result); + dest_data = PyUnicode_DATA(result); - result = PyUnicode_FromUnicode(result_buf, slicelength); - PyObject_FREE(result_buf); - return result; + for (cur = start, i = 0; i < slicelength; cur += step, i++) { + Py_UCS4 ch = PyUnicode_READ(src_kind, src_data, cur); + PyUnicode_WRITE(dest_kind, dest_data, i, ch); } + assert(_PyUnicode_CheckConsistency(result, 1)); + return result; } else { PyErr_SetString(PyExc_TypeError, "string indices must be integers"); return NULL; @@ -9390,16 +13203,17 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) /* Returns a new reference to a PyUnicode object, or NULL on failure. */ -static PyObject * -formatfloat(PyObject *v, int flags, int prec, int type) +static int +formatfloat(PyObject *v, int flags, int prec, int type, + PyObject **p_output, _PyUnicodeWriter *writer) { char *p; - PyObject *result; double x; + Py_ssize_t len; x = PyFloat_AsDouble(v); if (x == -1.0 && PyErr_Occurred()) - return NULL; + return -1; if (prec < 0) prec = 6; @@ -9407,54 +13221,170 @@ formatfloat(PyObject *v, int flags, int prec, int type) p = PyOS_double_to_string(x, type, prec, (flags & F_ALT) ? Py_DTSF_ALT : 0, NULL); if (p == NULL) - return NULL; - result = PyUnicode_FromStringAndSize(p, strlen(p)); + return -1; + len = strlen(p); + if (writer) { + if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1) { + PyMem_Free(p); + return -1; + } + unicode_write_cstr(writer->buffer, writer->pos, p, len); + writer->pos += len; + } + else + *p_output = _PyUnicode_FromASCII(p, len); PyMem_Free(p); - return result; + return 0; } +/* formatlong() emulates the format codes d, u, o, x and X, and + * the F_ALT flag, for Python's long (unbounded) ints. It's not used for + * Python's regular ints. + * Return value: a new PyUnicodeObject*, or NULL if error. + * The output string is of the form + * "-"? ("0x" | "0X")? digit+ + * "0x"/"0X" are present only for x and X conversions, with F_ALT + * set in flags. The case of hex digits will be correct, + * There will be at least prec digits, zero-filled on the left if + * necessary to get that many. + * val object to be converted + * flags bitmask of format flags; only F_ALT is looked at + * prec minimum number of digits; 0-fill on left if needed + * type a character in [duoxX]; u acts the same as d + * + * CAUTION: o, x and X conversions on regular ints can never + * produce a '-' sign, but can for Python's unbounded ints. + */ static PyObject* formatlong(PyObject *val, int flags, int prec, int type) { + PyObject *result = NULL; char *buf; - int len; - PyObject *str; /* temporary string object. */ - PyObject *result; + Py_ssize_t i; + int sign; /* 1 if '-', else 0 */ + int len; /* number of characters */ + Py_ssize_t llen; + int numdigits; /* len == numnondigits + numdigits */ + int numnondigits = 0; + + /* Avoid exceeding SSIZE_T_MAX */ + if (prec > INT_MAX-3) { + PyErr_SetString(PyExc_OverflowError, + "precision too large"); + return NULL; + } - str = _PyBytes_FormatLong(val, flags, prec, type, &buf, &len); - if (!str) + assert(PyLong_Check(val)); + + switch (type) { + case 'd': + case 'u': + /* Special-case boolean: we want 0/1 */ + if (PyBool_Check(val)) + result = PyNumber_ToBase(val, 10); + else + result = Py_TYPE(val)->tp_str(val); + break; + case 'o': + numnondigits = 2; + result = PyNumber_ToBase(val, 8); + break; + case 'x': + case 'X': + numnondigits = 2; + result = PyNumber_ToBase(val, 16); + break; + default: + assert(!"'type' not in [duoxX]"); + } + if (!result) return NULL; - result = PyUnicode_FromStringAndSize(buf, len); - Py_DECREF(str); + + assert(unicode_modifiable(result)); + assert(PyUnicode_IS_READY(result)); + assert(PyUnicode_IS_ASCII(result)); + + /* To modify the string in-place, there can only be one reference. */ + if (Py_REFCNT(result) != 1) { + PyErr_BadInternalCall(); + return NULL; + } + buf = PyUnicode_DATA(result); + llen = PyUnicode_GET_LENGTH(result); + if (llen > INT_MAX) { + PyErr_SetString(PyExc_ValueError, + "string too large in _PyBytes_FormatLong"); + return NULL; + } + len = (int)llen; + sign = buf[0] == '-'; + numnondigits += sign; + numdigits = len - numnondigits; + assert(numdigits > 0); + + /* Get rid of base marker unless F_ALT */ + if (((flags & F_ALT) == 0 && + (type == 'o' || type == 'x' || type == 'X'))) { + assert(buf[sign] == '0'); + assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' || + buf[sign+1] == 'o'); + numnondigits -= 2; + buf += 2; + len -= 2; + if (sign) + buf[0] = '-'; + assert(len == numnondigits + numdigits); + assert(numdigits > 0); + } + + /* Fill with leading zeroes to meet minimum width. */ + if (prec > numdigits) { + PyObject *r1 = PyBytes_FromStringAndSize(NULL, + numnondigits + prec); + char *b1; + if (!r1) { + Py_DECREF(result); + return NULL; + } + b1 = PyBytes_AS_STRING(r1); + for (i = 0; i < numnondigits; ++i) + *b1++ = *buf++; + for (i = 0; i < prec - numdigits; i++) + *b1++ = '0'; + for (i = 0; i < numdigits; i++) + *b1++ = *buf++; + *b1 = '\0'; + Py_DECREF(result); + result = r1; + buf = PyBytes_AS_STRING(result); + len = numnondigits + prec; + } + + /* Fix up case for hex conversions. */ + if (type == 'X') { + /* Need to convert all lower case letters to upper case. + and need to convert 0x to 0X (and -0x to -0X). */ + for (i = 0; i < len; i++) + if (buf[i] >= 'a' && buf[i] <= 'x') + buf[i] -= 'a'-'A'; + } + if (!PyUnicode_Check(result) || len != PyUnicode_GET_LENGTH(result)) { + PyObject *unicode; + unicode = _PyUnicode_FromASCII(buf, len); + Py_DECREF(result); + result = unicode; + } return result; } -static int -formatchar(Py_UNICODE *buf, - size_t buflen, - PyObject *v) +static Py_UCS4 +formatchar(PyObject *v) { /* presume that the buffer is at least 3 characters long */ if (PyUnicode_Check(v)) { - if (PyUnicode_GET_SIZE(v) == 1) { - buf[0] = PyUnicode_AS_UNICODE(v)[0]; - buf[1] = '\0'; - return 1; - } -#ifndef Py_UNICODE_WIDE - if (PyUnicode_GET_SIZE(v) == 2) { - /* Decode a valid surrogate pair */ - int c0 = PyUnicode_AS_UNICODE(v)[0]; - int c1 = PyUnicode_AS_UNICODE(v)[1]; - if (0xD800 <= c0 && c0 <= 0xDBFF && - 0xDC00 <= c1 && c1 <= 0xDFFF) { - buf[0] = c0; - buf[1] = c1; - buf[2] = '\0'; - return 2; - } + if (PyUnicode_GET_LENGTH(v) == 1) { + return PyUnicode_READ_CHAR(v, 0); } -#endif goto onError; } else { @@ -9464,45 +13394,35 @@ formatchar(Py_UNICODE *buf, if (x == -1 && PyErr_Occurred()) goto onError; - if (x < 0 || x > 0x10ffff) { + if (x < 0 || x > MAX_UNICODE) { PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); - return -1; + return (Py_UCS4) -1; } -#ifndef Py_UNICODE_WIDE - if (x > 0xffff) { - x -= 0x10000; - buf[0] = (Py_UNICODE)(0xD800 | (x >> 10)); - buf[1] = (Py_UNICODE)(0xDC00 | (x & 0x3FF)); - return 2; - } -#endif - buf[0] = (Py_UNICODE) x; - buf[1] = '\0'; - return 1; + return (Py_UCS4) x; } onError: PyErr_SetString(PyExc_TypeError, "%c requires int or char"); - return -1; + return (Py_UCS4) -1; } -/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) - FORMATBUFLEN is the length of the buffer in which chars are formatted. -*/ -#define FORMATBUFLEN (size_t)10 - -PyObject *PyUnicode_Format(PyObject *format, - PyObject *args) +PyObject * +PyUnicode_Format(PyObject *format, PyObject *args) { - Py_UNICODE *fmt, *res; - Py_ssize_t fmtcnt, rescnt, reslen, arglen, argidx; + Py_ssize_t fmtcnt, fmtpos, arglen, argidx; int args_owned = 0; - PyUnicodeObject *result = NULL; PyObject *dict = NULL; + PyObject *temp = NULL; + PyObject *second = NULL; PyObject *uformat; + void *fmt; + enum PyUnicode_Kind kind, fmtkind; + _PyUnicodeWriter writer; + Py_ssize_t sublen; + Py_UCS4 maxchar; if (format == NULL || args == NULL) { PyErr_BadInternalCall(); @@ -9511,14 +13431,17 @@ PyObject *PyUnicode_Format(PyObject *format, uformat = PyUnicode_FromObject(format); if (uformat == NULL) return NULL; - fmt = PyUnicode_AS_UNICODE(uformat); - fmtcnt = PyUnicode_GET_SIZE(uformat); + if (PyUnicode_READY(uformat) == -1) { + Py_DECREF(uformat); + return NULL; + } - reslen = rescnt = fmtcnt + 100; - result = _PyUnicode_New(reslen); - if (result == NULL) - goto onError; - res = PyUnicode_AS_UNICODE(result); + fmt = PyUnicode_DATA(uformat); + fmtkind = PyUnicode_KIND(uformat); + fmtcnt = PyUnicode_GET_LENGTH(uformat); + fmtpos = 0; + + _PyUnicodeWriter_Init(&writer, fmtcnt + 100); if (PyTuple_Check(args)) { arglen = PyTuple_Size(args); @@ -9532,35 +13455,46 @@ PyObject *PyUnicode_Format(PyObject *format, dict = args; while (--fmtcnt >= 0) { - if (*fmt != '%') { - if (--rescnt < 0) { - rescnt = fmtcnt + 100; - reslen += rescnt; - if (_PyUnicode_Resize(&result, reslen) < 0) - goto onError; - res = PyUnicode_AS_UNICODE(result) + reslen - rescnt; - --rescnt; - } - *res++ = *fmt++; + if (PyUnicode_READ(fmtkind, fmt, fmtpos) != '%') { + Py_ssize_t nonfmtpos; + nonfmtpos = fmtpos++; + while (fmtcnt >= 0 && + PyUnicode_READ(fmtkind, fmt, fmtpos) != '%') { + fmtpos++; + fmtcnt--; + } + if (fmtcnt < 0) + fmtpos--; + sublen = fmtpos - nonfmtpos; + maxchar = _PyUnicode_FindMaxChar(uformat, + nonfmtpos, nonfmtpos + sublen); + if (_PyUnicodeWriter_Prepare(&writer, sublen, maxchar) == -1) + goto onError; + + _PyUnicode_FastCopyCharacters(writer.buffer, writer.pos, + uformat, nonfmtpos, sublen); + writer.pos += sublen; } else { /* Got a format specifier */ int flags = 0; Py_ssize_t width = -1; int prec = -1; - Py_UNICODE c = '\0'; - Py_UNICODE fill; + Py_UCS4 c = '\0'; + Py_UCS4 fill; + int sign; + Py_UCS4 signchar; int isnumok; PyObject *v = NULL; - PyObject *temp = NULL; - Py_UNICODE *pbuf; - Py_UNICODE sign; - Py_ssize_t len; - Py_UNICODE formatbuf[FORMATBUFLEN]; /* For formatchar() */ - - fmt++; - if (*fmt == '(') { - Py_UNICODE *keystart; + void *pbuf = NULL; + Py_ssize_t pindex, len; + Py_UCS4 bufmaxchar; + Py_ssize_t buflen; + + fmtpos++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos); + if (c == '(') { + Py_ssize_t keystart; Py_ssize_t keylen; PyObject *key; int pcount = 1; @@ -9570,34 +13504,26 @@ PyObject *PyUnicode_Format(PyObject *format, "format requires a mapping"); goto onError; } - ++fmt; + ++fmtpos; --fmtcnt; - keystart = fmt; + keystart = fmtpos; /* Skip over balanced parentheses */ while (pcount > 0 && --fmtcnt >= 0) { - if (*fmt == ')') + c = PyUnicode_READ(fmtkind, fmt, fmtpos); + if (c == ')') --pcount; - else if (*fmt == '(') + else if (c == '(') ++pcount; - fmt++; + fmtpos++; } - keylen = fmt - keystart - 1; + keylen = fmtpos - keystart - 1; if (fmtcnt < 0 || pcount > 0) { PyErr_SetString(PyExc_ValueError, "incomplete format key"); goto onError; } -#if 0 - /* keys are converted to strings using UTF-8 and - then looked up since Python uses strings to hold - variables names etc. in its namespaces and we - wouldn't want to break common idioms. */ - key = PyUnicode_EncodeUTF8(keystart, - keylen, - NULL); -#else - key = PyUnicode_FromUnicode(keystart, keylen); -#endif + key = PyUnicode_Substring(uformat, + keystart, keystart + keylen); if (key == NULL) goto onError; if (args_owned) { @@ -9614,7 +13540,8 @@ PyObject *PyUnicode_Format(PyObject *format, argidx = -2; } while (--fmtcnt >= 0) { - switch (c = *fmt++) { + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); + switch (c) { case '-': flags |= F_LJUST; continue; case '+': flags |= F_SIGN; continue; case ' ': flags |= F_BLANK; continue; @@ -9640,14 +13567,17 @@ PyObject *PyUnicode_Format(PyObject *format, width = -width; } if (--fmtcnt >= 0) - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); } else if (c >= '0' && c <= '9') { width = c - '0'; while (--fmtcnt >= 0) { - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); if (c < '0' || c > '9') break; + /* Since c is unsigned, the RHS would end up as unsigned, + mixing signed and unsigned comparison. Since c is between + '0' and '9', casting to int is safe. */ if (width > (PY_SSIZE_T_MAX - ((int)c - '0')) / 10) { PyErr_SetString(PyExc_ValueError, "width too big"); @@ -9659,7 +13589,7 @@ PyObject *PyUnicode_Format(PyObject *format, if (c == '.') { prec = 0; if (--fmtcnt >= 0) - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); if (c == '*') { v = getnextarg(args, arglen, &argidx); if (v == NULL) @@ -9675,12 +13605,12 @@ PyObject *PyUnicode_Format(PyObject *format, if (prec < 0) prec = 0; if (--fmtcnt >= 0) - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); } else if (c >= '0' && c <= '9') { prec = c - '0'; while (--fmtcnt >= 0) { - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); if (c < '0' || c > '9') break; if (prec > (INT_MAX - ((int)c - '0')) / 10) { @@ -9695,7 +13625,7 @@ PyObject *PyUnicode_Format(PyObject *format, if (fmtcnt >= 0) { if (c == 'h' || c == 'l' || c == 'L') { if (--fmtcnt >= 0) - c = *fmt++; + c = PyUnicode_READ(fmtkind, fmt, fmtpos++); } } if (fmtcnt < 0) { @@ -9703,25 +13633,36 @@ PyObject *PyUnicode_Format(PyObject *format, "incomplete format"); goto onError; } - if (c != '%') { - v = getnextarg(args, arglen, &argidx); - if (v == NULL) + if (fmtcnt == 0) + writer.overallocate = 0; + + if (c == '%') { + if (_PyUnicodeWriter_Prepare(&writer, 1, '%') == -1) goto onError; + PyUnicode_WRITE(writer.kind, writer.data, writer.pos, '%'); + writer.pos += 1; + continue; } + + v = getnextarg(args, arglen, &argidx); + if (v == NULL) + goto onError; + sign = 0; + signchar = '\0'; fill = ' '; switch (c) { - case '%': - pbuf = formatbuf; - /* presume that buffer length is at least 1 */ - pbuf[0] = '%'; - len = 1; - break; - case 's': case 'r': case 'a': + if (PyLong_CheckExact(v) && width == -1 && prec == -1) { + /* Fast path */ + if (_PyLong_FormatWriter(&writer, v, 10, flags & F_ALT) == -1) + goto onError; + goto nextarg; + } + if (PyUnicode_CheckExact(v) && c == 's') { temp = v; Py_INCREF(temp); @@ -9733,21 +13674,7 @@ PyObject *PyUnicode_Format(PyObject *format, temp = PyObject_Repr(v); else temp = PyObject_ASCII(v); - if (temp == NULL) - goto onError; - if (PyUnicode_Check(temp)) - /* nothing to do */; - else { - Py_DECREF(temp); - PyErr_SetString(PyExc_TypeError, - "%s argument has non-string str()"); - goto onError; - } } - pbuf = PyUnicode_AS_UNICODE(temp); - len = PyUnicode_GET_SIZE(temp); - if (prec >= 0 && len > prec) - len = prec; break; case 'i': @@ -9756,6 +13683,32 @@ PyObject *PyUnicode_Format(PyObject *format, case 'o': case 'x': case 'X': + if (PyLong_CheckExact(v) + && width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + switch(c) + { + case 'd': + case 'i': + case 'u': + if (_PyLong_FormatWriter(&writer, v, 10, flags & F_ALT) == -1) + goto onError; + goto nextarg; + case 'x': + if (_PyLong_FormatWriter(&writer, v, 16, flags & F_ALT) == -1) + goto onError; + goto nextarg; + case 'o': + if (_PyLong_FormatWriter(&writer, v, 8, flags & F_ALT) == -1) + goto onError; + goto nextarg; + default: + break; + } + } + isnumok = 0; if (PyNumber_Check(v)) { PyObject *iobj=NULL; @@ -9770,13 +13723,9 @@ PyObject *PyUnicode_Format(PyObject *format, if (iobj!=NULL) { if (PyLong_Check(iobj)) { isnumok = 1; + sign = 1; temp = formatlong(iobj, flags, prec, (c == 'i'? 'd': c)); Py_DECREF(iobj); - if (!temp) - goto onError; - pbuf = PyUnicode_AS_UNICODE(temp); - len = PyUnicode_GET_SIZE(temp); - sign = 1; } else { Py_DECREF(iobj); @@ -9799,22 +13748,38 @@ PyObject *PyUnicode_Format(PyObject *format, case 'F': case 'g': case 'G': - temp = formatfloat(v, flags, prec, c); - if (!temp) - goto onError; - pbuf = PyUnicode_AS_UNICODE(temp); - len = PyUnicode_GET_SIZE(temp); + if (width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (formatfloat(v, flags, prec, c, NULL, &writer) == -1) + goto onError; + goto nextarg; + } + sign = 1; if (flags & F_ZERO) fill = '0'; + if (formatfloat(v, flags, prec, c, &temp, NULL) == -1) + temp = NULL; break; case 'c': - pbuf = formatbuf; - len = formatchar(pbuf, sizeof(formatbuf)/sizeof(Py_UNICODE), v); - if (len < 0) + { + Py_UCS4 ch = formatchar(v); + if (ch == (Py_UCS4) -1) goto onError; + if (width == -1 && prec == -1) { + /* Fast path */ + if (_PyUnicodeWriter_Prepare(&writer, 1, ch) == -1) + goto onError; + PyUnicode_WRITE(writer.kind, writer.data, writer.pos, ch); + writer.pos += 1; + goto nextarg; + } + temp = PyUnicode_FromOrdinal(ch); break; + } default: PyErr_Format(PyExc_ValueError, @@ -9822,90 +13787,139 @@ PyObject *PyUnicode_Format(PyObject *format, "at index %zd", (31<=c && c<=126) ? (char)c : '?', (int)c, - (Py_ssize_t)(fmt - 1 - - PyUnicode_AS_UNICODE(uformat))); + fmtpos - 1); + goto onError; + } + if (temp == NULL) + goto onError; + assert (PyUnicode_Check(temp)); + + if (width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + if (_PyUnicodeWriter_WriteStr(&writer, temp) == -1) + goto onError; + goto nextarg; + } + + if (PyUnicode_READY(temp) == -1) { + Py_CLEAR(temp); goto onError; } + kind = PyUnicode_KIND(temp); + pbuf = PyUnicode_DATA(temp); + len = PyUnicode_GET_LENGTH(temp); + + if (c == 's' || c == 'r' || c == 'a') { + if (prec >= 0 && len > prec) + len = prec; + } + + /* pbuf is initialized here. */ + pindex = 0; if (sign) { - if (*pbuf == '-' || *pbuf == '+') { - sign = *pbuf++; + Py_UCS4 ch = PyUnicode_READ(kind, pbuf, pindex); + if (ch == '-' || ch == '+') { + signchar = ch; len--; + pindex++; } else if (flags & F_SIGN) - sign = '+'; + signchar = '+'; else if (flags & F_BLANK) - sign = ' '; + signchar = ' '; else sign = 0; } if (width < len) width = len; - if (rescnt - (sign != 0) < width) { - reslen -= rescnt; - rescnt = width + fmtcnt + 100; - reslen += rescnt; - if (reslen < 0) { - Py_XDECREF(temp); - PyErr_NoMemory(); - goto onError; + + /* Compute the length and maximum character of the + written characters */ + bufmaxchar = 127; + if (!(flags & F_LJUST)) { + if (sign) { + if ((width-1) > len) + bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill); } - if (_PyUnicode_Resize(&result, reslen) < 0) { - Py_XDECREF(temp); - goto onError; + else { + if (width > len) + bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill); } - res = PyUnicode_AS_UNICODE(result) - + reslen - rescnt; } + maxchar = _PyUnicode_FindMaxChar(temp, 0, pindex+len); + bufmaxchar = MAX_MAXCHAR(bufmaxchar, maxchar); + + buflen = width; + if (sign && len == width) + buflen++; + + if (_PyUnicodeWriter_Prepare(&writer, buflen, bufmaxchar) == -1) + goto onError; + + /* Write characters */ if (sign) { - if (fill != ' ') - *res++ = sign; - rescnt--; + if (fill != ' ') { + PyUnicode_WRITE(writer.kind, writer.data, writer.pos, signchar); + writer.pos += 1; + } if (width > len) width--; } if ((flags & F_ALT) && (c == 'x' || c == 'X' || c == 'o')) { - assert(pbuf[0] == '0'); - assert(pbuf[1] == c); + assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); + assert(PyUnicode_READ(kind, pbuf, pindex + 1) == c); if (fill != ' ') { - *res++ = *pbuf++; - *res++ = *pbuf++; + PyUnicode_WRITE(writer.kind, writer.data, writer.pos, '0'); + PyUnicode_WRITE(writer.kind, writer.data, writer.pos+1, c); + writer.pos += 2; + pindex += 2; } - rescnt -= 2; width -= 2; if (width < 0) width = 0; len -= 2; } if (width > len && !(flags & F_LJUST)) { - do { - --rescnt; - *res++ = fill; - } while (--width > len); + sublen = width - len; + FILL(writer.kind, writer.data, fill, writer.pos, sublen); + writer.pos += sublen; + width = len; } if (fill == ' ') { - if (sign) - *res++ = sign; + if (sign) { + PyUnicode_WRITE(writer.kind, writer.data, writer.pos, signchar); + writer.pos += 1; + } if ((flags & F_ALT) && (c == 'x' || c == 'X' || c == 'o')) { - assert(pbuf[0] == '0'); - assert(pbuf[1] == c); - *res++ = *pbuf++; - *res++ = *pbuf++; + assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); + assert(PyUnicode_READ(kind, pbuf, pindex+1) == c); + PyUnicode_WRITE(writer.kind, writer.data, writer.pos, '0'); + PyUnicode_WRITE(writer.kind, writer.data, writer.pos+1, c); + writer.pos += 2; + pindex += 2; } } - Py_UNICODE_COPY(res, pbuf, len); - res += len; - rescnt -= len; - while (--width >= len) { - --rescnt; - *res++ = ' '; + + if (len) { + _PyUnicode_FastCopyCharacters(writer.buffer, writer.pos, + temp, pindex, len); + writer.pos += len; + } + if (width > len) { + sublen = width - len; + FILL(writer.kind, writer.data, ' ', writer.pos, sublen); + writer.pos += sublen; } + +nextarg: if (dict && (argidx < arglen) && c != '%') { PyErr_SetString(PyExc_TypeError, "not all arguments converted during string formatting"); - Py_XDECREF(temp); goto onError; } - Py_XDECREF(temp); + Py_CLEAR(temp); } /* '%' */ } /* until end */ if (argidx < arglen && !dict) { @@ -9914,17 +13928,19 @@ PyObject *PyUnicode_Format(PyObject *format, goto onError; } - if (_PyUnicode_Resize(&result, reslen - rescnt) < 0) - goto onError; if (args_owned) { Py_DECREF(args); } Py_DECREF(uformat); - return (PyObject *)result; + Py_XDECREF(temp); + Py_XDECREF(second); + return _PyUnicodeWriter_Finish(&writer); onError: - Py_XDECREF(result); Py_DECREF(uformat); + Py_XDECREF(temp); + Py_XDECREF(second); + _PyUnicodeWriter_Dealloc(&writer); if (args_owned) { Py_DECREF(args); } @@ -9947,8 +13963,10 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:str", kwlist, &x, &encoding, &errors)) return NULL; - if (x == NULL) - return (PyObject *)_PyUnicode_New(0); + if (x == NULL) { + Py_INCREF(unicode_empty); + return unicode_empty; + } if (encoding == NULL && errors == NULL) return PyObject_Str(x); else @@ -9958,31 +13976,101 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyUnicodeObject *tmp, *pnew; - Py_ssize_t n; + PyObject *unicode, *self; + Py_ssize_t length, char_size; + int share_wstr, share_utf8; + unsigned int kind; + void *data; assert(PyType_IsSubtype(type, &PyUnicode_Type)); - tmp = (PyUnicodeObject *)unicode_new(&PyUnicode_Type, args, kwds); - if (tmp == NULL) + + unicode = unicode_new(&PyUnicode_Type, args, kwds); + if (unicode == NULL) return NULL; - assert(PyUnicode_Check(tmp)); - pnew = (PyUnicodeObject *) type->tp_alloc(type, n = tmp->length); - if (pnew == NULL) { - Py_DECREF(tmp); + assert(_PyUnicode_CHECK(unicode)); + if (PyUnicode_READY(unicode) == -1) { + Py_DECREF(unicode); return NULL; } - pnew->str = (Py_UNICODE*) PyObject_MALLOC(sizeof(Py_UNICODE) * (n+1)); - if (pnew->str == NULL) { - _Py_ForgetReference((PyObject *)pnew); - PyObject_Del(pnew); - Py_DECREF(tmp); - return PyErr_NoMemory(); + + self = type->tp_alloc(type, 0); + if (self == NULL) { + Py_DECREF(unicode); + return NULL; } - Py_UNICODE_COPY(pnew->str, tmp->str, n+1); - pnew->length = n; - pnew->hash = tmp->hash; - Py_DECREF(tmp); - return (PyObject *)pnew; + kind = PyUnicode_KIND(unicode); + length = PyUnicode_GET_LENGTH(unicode); + + _PyUnicode_LENGTH(self) = length; +#ifdef Py_DEBUG + _PyUnicode_HASH(self) = -1; +#else + _PyUnicode_HASH(self) = _PyUnicode_HASH(unicode); +#endif + _PyUnicode_STATE(self).interned = 0; + _PyUnicode_STATE(self).kind = kind; + _PyUnicode_STATE(self).compact = 0; + _PyUnicode_STATE(self).ascii = _PyUnicode_STATE(unicode).ascii; + _PyUnicode_STATE(self).ready = 1; + _PyUnicode_WSTR(self) = NULL; + _PyUnicode_UTF8_LENGTH(self) = 0; + _PyUnicode_UTF8(self) = NULL; + _PyUnicode_WSTR_LENGTH(self) = 0; + _PyUnicode_DATA_ANY(self) = NULL; + + share_utf8 = 0; + share_wstr = 0; + if (kind == PyUnicode_1BYTE_KIND) { + char_size = 1; + if (PyUnicode_MAX_CHAR_VALUE(unicode) < 128) + share_utf8 = 1; + } + else if (kind == PyUnicode_2BYTE_KIND) { + char_size = 2; + if (sizeof(wchar_t) == 2) + share_wstr = 1; + } + else { + assert(kind == PyUnicode_4BYTE_KIND); + char_size = 4; + if (sizeof(wchar_t) == 4) + share_wstr = 1; + } + + /* Ensure we won't overflow the length. */ + if (length > (PY_SSIZE_T_MAX / char_size - 1)) { + PyErr_NoMemory(); + goto onError; + } + data = PyObject_MALLOC((length + 1) * char_size); + if (data == NULL) { + PyErr_NoMemory(); + goto onError; + } + + _PyUnicode_DATA_ANY(self) = data; + if (share_utf8) { + _PyUnicode_UTF8_LENGTH(self) = length; + _PyUnicode_UTF8(self) = data; + } + if (share_wstr) { + _PyUnicode_WSTR_LENGTH(self) = length; + _PyUnicode_WSTR(self) = (wchar_t *)data; + } + + Py_MEMCPY(data, PyUnicode_DATA(unicode), + kind * (length + 1)); + assert(_PyUnicode_CheckConsistency(self, 1)); +#ifdef Py_DEBUG + _PyUnicode_HASH(self) = _PyUnicode_HASH(unicode); +#endif + Py_DECREF(unicode); + return self; + +onError: + Py_DECREF(unicode); + Py_DECREF(self); + return NULL; } PyDoc_STRVAR(unicode_doc, @@ -10045,12 +14133,12 @@ PyTypeObject PyUnicode_Type = { /* Initialize the Unicode implementation */ -void _PyUnicode_Init(void) +int _PyUnicode_Init(void) { int i; /* XXX - move this array to unicodectype.c ? */ - Py_UNICODE linebreak[] = { + Py_UCS2 linebreak[] = { 0x000A, /* LINE FEED */ 0x000D, /* CARRIAGE RETURN */ 0x001C, /* FILE SEPARATOR */ @@ -10062,11 +14150,10 @@ void _PyUnicode_Init(void) }; /* Init the implementation */ - free_list = NULL; - numfree = 0; - unicode_empty = _PyUnicode_New(0); + unicode_empty = PyUnicode_New(0, 0); if (!unicode_empty) - return; + Py_FatalError("Can't create empty string"); + assert(_PyUnicode_CheckConsistency(unicode_empty, 1)); for (i = 0; i < 256; i++) unicode_latin1[i] = NULL; @@ -10075,8 +14162,8 @@ void _PyUnicode_Init(void) /* initialize the linebreak bloom filter */ bloom_linebreak = make_bloom_mask( - linebreak, sizeof(linebreak) / sizeof(linebreak[0]) - ); + PyUnicode_2BYTE_KIND, linebreak, + Py_ARRAY_LENGTH(linebreak)); PyType_Ready(&EncodingMapType); @@ -10085,6 +14172,15 @@ void _PyUnicode_Init(void) if (PyType_Ready(&PyFormatterIter_Type) < 0) Py_FatalError("Can't initialize formatter iter type"); + +#ifdef HAVE_MBCS + winver.dwOSVersionInfoSize = sizeof(winver); + if (!GetVersionEx((OSVERSIONINFO*)&winver)) { + PyErr_SetFromWindowsErr(0); + return -1; + } +#endif + return 0; } /* Finalize the Unicode implementation */ @@ -10092,21 +14188,7 @@ void _PyUnicode_Init(void) int PyUnicode_ClearFreeList(void) { - int freelist_size = numfree; - PyUnicodeObject *u; - - for (u = free_list; u != NULL;) { - PyUnicodeObject *v = u; - u = *(PyUnicodeObject **)u; - if (v->str) - PyObject_DEL(v->str); - Py_XDECREF(v->defenc); - PyObject_Del(v); - numfree--; - } - free_list = NULL; - assert(numfree == 0); - return freelist_size; + return 0; } void @@ -10123,17 +14205,22 @@ _PyUnicode_Fini(void) unicode_latin1[i] = NULL; } } + _PyUnicode_ClearStaticStrings(); (void)PyUnicode_ClearFreeList(); } void PyUnicode_InternInPlace(PyObject **p) { - register PyUnicodeObject *s = (PyUnicodeObject *)(*p); + register PyObject *s = *p; PyObject *t; +#ifdef Py_DEBUG + assert(s != NULL); + assert(_PyUnicode_CHECK(s)); +#else if (s == NULL || !PyUnicode_Check(s)) - Py_FatalError( - "PyUnicode_InternInPlace: unicode strings only please!"); + return; +#endif /* If it's a subclass, we don't really know what putting it in the interned dict might do. */ if (!PyUnicode_CheckExact(s)) @@ -10151,7 +14238,7 @@ PyUnicode_InternInPlace(PyObject **p) though the key is present in the dictionary, namely when this happens during a stack overflow. */ Py_ALLOW_RECURSION - t = PyDict_GetItem(interned, (PyObject *)s); + t = PyDict_GetItem(interned, s); Py_END_ALLOW_RECURSION if (t) { @@ -10162,7 +14249,7 @@ PyUnicode_InternInPlace(PyObject **p) } PyThreadState_GET()->recursion_critical = 1; - if (PyDict_SetItem(interned, (PyObject *)s, (PyObject *)s) < 0) { + if (PyDict_SetItem(interned, s, s) < 0) { PyErr_Clear(); PyThreadState_GET()->recursion_critical = 0; return; @@ -10171,7 +14258,7 @@ PyUnicode_InternInPlace(PyObject **p) /* The two references in interned are not counted by refcnt. The deallocator will take care of this */ Py_REFCNT(s) -= 2; - PyUnicode_CHECK_INTERNED(s) = SSTATE_INTERNED_MORTAL; + _PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL; } void @@ -10179,7 +14266,7 @@ PyUnicode_InternImmortal(PyObject **p) { PyUnicode_InternInPlace(p); if (PyUnicode_CHECK_INTERNED(*p) != SSTATE_INTERNED_IMMORTAL) { - PyUnicode_CHECK_INTERNED(*p) = SSTATE_INTERNED_IMMORTAL; + _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL; Py_INCREF(*p); } } @@ -10194,10 +14281,11 @@ PyUnicode_InternFromString(const char *cp) return s; } -void _Py_ReleaseInternedUnicodeStrings(void) +void +_Py_ReleaseInternedUnicodeStrings(void) { PyObject *keys; - PyUnicodeObject *s; + PyObject *s; Py_ssize_t i, n; Py_ssize_t immortal_size = 0, mortal_size = 0; @@ -10218,23 +14306,27 @@ void _Py_ReleaseInternedUnicodeStrings(void) fprintf(stderr, "releasing %" PY_FORMAT_SIZE_T "d interned strings\n", n); for (i = 0; i < n; i++) { - s = (PyUnicodeObject *) PyList_GET_ITEM(keys, i); - switch (s->state) { + s = PyList_GET_ITEM(keys, i); + if (PyUnicode_READY(s) == -1) { + assert(0 && "could not ready string"); + fprintf(stderr, "could not ready string\n"); + } + switch (PyUnicode_CHECK_INTERNED(s)) { case SSTATE_NOT_INTERNED: /* XXX Shouldn't happen */ break; case SSTATE_INTERNED_IMMORTAL: Py_REFCNT(s) += 1; - immortal_size += s->length; + immortal_size += PyUnicode_GET_LENGTH(s); break; case SSTATE_INTERNED_MORTAL: Py_REFCNT(s) += 2; - mortal_size += s->length; + mortal_size += PyUnicode_GET_LENGTH(s); break; default: Py_FatalError("Inconsistent interned string state."); } - s->state = SSTATE_NOT_INTERNED; + _PyUnicode_STATE(s).interned = SSTATE_NOT_INTERNED; } fprintf(stderr, "total size of all interned strings: " "%" PY_FORMAT_SIZE_T "d/%" PY_FORMAT_SIZE_T "d " @@ -10251,7 +14343,7 @@ void _Py_ReleaseInternedUnicodeStrings(void) typedef struct { PyObject_HEAD Py_ssize_t it_index; - PyUnicodeObject *it_seq; /* Set to NULL when iterator is exhausted */ + PyObject *it_seq; /* Set to NULL when iterator is exhausted */ } unicodeiterobject; static void @@ -10272,18 +14364,19 @@ unicodeiter_traverse(unicodeiterobject *it, visitproc visit, void *arg) static PyObject * unicodeiter_next(unicodeiterobject *it) { - PyUnicodeObject *seq; - PyObject *item; + PyObject *seq, *item; assert(it != NULL); seq = it->it_seq; if (seq == NULL) return NULL; - assert(PyUnicode_Check(seq)); + assert(_PyUnicode_CHECK(seq)); - if (it->it_index < PyUnicode_GET_SIZE(seq)) { - item = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(seq)+it->it_index, 1); + if (it->it_index < PyUnicode_GET_LENGTH(seq)) { + int kind = PyUnicode_KIND(seq); + void *data = PyUnicode_DATA(seq); + Py_UCS4 chr = PyUnicode_READ(kind, data, it->it_index); + item = PyUnicode_FromOrdinal(chr); if (item != NULL) ++it->it_index; return item; @@ -10299,15 +14392,49 @@ unicodeiter_len(unicodeiterobject *it) { Py_ssize_t len = 0; if (it->it_seq) - len = PyUnicode_GET_SIZE(it->it_seq) - it->it_index; + len = PyUnicode_GET_LENGTH(it->it_seq) - it->it_index; return PyLong_FromSsize_t(len); } PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); +static PyObject * +unicodeiter_reduce(unicodeiterobject *it) +{ + if (it->it_seq != NULL) { + return Py_BuildValue("N(O)n", _PyObject_GetBuiltin("iter"), + it->it_seq, it->it_index); + } else { + PyObject *u = PyUnicode_FromUnicode(NULL, 0); + if (u == NULL) + return NULL; + return Py_BuildValue("N(N)", _PyObject_GetBuiltin("iter"), u); + } +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); + +static PyObject * +unicodeiter_setstate(unicodeiterobject *it, PyObject *state) +{ + Py_ssize_t index = PyLong_AsSsize_t(state); + if (index == -1 && PyErr_Occurred()) + return NULL; + if (index < 0) + index = 0; + it->it_index = index; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); + static PyMethodDef unicodeiter_methods[] = { {"__length_hint__", (PyCFunction)unicodeiter_len, METH_NOARGS, length_hint_doc}, + {"__reduce__", (PyCFunction)unicodeiter_reduce, METH_NOARGS, + reduce_doc}, + {"__setstate__", (PyCFunction)unicodeiter_setstate, METH_O, + setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -10353,16 +14480,19 @@ unicode_iter(PyObject *seq) PyErr_BadInternalCall(); return NULL; } + if (PyUnicode_READY(seq) == -1) + return NULL; it = PyObject_GC_New(unicodeiterobject, &PyUnicodeIter_Type); if (it == NULL) return NULL; it->it_index = 0; Py_INCREF(seq); - it->it_seq = (PyUnicodeObject *)seq; + it->it_seq = seq; _PyObject_GC_TRACK(it); return (PyObject *)it; } + size_t Py_UNICODE_strlen(const Py_UNICODE *u) { @@ -10454,25 +14584,31 @@ Py_UNICODE_strrchr(const Py_UNICODE *s, Py_UNICODE c) } Py_UNICODE* -PyUnicode_AsUnicodeCopy(PyObject *object) +PyUnicode_AsUnicodeCopy(PyObject *unicode) { - PyUnicodeObject *unicode = (PyUnicodeObject *)object; - Py_UNICODE *copy; - Py_ssize_t size; + Py_UNICODE *u, *copy; + Py_ssize_t len, size; + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); + return NULL; + } + u = PyUnicode_AsUnicodeAndSize(unicode, &len); + if (u == NULL) + return NULL; /* Ensure we won't overflow the size. */ - if (PyUnicode_GET_SIZE(unicode) > ((PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) - 1)) { + if (len > ((PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) - 1)) { PyErr_NoMemory(); return NULL; } - size = PyUnicode_GET_SIZE(unicode) + 1; /* copy the nul character */ + size = len + 1; /* copy the null character */ size *= sizeof(Py_UNICODE); copy = PyMem_Malloc(size); if (copy == NULL) { PyErr_NoMemory(); return NULL; } - memcpy(copy, PyUnicode_AS_UNICODE(unicode), size); + memcpy(copy, u, size); return copy; } diff --git a/Objects/unicodetype_db.h b/Objects/unicodetype_db.h index 88a7912..46a92bb 100644 --- a/Objects/unicodetype_db.h +++ b/Objects/unicodetype_db.h @@ -8,184 +8,992 @@ const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { {0, 0, 0, 0, 0, 48}, {0, 0, 0, 0, 0, 1056}, {0, 0, 0, 0, 0, 1024}, - {0, 0, 0, 0, 0, 5638}, - {0, 0, 0, 1, 1, 5638}, - {0, 0, 0, 2, 2, 5638}, - {0, 0, 0, 3, 3, 5638}, - {0, 0, 0, 4, 4, 5638}, - {0, 0, 0, 5, 5, 5638}, - {0, 0, 0, 6, 6, 5638}, - {0, 0, 0, 7, 7, 5638}, - {0, 0, 0, 8, 8, 5638}, - {0, 0, 0, 9, 9, 5638}, - {0, 32, 0, 0, 0, 1921}, - {0, 0, 0, 0, 0, 1536}, - {65504, 0, 65504, 0, 0, 1801}, - {0, 0, 0, 0, 0, 1801}, - {0, 0, 0, 0, 2, 5124}, - {0, 0, 0, 0, 3, 5124}, - {743, 0, 743, 0, 0, 1801}, - {0, 0, 0, 0, 1, 5124}, {0, 0, 0, 0, 0, 5120}, - {121, 0, 121, 0, 0, 1801}, - {0, 1, 0, 0, 0, 1921}, - {65535, 0, 65535, 0, 0, 1801}, - {0, 65337, 0, 0, 0, 1921}, - {65304, 0, 65304, 0, 0, 1801}, - {0, 65415, 0, 0, 0, 1921}, - {65236, 0, 65236, 0, 0, 1801}, - {195, 0, 195, 0, 0, 1801}, - {0, 210, 0, 0, 0, 1921}, - {0, 206, 0, 0, 0, 1921}, - {0, 205, 0, 0, 0, 1921}, - {0, 79, 0, 0, 0, 1921}, - {0, 202, 0, 0, 0, 1921}, - {0, 203, 0, 0, 0, 1921}, - {0, 207, 0, 0, 0, 1921}, - {97, 0, 97, 0, 0, 1801}, - {0, 211, 0, 0, 0, 1921}, - {0, 209, 0, 0, 0, 1921}, - {163, 0, 163, 0, 0, 1801}, - {0, 213, 0, 0, 0, 1921}, - {130, 0, 130, 0, 0, 1801}, - {0, 214, 0, 0, 0, 1921}, - {0, 218, 0, 0, 0, 1921}, - {0, 217, 0, 0, 0, 1921}, - {0, 219, 0, 0, 0, 1921}, + {0, 0, 0, 0, 0, 3590}, + {0, 0, 0, 1, 1, 3590}, + {0, 0, 0, 2, 2, 3590}, + {0, 0, 0, 3, 3, 3590}, + {0, 0, 0, 4, 4, 3590}, + {0, 0, 0, 5, 5, 3590}, + {0, 0, 0, 6, 6, 3590}, + {0, 0, 0, 7, 7, 3590}, + {0, 0, 0, 8, 8, 3590}, + {0, 0, 0, 9, 9, 3590}, + {0, 32, 0, 0, 0, 10113}, + {0, 0, 0, 0, 0, 1536}, + {-32, 0, -32, 0, 0, 9993}, + {0, 0, 0, 0, 0, 9993}, + {0, 0, 0, 0, 0, 4096}, + {0, 0, 0, 0, 2, 3076}, + {0, 0, 0, 0, 3, 3076}, + {16777218, 17825792, 16777218, 0, 0, 26377}, + {0, 0, 0, 0, 0, 5632}, + {0, 0, 0, 0, 1, 3076}, + {0, 0, 0, 0, 0, 3072}, + {33554438, 18874371, 33554440, 0, 0, 26377}, + {121, 0, 121, 0, 0, 9993}, + {0, 1, 0, 0, 0, 10113}, + {-1, 0, -1, 0, 0, 9993}, + {16777228, 33554442, 16777228, 0, 0, 26497}, + {-232, 0, -232, 0, 0, 9993}, + {33554448, 18874381, 33554448, 0, 0, 26377}, + {0, -121, 0, 0, 0, 10113}, + {16777236, 17825810, 16777236, 0, 0, 26377}, + {195, 0, 195, 0, 0, 9993}, + {0, 210, 0, 0, 0, 10113}, + {0, 206, 0, 0, 0, 10113}, + {0, 205, 0, 0, 0, 10113}, + {0, 79, 0, 0, 0, 10113}, + {0, 202, 0, 0, 0, 10113}, + {0, 203, 0, 0, 0, 10113}, + {0, 207, 0, 0, 0, 10113}, + {97, 0, 97, 0, 0, 9993}, + {0, 211, 0, 0, 0, 10113}, + {0, 209, 0, 0, 0, 10113}, + {163, 0, 163, 0, 0, 9993}, + {0, 213, 0, 0, 0, 10113}, + {130, 0, 130, 0, 0, 9993}, + {0, 214, 0, 0, 0, 10113}, + {0, 218, 0, 0, 0, 10113}, + {0, 217, 0, 0, 0, 10113}, + {0, 219, 0, 0, 0, 10113}, {0, 0, 0, 0, 0, 1793}, - {56, 0, 56, 0, 0, 1801}, - {0, 2, 1, 0, 0, 1921}, - {65535, 1, 0, 0, 0, 1857}, - {65534, 0, 65535, 0, 0, 1801}, - {65457, 0, 65457, 0, 0, 1801}, - {0, 65439, 0, 0, 0, 1921}, - {0, 65480, 0, 0, 0, 1921}, - {0, 65406, 0, 0, 0, 1921}, - {0, 10795, 0, 0, 0, 1921}, - {0, 65373, 0, 0, 0, 1921}, - {0, 10792, 0, 0, 0, 1921}, - {10815, 0, 10815, 0, 0, 1801}, - {0, 65341, 0, 0, 0, 1921}, - {0, 69, 0, 0, 0, 1921}, - {0, 71, 0, 0, 0, 1921}, - {10783, 0, 10783, 0, 0, 1801}, - {10780, 0, 10780, 0, 0, 1801}, - {10782, 0, 10782, 0, 0, 1801}, - {65326, 0, 65326, 0, 0, 1801}, - {65330, 0, 65330, 0, 0, 1801}, - {65331, 0, 65331, 0, 0, 1801}, - {65334, 0, 65334, 0, 0, 1801}, - {65333, 0, 65333, 0, 0, 1801}, - {65329, 0, 65329, 0, 0, 1801}, - {42893, 613, 42893, 0, 0, 3849}, - {65327, 0, 65327, 0, 0, 1801}, - {65325, 0, 65325, 0, 0, 1801}, - {10743, 0, 10743, 0, 0, 1801}, - {10749, 0, 10749, 0, 0, 1801}, - {65323, 0, 65323, 0, 0, 1801}, - {65322, 0, 65322, 0, 0, 1801}, - {10727, 0, 10727, 0, 0, 1801}, - {65318, 0, 65318, 0, 0, 1801}, - {65467, 0, 65467, 0, 0, 1801}, - {65319, 0, 65319, 0, 0, 1801}, - {65465, 0, 65465, 0, 0, 1801}, - {65317, 0, 65317, 0, 0, 1801}, - {84, 0, 84, 0, 0, 1536}, - {0, 0, 0, 0, 0, 1025}, - {0, 38, 0, 0, 0, 1921}, - {0, 37, 0, 0, 0, 1921}, - {0, 64, 0, 0, 0, 1921}, - {0, 63, 0, 0, 0, 1921}, - {65498, 0, 65498, 0, 0, 1801}, - {65499, 0, 65499, 0, 0, 1801}, - {65505, 0, 65505, 0, 0, 1801}, - {65472, 0, 65472, 0, 0, 1801}, - {65473, 0, 65473, 0, 0, 1801}, - {0, 8, 0, 0, 0, 1921}, - {65474, 0, 65474, 0, 0, 1801}, - {65479, 0, 65479, 0, 0, 1801}, - {0, 0, 0, 0, 0, 1921}, - {65489, 0, 65489, 0, 0, 1801}, - {65482, 0, 65482, 0, 0, 1801}, - {65528, 0, 65528, 0, 0, 1801}, - {65450, 0, 65450, 0, 0, 1801}, - {65456, 0, 65456, 0, 0, 1801}, - {7, 0, 7, 0, 0, 1801}, - {0, 65476, 0, 0, 0, 1921}, - {65440, 0, 65440, 0, 0, 1801}, - {0, 65529, 0, 0, 0, 1921}, - {0, 80, 0, 0, 0, 1921}, - {0, 15, 0, 0, 0, 1921}, - {65521, 0, 65521, 0, 0, 1801}, - {0, 48, 0, 0, 0, 1921}, - {65488, 0, 65488, 0, 0, 1801}, + {56, 0, 56, 0, 0, 9993}, + {0, 2, 1, 0, 0, 10113}, + {-1, 1, 0, 0, 0, 10049}, + {-2, 0, -1, 0, 0, 9993}, + {-79, 0, -79, 0, 0, 9993}, + {33554456, 18874389, 33554456, 0, 0, 26377}, + {0, -97, 0, 0, 0, 10113}, + {0, -56, 0, 0, 0, 10113}, + {0, -130, 0, 0, 0, 10113}, + {0, 10795, 0, 0, 0, 10113}, + {0, -163, 0, 0, 0, 10113}, + {0, 10792, 0, 0, 0, 10113}, + {10815, 0, 10815, 0, 0, 9993}, + {0, -195, 0, 0, 0, 10113}, + {0, 69, 0, 0, 0, 10113}, + {0, 71, 0, 0, 0, 10113}, + {10783, 0, 10783, 0, 0, 9993}, + {10780, 0, 10780, 0, 0, 9993}, + {10782, 0, 10782, 0, 0, 9993}, + {-210, 0, -210, 0, 0, 9993}, + {-206, 0, -206, 0, 0, 9993}, + {-205, 0, -205, 0, 0, 9993}, + {-202, 0, -202, 0, 0, 9993}, + {-203, 0, -203, 0, 0, 9993}, + {-207, 0, -207, 0, 0, 9993}, + {42280, 0, 42280, 0, 0, 9993}, + {42308, 0, 42308, 0, 0, 9993}, + {-209, 0, -209, 0, 0, 9993}, + {-211, 0, -211, 0, 0, 9993}, + {10743, 0, 10743, 0, 0, 9993}, + {10749, 0, 10749, 0, 0, 9993}, + {-213, 0, -213, 0, 0, 9993}, + {-214, 0, -214, 0, 0, 9993}, + {10727, 0, 10727, 0, 0, 9993}, + {-218, 0, -218, 0, 0, 9993}, + {-69, 0, -69, 0, 0, 9993}, + {-217, 0, -217, 0, 0, 9993}, + {-71, 0, -71, 0, 0, 9993}, + {-219, 0, -219, 0, 0, 9993}, + {0, 0, 0, 0, 0, 14089}, + {0, 0, 0, 0, 0, 5889}, + {16777244, 17825818, 16777244, 0, 0, 30216}, + {0, 0, 0, 0, 0, 13321}, + {0, 38, 0, 0, 0, 10113}, + {0, 37, 0, 0, 0, 10113}, + {0, 64, 0, 0, 0, 10113}, + {0, 63, 0, 0, 0, 10113}, + {50331681, 19922973, 50331681, 0, 0, 26377}, + {-38, 0, -38, 0, 0, 9993}, + {-37, 0, -37, 0, 0, 9993}, + {50331688, 19922980, 50331688, 0, 0, 26377}, + {16777261, 17825835, 16777261, 0, 0, 26377}, + {-64, 0, -64, 0, 0, 9993}, + {-63, 0, -63, 0, 0, 9993}, + {0, 8, 0, 0, 0, 10113}, + {16777264, 17825838, 16777264, 0, 0, 26377}, + {16777267, 17825841, 16777267, 0, 0, 26377}, + {0, 0, 0, 0, 0, 10113}, + {16777270, 17825844, 16777270, 0, 0, 26377}, + {16777273, 17825847, 16777273, 0, 0, 26377}, + {-8, 0, -8, 0, 0, 9993}, + {16777276, 17825850, 16777276, 0, 0, 26377}, + {16777279, 17825853, 16777279, 0, 0, 26377}, + {7, 0, 7, 0, 0, 9993}, + {0, -60, 0, 0, 0, 10113}, + {16777282, 17825856, 16777282, 0, 0, 26377}, + {0, -7, 0, 0, 0, 10113}, + {0, 80, 0, 0, 0, 10113}, + {-80, 0, -80, 0, 0, 9993}, + {0, 15, 0, 0, 0, 10113}, + {-15, 0, -15, 0, 0, 9993}, + {0, 48, 0, 0, 0, 10113}, + {-48, 0, -48, 0, 0, 9993}, + {33554502, 18874435, 33554504, 0, 0, 26377}, {0, 0, 0, 0, 0, 1537}, - {0, 7264, 0, 0, 0, 1921}, - {0, 0, 0, 0, 1, 5636}, - {0, 0, 0, 0, 2, 5636}, - {0, 0, 0, 0, 3, 5636}, - {0, 0, 0, 0, 4, 5636}, - {0, 0, 0, 0, 5, 5636}, - {0, 0, 0, 0, 6, 5636}, - {0, 0, 0, 0, 7, 5636}, - {0, 0, 0, 0, 8, 5636}, - {0, 0, 0, 0, 9, 5636}, - {0, 0, 0, 0, 0, 5888}, - {42877, 7545, 42877, 0, 0, 3849}, - {3814, 0, 3814, 0, 0, 1801}, - {65477, 0, 65477, 0, 0, 1801}, - {0, 57921, 0, 0, 0, 1921}, - {8, 0, 8, 0, 0, 1801}, - {0, 65528, 0, 0, 0, 1921}, - {74, 0, 74, 0, 0, 1801}, - {86, 0, 86, 0, 0, 1801}, - {100, 0, 100, 0, 0, 1801}, - {128, 0, 128, 0, 0, 1801}, - {112, 0, 112, 0, 0, 1801}, - {126, 0, 126, 0, 0, 1801}, - {0, 65528, 0, 0, 0, 1857}, - {9, 0, 9, 0, 0, 1801}, - {0, 65462, 0, 0, 0, 1921}, - {0, 65527, 0, 0, 0, 1857}, - {58331, 0, 58331, 0, 0, 1801}, - {0, 65450, 0, 0, 0, 1921}, - {0, 65436, 0, 0, 0, 1921}, - {0, 65424, 0, 0, 0, 1921}, - {0, 65408, 0, 0, 0, 1921}, - {0, 65410, 0, 0, 0, 1921}, - {0, 0, 0, 0, 0, 5124}, - {0, 0, 0, 0, 4, 5124}, - {0, 0, 0, 0, 5, 5124}, - {0, 0, 0, 0, 6, 5124}, - {0, 0, 0, 0, 7, 5124}, - {0, 0, 0, 0, 8, 5124}, - {0, 0, 0, 0, 9, 5124}, + {0, 7264, 0, 0, 0, 10113}, + {0, 0, 0, 0, 1, 3588}, + {0, 0, 0, 0, 2, 3588}, + {0, 0, 0, 0, 3, 3588}, + {0, 0, 0, 0, 4, 3588}, + {0, 0, 0, 0, 5, 3588}, + {0, 0, 0, 0, 6, 3588}, + {0, 0, 0, 0, 7, 3588}, + {0, 0, 0, 0, 8, 3588}, + {0, 0, 0, 0, 9, 3588}, + {0, 0, 0, 0, 0, 3840}, + {35332, 0, 35332, 0, 0, 9993}, + {3814, 0, 3814, 0, 0, 9993}, + {33554509, 18874442, 33554509, 0, 0, 26377}, + {33554514, 18874447, 33554514, 0, 0, 26377}, + {33554519, 18874452, 33554519, 0, 0, 26377}, + {33554524, 18874457, 33554524, 0, 0, 26377}, + {33554529, 18874462, 33554529, 0, 0, 26377}, + {16777317, 17825891, 16777317, 0, 0, 26377}, + {16777321, 18874470, 16777321, 0, 0, 26497}, + {8, 0, 8, 0, 0, 9993}, + {0, -8, 0, 0, 0, 10113}, + {33554541, 18874474, 33554541, 0, 0, 26377}, + {50331763, 19923055, 50331763, 0, 0, 26377}, + {50331770, 19923062, 50331770, 0, 0, 26377}, + {50331777, 19923069, 50331777, 0, 0, 26377}, + {74, 0, 74, 0, 0, 9993}, + {86, 0, 86, 0, 0, 9993}, + {100, 0, 100, 0, 0, 9993}, + {128, 0, 128, 0, 0, 9993}, + {112, 0, 112, 0, 0, 9993}, + {126, 0, 126, 0, 0, 9993}, + {33554567, 18874500, 16777353, 0, 0, 26377}, + {33554573, 18874506, 16777359, 0, 0, 26377}, + {33554579, 18874512, 16777365, 0, 0, 26377}, + {33554585, 18874518, 16777371, 0, 0, 26377}, + {33554591, 18874524, 16777377, 0, 0, 26377}, + {33554597, 18874530, 16777383, 0, 0, 26377}, + {33554603, 18874536, 16777389, 0, 0, 26377}, + {33554609, 18874542, 16777395, 0, 0, 26377}, + {33554615, 18874548, 16777401, 0, 0, 26433}, + {33554621, 18874554, 16777407, 0, 0, 26433}, + {33554627, 18874560, 16777413, 0, 0, 26433}, + {33554633, 18874566, 16777419, 0, 0, 26433}, + {33554639, 18874572, 16777425, 0, 0, 26433}, + {33554645, 18874578, 16777431, 0, 0, 26433}, + {33554651, 18874584, 16777437, 0, 0, 26433}, + {33554657, 18874590, 16777443, 0, 0, 26433}, + {33554663, 18874596, 16777449, 0, 0, 26377}, + {33554669, 18874602, 16777455, 0, 0, 26377}, + {33554675, 18874608, 16777461, 0, 0, 26377}, + {33554681, 18874614, 16777467, 0, 0, 26377}, + {33554687, 18874620, 16777473, 0, 0, 26377}, + {33554693, 18874626, 16777479, 0, 0, 26377}, + {33554699, 18874632, 16777485, 0, 0, 26377}, + {33554705, 18874638, 16777491, 0, 0, 26377}, + {33554711, 18874644, 16777497, 0, 0, 26433}, + {33554717, 18874650, 16777503, 0, 0, 26433}, + {33554723, 18874656, 16777509, 0, 0, 26433}, + {33554729, 18874662, 16777515, 0, 0, 26433}, + {33554735, 18874668, 16777521, 0, 0, 26433}, + {33554741, 18874674, 16777527, 0, 0, 26433}, + {33554747, 18874680, 16777533, 0, 0, 26433}, + {33554753, 18874686, 16777539, 0, 0, 26433}, + {33554759, 18874692, 16777545, 0, 0, 26377}, + {33554765, 18874698, 16777551, 0, 0, 26377}, + {33554771, 18874704, 16777557, 0, 0, 26377}, + {33554777, 18874710, 16777563, 0, 0, 26377}, + {33554783, 18874716, 16777569, 0, 0, 26377}, + {33554789, 18874722, 16777575, 0, 0, 26377}, + {33554795, 18874728, 16777581, 0, 0, 26377}, + {33554801, 18874734, 16777587, 0, 0, 26377}, + {33554807, 18874740, 16777593, 0, 0, 26433}, + {33554813, 18874746, 16777599, 0, 0, 26433}, + {33554819, 18874752, 16777605, 0, 0, 26433}, + {33554825, 18874758, 16777611, 0, 0, 26433}, + {33554831, 18874764, 16777617, 0, 0, 26433}, + {33554837, 18874770, 16777623, 0, 0, 26433}, + {33554843, 18874776, 16777629, 0, 0, 26433}, + {33554849, 18874782, 16777635, 0, 0, 26433}, + {33554855, 18874788, 33554857, 0, 0, 26377}, + {33554862, 18874795, 16777648, 0, 0, 26377}, + {33554868, 18874801, 33554870, 0, 0, 26377}, + {33554875, 18874808, 33554875, 0, 0, 26377}, + {50332097, 19923389, 50332100, 0, 0, 26377}, + {0, -74, 0, 0, 0, 10113}, + {33554890, 18874823, 16777676, 0, 0, 26433}, + {16777679, 17826253, 16777679, 0, 0, 26377}, + {33554899, 18874832, 33554901, 0, 0, 26377}, + {33554906, 18874839, 16777692, 0, 0, 26377}, + {33554912, 18874845, 33554914, 0, 0, 26377}, + {33554919, 18874852, 33554919, 0, 0, 26377}, + {50332141, 19923433, 50332144, 0, 0, 26377}, + {0, -86, 0, 0, 0, 10113}, + {33554934, 18874867, 16777720, 0, 0, 26433}, + {50332157, 19923449, 50332157, 0, 0, 26377}, + {50332164, 19923456, 50332164, 0, 0, 26377}, + {33554954, 18874887, 33554954, 0, 0, 26377}, + {50332176, 19923468, 50332176, 0, 0, 26377}, + {0, -100, 0, 0, 0, 10113}, + {50332183, 19923475, 50332183, 0, 0, 26377}, + {50332190, 19923482, 50332190, 0, 0, 26377}, + {33554980, 18874913, 33554980, 0, 0, 26377}, + {33554985, 18874918, 33554985, 0, 0, 26377}, + {50332207, 19923499, 50332207, 0, 0, 26377}, + {0, -112, 0, 0, 0, 10113}, + {33554997, 18874930, 33554999, 0, 0, 26377}, + {33555004, 18874937, 16777790, 0, 0, 26377}, + {33555010, 18874943, 33555012, 0, 0, 26377}, + {33555017, 18874950, 33555017, 0, 0, 26377}, + {50332239, 19923531, 50332242, 0, 0, 26377}, + {0, -128, 0, 0, 0, 10113}, + {0, -126, 0, 0, 0, 10113}, + {33555032, 18874965, 16777818, 0, 0, 26433}, + {0, 0, 0, 0, 0, 3076}, + {0, 0, 0, 0, 4, 3076}, + {0, 0, 0, 0, 5, 3076}, + {0, 0, 0, 0, 6, 3076}, + {0, 0, 0, 0, 7, 3076}, + {0, 0, 0, 0, 8, 3076}, + {0, 0, 0, 0, 9, 3076}, {0, 0, 0, 0, 0, 1792}, - {0, 58019, 0, 0, 0, 1921}, - {0, 57153, 0, 0, 0, 1921}, - {0, 57274, 0, 0, 0, 1921}, - {0, 28, 0, 0, 0, 1921}, - {65508, 0, 65508, 0, 0, 1801}, - {0, 16, 0, 0, 0, 5888}, - {65520, 0, 65520, 0, 0, 5888}, - {0, 26, 0, 0, 0, 1024}, - {65510, 0, 65510, 0, 0, 1024}, - {0, 54793, 0, 0, 0, 1921}, - {0, 61722, 0, 0, 0, 1921}, - {0, 54809, 0, 0, 0, 1921}, - {54741, 0, 54741, 0, 0, 1801}, - {54744, 0, 54744, 0, 0, 1801}, - {0, 54756, 0, 0, 0, 1921}, - {0, 54787, 0, 0, 0, 1921}, - {0, 54753, 0, 0, 0, 1921}, - {0, 54754, 0, 0, 0, 1921}, - {0, 54721, 0, 0, 0, 1921}, - {58272, 0, 58272, 0, 0, 1801}, - {0, 0, 0, 0, 0, 5889}, - {42877, 7545, 42877, 0, 0, 3969}, - {42893, 613, 42893, 0, 0, 3969}, - {0, 40, 0, 0, 0, 1921}, - {65496, 0, 65496, 0, 0, 1801}, + {0, -7517, 0, 0, 0, 10113}, + {0, -8383, 0, 0, 0, 10113}, + {0, -8262, 0, 0, 0, 10113}, + {0, 28, 0, 0, 0, 10113}, + {-28, 0, -28, 0, 0, 9993}, + {0, 16, 0, 0, 0, 12160}, + {-16, 0, -16, 0, 0, 12040}, + {0, 26, 0, 0, 0, 9344}, + {-26, 0, -26, 0, 0, 9224}, + {0, -10743, 0, 0, 0, 10113}, + {0, -3814, 0, 0, 0, 10113}, + {0, -10727, 0, 0, 0, 10113}, + {-10795, 0, -10795, 0, 0, 9993}, + {-10792, 0, -10792, 0, 0, 9993}, + {0, -10780, 0, 0, 0, 10113}, + {0, -10749, 0, 0, 0, 10113}, + {0, -10783, 0, 0, 0, 10113}, + {0, -10782, 0, 0, 0, 10113}, + {0, -10815, 0, 0, 0, 10113}, + {-7264, 0, -7264, 0, 0, 9993}, + {0, 0, 0, 0, 0, 5121}, + {0, 0, 0, 0, 0, 3841}, + {0, -35332, 0, 0, 0, 10113}, + {0, -42280, 0, 0, 0, 10113}, + {0, -42308, 0, 0, 0, 10113}, + {33555038, 18874971, 33555040, 0, 0, 26377}, + {33555045, 18874978, 33555047, 0, 0, 26377}, + {33555052, 18874985, 33555054, 0, 0, 26377}, + {50332276, 19923568, 50332279, 0, 0, 26377}, + {50332286, 19923578, 50332289, 0, 0, 26377}, + {33555079, 18875012, 33555081, 0, 0, 26377}, + {33555086, 18875019, 33555088, 0, 0, 26377}, + {33555093, 18875026, 33555095, 0, 0, 26377}, + {33555100, 18875033, 33555102, 0, 0, 26377}, + {33555107, 18875040, 33555109, 0, 0, 26377}, + {33555114, 18875047, 33555116, 0, 0, 26377}, + {33555121, 18875054, 33555123, 0, 0, 26377}, + {0, 0, 0, 0, 0, 1025}, + {0, 0, 0, 0, 0, 5633}, + {0, 40, 0, 0, 0, 10113}, + {-40, 0, -40, 0, 0, 9993}, +}; + +/* extended case mappings */ + +const Py_UCS4 _PyUnicode_ExtendedCase[] = { + 181, + 956, + 924, + 223, + 115, + 115, + 83, + 83, + 83, + 115, + 105, + 775, + 304, + 329, + 700, + 110, + 700, + 78, + 383, + 115, + 83, + 496, + 106, + 780, + 74, + 780, + 837, + 953, + 921, + 912, + 953, + 776, + 769, + 921, + 776, + 769, + 944, + 965, + 776, + 769, + 933, + 776, + 769, + 962, + 963, + 931, + 976, + 946, + 914, + 977, + 952, + 920, + 981, + 966, + 934, + 982, + 960, + 928, + 1008, + 954, + 922, + 1009, + 961, + 929, + 1013, + 949, + 917, + 1415, + 1381, + 1410, + 1333, + 1362, + 1333, + 1410, + 7830, + 104, + 817, + 72, + 817, + 7831, + 116, + 776, + 84, + 776, + 7832, + 119, + 778, + 87, + 778, + 7833, + 121, + 778, + 89, + 778, + 7834, + 97, + 702, + 65, + 702, + 7835, + 7777, + 7776, + 223, + 115, + 115, + 7838, + 8016, + 965, + 787, + 933, + 787, + 8018, + 965, + 787, + 768, + 933, + 787, + 768, + 8020, + 965, + 787, + 769, + 933, + 787, + 769, + 8022, + 965, + 787, + 834, + 933, + 787, + 834, + 8064, + 7936, + 953, + 7944, + 921, + 8072, + 8065, + 7937, + 953, + 7945, + 921, + 8073, + 8066, + 7938, + 953, + 7946, + 921, + 8074, + 8067, + 7939, + 953, + 7947, + 921, + 8075, + 8068, + 7940, + 953, + 7948, + 921, + 8076, + 8069, + 7941, + 953, + 7949, + 921, + 8077, + 8070, + 7942, + 953, + 7950, + 921, + 8078, + 8071, + 7943, + 953, + 7951, + 921, + 8079, + 8064, + 7936, + 953, + 7944, + 921, + 8072, + 8065, + 7937, + 953, + 7945, + 921, + 8073, + 8066, + 7938, + 953, + 7946, + 921, + 8074, + 8067, + 7939, + 953, + 7947, + 921, + 8075, + 8068, + 7940, + 953, + 7948, + 921, + 8076, + 8069, + 7941, + 953, + 7949, + 921, + 8077, + 8070, + 7942, + 953, + 7950, + 921, + 8078, + 8071, + 7943, + 953, + 7951, + 921, + 8079, + 8080, + 7968, + 953, + 7976, + 921, + 8088, + 8081, + 7969, + 953, + 7977, + 921, + 8089, + 8082, + 7970, + 953, + 7978, + 921, + 8090, + 8083, + 7971, + 953, + 7979, + 921, + 8091, + 8084, + 7972, + 953, + 7980, + 921, + 8092, + 8085, + 7973, + 953, + 7981, + 921, + 8093, + 8086, + 7974, + 953, + 7982, + 921, + 8094, + 8087, + 7975, + 953, + 7983, + 921, + 8095, + 8080, + 7968, + 953, + 7976, + 921, + 8088, + 8081, + 7969, + 953, + 7977, + 921, + 8089, + 8082, + 7970, + 953, + 7978, + 921, + 8090, + 8083, + 7971, + 953, + 7979, + 921, + 8091, + 8084, + 7972, + 953, + 7980, + 921, + 8092, + 8085, + 7973, + 953, + 7981, + 921, + 8093, + 8086, + 7974, + 953, + 7982, + 921, + 8094, + 8087, + 7975, + 953, + 7983, + 921, + 8095, + 8096, + 8032, + 953, + 8040, + 921, + 8104, + 8097, + 8033, + 953, + 8041, + 921, + 8105, + 8098, + 8034, + 953, + 8042, + 921, + 8106, + 8099, + 8035, + 953, + 8043, + 921, + 8107, + 8100, + 8036, + 953, + 8044, + 921, + 8108, + 8101, + 8037, + 953, + 8045, + 921, + 8109, + 8102, + 8038, + 953, + 8046, + 921, + 8110, + 8103, + 8039, + 953, + 8047, + 921, + 8111, + 8096, + 8032, + 953, + 8040, + 921, + 8104, + 8097, + 8033, + 953, + 8041, + 921, + 8105, + 8098, + 8034, + 953, + 8042, + 921, + 8106, + 8099, + 8035, + 953, + 8043, + 921, + 8107, + 8100, + 8036, + 953, + 8044, + 921, + 8108, + 8101, + 8037, + 953, + 8045, + 921, + 8109, + 8102, + 8038, + 953, + 8046, + 921, + 8110, + 8103, + 8039, + 953, + 8047, + 921, + 8111, + 8114, + 8048, + 953, + 8122, + 921, + 8122, + 837, + 8115, + 945, + 953, + 913, + 921, + 8124, + 8116, + 940, + 953, + 902, + 921, + 902, + 837, + 8118, + 945, + 834, + 913, + 834, + 8119, + 945, + 834, + 953, + 913, + 834, + 921, + 913, + 834, + 837, + 8115, + 945, + 953, + 913, + 921, + 8124, + 8126, + 953, + 921, + 8130, + 8052, + 953, + 8138, + 921, + 8138, + 837, + 8131, + 951, + 953, + 919, + 921, + 8140, + 8132, + 942, + 953, + 905, + 921, + 905, + 837, + 8134, + 951, + 834, + 919, + 834, + 8135, + 951, + 834, + 953, + 919, + 834, + 921, + 919, + 834, + 837, + 8131, + 951, + 953, + 919, + 921, + 8140, + 8146, + 953, + 776, + 768, + 921, + 776, + 768, + 8147, + 953, + 776, + 769, + 921, + 776, + 769, + 8150, + 953, + 834, + 921, + 834, + 8151, + 953, + 776, + 834, + 921, + 776, + 834, + 8162, + 965, + 776, + 768, + 933, + 776, + 768, + 8163, + 965, + 776, + 769, + 933, + 776, + 769, + 8164, + 961, + 787, + 929, + 787, + 8166, + 965, + 834, + 933, + 834, + 8167, + 965, + 776, + 834, + 933, + 776, + 834, + 8178, + 8060, + 953, + 8186, + 921, + 8186, + 837, + 8179, + 969, + 953, + 937, + 921, + 8188, + 8180, + 974, + 953, + 911, + 921, + 911, + 837, + 8182, + 969, + 834, + 937, + 834, + 8183, + 969, + 834, + 953, + 937, + 834, + 921, + 937, + 834, + 837, + 8179, + 969, + 953, + 937, + 921, + 8188, + 64256, + 102, + 102, + 70, + 70, + 70, + 102, + 64257, + 102, + 105, + 70, + 73, + 70, + 105, + 64258, + 102, + 108, + 70, + 76, + 70, + 108, + 64259, + 102, + 102, + 105, + 70, + 70, + 73, + 70, + 102, + 105, + 64260, + 102, + 102, + 108, + 70, + 70, + 76, + 70, + 102, + 108, + 64261, + 115, + 116, + 83, + 84, + 83, + 116, + 64262, + 115, + 116, + 83, + 84, + 83, + 116, + 64275, + 1396, + 1398, + 1348, + 1350, + 1348, + 1398, + 64276, + 1396, + 1381, + 1348, + 1333, + 1348, + 1381, + 64277, + 1396, + 1387, + 1348, + 1339, + 1348, + 1387, + 64278, + 1406, + 1398, + 1358, + 1350, + 1358, + 1398, + 64279, + 1396, + 1389, + 1348, + 1341, + 1348, + 1389, }; /* type indexes */ @@ -195,72 +1003,72 @@ static unsigned char index1[] = { 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 34, 35, 36, 37, 38, 39, 34, 34, 34, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 64, 64, 65, 66, 67, 64, - 64, 64, 64, 68, 69, 64, 64, 64, 64, 64, 64, 70, 17, 71, 72, 73, 74, 75, - 76, 64, 77, 78, 79, 80, 81, 82, 83, 64, 64, 84, 85, 34, 34, 34, 34, 34, - 34, 86, 34, 34, 34, 34, 34, 87, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 64, 64, 64, 68, 69, 64, 64, 64, 64, 64, 64, 70, 71, 72, 73, 74, 75, 76, + 77, 64, 78, 79, 80, 81, 82, 83, 84, 64, 64, 85, 86, 34, 34, 34, 34, 34, + 34, 87, 34, 34, 34, 34, 34, 88, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 88, 89, 90, 91, 34, 34, 34, 92, 34, 34, - 34, 93, 94, 34, 34, 34, 34, 34, 95, 34, 34, 34, 96, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 97, 98, 99, 34, 34, 34, 34, 34, 34, 100, 101, 34, 34, - 34, 34, 34, 34, 34, 34, 102, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 103, 34, 34, 34, 34, 34, 34, 34, 34, 104, 34, 34, 34, 34, - 100, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 103, 34, 34, 34, 34, 34, 34, 105, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 106, 107, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 108, 109, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 110, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 111, 34, 34, 112, 113, 114, 115, 116, 117, 118, 119, 120, - 121, 122, 123, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 89, 90, 91, 92, 34, 34, 34, 93, 34, 34, + 34, 94, 95, 34, 34, 34, 34, 34, 96, 34, 34, 34, 97, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 98, 99, 100, 34, 34, 34, 34, 34, 34, 101, 102, 34, + 34, 34, 34, 34, 34, 34, 34, 103, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 104, 34, 34, 34, 34, 34, 34, 34, 34, 105, 34, 34, 34, 34, + 101, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 104, 34, 34, 34, 34, 34, 34, 106, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 107, 108, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 109, 110, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 111, 112, 34, 34, 34, 34, 34, + 34, 34, 34, 113, 34, 34, 114, 115, 116, 117, 118, 119, 120, 121, 122, + 123, 124, 125, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 124, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 126, 127, 128, - 129, 130, 131, 132, 34, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 17, 143, 144, 145, 146, 147, 17, 17, 17, 17, 17, 17, 148, 17, 149, 17, - 150, 17, 151, 17, 152, 17, 17, 17, 153, 17, 17, 17, 154, 155, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 34, 34, 34, 34, 34, 34, 156, 17, 157, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 34, 34, 34, 34, 34, 34, 34, 34, 158, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 34, 34, 34, 34, 159, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 160, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 64, 161, 162, 163, 164, 17, 165, 17, 166, 167, 168, 169, 170, 171, - 172, 173, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 174, 175, 176, - 177, 178, 17, 179, 180, 181, 182, 183, 184, 185, 186, 65, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 187, 188, 189, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 86, 190, 34, 191, - 192, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 126, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 128, 129, 130, + 131, 132, 133, 134, 34, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, + 71, 145, 146, 147, 148, 149, 71, 71, 71, 71, 71, 71, 150, 71, 151, 152, + 153, 71, 154, 71, 155, 71, 71, 71, 156, 71, 71, 71, 157, 158, 159, 160, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 161, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 34, 34, 34, 34, 34, 34, 162, 71, + 163, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 34, 34, 34, 34, 34, 34, 34, 34, 164, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 34, 34, 34, 34, 165, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 166, 167, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 168, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 64, 169, 170, 171, 172, 71, 173, 71, 174, 175, 176, 177, 178, + 179, 180, 181, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 182, 183, 71, 71, 184, + 185, 186, 187, 188, 71, 189, 190, 191, 192, 193, 194, 195, 196, 65, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 197, 198, + 199, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 87, 200, + 34, 201, 202, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 193, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 194, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 203, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 204, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 195, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 205, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 196, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 206, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, @@ -268,861 +1076,862 @@ static unsigned char index1[] = { 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 197, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 207, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, - 34, 34, 198, 34, 199, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 34, 193, 34, 34, 199, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 200, 17, 201, 202, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 203, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, - 125, 125, 125, 125, 125, 125, 125, 125, 125, 203, + 34, 34, 34, 34, 208, 34, 209, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 34, 203, 34, 34, 209, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 210, 71, 211, 212, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 213, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 213, }; -static unsigned char index2[] = { +static unsigned short index2[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 3, 3, 3, 2, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 5, 5, 5, 5, 17, 5, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 5, 5, - 5, 5, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 19, 5, 5, - 1, 5, 5, 5, 5, 20, 21, 5, 22, 5, 17, 5, 23, 19, 5, 24, 24, 24, 5, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 5, 16, 16, 16, 16, 16, 16, 16, 19, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 5, - 18, 18, 18, 18, 18, 18, 18, 25, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 28, 29, 26, 27, 26, 27, 26, 27, 19, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 19, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 30, 26, 27, 26, 27, 26, 27, 31, 32, 33, 26, 27, 26, 27, 34, 26, - 27, 35, 35, 26, 27, 19, 36, 37, 38, 26, 27, 35, 39, 40, 41, 42, 26, 27, - 43, 19, 41, 44, 45, 46, 26, 27, 26, 27, 26, 27, 47, 26, 27, 47, 19, 19, - 26, 27, 47, 26, 27, 48, 48, 26, 27, 26, 27, 49, 26, 27, 19, 50, 26, 27, - 19, 51, 50, 50, 50, 50, 52, 53, 54, 52, 53, 54, 52, 53, 54, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 55, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 19, 52, 53, 54, - 26, 27, 56, 57, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 58, 19, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 19, 19, 19, 19, 19, 19, 59, 26, - 27, 60, 61, 62, 62, 26, 27, 63, 64, 65, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 66, 67, 68, 69, 70, 19, 71, 71, 19, 72, 19, 73, 19, 19, 19, 19, - 71, 19, 19, 74, 19, 75, 19, 19, 76, 77, 19, 78, 19, 19, 19, 77, 19, 79, - 80, 19, 19, 81, 19, 19, 19, 19, 19, 19, 19, 82, 19, 19, 83, 19, 19, 83, - 19, 19, 19, 19, 83, 84, 85, 85, 86, 19, 19, 19, 19, 19, 87, 19, 50, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 5, 5, 5, 5, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 50, 50, 50, - 50, 50, 5, 5, 5, 5, 5, 5, 5, 50, 5, 50, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 1, 1, 1, 1, 3, 3, 3, 2, 4, 5, 5, 5, 5, 5, 5, 6, 5, 5, 5, 5, 5, 5, 6, 5, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 6, 5, 5, 5, 5, 5, 5, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 88, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 26, 27, 26, 27, 50, 5, 26, 27, 0, 0, 89, - 45, 45, 45, 5, 0, 0, 0, 0, 0, 5, 5, 90, 17, 91, 91, 91, 0, 92, 0, 93, 93, - 19, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 0, 16, 16, 16, 16, 16, 16, 16, 16, 16, 94, 95, 95, 95, 19, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 96, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 97, 98, 98, 99, 100, 101, 102, 102, 102, 103, - 104, 105, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 106, 107, 108, 19, 109, 110, 5, 26, 27, - 111, 26, 27, 19, 58, 58, 58, 112, 112, 112, 112, 112, 112, 112, 112, 112, - 112, 112, 112, 112, 112, 112, 112, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 5, 17, 17, 17, 17, 17, 5, 5, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 113, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 114, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 0, 0, 50, 5, 5, 5, 5, 5, 5, 0, 116, 116, 116, 116, 116, - 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, - 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, - 116, 116, 116, 116, 116, 19, 0, 5, 5, 0, 0, 0, 0, 0, 0, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 5, 17, 5, 17, 17, 5, 17, 17, 5, 17, 0, 0, 0, 0, 0, 0, - 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 50, 50, 50, 5, 5, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 5, 0, 0, 5, 5, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 5, 5, 5, 5, 50, 50, 17, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, 50, 17, - 17, 17, 17, 17, 17, 17, 1, 5, 17, 17, 17, 17, 17, 17, 50, 50, 17, 17, 5, - 17, 17, 17, 17, 50, 50, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 50, 50, 50, - 5, 5, 50, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 1, 50, 17, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 50, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 50, 50, 5, 5, 5, 5, 50, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, - 17, 17, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 50, 17, 17, 17, 50, 17, - 17, 17, 17, 17, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 17, 17, 17, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 17, 17, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 50, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 50, 17, 17, 17, - 17, 17, 17, 17, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 5, 5, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 5, 50, 50, 50, 50, 50, 50, 50, 0, 50, - 50, 50, 50, 50, 50, 50, 0, 17, 17, 17, 0, 50, 50, 50, 50, 50, 50, 50, 50, - 0, 0, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 0, 50, - 0, 0, 0, 50, 50, 50, 50, 0, 0, 17, 50, 17, 17, 17, 17, 17, 17, 17, 0, 0, - 17, 17, 0, 0, 17, 17, 17, 50, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 50, - 50, 0, 50, 50, 50, 17, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 50, - 50, 5, 5, 24, 24, 24, 24, 24, 24, 5, 5, 0, 0, 0, 0, 0, 17, 17, 17, 0, 50, - 50, 50, 50, 50, 50, 0, 0, 0, 0, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, - 50, 50, 50, 50, 50, 0, 50, 50, 0, 50, 50, 0, 50, 50, 0, 0, 17, 0, 17, 17, - 17, 17, 17, 0, 0, 0, 0, 17, 17, 0, 0, 17, 17, 17, 0, 0, 0, 17, 0, 0, 0, - 0, 0, 0, 0, 50, 50, 50, 50, 0, 50, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 17, 17, 50, 50, 50, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 17, 17, 17, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 0, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 0, 50, 50, 50, - 50, 50, 0, 0, 17, 50, 17, 17, 17, 17, 17, 17, 17, 17, 0, 17, 17, 17, 0, - 17, 17, 17, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, - 50, 17, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 5, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 17, 0, 50, 50, 50, 50, 50, 50, - 50, 50, 0, 0, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, - 50, 0, 50, 50, 0, 50, 50, 50, 50, 50, 0, 0, 17, 50, 17, 17, 17, 17, 17, - 17, 17, 0, 0, 17, 17, 0, 0, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, - 0, 0, 0, 0, 50, 50, 0, 50, 50, 50, 17, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 5, 50, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 50, 0, 50, 50, 50, 50, 50, 50, 0, 0, 0, 50, 50, 50, 0, 50, 50, 50, - 50, 0, 0, 0, 50, 50, 0, 50, 0, 50, 50, 0, 0, 0, 50, 50, 0, 0, 0, 50, 50, - 50, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, - 17, 17, 17, 17, 17, 0, 0, 0, 17, 17, 17, 0, 17, 17, 17, 17, 0, 0, 50, 0, - 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 24, 24, 24, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, - 0, 0, 17, 17, 17, 0, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 0, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, - 50, 50, 50, 0, 0, 0, 50, 17, 17, 17, 17, 17, 17, 17, 0, 17, 17, 17, 0, - 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 17, 17, 0, 50, 50, 0, 0, 0, 0, 0, 0, - 50, 50, 17, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, - 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 5, 0, 0, 17, 17, 0, 50, 50, 50, 50, - 50, 50, 50, 50, 0, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 0, 0, 17, 50, 17, 17, 17, - 17, 17, 17, 17, 0, 17, 17, 17, 0, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, - 17, 17, 0, 0, 0, 0, 0, 0, 0, 50, 0, 50, 50, 17, 17, 0, 0, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 0, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 17, 17, 0, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 0, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 0, 0, 50, 17, 17, 17, 17, 17, 17, 17, 0, 17, 17, 17, 0, 17, - 17, 17, 17, 50, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 50, - 50, 17, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 24, 24, 24, 24, 24, - 24, 0, 0, 0, 5, 50, 50, 50, 50, 50, 50, 0, 0, 17, 17, 0, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 0, 0, 50, - 50, 50, 50, 50, 50, 50, 0, 0, 0, 17, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, - 0, 17, 0, 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 50, 117, 17, 17, 17, - 17, 17, 17, 17, 0, 0, 0, 0, 5, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, - 17, 17, 17, 17, 17, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 5, 5, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 0, 50, 0, 0, 50, 50, 0, 50, 0, 0, - 50, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 0, - 50, 50, 50, 0, 50, 0, 50, 0, 0, 50, 50, 0, 50, 50, 50, 50, 17, 50, 117, - 17, 17, 17, 17, 17, 17, 0, 17, 17, 50, 0, 0, 50, 50, 50, 50, 50, 0, 50, - 0, 17, 17, 17, 17, 17, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, - 0, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 17, 17, 5, 5, 5, 5, 5, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, 17, - 5, 17, 5, 17, 5, 5, 5, 5, 17, 17, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, - 0, 0, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 5, 17, 17, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 0, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 0, 5, 5, 5, 5, 5, 5, 5, 5, 17, 5, 5, 5, 5, 5, 5, 0, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 50, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 5, 5, 5, 5, 5, 5, 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 50, 50, - 50, 50, 17, 17, 17, 50, 17, 17, 17, 50, 50, 17, 17, 17, 17, 17, 17, 17, - 50, 50, 50, 17, 17, 17, 17, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 50, 17, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 17, 17, 17, 17, 5, 5, 118, 118, 118, 118, 118, - 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, - 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, 118, - 118, 118, 118, 118, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 5, 50, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 0, 0, 50, 50, - 50, 50, 50, 50, 50, 0, 50, 0, 50, 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 0, 50, 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 0, 50, 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 50, 0, 50, - 0, 50, 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 17, 17, 17, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 119, 120, 121, 122, 123, 124, 125, 126, 127, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 5, 5, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 2, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, 5, 0, 0, 0, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 5, 5, 5, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, - 50, 50, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 5, 5, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 0, 17, 17, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 1, 1, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 5, 5, 5, 50, 5, 5, 5, 5, 50, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, - 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 17, 17, 17, 2, 0, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 17, 50, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 0, 0, 0, 0, 5, 0, 0, 0, 5, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 50, - 50, 50, 50, 50, 50, 50, 17, 17, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 119, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 17, 17, 17, 17, 17, 0, 0, 5, 5, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 0, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 17, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, - 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 50, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, - 14, 15, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 17, 17, 17, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 0, 0, 0, 50, 50, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, - 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 5, 5, 5, 5, - 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 50, 50, 50, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 17, 5, 17, + 17, 17, 17, 17, 5, 5, 5, 6, 18, 6, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 5, 5, + 5, 5, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 5, 5, 5, 5, 5, 5, 5, 6, 5, 20, 5, 5, + 21, 5, 6, 5, 5, 22, 23, 6, 24, 5, 25, 6, 26, 20, 5, 27, 27, 27, 5, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 50, 50, 50, 50, 17, 50, 50, 50, 50, 17, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 17, 17, 17, 17, 5, 17, 17, 17, 17, 17, 17, 17, 28, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 50, 129, - 19, 19, 19, 130, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, - 17, 17, 17, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 19, 19, 19, 19, 19, 131, 19, 19, 132, - 19, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 133, 133, 133, 133, 133, 133, 133, 133, 134, - 134, 134, 134, 134, 134, 134, 134, 133, 133, 133, 133, 133, 133, 0, 0, - 134, 134, 134, 134, 134, 134, 0, 0, 133, 133, 133, 133, 133, 133, 133, - 133, 134, 134, 134, 134, 134, 134, 134, 134, 133, 133, 133, 133, 133, - 133, 133, 133, 134, 134, 134, 134, 134, 134, 134, 134, 133, 133, 133, - 133, 133, 133, 0, 0, 134, 134, 134, 134, 134, 134, 0, 0, 19, 133, 19, - 133, 19, 133, 19, 133, 0, 134, 0, 134, 0, 134, 0, 134, 133, 133, 133, - 133, 133, 133, 133, 133, 134, 134, 134, 134, 134, 134, 134, 134, 135, - 135, 136, 136, 136, 136, 137, 137, 138, 138, 139, 139, 140, 140, 0, 0, - 133, 133, 133, 133, 133, 133, 133, 133, 141, 141, 141, 141, 141, 141, - 141, 141, 133, 133, 133, 133, 133, 133, 133, 133, 141, 141, 141, 141, - 141, 141, 141, 141, 133, 133, 133, 133, 133, 133, 133, 133, 141, 141, - 141, 141, 141, 141, 141, 141, 133, 133, 19, 142, 19, 0, 19, 19, 134, 134, - 143, 143, 144, 5, 145, 5, 5, 5, 19, 142, 19, 0, 19, 19, 146, 146, 146, - 146, 144, 5, 5, 5, 133, 133, 19, 19, 0, 0, 19, 19, 134, 134, 147, 147, 0, - 5, 5, 5, 133, 133, 19, 19, 19, 108, 19, 19, 134, 134, 148, 148, 111, 5, - 5, 5, 0, 0, 19, 142, 19, 0, 19, 19, 149, 149, 150, 150, 144, 5, 5, 0, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3, 3, 1, 1, 1, 1, 1, 2, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 17, 17, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 17, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 2, 1, - 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 151, 50, 0, 0, 152, 153, - 154, 155, 156, 157, 5, 5, 5, 5, 5, 50, 151, 23, 20, 21, 152, 153, 154, - 155, 156, 157, 5, 5, 5, 5, 5, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 5, 5, 5, 5, 17, 5, 5, 5, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 102, 5, 5, 5, 5, 102, - 5, 5, 19, 102, 102, 102, 19, 19, 102, 102, 102, 19, 5, 102, 5, 5, 158, - 102, 102, 102, 102, 102, 5, 5, 5, 5, 5, 5, 102, 5, 159, 5, 102, 5, 160, - 161, 102, 102, 158, 19, 102, 102, 162, 102, 19, 50, 50, 50, 50, 19, 5, 5, - 19, 19, 102, 102, 5, 5, 5, 5, 5, 102, 19, 19, 19, 19, 5, 5, 5, 5, 163, 5, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 164, 164, - 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, - 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, 165, - 165, 165, 128, 128, 128, 26, 27, 128, 128, 128, 128, 24, 0, 0, 0, 0, 0, - 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 19, 19, 19, 19, 19, 19, 19, 29, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 32, 33, 30, 31, 30, 31, 30, 31, 20, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 34, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 35, 30, 31, 30, 31, 30, 31, 36, 37, 38, 30, 31, 30, 31, 39, + 30, 31, 40, 40, 30, 31, 20, 41, 42, 43, 30, 31, 40, 44, 45, 46, 47, 30, + 31, 48, 20, 46, 49, 50, 51, 30, 31, 30, 31, 30, 31, 52, 30, 31, 52, 20, + 20, 30, 31, 52, 30, 31, 53, 53, 30, 31, 30, 31, 54, 30, 31, 20, 55, 30, + 31, 20, 56, 55, 55, 55, 55, 57, 58, 59, 57, 58, 59, 57, 58, 59, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 60, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 61, 57, 58, + 59, 30, 31, 62, 63, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 64, 20, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 20, 20, 20, 20, 20, 20, 65, + 30, 31, 66, 67, 68, 68, 30, 31, 69, 70, 71, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 72, 73, 74, 75, 76, 20, 77, 77, 20, 78, 20, 79, 20, 20, 20, + 20, 77, 20, 20, 80, 20, 81, 82, 20, 83, 84, 20, 85, 20, 20, 20, 84, 20, + 86, 87, 20, 20, 88, 20, 20, 20, 20, 20, 20, 20, 89, 20, 20, 90, 20, 20, + 90, 20, 20, 20, 20, 90, 91, 92, 92, 93, 20, 20, 20, 20, 20, 94, 20, 55, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 96, 96, 96, 96, 96, 96, 96, 95, 95, 6, 6, 6, 6, 96, 96, 96, 96, 96, 96, + 96, 96, 96, 96, 96, 96, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 95, 95, + 95, 95, 95, 6, 6, 6, 6, 6, 6, 6, 96, 6, 96, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 6, 6, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 97, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 30, 31, 30, 31, 96, 6, 30, 31, 0, 0, + 98, 50, 50, 50, 5, 0, 0, 0, 0, 0, 6, 6, 99, 25, 100, 100, 100, 0, 101, 0, + 102, 102, 103, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 0, 17, 17, 17, 17, 17, 17, 17, 17, 17, 104, 105, 105, 105, + 106, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 107, 19, 19, 19, 19, 19, 19, 19, 19, 19, 108, 109, 109, 110, 111, 112, + 113, 113, 113, 114, 115, 116, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 117, 118, 119, 20, + 120, 121, 5, 30, 31, 122, 30, 31, 20, 64, 64, 64, 123, 123, 123, 123, + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 124, 124, 124, 124, 124, 124, 124, 124, 124, + 124, 124, 124, 124, 124, 124, 124, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 5, 25, 25, 25, 25, 25, 6, 6, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 125, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 126, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 0, 0, 96, 5, 5, 5, 5, 5, 5, + 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 129, 0, 5, 5, 0, 0, 0, + 0, 5, 0, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 5, 25, 5, 25, 25, 5, 25, 25, + 5, 25, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 0, 0, 0, 55, 55, 55, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 21, 21, + 21, 21, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 5, 0, 0, 5, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 96, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, 5, 5, 5, 5, 55, 55, 25, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 5, 55, 25, 25, 25, 25, 25, 25, 25, 21, 5, 25, 25, 25, + 25, 25, 25, 96, 96, 25, 25, 5, 25, 25, 25, 25, 55, 55, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 55, 55, 55, 5, 5, 55, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 0, 21, 55, 25, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 25, 25, 25, 25, 25, 25, 25, 25, 25, 96, 96, 5, 5, 5, 5, 96, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 25, 25, 25, 25, 96, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 96, 25, 25, 25, 96, 25, 25, 25, 25, 25, 0, 0, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 25, + 25, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 0, 25, 25, 25, 18, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 18, 25, 55, 18, 18, 18, + 25, 25, 25, 25, 25, 25, 25, 25, 18, 18, 18, 18, 25, 18, 18, 55, 25, 25, + 25, 25, 25, 25, 25, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 25, 5, 5, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 5, 96, 55, 55, 55, 55, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 0, 25, 18, 18, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 0, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, + 55, 0, 0, 0, 55, 55, 55, 55, 0, 0, 25, 55, 18, 18, 18, 25, 25, 25, 25, 0, + 0, 18, 18, 0, 0, 18, 18, 25, 55, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, + 55, 55, 0, 55, 55, 55, 25, 25, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 55, 55, 5, 5, 27, 27, 27, 27, 27, 27, 5, 5, 0, 0, 0, 0, 0, 25, 25, 18, 0, + 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, 55, 0, 55, 55, 0, 0, 25, 0, 18, + 18, 18, 25, 25, 0, 0, 0, 0, 25, 25, 0, 0, 25, 25, 25, 0, 0, 0, 25, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 0, 55, 0, 0, 0, 0, 0, 0, 0, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 25, 25, 55, 55, 55, 25, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 25, 25, 18, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, + 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, 55, + 55, 55, 55, 55, 0, 0, 25, 55, 18, 18, 18, 25, 25, 25, 25, 25, 0, 25, 25, + 18, 0, 18, 18, 25, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 25, 25, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 5, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 18, 18, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, + 55, 55, 0, 55, 55, 0, 55, 55, 55, 55, 55, 0, 0, 25, 55, 18, 25, 18, 25, + 25, 25, 25, 0, 0, 18, 18, 0, 0, 18, 18, 25, 0, 0, 0, 0, 0, 0, 0, 0, 25, + 18, 0, 0, 0, 0, 55, 55, 0, 55, 55, 55, 25, 25, 0, 0, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 5, 55, 27, 27, 27, 27, 27, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 25, 55, 0, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, 55, 0, 55, 55, 55, + 55, 0, 0, 0, 55, 55, 0, 55, 0, 55, 55, 0, 0, 0, 55, 55, 0, 0, 0, 55, 55, + 55, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 18, 18, 25, 18, 18, 0, 0, 0, 18, 18, 18, 0, 18, 18, 18, 25, 0, 0, 55, 0, + 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 27, 27, 27, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, + 0, 0, 18, 18, 18, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, + 55, 55, 55, 0, 0, 0, 55, 25, 25, 25, 18, 18, 18, 18, 0, 25, 25, 25, 0, + 25, 25, 25, 25, 0, 0, 0, 0, 0, 0, 0, 25, 25, 0, 55, 55, 0, 0, 0, 0, 0, 0, + 55, 55, 25, 25, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, + 0, 0, 0, 27, 27, 27, 27, 27, 27, 27, 5, 0, 0, 18, 18, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 0, 25, 55, 18, 25, 18, + 18, 18, 18, 18, 0, 25, 18, 18, 0, 18, 18, 25, 25, 0, 0, 0, 0, 0, 0, 0, + 18, 18, 0, 0, 0, 0, 0, 0, 0, 55, 0, 55, 55, 25, 25, 0, 0, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 0, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 18, 18, 0, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 55, 18, 18, 18, 25, 25, 25, 25, 0, 18, 18, 18, 0, + 18, 18, 18, 25, 55, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 25, 25, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 27, 27, 27, + 27, 27, 27, 0, 0, 0, 5, 55, 55, 55, 55, 55, 55, 0, 0, 18, 18, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 25, 0, 0, 0, 0, 18, 18, 18, 25, + 25, 25, 0, 25, 0, 18, 18, 18, 18, 18, 18, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 18, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 55, 130, 25, 25, + 25, 25, 25, 25, 25, 0, 0, 0, 0, 5, 55, 55, 55, 55, 55, 55, 96, 25, 25, + 25, 25, 25, 25, 25, 25, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 5, 5, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 0, 55, 0, 0, 55, 55, 0, 55, + 0, 0, 55, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 0, 55, 0, 55, 0, 0, 55, 55, 0, 55, 55, 55, 55, 25, 55, + 130, 25, 25, 25, 25, 25, 25, 0, 25, 25, 55, 0, 0, 55, 55, 55, 55, 55, 0, + 96, 0, 25, 25, 25, 25, 25, 25, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 0, 0, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 25, 25, 5, 5, 5, 5, 5, 5, 7, + 8, 9, 10, 11, 12, 13, 14, 15, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 5, 25, 5, 25, 5, 25, 5, 5, 5, 5, 18, 18, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 0, 0, 0, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 18, 25, 25, 25, 25, 25, 5, 25, 25, 55, 55, 55, 55, 55, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 0, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 0, 5, 5, 5, 5, 5, 5, 5, 5, 25, 5, 5, 5, + 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 18, 18, 25, 25, 25, 25, 18, + 25, 25, 25, 25, 25, 25, 18, 25, 25, 18, 18, 25, 25, 55, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 5, 5, 5, 5, 5, 5, 55, 55, 55, 55, 55, 55, 18, 18, 25, + 25, 55, 55, 55, 55, 25, 25, 25, 55, 18, 18, 18, 55, 55, 18, 18, 18, 18, + 18, 18, 18, 55, 55, 55, 25, 25, 25, 25, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 25, 18, 18, 25, 25, 18, 18, 18, 18, 18, 18, 25, 55, + 18, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 18, 18, 25, 5, 5, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 0, 131, 0, 0, 0, 0, 0, 131, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 5, 96, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, + 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 25, 25, 25, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 132, 133, 134, 135, 136, 137, 138, 139, 140, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 5, 5, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 2, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 5, 5, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 5, 5, 5, 141, 141, 141, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 25, 25, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, + 25, 25, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 25, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 0, 25, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 25, 18, 25, 25, 25, + 25, 25, 25, 25, 18, 18, 18, 18, 18, 18, 18, 18, 25, 18, 18, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 5, 5, 5, 96, 5, 5, 5, 5, 55, 25, 0, 0, 7, + 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 25, 25, 25, 2, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 96, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 55, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 25, 25, 25, + 18, 18, 18, 18, 25, 25, 18, 18, 18, 0, 0, 0, 0, 18, 18, 25, 18, 18, 18, + 18, 18, 18, 25, 25, 25, 0, 0, 0, 0, 5, 0, 0, 0, 5, 5, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 0, 0, 0, 0, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 55, 55, 55, 55, 55, 55, 55, 18, 18, 0, 0, 0, 0, 0, 0, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 132, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 25, 25, 18, 18, 18, 0, 0, 5, 5, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 18, 25, 18, 25, + 25, 25, 25, 25, 25, 25, 0, 25, 18, 25, 18, 18, 25, 25, 25, 25, 25, 25, + 25, 25, 18, 18, 18, 18, 18, 18, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 0, 0, 25, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 96, 5, + 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 25, 25, 18, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 18, 25, 25, 25, 25, 25, 18, 25, + 18, 18, 18, 18, 18, 25, 18, 18, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 25, 25, 25, 25, 25, 25, 25, 25, 25, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 0, 0, 0, 25, 25, 18, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 18, 25, 25, 25, 25, 18, 18, 25, 25, 18, 25, 18, 18, 55, 55, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 18, 25, + 25, 18, 18, 18, 25, 18, 25, 25, 25, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, + 5, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 18, 18, 18, 18, 18, 18, 18, 18, 25, 25, 25, 25, 25, 25, 25, 25, 18, + 18, 25, 25, 0, 0, 0, 5, 5, 5, 5, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 0, 0, 0, 55, 55, 55, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 96, 96, 96, 96, 96, 96, 5, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 25, 25, 25, 5, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 18, 25, 25, 25, 25, 25, 25, 25, 55, 55, 55, 55, 25, 55, 55, 55, + 55, 18, 18, 25, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 95, 142, 20, 20, 20, 143, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 95, 95, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 25, 25, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 144, + 145, 146, 147, 148, 149, 20, 20, 150, 20, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 151, 151, + 151, 151, 151, 151, 151, 151, 152, 152, 152, 152, 152, 152, 152, 152, + 151, 151, 151, 151, 151, 151, 0, 0, 152, 152, 152, 152, 152, 152, 0, 0, + 151, 151, 151, 151, 151, 151, 151, 151, 152, 152, 152, 152, 152, 152, + 152, 152, 151, 151, 151, 151, 151, 151, 151, 151, 152, 152, 152, 152, + 152, 152, 152, 152, 151, 151, 151, 151, 151, 151, 0, 0, 152, 152, 152, + 152, 152, 152, 0, 0, 153, 151, 154, 151, 155, 151, 156, 151, 0, 152, 0, + 152, 0, 152, 0, 152, 151, 151, 151, 151, 151, 151, 151, 151, 152, 152, + 152, 152, 152, 152, 152, 152, 157, 157, 158, 158, 158, 158, 159, 159, + 160, 160, 161, 161, 162, 162, 0, 0, 163, 164, 165, 166, 167, 168, 169, + 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 151, + 151, 211, 212, 213, 0, 214, 215, 152, 152, 216, 216, 217, 6, 218, 6, 6, + 6, 219, 220, 221, 0, 222, 223, 224, 224, 224, 224, 225, 6, 6, 6, 151, + 151, 226, 227, 0, 0, 228, 229, 152, 152, 230, 230, 0, 6, 6, 6, 151, 151, + 231, 232, 233, 119, 234, 235, 152, 152, 236, 236, 122, 6, 6, 6, 0, 0, + 237, 238, 239, 0, 240, 241, 242, 242, 243, 243, 244, 6, 6, 0, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 21, 21, 21, 21, 21, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 5, 5, 6, 3, 3, 21, 21, 21, 21, 21, 2, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 18, 18, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 18, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 2, 21, + 21, 21, 21, 21, 0, 0, 0, 0, 0, 21, 21, 21, 21, 21, 21, 245, 95, 0, 0, + 246, 247, 248, 249, 250, 251, 5, 5, 5, 5, 5, 95, 245, 26, 22, 23, 246, + 247, 248, 249, 250, 251, 5, 5, 5, 5, 5, 0, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 95, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 6, 6, 6, 6, 25, 6, 6, 6, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 113, 5, 5, + 5, 5, 113, 5, 5, 20, 113, 113, 113, 20, 20, 113, 113, 113, 20, 5, 113, 5, + 5, 252, 113, 113, 113, 113, 113, 5, 5, 5, 5, 5, 5, 113, 5, 253, 5, 113, + 5, 254, 255, 113, 113, 252, 20, 113, 113, 256, 113, 20, 55, 55, 55, 55, + 20, 5, 5, 20, 20, 113, 113, 5, 5, 5, 5, 5, 113, 20, 20, 20, 20, 5, 5, 5, + 5, 257, 5, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + 258, 258, 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, + 259, 259, 259, 259, 141, 141, 141, 30, 31, 141, 141, 141, 141, 27, 0, 0, + 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, @@ -1135,632 +1944,685 @@ static unsigned char index2[] = { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 20, 21, 152, 153, 154, 155, 156, 157, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, 20, 21, 152, 153, 154, 155, - 156, 157, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, 20, 21, 152, - 153, 154, 155, 156, 157, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, - 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 166, 167, 167, - 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, - 167, 167, 167, 167, 167, 167, 167, 167, 167, 167, 151, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 23, 20, 21, 152, 153, 154, 155, 156, 157, 24, - 151, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 22, 23, 246, 247, 248, 249, 250, + 251, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 22, 23, 246, 247, + 248, 249, 250, 251, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 22, + 23, 246, 247, 248, 249, 250, 251, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, 260, 261, + 261, 261, 261, 261, 261, 261, 261, 261, 261, 261, 261, 261, 261, 261, + 261, 261, 261, 261, 261, 261, 261, 261, 261, 261, 261, 245, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 26, 22, 23, 246, 247, 248, 249, 250, 251, 27, + 245, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 26, + 22, 23, 246, 247, 248, 249, 250, 251, 27, 26, 22, 23, 246, 247, 248, 249, + 250, 251, 27, 26, 22, 23, 246, 247, 248, 249, 250, 251, 27, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 23, - 20, 21, 152, 153, 154, 155, 156, 157, 24, 23, 20, 21, 152, 153, 154, 155, - 156, 157, 24, 23, 20, 21, 152, 153, 154, 155, 156, 157, 24, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 0, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, - 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 115, 0, - 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, - 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, - 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, 116, - 116, 116, 116, 116, 116, 0, 26, 27, 168, 169, 170, 171, 172, 26, 27, 26, - 27, 26, 27, 173, 174, 175, 176, 19, 26, 27, 19, 26, 27, 19, 19, 19, 19, - 19, 19, 50, 177, 177, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 19, 5, 5, 5, - 5, 5, 5, 26, 27, 26, 27, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 24, - 5, 5, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, - 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, - 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 17, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, - 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, - 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, - 0, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, - 50, 50, 50, 50, 50, 0, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 89, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 30, 31, 262, 263, + 264, 265, 266, 30, 31, 30, 31, 30, 31, 267, 268, 269, 270, 20, 30, 31, + 20, 30, 31, 20, 20, 20, 20, 20, 95, 95, 271, 271, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 20, 5, 5, 5, 5, 5, 5, 30, 31, 30, 31, 25, 25, 25, 30, 31, + 0, 0, 0, 0, 0, 5, 5, 5, 5, 27, 5, 5, 272, 272, 272, 272, 272, 272, 272, + 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, + 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, 272, + 272, 272, 272, 0, 272, 0, 0, 0, 0, 0, 272, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, + 0, 96, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 273, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, - 0, 0, 2, 5, 5, 5, 5, 50, 50, 128, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 17, 17, 17, 17, 17, 17, 5, 50, 50, 50, 50, 50, 5, 5, 128, 128, - 128, 50, 50, 5, 5, 5, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 0, 0, 17, 17, 5, 5, 50, 50, 50, 5, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, 50, 50, 50, 50, 0, 0, 0, - 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 5, 5, 24, 24, 24, 24, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, - 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 0, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 179, 50, 50, 179, - 50, 50, 50, 179, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, - 179, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 179, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, - 50, 179, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 179, - 179, 179, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 179, 179, 179, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, - 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 179, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 179, 179, 50, 179, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, - 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, 5, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, 5, 5, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 50, 17, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, - 0, 0, 17, 17, 5, 50, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, - 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 0, 0, 0, 0, 0, 0, 0, 0, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 17, 17, 5, 5, 5, 5, 5, 5, 0, 0, - 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, 5, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 19, 19, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 50, 19, 19, 19, 19, 19, 19, 19, 19, 26, 27, 26, 27, 180, 26, 27, - 26, 27, 26, 27, 26, 27, 26, 27, 50, 5, 5, 26, 27, 181, 19, 0, 26, 27, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 27, 26, 27, 26, 27, 26, 27, - 26, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 50, 50, 50, 50, 50, 50, 50, 17, 50, 50, - 50, 17, 50, 50, 50, 50, 17, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 5, 5, - 5, 5, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, 5, 5, - 5, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 6, 7, 8, 9, 10, 11, 12, - 13, 14, 15, 0, 0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 50, 50, 50, 50, 50, 50, 5, 5, 5, 50, 0, 0, 0, - 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 5, 5, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 5, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 17, 17, 17, 17, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 50, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 17, 50, 50, 50, 50, 50, - 50, 50, 50, 17, 17, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 5, 5, - 5, 5, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 5, 5, 5, 50, 17, 0, 0, 0, 0, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 17, 50, 17, 17, 17, 50, 50, 17, 17, 50, 50, - 50, 50, 50, 17, 17, 50, 17, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 50, 50, 50, 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 0, 0, 50, 50, - 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 0, - 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 5, 17, 17, 0, 0, - 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 2, 5, 5, 5, 5, 96, 55, 141, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 25, 25, 25, 25, 18, 18, 5, 96, + 96, 96, 96, 96, 5, 5, 141, 141, 141, 96, 55, 5, 5, 5, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 25, 25, 6, 6, 96, 96, 55, + 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 5, 96, 96, 96, 55, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 5, 5, 27, 27, 27, 27, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 27, 27, 27, 27, 27, 27, 27, 27, 5, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 55, 55, 55, + 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 274, 55, 55, 274, 55, 55, 55, 274, 55, 274, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 274, 55, 274, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 274, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, + 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 274, 55, 274, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 274, 274, 274, 55, 55, + 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 274, 274, 274, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, + 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 274, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 274, 274, 274, 55, 274, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 274, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, + 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 96, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 96, 96, 96, 96, 96, 96, 5, 5, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 96, 5, 5, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 55, 55, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 55, 25, 6, 6, 6, 5, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 5, 96, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, + 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 0, 0, 0, 0, 0, 0, 0, 25, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 25, 25, 5, 5, 5, 5, 5, 5, 0, 0, + 0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 6, 6, 6, 6, 6, 96, 96, 96, 96, 96, 96, 96, 96, 96, 6, 6, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 20, 20, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 95, 20, 20, 20, 20, 20, 20, 20, 20, 30, 31, 30, 31, 275, 30, 31, + 30, 31, 30, 31, 30, 31, 30, 31, 96, 6, 6, 30, 31, 276, 20, 0, 30, 31, 30, + 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 31, 30, 31, 30, 31, 30, 31, + 30, 31, 277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 95, 20, 55, 55, 55, 55, 55, 55, 55, 25, + 55, 55, 55, 25, 55, 55, 55, 55, 25, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 18, 18, 25, 25, + 18, 5, 5, 5, 5, 0, 0, 0, 0, 27, 27, 27, 27, 27, 27, 5, 5, 5, 5, 0, 0, 0, + 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 18, 18, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 55, 55, 55, 55, 55, 55, 5, 5, 5, 55, + 0, 0, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 25, 25, 25, 25, 25, 25, 25, 25, 5, 5, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 18, 18, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 25, + 25, 25, 18, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 18, 18, 25, + 25, 25, 25, 18, 18, 25, 18, 18, 18, 18, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 0, 96, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 5, 5, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 25, 25, 25, 25, 25, 18, 18, 25, + 25, 18, 18, 25, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 25, 55, 55, + 55, 55, 55, 55, 55, 55, 25, 18, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 0, 0, 5, 5, 5, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 96, 55, 55, 55, 55, 55, 55, 5, 5, 5, 55, 18, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 55, 25, 25, 25, 55, 55, + 25, 25, 55, 55, 55, 55, 55, 25, 25, 55, 25, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 96, 5, 5, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 18, 25, 25, 18, 18, 5, 5, 55, 96, 96, + 18, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, + 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 18, 18, 25, + 18, 18, 25, 18, 18, 5, 18, 25, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, - 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 179, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 19, - 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 19, 19, 19, 19, 0, 0, 0, 0, - 0, 50, 17, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 0, 50, 0, 50, 50, - 0, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 89, 89, 89, 89, 89, 89, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 89, 89, 5, 5, 0, 0, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 17, - 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 17, 17, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 17, 17, - 17, 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 0, 5, 5, 5, 5, 0, 0, 0, 0, 89, 50, 89, 50, 89, 0, 89, 50, 89, 50, 89, 50, - 89, 50, 89, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 0, 0, 1, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 5, 5, 5, 5, 5, 5, 5, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 5, 5, 5, 5, 17, 5, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, - 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 117, 117, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 0, 0, 0, 50, 50, 50, 50, 50, 50, 0, 0, 50, 50, 50, - 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 0, 0, 50, 50, 50, 0, 0, 0, 5, - 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 5, 5, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 0, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 5, 5, 5, 0, 0, - 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 24, 24, 24, 24, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 24, 0, 0, 0, 0, 0, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 17, 0, 0, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 24, 24, 24, 24, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 128, 50, 50, 50, 50, 50, 50, 50, 50, 128, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 5, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, - 0, 50, 50, 50, 50, 50, 50, 50, 50, 5, 128, 128, 128, 128, 128, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 182, 182, 182, 182, 182, - 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, - 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, - 182, 182, 182, 182, 182, 182, 182, 183, 183, 183, 183, 183, 183, 183, - 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, - 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, - 183, 183, 183, 183, 183, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, - 50, 50, 0, 0, 50, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 50, 50, 0, 0, 0, - 50, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 5, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 24, 24, 24, 24, 24, 24, 0, 0, 0, 5, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 50, 17, 17, 17, 0, 17, 17, 0, 0, 0, 0, 0, 17, 17, - 17, 17, 50, 50, 50, 50, 0, 50, 50, 50, 0, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 0, 0, 0, 0, 17, 17, 17, 0, 0, 0, 0, 17, 23, 20, 21, 152, 24, 24, 24, - 24, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, - 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 24, 24, 5, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 5, - 5, 5, 5, 5, 5, 5, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 23, 20, 21, 152, 153, 154, 155, 156, 157, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 17, 17, 17, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 5, 5, 5, 5, 5, 5, - 5, 0, 0, 0, 0, 23, 20, 21, 152, 153, 154, 155, 156, 157, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 17, 17, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 5, 5, 1, 5, - 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, + 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 274, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 278, 279, 280, 281, 282, 283, 284, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 285, 286, 287, 288, 289, 0, 0, 0, 0, 0, 55, 25, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, 55, 0, 55, 55, 0, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 290, 290, 290, 290, 290, 290, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 290, 290, 5, 5, 0, 0, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 5, 5, 5, 6, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 25, 25, + 25, 25, 25, 25, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 18, 18, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 18, 18, + 18, 5, 5, 6, 0, 5, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 0, 5, 5, 5, 5, 0, 0, 0, 0, 290, 55, 290, 55, 290, 0, 290, 55, 290, 55, + 290, 55, 290, 55, 290, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 0, 0, 21, 0, 5, 5, 5, 5, 5, 5, 6, 5, 5, 5, 5, 5, + 5, 6, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 6, 5, 5, 5, 5, 5, 5, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 5, 5, 5, 6, 18, 6, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 96, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 291, 291, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, + 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, + 55, 0, 0, 0, 5, 5, 5, 6, 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 21, 21, 21, 5, 5, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 5, 5, 5, 0, 0, 0, 0, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 0, 0, 0, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 27, 27, 27, 27, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 27, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 25, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 0, 27, 27, 27, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 141, 55, 55, 55, + 55, 55, 55, 55, 55, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 5, 141, + 141, 141, 141, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, + 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, + 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 292, 293, 293, + 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, + 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, + 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 55, 55, 0, 0, 0, 55, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 5, 27, 27, 27, + 27, 27, 27, 27, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 27, 27, 27, 27, + 27, 27, 0, 0, 0, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 158, 158, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 158, 158, 128, 128, - 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 55, + 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 25, 25, 25, 0, 25, + 25, 0, 0, 0, 0, 0, 25, 25, 25, 25, 55, 55, 55, 55, 0, 55, 55, 55, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 25, 25, 25, 0, 0, 0, 0, 25, + 26, 22, 23, 246, 27, 27, 27, 27, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 27, 27, 5, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 27, 27, + 27, 27, 27, 27, 27, 27, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 27, 27, 27, 27, 27, 27, 27, + 27, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 22, 23, 246, 247, 248, 249, 250, 251, 27, + 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 27, 27, 0, 18, 25, 18, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 26, 22, 23, 246, 247, 248, + 249, 250, 251, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 25, 25, 18, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 18, 18, 18, 25, 25, 25, + 25, 18, 18, 25, 25, 5, 5, 21, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 25, 25, 25, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 25, 25, 25, 25, 25, + 18, 25, 25, 25, 25, 25, 25, 25, 25, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 18, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 18, 18, 18, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 18, 18, 55, 55, 55, 55, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, + 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 25, 18, 25, 18, 18, 25, 25, 25, 25, 25, + 25, 18, 25, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 252, 252, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 141, 252, 252, 141, 141, 141, 141, 141, 141, 141, 141, 141, + 141, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, + 25, 25, 25, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1775,272 +2637,286 @@ static unsigned char index2[] = { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 17, 17, 17, 17, 17, - 5, 5, 5, 17, 17, 17, 17, 17, 17, 1, 1, 1, 1, 1, 1, 1, 1, 17, 17, 17, 17, - 17, 17, 17, 17, 5, 5, 17, 17, 17, 17, 17, 17, 17, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 17, 17, - 17, 17, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 18, 18, 25, 25, 25, + 5, 5, 5, 18, 18, 18, 18, 18, 18, 21, 21, 21, 21, 21, 21, 21, 21, 25, 25, + 25, 25, 25, 25, 25, 25, 5, 5, 25, 25, 25, 25, 25, 25, 25, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 25, 25, 25, 25, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 17, 17, 17, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 25, 25, 25, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 19, 19, 19, 19, 19, 19, 19, 0, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 102, 0, 102, 102, 0, 0, 102, 0, 0, 102, 102, 0, 0, - 102, 102, 102, 102, 0, 102, 102, 102, 102, 102, 102, 102, 102, 19, 19, - 19, 19, 0, 19, 0, 19, 19, 19, 19, 19, 19, 19, 0, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 102, 102, 0, 102, 102, 102, 102, - 0, 0, 102, 102, 102, 102, 102, 102, 102, 102, 0, 102, 102, 102, 102, 102, - 102, 102, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 102, 102, 0, 102, 102, 102, - 102, 0, 102, 102, 102, 102, 102, 0, 102, 0, 0, 0, 102, 102, 102, 102, - 102, 102, 102, 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 0, 0, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 5, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 5, 19, 19, 19, 19, 19, 19, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 5, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 5, 19, 19, 19, 19, 19, 19, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 5, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 5, 19, 19, 19, 19, - 19, 19, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 5, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 5, 19, 19, 19, 19, 19, 19, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, 102, - 102, 102, 102, 102, 102, 102, 5, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, - 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 5, 19, 19, - 19, 19, 19, 19, 102, 19, 0, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 20, 20, 20, 20, + 20, 20, 20, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 113, 0, 113, 113, 0, 0, 113, 0, 0, + 113, 113, 0, 0, 113, 113, 113, 113, 0, 113, 113, 113, 113, 113, 113, 113, + 113, 20, 20, 20, 20, 0, 20, 0, 20, 20, 20, 20, 20, 20, 20, 0, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 113, 113, 0, 113, + 113, 113, 113, 0, 0, 113, 113, 113, 113, 113, 113, 113, 113, 0, 113, 113, + 113, 113, 113, 113, 113, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 113, 113, 0, + 113, 113, 113, 113, 0, 113, 113, 113, 113, 113, 0, 113, 0, 0, 0, 113, + 113, 113, 113, 113, 113, 113, 0, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 0, 0, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 5, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 5, 20, 20, 20, 20, + 20, 20, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 5, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 5, 20, 20, 20, 20, 20, 20, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 5, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 5, 20, 20, + 20, 20, 20, 20, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 5, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 5, 20, 20, 20, 20, 20, 20, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, + 113, 113, 113, 113, 113, 113, 113, 113, 5, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 5, 20, 20, 20, 20, 20, 20, 113, 20, 0, 0, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 0, 55, 0, 0, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 55, + 55, 55, 55, 0, 55, 0, 55, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 55, 0, 55, 0, + 55, 0, 55, 55, 55, 0, 55, 55, 0, 55, 0, 0, 55, 0, 55, 0, 55, 0, 55, 0, + 55, 0, 55, 55, 0, 55, 0, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, + 55, 0, 55, 55, 55, 55, 0, 55, 55, 55, 55, 0, 55, 0, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 55, 55, 55, 0, 55, 55, 55, 55, 55, 0, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 151, 23, 20, 21, 152, 153, - 154, 155, 156, 157, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 245, 26, 22, 23, 246, 247, 248, 249, + 250, 251, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, - 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, + 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, - 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 0, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, + 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, + 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 0, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, - 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 5, 5, 5, 5, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 0, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 5, 5, 5, 0, 5, 0, 5, 0, 5, 0, 5, - 5, 5, 0, 5, 5, 5, 5, 5, 5, 0, 0, 5, 5, 5, 5, 0, 5, 0, 0, 5, 5, 5, 5, 0, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, - 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 179, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 179, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, - 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, + 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 274, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, }; /* Returns the numeric value as double for Unicode characters @@ -2100,6 +2976,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1018A: case 0x104A0: case 0x11066: + case 0x110F0: + case 0x11136: + case 0x111D0: + case 0x116C0: case 0x1D7CE: case 0x1D7D8: case 0x1D7E2: @@ -2187,6 +3067,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10E60: case 0x11052: case 0x11067: + case 0x110F1: + case 0x11137: + case 0x111D1: + case 0x116C1: case 0x12415: case 0x1241E: case 0x1242C: @@ -2263,6 +3147,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x2793: case 0x3038: case 0x3229: + case 0x3248: case 0x3289: case 0x4EC0: case 0x5341: @@ -2483,6 +3368,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10E61: case 0x11053: case 0x11068: + case 0x110F2: + case 0x11138: + case 0x111D2: + case 0x116C2: case 0x12400: case 0x12416: case 0x1241F: @@ -2515,6 +3404,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x249B: case 0x24F4: case 0x3039: + case 0x3249: case 0x5344: case 0x5EFF: case 0x10111: @@ -2626,6 +3516,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10E62: case 0x11054: case 0x11069: + case 0x110F3: + case 0x11139: + case 0x111D3: + case 0x116C3: case 0x12401: case 0x12408: case 0x12417: @@ -2671,6 +3565,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) return (double) 3.0/8.0; case 0x1374: case 0x303A: + case 0x324A: case 0x325A: case 0x5345: case 0x10112: @@ -2770,6 +3665,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10E63: case 0x11055: case 0x1106A: + case 0x110F4: + case 0x1113A: + case 0x111D4: + case 0x116C4: case 0x12402: case 0x12409: case 0x1240F: @@ -2799,6 +3698,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x2158: return (double) 4.0/5.0; case 0x1375: + case 0x324B: case 0x32B5: case 0x534C: case 0x10113: @@ -2900,6 +3800,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10E64: case 0x11056: case 0x1106B: + case 0x110F5: + case 0x1113B: + case 0x111D5: + case 0x116C5: case 0x12403: case 0x1240A: case 0x12410: @@ -2931,6 +3835,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x216C: case 0x217C: case 0x2186: + case 0x324C: case 0x32BF: case 0x10114: case 0x10144: @@ -3034,6 +3939,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10E65: case 0x11057: case 0x1106C: + case 0x110F6: + case 0x1113C: + case 0x111D6: + case 0x116C6: case 0x12404: case 0x1240B: case 0x12411: @@ -3051,6 +3960,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x20AEA: return (double) 6.0; case 0x1377: + case 0x324D: case 0x10115: case 0x10E6E: case 0x11060: @@ -3124,6 +4034,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10E66: case 0x11058: case 0x1106D: + case 0x110F7: + case 0x1113D: + case 0x111D7: + case 0x116C7: case 0x12405: case 0x1240C: case 0x12412: @@ -3146,6 +4060,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x215E: return (double) 7.0/8.0; case 0x1378: + case 0x324E: case 0x10116: case 0x10E6F: case 0x11061: @@ -3217,6 +4132,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10E67: case 0x11059: case 0x1106E: + case 0x110F8: + case 0x1113E: + case 0x111D8: + case 0x116C8: case 0x12406: case 0x1240D: case 0x12413: @@ -3233,6 +4152,7 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x1F109: return (double) 8.0; case 0x1379: + case 0x324F: case 0x10117: case 0x10E70: case 0x11062: @@ -3305,6 +4225,10 @@ double _PyUnicode_ToNumeric(Py_UCS4 ch) case 0x10E68: case 0x1105A: case 0x1106F: + case 0x110F9: + case 0x1113F: + case 0x111D9: + case 0x116C9: case 0x12407: case 0x1240E: case 0x12414: diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 13323cf..e07adb2 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -156,28 +156,32 @@ weakref_hash(PyWeakReference *self) static PyObject * weakref_repr(PyWeakReference *self) { - char buffer[256]; - if (PyWeakref_GET_OBJECT(self) == Py_None) { - PyOS_snprintf(buffer, sizeof(buffer), "<weakref at %p; dead>", self); + PyObject *name, *repr; + _Py_IDENTIFIER(__name__); + + if (PyWeakref_GET_OBJECT(self) == Py_None) + return PyUnicode_FromFormat("<weakref at %p; dead>", self); + + name = _PyObject_GetAttrId(PyWeakref_GET_OBJECT(self), &PyId___name__); + if (name == NULL || !PyUnicode_Check(name)) { + if (name == NULL) + PyErr_Clear(); + repr = PyUnicode_FromFormat( + "<weakref at %p; to '%s' at %p>", + self, + Py_TYPE(PyWeakref_GET_OBJECT(self))->tp_name, + PyWeakref_GET_OBJECT(self)); } else { - char *name = NULL; - PyObject *nameobj = PyObject_GetAttrString(PyWeakref_GET_OBJECT(self), - "__name__"); - if (nameobj == NULL) - PyErr_Clear(); - else if (PyUnicode_Check(nameobj)) - name = _PyUnicode_AsString(nameobj); - PyOS_snprintf(buffer, sizeof(buffer), - name ? "<weakref at %p; to '%.50s' at %p (%s)>" - : "<weakref at %p; to '%.50s' at %p>", - self, - Py_TYPE(PyWeakref_GET_OBJECT(self))->tp_name, - PyWeakref_GET_OBJECT(self), - name); - Py_XDECREF(nameobj); + repr = PyUnicode_FromFormat( + "<weakref at %p; to '%s' at %p (%U)>", + self, + Py_TYPE(PyWeakref_GET_OBJECT(self))->tp_name, + PyWeakref_GET_OBJECT(self), + name); } - return PyUnicode_FromString(buffer); + Py_XDECREF(name); + return repr; } /* Weak references only support equality, not ordering. Two weak references @@ -190,8 +194,7 @@ weakref_richcompare(PyWeakReference* self, PyWeakReference* other, int op) if ((op != Py_EQ && op != Py_NE) || !PyWeakref_Check(self) || !PyWeakref_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } if (PyWeakref_GET_OBJECT(self) == Py_None || PyWeakref_GET_OBJECT(other) == Py_None) { @@ -438,8 +441,9 @@ proxy_checkref(PyWeakReference *proxy) #define WRAP_METHOD(method, special) \ static PyObject * \ method(PyObject *proxy) { \ + _Py_IDENTIFIER(special); \ UNWRAP(proxy); \ - return PyObject_CallMethod(proxy, special, ""); \ + return _PyObject_CallMethodId(proxy, &PyId_##special, ""); \ } @@ -452,12 +456,11 @@ WRAP_TERNARY(proxy_call, PyEval_CallObjectWithKeywords) static PyObject * proxy_repr(PyWeakReference *proxy) { - char buf[160]; - PyOS_snprintf(buf, sizeof(buf), - "<weakproxy at %p to %.100s at %p>", proxy, - Py_TYPE(PyWeakref_GET_OBJECT(proxy))->tp_name, - PyWeakref_GET_OBJECT(proxy)); - return PyUnicode_FromString(buf); + return PyUnicode_FromFormat( + "<weakproxy at %p to %s at %p>", + proxy, + Py_TYPE(PyWeakref_GET_OBJECT(proxy))->tp_name, + PyWeakref_GET_OBJECT(proxy)); } @@ -583,7 +586,7 @@ proxy_iternext(PyWeakReference *proxy) } -WRAP_METHOD(proxy_bytes, "__bytes__") +WRAP_METHOD(proxy_bytes, __bytes__) static PyMethodDef proxy_methods[] = { |