diff options
Diffstat (limited to 'Objects/unicodeobject.c')
| -rw-r--r-- | Objects/unicodeobject.c | 18164 |
1 files changed, 5599 insertions, 12565 deletions
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1ec2acc..a859fa0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1,7 +1,8 @@ /* Unicode implementation based on original code by Fredrik Lundh, -modified by Marc-Andre Lemburg <mal@lemburg.com>. +modified by Marc-Andre Lemburg <mal@lemburg.com> according to the +Unicode Integration Proposal (see file Misc/unicode.txt). Major speed upgrades to the method implementations at the Reykjavik NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke. @@ -40,43 +41,44 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "pycore_fileutils.h" -#include "pycore_initconfig.h" -#include "pycore_object.h" -#include "pycore_pathconfig.h" -#include "pycore_pylifecycle.h" -#include "pycore_pystate.h" + +#include "unicodeobject.h" #include "ucnhash.h" -#include "bytes_methods.h" -#include "stringlib/eq.h" #ifdef MS_WINDOWS #include <windows.h> #endif -/* Uncomment to display statistics on interned strings at exit when - using Valgrind or Insecure++. */ -/* #define INTERNED_STATS 1 */ +/* Limit for the Unicode object free list */ + +#define PyUnicode_MAXFREELIST 1024 + +/* Limit for the Unicode object free list stay alive optimization. + + The implementation will keep allocated Unicode memory intact for + all objects on the free list having a size less than this + limit. This reduces malloc() overhead for small Unicode objects. + + At worst this will result in PyUnicode_MAXFREELIST * + (sizeof(PyUnicodeObject) + KEEPALIVE_SIZE_LIMIT + + malloc()-overhead) bytes of unused garbage. + + Setting the limit to 0 effectively turns the feature off. + Note: This is an experimental feature ! If you get core dumps when + using Unicode objects, turn this feature off. -/*[clinic input] -class str "PyObject *" "&PyUnicode_Type" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4884c934de622cf6]*/ +*/ -/*[python input] -class Py_UCS4_converter(CConverter): - type = 'Py_UCS4' - converter = 'convert_uc' +#define KEEPALIVE_SIZE_LIMIT 9 - def converter_init(self): - if self.default is not unspecified: - self.c_default = ascii(self.default) - if len(self.c_default) > 4 or self.c_default[0] != "'": - self.c_default = hex(ord(self.default)) +/* Endianness switches; defaults to little endian */ -[python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=88f5dd06cd8e7a61]*/ +#ifdef WORDS_BIGENDIAN +# define BYTEORDER_IS_BIG_ENDIAN +#else +# define BYTEORDER_IS_LITTLE_ENDIAN +#endif /* --- Globals ------------------------------------------------------------ @@ -91,197 +93,37 @@ NOTE: In the interpreter's initialization phase, some globals are currently extern "C" { #endif -/* Maximum code point of Unicode 6.0: 0x10ffff (1,114,111) */ -#define MAX_UNICODE 0x10ffff - -#ifdef Py_DEBUG -# define _PyUnicode_CHECK(op) _PyUnicode_CheckConsistency(op, 0) -#else -# define _PyUnicode_CHECK(op) PyUnicode_Check(op) -#endif - -#define _PyUnicode_UTF8(op) \ - (((PyCompactUnicodeObject*)(op))->utf8) -#define PyUnicode_UTF8(op) \ - (assert(_PyUnicode_CHECK(op)), \ - assert(PyUnicode_IS_READY(op)), \ - PyUnicode_IS_COMPACT_ASCII(op) ? \ - ((char*)((PyASCIIObject*)(op) + 1)) : \ - _PyUnicode_UTF8(op)) -#define _PyUnicode_UTF8_LENGTH(op) \ - (((PyCompactUnicodeObject*)(op))->utf8_length) -#define PyUnicode_UTF8_LENGTH(op) \ - (assert(_PyUnicode_CHECK(op)), \ - assert(PyUnicode_IS_READY(op)), \ - PyUnicode_IS_COMPACT_ASCII(op) ? \ - ((PyASCIIObject*)(op))->length : \ - _PyUnicode_UTF8_LENGTH(op)) -#define _PyUnicode_WSTR(op) \ - (((PyASCIIObject*)(op))->wstr) -#define _PyUnicode_WSTR_LENGTH(op) \ - (((PyCompactUnicodeObject*)(op))->wstr_length) -#define _PyUnicode_LENGTH(op) \ - (((PyASCIIObject *)(op))->length) -#define _PyUnicode_STATE(op) \ - (((PyASCIIObject *)(op))->state) -#define _PyUnicode_HASH(op) \ - (((PyASCIIObject *)(op))->hash) -#define _PyUnicode_KIND(op) \ - (assert(_PyUnicode_CHECK(op)), \ - ((PyASCIIObject *)(op))->state.kind) -#define _PyUnicode_GET_LENGTH(op) \ - (assert(_PyUnicode_CHECK(op)), \ - ((PyASCIIObject *)(op))->length) -#define _PyUnicode_DATA_ANY(op) \ - (((PyUnicodeObject*)(op))->data.any) - -#undef PyUnicode_READY -#define PyUnicode_READY(op) \ - (assert(_PyUnicode_CHECK(op)), \ - (PyUnicode_IS_READY(op) ? \ - 0 : \ - _PyUnicode_Ready(op))) - -#define _PyUnicode_SHARE_UTF8(op) \ - (assert(_PyUnicode_CHECK(op)), \ - assert(!PyUnicode_IS_COMPACT_ASCII(op)), \ - (_PyUnicode_UTF8(op) == PyUnicode_DATA(op))) -#define _PyUnicode_SHARE_WSTR(op) \ - (assert(_PyUnicode_CHECK(op)), \ - (_PyUnicode_WSTR(unicode) == PyUnicode_DATA(op))) - -/* true if the Unicode object has an allocated UTF-8 memory block - (not shared with other data) */ -#define _PyUnicode_HAS_UTF8_MEMORY(op) \ - ((!PyUnicode_IS_COMPACT_ASCII(op) \ - && _PyUnicode_UTF8(op) \ - && _PyUnicode_UTF8(op) != PyUnicode_DATA(op))) - -/* true if the Unicode object has an allocated wstr memory block - (not shared with other data) */ -#define _PyUnicode_HAS_WSTR_MEMORY(op) \ - ((_PyUnicode_WSTR(op) && \ - (!PyUnicode_IS_READY(op) || \ - _PyUnicode_WSTR(op) != PyUnicode_DATA(op)))) - -/* Generic helper macro to convert characters of different types. - from_type and to_type have to be valid type names, begin and end - are pointers to the source characters which should be of type - "from_type *". to is a pointer of type "to_type *" and points to the - buffer where the result characters are written to. */ -#define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \ - do { \ - to_type *_to = (to_type *)(to); \ - const from_type *_iter = (from_type *)(begin); \ - const from_type *_end = (from_type *)(end); \ - Py_ssize_t n = (_end) - (_iter); \ - const from_type *_unrolled_end = \ - _iter + _Py_SIZE_ROUND_DOWN(n, 4); \ - while (_iter < (_unrolled_end)) { \ - _to[0] = (to_type) _iter[0]; \ - _to[1] = (to_type) _iter[1]; \ - _to[2] = (to_type) _iter[2]; \ - _to[3] = (to_type) _iter[3]; \ - _iter += 4; _to += 4; \ - } \ - while (_iter < (_end)) \ - *_to++ = (to_type) *_iter++; \ - } while (0) - -#ifdef MS_WINDOWS - /* On Windows, overallocate by 50% is the best factor */ -# define OVERALLOCATE_FACTOR 2 -#else - /* On Linux, overallocate by 25% is the best factor */ -# define OVERALLOCATE_FACTOR 4 -#endif - -/* This dictionary holds all interned unicode strings. Note that references - to strings in this dictionary are *not* counted in the string's ob_refcnt. - When the interned string reaches a refcnt of 0 the string deallocation - function will delete the reference from this dictionary. - - Another way to look at this is that to say that the actual reference - count of a string is: s->ob_refcnt + (s->state ? 2 : 0) -*/ -static PyObject *interned = NULL; +/* Free list for Unicode objects */ +static PyUnicodeObject *free_list = NULL; +static int numfree = 0; /* The empty Unicode object is shared to improve performance. */ -static PyObject *unicode_empty = NULL; +static PyUnicodeObject *unicode_empty = NULL; -#define _Py_INCREF_UNICODE_EMPTY() \ +#define _Py_RETURN_UNICODE_EMPTY() \ do { \ if (unicode_empty != NULL) \ Py_INCREF(unicode_empty); \ else { \ - unicode_empty = PyUnicode_New(0, 0); \ - if (unicode_empty != NULL) { \ + unicode_empty = _PyUnicode_New(0); \ + if (unicode_empty != NULL) \ Py_INCREF(unicode_empty); \ - assert(_PyUnicode_CheckConsistency(unicode_empty, 1)); \ - } \ } \ + return (PyObject *)unicode_empty; \ } while (0) -#define _Py_RETURN_UNICODE_EMPTY() \ - do { \ - _Py_INCREF_UNICODE_EMPTY(); \ - return unicode_empty; \ - } while (0) - -static inline void -unicode_fill(enum PyUnicode_Kind kind, void *data, Py_UCS4 value, - Py_ssize_t start, Py_ssize_t length) -{ - assert(0 <= start); - assert(kind != PyUnicode_WCHAR_KIND); - switch (kind) { - case PyUnicode_1BYTE_KIND: { - assert(value <= 0xff); - Py_UCS1 ch = (unsigned char)value; - Py_UCS1 *to = (Py_UCS1 *)data + start; - memset(to, ch, length); - break; - } - case PyUnicode_2BYTE_KIND: { - assert(value <= 0xffff); - Py_UCS2 ch = (Py_UCS2)value; - Py_UCS2 *to = (Py_UCS2 *)data + start; - const Py_UCS2 *end = to + length; - for (; to < end; ++to) *to = ch; - break; - } - case PyUnicode_4BYTE_KIND: { - assert(value <= MAX_UNICODE); - Py_UCS4 ch = value; - Py_UCS4 * to = (Py_UCS4 *)data + start; - const Py_UCS4 *end = to + length; - for (; to < end; ++to) *to = ch; - break; - } - default: Py_UNREACHABLE(); - } -} - +/* Single character Unicode strings in the Latin-1 range are being + shared as well. */ +static PyUnicodeObject *unicode_latin1[256] = {NULL}; -/* Forward declaration */ -static inline int -_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch); -static inline void -_PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer); -static PyObject * -unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, - const char *errors); -static PyObject * -unicode_decode_utf8(const char *s, Py_ssize_t size, - _Py_error_handler error_handler, const char *errors, - Py_ssize_t *consumed); +/* Default encoding to use and assume when NULL is passed as encoding + parameter; it is initialized by _PyUnicode_Init(). -/* List of static strings. */ -static _Py_Identifier *static_strings = NULL; + Always use the PyUnicode_SetDefaultEncoding() and + PyUnicode_GetDefaultEncoding() APIs to access this global. -/* Single character Unicode strings in the Latin-1 range are being - shared as well. */ -static PyObject *unicode_latin1[256] = {NULL}; +*/ +static char unicode_default_encoding[100 + 1] = "ascii"; /* Fast detection of the most frequent whitespace characters */ const unsigned char _Py_ascii_whitespace[] = { @@ -314,34 +156,8 @@ const unsigned char _Py_ascii_whitespace[] = { 0, 0, 0, 0, 0, 0, 0, 0 }; -/* forward */ -static PyUnicodeObject *_PyUnicode_New(Py_ssize_t length); -static PyObject* get_latin1_char(unsigned char ch); -static int unicode_modifiable(PyObject *unicode); - - -static PyObject * -_PyUnicode_FromUCS1(const Py_UCS1 *s, Py_ssize_t size); -static PyObject * -_PyUnicode_FromUCS2(const Py_UCS2 *s, Py_ssize_t size); -static PyObject * -_PyUnicode_FromUCS4(const Py_UCS4 *s, Py_ssize_t size); - -static PyObject * -unicode_encode_call_errorhandler(const char *errors, - PyObject **errorHandler,const char *encoding, const char *reason, - PyObject *unicode, PyObject **exceptionObject, - Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos); - -static void -raise_encode_exception(PyObject **exceptionObject, - const char *encoding, - PyObject *unicode, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason); - /* Same for linebreaks */ -static const unsigned char ascii_linebreak[] = { +static unsigned char ascii_linebreak[] = { 0, 0, 0, 0, 0, 0, 0, 0, /* 0x000A, * LINE FEED */ /* 0x000B, * LINE TABULATION */ @@ -368,110 +184,7 @@ static const unsigned char ascii_linebreak[] = { 0, 0, 0, 0, 0, 0, 0, 0 }; -static int convert_uc(PyObject *obj, void *addr); - -#include "clinic/unicodeobject.c.h" - -_Py_error_handler -_Py_GetErrorHandler(const char *errors) -{ - if (errors == NULL || strcmp(errors, "strict") == 0) { - return _Py_ERROR_STRICT; - } - if (strcmp(errors, "surrogateescape") == 0) { - return _Py_ERROR_SURROGATEESCAPE; - } - if (strcmp(errors, "replace") == 0) { - return _Py_ERROR_REPLACE; - } - if (strcmp(errors, "ignore") == 0) { - return _Py_ERROR_IGNORE; - } - if (strcmp(errors, "backslashreplace") == 0) { - return _Py_ERROR_BACKSLASHREPLACE; - } - if (strcmp(errors, "surrogatepass") == 0) { - return _Py_ERROR_SURROGATEPASS; - } - if (strcmp(errors, "xmlcharrefreplace") == 0) { - return _Py_ERROR_XMLCHARREFREPLACE; - } - return _Py_ERROR_OTHER; -} - - -static _Py_error_handler -get_error_handler_wide(const wchar_t *errors) -{ - if (errors == NULL || wcscmp(errors, L"strict") == 0) { - return _Py_ERROR_STRICT; - } - if (wcscmp(errors, L"surrogateescape") == 0) { - return _Py_ERROR_SURROGATEESCAPE; - } - if (wcscmp(errors, L"replace") == 0) { - return _Py_ERROR_REPLACE; - } - if (wcscmp(errors, L"ignore") == 0) { - return _Py_ERROR_IGNORE; - } - if (wcscmp(errors, L"backslashreplace") == 0) { - return _Py_ERROR_BACKSLASHREPLACE; - } - if (wcscmp(errors, L"surrogatepass") == 0) { - return _Py_ERROR_SURROGATEPASS; - } - if (wcscmp(errors, L"xmlcharrefreplace") == 0) { - return _Py_ERROR_XMLCHARREFREPLACE; - } - return _Py_ERROR_OTHER; -} - - -static inline int -unicode_check_encoding_errors(const char *encoding, const char *errors) -{ - if (encoding == NULL && errors == NULL) { - return 0; - } - PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); -#ifndef Py_DEBUG - /* In release mode, only check in development mode (-X dev) */ - if (!interp->config.dev_mode) { - return 0; - } -#else - /* Always check in debug mode */ -#endif - - /* Avoid calling _PyCodec_Lookup() and PyCodec_LookupError() before the - codec registry is ready: before_PyUnicode_InitEncodings() is called. */ - if (!interp->fs_codec.encoding) { - return 0; - } - - if (encoding != NULL) { - PyObject *handler = _PyCodec_Lookup(encoding); - if (handler == NULL) { - return -1; - } - Py_DECREF(handler); - } - - if (errors != NULL) { - PyObject *handler = PyCodec_LookupError(errors); - if (handler == NULL) { - return -1; - } - Py_DECREF(handler); - } - return 0; -} - - -/* The max unicode value is always 0x10FFFF while using the PEP-393 API. - This function is kept for backward compatibility with the old API. */ Py_UNICODE PyUnicode_GetMax(void) { @@ -484,349 +197,6 @@ PyUnicode_GetMax(void) #endif } -int -_PyUnicode_CheckConsistency(PyObject *op, int check_content) -{ -#define CHECK(expr) \ - do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) - - PyASCIIObject *ascii; - unsigned int kind; - - assert(op != NULL); - CHECK(PyUnicode_Check(op)); - - ascii = (PyASCIIObject *)op; - kind = ascii->state.kind; - - if (ascii->state.ascii == 1 && ascii->state.compact == 1) { - CHECK(kind == PyUnicode_1BYTE_KIND); - CHECK(ascii->state.ready == 1); - } - else { - PyCompactUnicodeObject *compact = (PyCompactUnicodeObject *)op; - void *data; - - if (ascii->state.compact == 1) { - data = compact + 1; - CHECK(kind == PyUnicode_1BYTE_KIND - || kind == PyUnicode_2BYTE_KIND - || kind == PyUnicode_4BYTE_KIND); - CHECK(ascii->state.ascii == 0); - CHECK(ascii->state.ready == 1); - CHECK(compact->utf8 != data); - } - else { - PyUnicodeObject *unicode = (PyUnicodeObject *)op; - - data = unicode->data.any; - if (kind == PyUnicode_WCHAR_KIND) { - CHECK(ascii->length == 0); - CHECK(ascii->hash == -1); - CHECK(ascii->state.compact == 0); - CHECK(ascii->state.ascii == 0); - CHECK(ascii->state.ready == 0); - CHECK(ascii->state.interned == SSTATE_NOT_INTERNED); - CHECK(ascii->wstr != NULL); - CHECK(data == NULL); - CHECK(compact->utf8 == NULL); - } - else { - CHECK(kind == PyUnicode_1BYTE_KIND - || kind == PyUnicode_2BYTE_KIND - || kind == PyUnicode_4BYTE_KIND); - CHECK(ascii->state.compact == 0); - CHECK(ascii->state.ready == 1); - CHECK(data != NULL); - if (ascii->state.ascii) { - CHECK(compact->utf8 == data); - CHECK(compact->utf8_length == ascii->length); - } - else - CHECK(compact->utf8 != data); - } - } - if (kind != PyUnicode_WCHAR_KIND) { - if ( -#if SIZEOF_WCHAR_T == 2 - kind == PyUnicode_2BYTE_KIND -#else - kind == PyUnicode_4BYTE_KIND -#endif - ) - { - CHECK(ascii->wstr == data); - CHECK(compact->wstr_length == ascii->length); - } else - CHECK(ascii->wstr != data); - } - - if (compact->utf8 == NULL) - CHECK(compact->utf8_length == 0); - if (ascii->wstr == NULL) - CHECK(compact->wstr_length == 0); - } - - /* check that the best kind is used: O(n) operation */ - if (check_content && kind != PyUnicode_WCHAR_KIND) { - Py_ssize_t i; - Py_UCS4 maxchar = 0; - void *data; - Py_UCS4 ch; - - data = PyUnicode_DATA(ascii); - for (i=0; i < ascii->length; i++) - { - ch = PyUnicode_READ(kind, data, i); - if (ch > maxchar) - maxchar = ch; - } - if (kind == PyUnicode_1BYTE_KIND) { - if (ascii->state.ascii == 0) { - CHECK(maxchar >= 128); - CHECK(maxchar <= 255); - } - else - CHECK(maxchar < 128); - } - else if (kind == PyUnicode_2BYTE_KIND) { - CHECK(maxchar >= 0x100); - CHECK(maxchar <= 0xFFFF); - } - else { - CHECK(maxchar >= 0x10000); - CHECK(maxchar <= MAX_UNICODE); - } - CHECK(PyUnicode_READ(kind, data, ascii->length) == 0); - } - return 1; - -#undef CHECK -} - - -static PyObject* -unicode_result_wchar(PyObject *unicode) -{ -#ifndef Py_DEBUG - Py_ssize_t len; - - len = _PyUnicode_WSTR_LENGTH(unicode); - if (len == 0) { - Py_DECREF(unicode); - _Py_RETURN_UNICODE_EMPTY(); - } - - if (len == 1) { - wchar_t ch = _PyUnicode_WSTR(unicode)[0]; - if ((Py_UCS4)ch < 256) { - PyObject *latin1_char = get_latin1_char((unsigned char)ch); - Py_DECREF(unicode); - return latin1_char; - } - } - - if (_PyUnicode_Ready(unicode) < 0) { - Py_DECREF(unicode); - return NULL; - } -#else - assert(Py_REFCNT(unicode) == 1); - - /* don't make the result ready in debug mode to ensure that the caller - makes the string ready before using it */ - assert(_PyUnicode_CheckConsistency(unicode, 1)); -#endif - return unicode; -} - -static PyObject* -unicode_result_ready(PyObject *unicode) -{ - Py_ssize_t length; - - length = PyUnicode_GET_LENGTH(unicode); - if (length == 0) { - if (unicode != unicode_empty) { - Py_DECREF(unicode); - _Py_RETURN_UNICODE_EMPTY(); - } - return unicode_empty; - } - - if (length == 1) { - void *data = PyUnicode_DATA(unicode); - int kind = PyUnicode_KIND(unicode); - Py_UCS4 ch = PyUnicode_READ(kind, data, 0); - if (ch < 256) { - PyObject *latin1_char = unicode_latin1[ch]; - if (latin1_char != NULL) { - if (unicode != latin1_char) { - Py_INCREF(latin1_char); - Py_DECREF(unicode); - } - return latin1_char; - } - else { - assert(_PyUnicode_CheckConsistency(unicode, 1)); - Py_INCREF(unicode); - unicode_latin1[ch] = unicode; - return unicode; - } - } - } - - assert(_PyUnicode_CheckConsistency(unicode, 1)); - return unicode; -} - -static PyObject* -unicode_result(PyObject *unicode) -{ - assert(_PyUnicode_CHECK(unicode)); - if (PyUnicode_IS_READY(unicode)) - return unicode_result_ready(unicode); - else - return unicode_result_wchar(unicode); -} - -static PyObject* -unicode_result_unchanged(PyObject *unicode) -{ - if (PyUnicode_CheckExact(unicode)) { - if (PyUnicode_READY(unicode) == -1) - return NULL; - Py_INCREF(unicode); - return unicode; - } - else - /* Subtype -- return genuine unicode string with the same value. */ - return _PyUnicode_Copy(unicode); -} - -/* Implementation of the "backslashreplace" error handler for 8-bit encodings: - ASCII, Latin1, UTF-8, etc. */ -static char* -backslashreplace(_PyBytesWriter *writer, char *str, - PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) -{ - Py_ssize_t size, i; - Py_UCS4 ch; - enum PyUnicode_Kind kind; - void *data; - - assert(PyUnicode_IS_READY(unicode)); - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - - size = 0; - /* determine replacement size */ - for (i = collstart; i < collend; ++i) { - Py_ssize_t incr; - - ch = PyUnicode_READ(kind, data, i); - if (ch < 0x100) - incr = 2+2; - else if (ch < 0x10000) - incr = 2+4; - else { - assert(ch <= MAX_UNICODE); - incr = 2+8; - } - if (size > PY_SSIZE_T_MAX - incr) { - PyErr_SetString(PyExc_OverflowError, - "encoded result is too long for a Python string"); - return NULL; - } - size += incr; - } - - str = _PyBytesWriter_Prepare(writer, str, size); - if (str == NULL) - return NULL; - - /* generate replacement */ - for (i = collstart; i < collend; ++i) { - ch = PyUnicode_READ(kind, data, i); - *str++ = '\\'; - if (ch >= 0x00010000) { - *str++ = 'U'; - *str++ = Py_hexdigits[(ch>>28)&0xf]; - *str++ = Py_hexdigits[(ch>>24)&0xf]; - *str++ = Py_hexdigits[(ch>>20)&0xf]; - *str++ = Py_hexdigits[(ch>>16)&0xf]; - *str++ = Py_hexdigits[(ch>>12)&0xf]; - *str++ = Py_hexdigits[(ch>>8)&0xf]; - } - else if (ch >= 0x100) { - *str++ = 'u'; - *str++ = Py_hexdigits[(ch>>12)&0xf]; - *str++ = Py_hexdigits[(ch>>8)&0xf]; - } - else - *str++ = 'x'; - *str++ = Py_hexdigits[(ch>>4)&0xf]; - *str++ = Py_hexdigits[ch&0xf]; - } - return str; -} - -/* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings: - ASCII, Latin1, UTF-8, etc. */ -static char* -xmlcharrefreplace(_PyBytesWriter *writer, char *str, - PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) -{ - Py_ssize_t size, i; - Py_UCS4 ch; - enum PyUnicode_Kind kind; - void *data; - - assert(PyUnicode_IS_READY(unicode)); - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - - size = 0; - /* determine replacement size */ - for (i = collstart; i < collend; ++i) { - Py_ssize_t incr; - - ch = PyUnicode_READ(kind, data, i); - if (ch < 10) - incr = 2+1+1; - else if (ch < 100) - incr = 2+2+1; - else if (ch < 1000) - incr = 2+3+1; - else if (ch < 10000) - incr = 2+4+1; - else if (ch < 100000) - incr = 2+5+1; - else if (ch < 1000000) - incr = 2+6+1; - else { - assert(ch <= MAX_UNICODE); - incr = 2+7+1; - } - if (size > PY_SSIZE_T_MAX - incr) { - PyErr_SetString(PyExc_OverflowError, - "encoded result is too long for a Python string"); - return NULL; - } - size += incr; - } - - str = _PyBytesWriter_Prepare(writer, str, size); - if (str == NULL) - return NULL; - - /* generate replacement */ - for (i = collstart; i < collend; ++i) { - str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); - } - return str; -} - /* --- Bloom Filters ----------------------------------------------------- */ /* stuff to implement simple "bloom filters" for Unicode characters. @@ -849,392 +219,149 @@ xmlcharrefreplace(_PyBytesWriter *writer, char *str, static BLOOM_MASK bloom_linebreak = ~(BLOOM_MASK)0; +#define BLOOM_ADD(mask, ch) ((mask |= (1UL << ((ch) & (BLOOM_WIDTH - 1))))) #define BLOOM(mask, ch) ((mask & (1UL << ((ch) & (BLOOM_WIDTH - 1))))) #define BLOOM_LINEBREAK(ch) \ ((ch) < 128U ? ascii_linebreak[(ch)] : \ (BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK(ch))) -static inline BLOOM_MASK -make_bloom_mask(int kind, void* ptr, Py_ssize_t len) -{ -#define BLOOM_UPDATE(TYPE, MASK, PTR, LEN) \ - do { \ - TYPE *data = (TYPE *)PTR; \ - TYPE *end = data + LEN; \ - Py_UCS4 ch; \ - for (; data != end; data++) { \ - ch = *data; \ - MASK |= (1UL << (ch & (BLOOM_WIDTH - 1))); \ - } \ - break; \ - } while (0) - +Py_LOCAL_INLINE(BLOOM_MASK) make_bloom_mask(Py_UNICODE* ptr, Py_ssize_t len) +{ /* calculate simple bloom-style bitmask for a given unicode string */ BLOOM_MASK mask; + Py_ssize_t i; mask = 0; - switch (kind) { - case PyUnicode_1BYTE_KIND: - BLOOM_UPDATE(Py_UCS1, mask, ptr, len); - break; - case PyUnicode_2BYTE_KIND: - BLOOM_UPDATE(Py_UCS2, mask, ptr, len); - break; - case PyUnicode_4BYTE_KIND: - BLOOM_UPDATE(Py_UCS4, mask, ptr, len); - break; - default: - Py_UNREACHABLE(); - } - return mask; + for (i = 0; i < len; i++) + BLOOM_ADD(mask, ptr[i]); -#undef BLOOM_UPDATE + return mask; } -static int -ensure_unicode(PyObject *obj) +Py_LOCAL_INLINE(int) unicode_member(Py_UNICODE chr, Py_UNICODE* set, Py_ssize_t setlen) { - if (!PyUnicode_Check(obj)) { - PyErr_Format(PyExc_TypeError, - "must be str, not %.100s", - Py_TYPE(obj)->tp_name); - return -1; - } - return PyUnicode_READY(obj); -} - -/* Compilation of templated routines */ - -#include "stringlib/asciilib.h" -#include "stringlib/fastsearch.h" -#include "stringlib/partition.h" -#include "stringlib/split.h" -#include "stringlib/count.h" -#include "stringlib/find.h" -#include "stringlib/find_max_char.h" -#include "stringlib/undef.h" - -#include "stringlib/ucs1lib.h" -#include "stringlib/fastsearch.h" -#include "stringlib/partition.h" -#include "stringlib/split.h" -#include "stringlib/count.h" -#include "stringlib/find.h" -#include "stringlib/replace.h" -#include "stringlib/find_max_char.h" -#include "stringlib/undef.h" - -#include "stringlib/ucs2lib.h" -#include "stringlib/fastsearch.h" -#include "stringlib/partition.h" -#include "stringlib/split.h" -#include "stringlib/count.h" -#include "stringlib/find.h" -#include "stringlib/replace.h" -#include "stringlib/find_max_char.h" -#include "stringlib/undef.h" - -#include "stringlib/ucs4lib.h" -#include "stringlib/fastsearch.h" -#include "stringlib/partition.h" -#include "stringlib/split.h" -#include "stringlib/count.h" -#include "stringlib/find.h" -#include "stringlib/replace.h" -#include "stringlib/find_max_char.h" -#include "stringlib/undef.h" - -#include "stringlib/unicodedefs.h" -#include "stringlib/fastsearch.h" -#include "stringlib/count.h" -#include "stringlib/find.h" -#include "stringlib/undef.h" + Py_ssize_t i; -/* --- Unicode Object ----------------------------------------------------- */ + for (i = 0; i < setlen; i++) + if (set[i] == chr) + return 1; -static inline Py_ssize_t -findchar(const void *s, int kind, - Py_ssize_t size, Py_UCS4 ch, - int direction) -{ - switch (kind) { - case PyUnicode_1BYTE_KIND: - if ((Py_UCS1) ch != ch) - return -1; - if (direction > 0) - return ucs1lib_find_char((Py_UCS1 *) s, size, (Py_UCS1) ch); - else - return ucs1lib_rfind_char((Py_UCS1 *) s, size, (Py_UCS1) ch); - case PyUnicode_2BYTE_KIND: - if ((Py_UCS2) ch != ch) - return -1; - if (direction > 0) - return ucs2lib_find_char((Py_UCS2 *) s, size, (Py_UCS2) ch); - else - return ucs2lib_rfind_char((Py_UCS2 *) s, size, (Py_UCS2) ch); - case PyUnicode_4BYTE_KIND: - if (direction > 0) - return ucs4lib_find_char((Py_UCS4 *) s, size, ch); - else - return ucs4lib_rfind_char((Py_UCS4 *) s, size, ch); - default: - Py_UNREACHABLE(); - } + return 0; } -#ifdef Py_DEBUG -/* Fill the data of a Unicode string with invalid characters to detect bugs - earlier. +#define BLOOM_MEMBER(mask, chr, set, setlen) \ + BLOOM(mask, chr) && unicode_member(chr, set, setlen) - _PyUnicode_CheckConsistency(str, 1) detects invalid characters, at least for - ASCII and UCS-4 strings. U+00FF is invalid in ASCII and U+FFFFFFFF is an - invalid character in Unicode 6.0. */ -static void -unicode_fill_invalid(PyObject *unicode, Py_ssize_t old_length) -{ - int kind = PyUnicode_KIND(unicode); - Py_UCS1 *data = PyUnicode_1BYTE_DATA(unicode); - Py_ssize_t length = _PyUnicode_LENGTH(unicode); - if (length <= old_length) - return; - memset(data + old_length * kind, 0xff, (length - old_length) * kind); -} -#endif +/* --- Unicode Object ----------------------------------------------------- */ -static PyObject* -resize_compact(PyObject *unicode, Py_ssize_t length) +static +int unicode_resize(register PyUnicodeObject *unicode, + Py_ssize_t length) { - Py_ssize_t char_size; - Py_ssize_t struct_size; - Py_ssize_t new_size; - int share_wstr; - PyObject *new_unicode; -#ifdef Py_DEBUG - Py_ssize_t old_length = _PyUnicode_LENGTH(unicode); -#endif - - assert(unicode_modifiable(unicode)); - assert(PyUnicode_IS_READY(unicode)); - assert(PyUnicode_IS_COMPACT(unicode)); - - char_size = PyUnicode_KIND(unicode); - if (PyUnicode_IS_ASCII(unicode)) - struct_size = sizeof(PyASCIIObject); - else - struct_size = sizeof(PyCompactUnicodeObject); - share_wstr = _PyUnicode_SHARE_WSTR(unicode); + void *oldstr; - if (length > ((PY_SSIZE_T_MAX - struct_size) / char_size - 1)) { - PyErr_NoMemory(); - return NULL; - } - new_size = (struct_size + (length + 1) * char_size); + /* Shortcut if there's nothing much to do. */ + if (unicode->length == length) + goto reset; - if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { - PyObject_DEL(_PyUnicode_UTF8(unicode)); - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; - } - _Py_DEC_REFTOTAL; - _Py_ForgetReference(unicode); + /* Resizing shared object (unicode_empty or single character + objects) in-place is not allowed. Use PyUnicode_Resize() + instead ! */ - new_unicode = (PyObject *)PyObject_REALLOC(unicode, new_size); - if (new_unicode == NULL) { - _Py_NewReference(unicode); - PyErr_NoMemory(); - return NULL; - } - unicode = new_unicode; - _Py_NewReference(unicode); - - _PyUnicode_LENGTH(unicode) = length; - if (share_wstr) { - _PyUnicode_WSTR(unicode) = PyUnicode_DATA(unicode); - if (!PyUnicode_IS_ASCII(unicode)) - _PyUnicode_WSTR_LENGTH(unicode) = length; - } - else if (_PyUnicode_HAS_WSTR_MEMORY(unicode)) { - PyObject_DEL(_PyUnicode_WSTR(unicode)); - _PyUnicode_WSTR(unicode) = NULL; - if (!PyUnicode_IS_ASCII(unicode)) - _PyUnicode_WSTR_LENGTH(unicode) = 0; + if (unicode == unicode_empty || + (unicode->length == 1 && + unicode->str[0] < 256U && + unicode_latin1[unicode->str[0]] == unicode)) { + PyErr_SetString(PyExc_SystemError, + "can't resize shared unicode objects"); + return -1; } -#ifdef Py_DEBUG - unicode_fill_invalid(unicode, old_length); -#endif - PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode), - length, 0); - assert(_PyUnicode_CheckConsistency(unicode, 0)); - return unicode; -} - -static int -resize_inplace(PyObject *unicode, Py_ssize_t length) -{ - wchar_t *wstr; - Py_ssize_t new_size; - assert(!PyUnicode_IS_COMPACT(unicode)); - assert(Py_REFCNT(unicode) == 1); - - if (PyUnicode_IS_READY(unicode)) { - Py_ssize_t char_size; - int share_wstr, share_utf8; - void *data; -#ifdef Py_DEBUG - Py_ssize_t old_length = _PyUnicode_LENGTH(unicode); -#endif - - data = _PyUnicode_DATA_ANY(unicode); - char_size = PyUnicode_KIND(unicode); - share_wstr = _PyUnicode_SHARE_WSTR(unicode); - share_utf8 = _PyUnicode_SHARE_UTF8(unicode); - - if (length > (PY_SSIZE_T_MAX / char_size - 1)) { - PyErr_NoMemory(); - return -1; - } - new_size = (length + 1) * char_size; - if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode)) - { - PyObject_DEL(_PyUnicode_UTF8(unicode)); - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; - } - - data = (PyObject *)PyObject_REALLOC(data, new_size); - if (data == NULL) { - PyErr_NoMemory(); - return -1; - } - _PyUnicode_DATA_ANY(unicode) = data; - if (share_wstr) { - _PyUnicode_WSTR(unicode) = data; - _PyUnicode_WSTR_LENGTH(unicode) = length; - } - if (share_utf8) { - _PyUnicode_UTF8(unicode) = data; - _PyUnicode_UTF8_LENGTH(unicode) = length; - } - _PyUnicode_LENGTH(unicode) = length; - PyUnicode_WRITE(PyUnicode_KIND(unicode), data, length, 0); -#ifdef Py_DEBUG - unicode_fill_invalid(unicode, old_length); -#endif - if (share_wstr || _PyUnicode_WSTR(unicode) == NULL) { - assert(_PyUnicode_CheckConsistency(unicode, 0)); - return 0; - } - } - assert(_PyUnicode_WSTR(unicode) != NULL); + /* We allocate one more byte to make sure the string is Ux0000 terminated. + The overallocation is also used by fastsearch, which assumes that it's + safe to look at str[length] (without making any assumptions about what + it contains). */ - /* check for integer overflow */ - if (length > PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(wchar_t) - 1) { - PyErr_NoMemory(); - return -1; - } - new_size = sizeof(wchar_t) * (length + 1); - wstr = _PyUnicode_WSTR(unicode); - wstr = PyObject_REALLOC(wstr, new_size); - if (!wstr) { + oldstr = unicode->str; + unicode->str = PyObject_REALLOC(unicode->str, + sizeof(Py_UNICODE) * (length + 1)); + if (!unicode->str) { + unicode->str = (Py_UNICODE *)oldstr; PyErr_NoMemory(); return -1; } - _PyUnicode_WSTR(unicode) = wstr; - _PyUnicode_WSTR(unicode)[length] = 0; - _PyUnicode_WSTR_LENGTH(unicode) = length; - assert(_PyUnicode_CheckConsistency(unicode, 0)); - return 0; -} - -static PyObject* -resize_copy(PyObject *unicode, Py_ssize_t length) -{ - Py_ssize_t copy_length; - if (_PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND) { - PyObject *copy; - - assert(PyUnicode_IS_READY(unicode)); + unicode->str[length] = 0; + unicode->length = length; - copy = PyUnicode_New(length, PyUnicode_MAX_CHAR_VALUE(unicode)); - if (copy == NULL) - return NULL; - - copy_length = Py_MIN(length, PyUnicode_GET_LENGTH(unicode)); - _PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, copy_length); - return copy; + reset: + /* Reset the object caches */ + if (unicode->defenc) { + Py_CLEAR(unicode->defenc); } - else { - PyObject *w; + unicode->hash = -1; - w = (PyObject*)_PyUnicode_New(length); - if (w == NULL) - return NULL; - copy_length = _PyUnicode_WSTR_LENGTH(unicode); - copy_length = Py_MIN(copy_length, length); - memcpy(_PyUnicode_WSTR(w), _PyUnicode_WSTR(unicode), - copy_length * sizeof(wchar_t)); - return w; - } + return 0; } /* We allocate one more byte to make sure the string is - Ux0000 terminated; some code (e.g. new_identifier) - relies on that. + Ux0000 terminated; some code relies on that. XXX This allocator could further be enhanced by assuring that the free list never reduces its size below 1. */ -static PyUnicodeObject * -_PyUnicode_New(Py_ssize_t length) +static +PyUnicodeObject *_PyUnicode_New(Py_ssize_t length) { - PyUnicodeObject *unicode; - size_t new_size; + register PyUnicodeObject *unicode; /* Optimization for empty strings */ if (length == 0 && unicode_empty != NULL) { Py_INCREF(unicode_empty); - return (PyUnicodeObject*)unicode_empty; + return unicode_empty; } /* Ensure we won't overflow the size. */ - if (length > ((PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(Py_UNICODE)) - 1)) { + if (length > ((PY_SSIZE_T_MAX / sizeof(Py_UNICODE)) - 1)) { return (PyUnicodeObject *)PyErr_NoMemory(); } - if (length < 0) { - PyErr_SetString(PyExc_SystemError, - "Negative size passed to _PyUnicode_New"); - return NULL; + + /* Unicode freelist & memory allocation */ + if (free_list) { + unicode = free_list; + free_list = *(PyUnicodeObject **)unicode; + numfree--; + if (unicode->str) { + /* Keep-Alive optimization: we only upsize the buffer, + never downsize it. */ + if ((unicode->length < length) && + unicode_resize(unicode, length) < 0) { + PyObject_DEL(unicode->str); + unicode->str = NULL; + } + } + else { + size_t new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); + unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size); + } + (void)PyObject_INIT(unicode, &PyUnicode_Type); + } + else { + size_t new_size; + unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type); + if (unicode == NULL) + return NULL; + new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); + unicode->str = (Py_UNICODE*) PyObject_MALLOC(new_size); } - unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type); - if (unicode == NULL) - return NULL; - new_size = sizeof(Py_UNICODE) * ((size_t)length + 1); - - _PyUnicode_WSTR_LENGTH(unicode) = length; - _PyUnicode_HASH(unicode) = -1; - _PyUnicode_STATE(unicode).interned = 0; - _PyUnicode_STATE(unicode).kind = 0; - _PyUnicode_STATE(unicode).compact = 0; - _PyUnicode_STATE(unicode).ready = 0; - _PyUnicode_STATE(unicode).ascii = 0; - _PyUnicode_DATA_ANY(unicode) = NULL; - _PyUnicode_LENGTH(unicode) = 0; - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; - - _PyUnicode_WSTR(unicode) = (Py_UNICODE*) PyObject_MALLOC(new_size); - if (!_PyUnicode_WSTR(unicode)) { - Py_DECREF(unicode); + if (!unicode->str) { PyErr_NoMemory(); - return NULL; + goto onError; } - /* Initialize the first element to guard against cases where * the caller fails before initializing str -- unicode_resize() * reads str[0], and the Keep-Alive optimization can keep memory @@ -1242,1804 +369,724 @@ _PyUnicode_New(Py_ssize_t length) * We don't want unicode_resize to read uninitialized memory in * that case. */ - _PyUnicode_WSTR(unicode)[0] = 0; - _PyUnicode_WSTR(unicode)[length] = 0; - - assert(_PyUnicode_CheckConsistency((PyObject *)unicode, 0)); + unicode->str[0] = 0; + unicode->str[length] = 0; + unicode->length = length; + unicode->hash = -1; + unicode->defenc = NULL; return unicode; -} - -static const char* -unicode_kind_name(PyObject *unicode) -{ - /* don't check consistency: unicode_kind_name() is called from - _PyUnicode_Dump() */ - if (!PyUnicode_IS_COMPACT(unicode)) - { - if (!PyUnicode_IS_READY(unicode)) - return "wstr"; - switch (PyUnicode_KIND(unicode)) - { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(unicode)) - return "legacy ascii"; - else - return "legacy latin1"; - case PyUnicode_2BYTE_KIND: - return "legacy UCS2"; - case PyUnicode_4BYTE_KIND: - return "legacy UCS4"; - default: - return "<legacy invalid kind>"; - } - } - assert(PyUnicode_IS_READY(unicode)); - switch (PyUnicode_KIND(unicode)) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(unicode)) - return "ascii"; - else - return "latin1"; - case PyUnicode_2BYTE_KIND: - return "UCS2"; - case PyUnicode_4BYTE_KIND: - return "UCS4"; - default: - return "<invalid compact kind>"; - } -} - -#ifdef Py_DEBUG -/* Functions wrapping macros for use in debugger */ -char *_PyUnicode_utf8(void *unicode_raw){ - PyObject *unicode = _PyObject_CAST(unicode_raw); - return PyUnicode_UTF8(unicode); -} - -void *_PyUnicode_compact_data(void *unicode_raw) { - PyObject *unicode = _PyObject_CAST(unicode_raw); - return _PyUnicode_COMPACT_DATA(unicode); -} -void *_PyUnicode_data(void *unicode_raw) { - PyObject *unicode = _PyObject_CAST(unicode_raw); - printf("obj %p\n", (void*)unicode); - printf("compact %d\n", PyUnicode_IS_COMPACT(unicode)); - printf("compact ascii %d\n", PyUnicode_IS_COMPACT_ASCII(unicode)); - printf("ascii op %p\n", ((void*)((PyASCIIObject*)(unicode) + 1))); - printf("compact op %p\n", ((void*)((PyCompactUnicodeObject*)(unicode) + 1))); - printf("compact data %p\n", _PyUnicode_COMPACT_DATA(unicode)); - return PyUnicode_DATA(unicode); -} - -void -_PyUnicode_Dump(PyObject *op) -{ - PyASCIIObject *ascii = (PyASCIIObject *)op; - PyCompactUnicodeObject *compact = (PyCompactUnicodeObject *)op; - PyUnicodeObject *unicode = (PyUnicodeObject *)op; - void *data; - - if (ascii->state.compact) - { - if (ascii->state.ascii) - data = (ascii + 1); - else - data = (compact + 1); - } - else - data = unicode->data.any; - printf("%s: len=%" PY_FORMAT_SIZE_T "u, ", - unicode_kind_name(op), ascii->length); - - if (ascii->wstr == data) - printf("shared "); - printf("wstr=%p", (void *)ascii->wstr); - - if (!(ascii->state.ascii == 1 && ascii->state.compact == 1)) { - printf(" (%" PY_FORMAT_SIZE_T "u), ", compact->wstr_length); - if (!ascii->state.compact && compact->utf8 == unicode->data.any) - printf("shared "); - printf("utf8=%p (%" PY_FORMAT_SIZE_T "u)", - (void *)compact->utf8, compact->utf8_length); - } - printf(", data=%p\n", data); -} -#endif - -PyObject * -PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) -{ - PyObject *obj; - PyCompactUnicodeObject *unicode; - void *data; - enum PyUnicode_Kind kind; - int is_sharing, is_ascii; - Py_ssize_t char_size; - Py_ssize_t struct_size; - - /* Optimization for empty strings */ - if (size == 0 && unicode_empty != NULL) { - Py_INCREF(unicode_empty); - return unicode_empty; - } - - is_ascii = 0; - is_sharing = 0; - struct_size = sizeof(PyCompactUnicodeObject); - if (maxchar < 128) { - kind = PyUnicode_1BYTE_KIND; - char_size = 1; - is_ascii = 1; - struct_size = sizeof(PyASCIIObject); - } - else if (maxchar < 256) { - kind = PyUnicode_1BYTE_KIND; - char_size = 1; - } - else if (maxchar < 65536) { - kind = PyUnicode_2BYTE_KIND; - char_size = 2; - if (sizeof(wchar_t) == 2) - is_sharing = 1; - } - else { - if (maxchar > MAX_UNICODE) { - PyErr_SetString(PyExc_SystemError, - "invalid maximum character passed to PyUnicode_New"); - return NULL; - } - kind = PyUnicode_4BYTE_KIND; - char_size = 4; - if (sizeof(wchar_t) == 4) - is_sharing = 1; - } - - /* Ensure we won't overflow the size. */ - if (size < 0) { - PyErr_SetString(PyExc_SystemError, - "Negative size passed to PyUnicode_New"); - return NULL; - } - if (size > ((PY_SSIZE_T_MAX - struct_size) / char_size - 1)) - return PyErr_NoMemory(); - - /* Duplicated allocation code from _PyObject_New() instead of a call to - * PyObject_New() so we are able to allocate space for the object and - * it's data buffer. - */ - obj = (PyObject *) PyObject_MALLOC(struct_size + (size + 1) * char_size); - if (obj == NULL) - return PyErr_NoMemory(); - obj = PyObject_INIT(obj, &PyUnicode_Type); - if (obj == NULL) - return NULL; - - unicode = (PyCompactUnicodeObject *)obj; - if (is_ascii) - data = ((PyASCIIObject*)obj) + 1; - else - data = unicode + 1; - _PyUnicode_LENGTH(unicode) = size; - _PyUnicode_HASH(unicode) = -1; - _PyUnicode_STATE(unicode).interned = 0; - _PyUnicode_STATE(unicode).kind = kind; - _PyUnicode_STATE(unicode).compact = 1; - _PyUnicode_STATE(unicode).ready = 1; - _PyUnicode_STATE(unicode).ascii = is_ascii; - if (is_ascii) { - ((char*)data)[size] = 0; - _PyUnicode_WSTR(unicode) = NULL; - } - else if (kind == PyUnicode_1BYTE_KIND) { - ((char*)data)[size] = 0; - _PyUnicode_WSTR(unicode) = NULL; - _PyUnicode_WSTR_LENGTH(unicode) = 0; - unicode->utf8 = NULL; - unicode->utf8_length = 0; - } - else { - unicode->utf8 = NULL; - unicode->utf8_length = 0; - if (kind == PyUnicode_2BYTE_KIND) - ((Py_UCS2*)data)[size] = 0; - else /* kind == PyUnicode_4BYTE_KIND */ - ((Py_UCS4*)data)[size] = 0; - if (is_sharing) { - _PyUnicode_WSTR_LENGTH(unicode) = size; - _PyUnicode_WSTR(unicode) = (wchar_t *)data; - } - else { - _PyUnicode_WSTR_LENGTH(unicode) = 0; - _PyUnicode_WSTR(unicode) = NULL; - } - } -#ifdef Py_DEBUG - unicode_fill_invalid((PyObject*)unicode, 0); -#endif - assert(_PyUnicode_CheckConsistency((PyObject*)unicode, 0)); - return obj; -} - -#if SIZEOF_WCHAR_T == 2 -/* Helper function to convert a 16-bits wchar_t representation to UCS4, this - will decode surrogate pairs, the other conversions are implemented as macros - for efficiency. - - This function assumes that unicode can hold one more code point than wstr - characters for a terminating null character. */ -static void -unicode_convert_wchar_to_ucs4(const wchar_t *begin, const wchar_t *end, - PyObject *unicode) -{ - const wchar_t *iter; - Py_UCS4 *ucs4_out; - - assert(unicode != NULL); - assert(_PyUnicode_CHECK(unicode)); - assert(_PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); - ucs4_out = PyUnicode_4BYTE_DATA(unicode); - - for (iter = begin; iter < end; ) { - assert(ucs4_out < (PyUnicode_4BYTE_DATA(unicode) + - _PyUnicode_GET_LENGTH(unicode))); - if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) - && (iter+1) < end - && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) - { - *ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); - iter += 2; - } - else { - *ucs4_out++ = *iter; - iter++; - } - } - assert(ucs4_out == (PyUnicode_4BYTE_DATA(unicode) + - _PyUnicode_GET_LENGTH(unicode))); - -} -#endif -static int -unicode_check_modifiable(PyObject *unicode) -{ - if (!unicode_modifiable(unicode)) { - PyErr_SetString(PyExc_SystemError, - "Cannot modify a string currently used"); - return -1; - } - return 0; + onError: + /* XXX UNREF/NEWREF interface should be more symmetrical */ + _Py_DEC_REFTOTAL; + _Py_ForgetReference((PyObject *)unicode); + PyObject_Del(unicode); + return NULL; } -static int -_copy_characters(PyObject *to, Py_ssize_t to_start, - PyObject *from, Py_ssize_t from_start, - Py_ssize_t how_many, int check_maxchar) +static +void unicode_dealloc(register PyUnicodeObject *unicode) { - unsigned int from_kind, to_kind; - void *from_data, *to_data; - - assert(0 <= how_many); - assert(0 <= from_start); - assert(0 <= to_start); - assert(PyUnicode_Check(from)); - assert(PyUnicode_IS_READY(from)); - assert(from_start + how_many <= PyUnicode_GET_LENGTH(from)); - - assert(PyUnicode_Check(to)); - assert(PyUnicode_IS_READY(to)); - assert(to_start + how_many <= PyUnicode_GET_LENGTH(to)); - - if (how_many == 0) - return 0; - - from_kind = PyUnicode_KIND(from); - from_data = PyUnicode_DATA(from); - to_kind = PyUnicode_KIND(to); - to_data = PyUnicode_DATA(to); - -#ifdef Py_DEBUG - if (!check_maxchar - && PyUnicode_MAX_CHAR_VALUE(from) > PyUnicode_MAX_CHAR_VALUE(to)) - { - const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to); - Py_UCS4 ch; - Py_ssize_t i; - for (i=0; i < how_many; i++) { - ch = PyUnicode_READ(from_kind, from_data, from_start + i); - assert(ch <= to_maxchar); + if (PyUnicode_CheckExact(unicode) && + numfree < PyUnicode_MAXFREELIST) { + /* Keep-Alive optimization */ + if (unicode->length >= KEEPALIVE_SIZE_LIMIT) { + PyObject_DEL(unicode->str); + unicode->str = NULL; + unicode->length = 0; } - } -#endif - - if (from_kind == to_kind) { - if (check_maxchar - && !PyUnicode_IS_ASCII(from) && PyUnicode_IS_ASCII(to)) - { - /* Writing Latin-1 characters into an ASCII string requires to - check that all written characters are pure ASCII */ - Py_UCS4 max_char; - max_char = ucs1lib_find_max_char(from_data, - (Py_UCS1*)from_data + how_many); - if (max_char >= 128) - return -1; + if (unicode->defenc) { + Py_CLEAR(unicode->defenc); } - memcpy((char*)to_data + to_kind * to_start, - (char*)from_data + from_kind * from_start, - to_kind * how_many); - } - else if (from_kind == PyUnicode_1BYTE_KIND - && to_kind == PyUnicode_2BYTE_KIND) - { - _PyUnicode_CONVERT_BYTES( - Py_UCS1, Py_UCS2, - PyUnicode_1BYTE_DATA(from) + from_start, - PyUnicode_1BYTE_DATA(from) + from_start + how_many, - PyUnicode_2BYTE_DATA(to) + to_start - ); - } - else if (from_kind == PyUnicode_1BYTE_KIND - && to_kind == PyUnicode_4BYTE_KIND) - { - _PyUnicode_CONVERT_BYTES( - Py_UCS1, Py_UCS4, - PyUnicode_1BYTE_DATA(from) + from_start, - PyUnicode_1BYTE_DATA(from) + from_start + how_many, - PyUnicode_4BYTE_DATA(to) + to_start - ); - } - else if (from_kind == PyUnicode_2BYTE_KIND - && to_kind == PyUnicode_4BYTE_KIND) - { - _PyUnicode_CONVERT_BYTES( - Py_UCS2, Py_UCS4, - PyUnicode_2BYTE_DATA(from) + from_start, - PyUnicode_2BYTE_DATA(from) + from_start + how_many, - PyUnicode_4BYTE_DATA(to) + to_start - ); + /* Add to free list */ + *(PyUnicodeObject **)unicode = free_list; + free_list = unicode; + numfree++; } else { - assert (PyUnicode_MAX_CHAR_VALUE(from) > PyUnicode_MAX_CHAR_VALUE(to)); - - if (!check_maxchar) { - if (from_kind == PyUnicode_2BYTE_KIND - && to_kind == PyUnicode_1BYTE_KIND) - { - _PyUnicode_CONVERT_BYTES( - Py_UCS2, Py_UCS1, - PyUnicode_2BYTE_DATA(from) + from_start, - PyUnicode_2BYTE_DATA(from) + from_start + how_many, - PyUnicode_1BYTE_DATA(to) + to_start - ); - } - else if (from_kind == PyUnicode_4BYTE_KIND - && to_kind == PyUnicode_1BYTE_KIND) - { - _PyUnicode_CONVERT_BYTES( - Py_UCS4, Py_UCS1, - PyUnicode_4BYTE_DATA(from) + from_start, - PyUnicode_4BYTE_DATA(from) + from_start + how_many, - PyUnicode_1BYTE_DATA(to) + to_start - ); - } - else if (from_kind == PyUnicode_4BYTE_KIND - && to_kind == PyUnicode_2BYTE_KIND) - { - _PyUnicode_CONVERT_BYTES( - Py_UCS4, Py_UCS2, - PyUnicode_4BYTE_DATA(from) + from_start, - PyUnicode_4BYTE_DATA(from) + from_start + how_many, - PyUnicode_2BYTE_DATA(to) + to_start - ); - } - else { - Py_UNREACHABLE(); - } - } - else { - const Py_UCS4 to_maxchar = PyUnicode_MAX_CHAR_VALUE(to); - Py_UCS4 ch; - Py_ssize_t i; - - for (i=0; i < how_many; i++) { - ch = PyUnicode_READ(from_kind, from_data, from_start + i); - if (ch > to_maxchar) - return -1; - PyUnicode_WRITE(to_kind, to_data, to_start + i, ch); - } - } + PyObject_DEL(unicode->str); + Py_XDECREF(unicode->defenc); + Py_TYPE(unicode)->tp_free((PyObject *)unicode); } - return 0; } -void -_PyUnicode_FastCopyCharacters( - PyObject *to, Py_ssize_t to_start, - PyObject *from, Py_ssize_t from_start, Py_ssize_t how_many) -{ - (void)_copy_characters(to, to_start, from, from_start, how_many, 0); -} - -Py_ssize_t -PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start, - PyObject *from, Py_ssize_t from_start, - Py_ssize_t how_many) +static +int _PyUnicode_Resize(PyUnicodeObject **unicode, Py_ssize_t length) { - int err; + register PyUnicodeObject *v; - if (!PyUnicode_Check(from) || !PyUnicode_Check(to)) { + /* Argument checks */ + if (unicode == NULL) { PyErr_BadInternalCall(); return -1; } - - if (PyUnicode_READY(from) == -1) - return -1; - if (PyUnicode_READY(to) == -1) - return -1; - - if ((size_t)from_start > (size_t)PyUnicode_GET_LENGTH(from)) { - PyErr_SetString(PyExc_IndexError, "string index out of range"); - return -1; - } - if ((size_t)to_start > (size_t)PyUnicode_GET_LENGTH(to)) { - PyErr_SetString(PyExc_IndexError, "string index out of range"); - return -1; - } - if (how_many < 0) { - PyErr_SetString(PyExc_SystemError, "how_many cannot be negative"); - return -1; - } - how_many = Py_MIN(PyUnicode_GET_LENGTH(from)-from_start, how_many); - if (to_start + how_many > PyUnicode_GET_LENGTH(to)) { - PyErr_Format(PyExc_SystemError, - "Cannot write %zi characters at %zi " - "in a string of %zi characters", - how_many, to_start, PyUnicode_GET_LENGTH(to)); - return -1; - } - - if (how_many == 0) - return 0; - - if (unicode_check_modifiable(to)) - return -1; - - err = _copy_characters(to, to_start, from, from_start, how_many, 1); - if (err) { - PyErr_Format(PyExc_SystemError, - "Cannot copy %s characters " - "into a string of %s characters", - unicode_kind_name(from), - unicode_kind_name(to)); + v = *unicode; + if (v == NULL || !PyUnicode_Check(v) || length < 0) { + PyErr_BadInternalCall(); return -1; } - return how_many; -} - -/* Find the maximum code point and count the number of surrogate pairs so a - correct string length can be computed before converting a string to UCS4. - This function counts single surrogates as a character and not as a pair. - - Return 0 on success, or -1 on error. */ -static int -find_maxchar_surrogates(const wchar_t *begin, const wchar_t *end, - Py_UCS4 *maxchar, Py_ssize_t *num_surrogates) -{ - const wchar_t *iter; - Py_UCS4 ch; - - assert(num_surrogates != NULL && maxchar != NULL); - *num_surrogates = 0; - *maxchar = 0; - - for (iter = begin; iter < end; ) { -#if SIZEOF_WCHAR_T == 2 - if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0]) - && (iter+1) < end - && Py_UNICODE_IS_LOW_SURROGATE(iter[1])) - { - ch = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]); - ++(*num_surrogates); - iter += 2; - } - else -#endif - { - ch = *iter; - iter++; - } - if (ch > *maxchar) { - *maxchar = ch; - if (*maxchar > MAX_UNICODE) { - PyErr_Format(PyExc_ValueError, - "character U+%x is not in range [U+0000; U+10ffff]", - ch); - return -1; - } + if (v->length == 0) { + if (length == 0) { + return 0; } + *unicode = _PyUnicode_New(length); + Py_DECREF(v); + return (*unicode == NULL) ? -1 : 0; } - return 0; -} - -int -_PyUnicode_Ready(PyObject *unicode) -{ - wchar_t *end; - Py_UCS4 maxchar = 0; - Py_ssize_t num_surrogates; -#if SIZEOF_WCHAR_T == 2 - Py_ssize_t length_wo_surrogates; -#endif - - /* _PyUnicode_Ready() is only intended for old-style API usage where - strings were created using _PyObject_New() and where no canonical - representation (the str field) has been set yet aka strings - which are not yet ready. */ - assert(_PyUnicode_CHECK(unicode)); - assert(_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND); - assert(_PyUnicode_WSTR(unicode) != NULL); - assert(_PyUnicode_DATA_ANY(unicode) == NULL); - assert(_PyUnicode_UTF8(unicode) == NULL); - /* Actually, it should neither be interned nor be anything else: */ - assert(_PyUnicode_STATE(unicode).interned == SSTATE_NOT_INTERNED); - - end = _PyUnicode_WSTR(unicode) + _PyUnicode_WSTR_LENGTH(unicode); - if (find_maxchar_surrogates(_PyUnicode_WSTR(unicode), end, - &maxchar, &num_surrogates) == -1) + if (Py_REFCNT(v) != 1) { + PyErr_BadInternalCall(); return -1; - - if (maxchar < 256) { - _PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(_PyUnicode_WSTR_LENGTH(unicode) + 1); - if (!_PyUnicode_DATA_ANY(unicode)) { - PyErr_NoMemory(); - return -1; - } - _PyUnicode_CONVERT_BYTES(wchar_t, unsigned char, - _PyUnicode_WSTR(unicode), end, - PyUnicode_1BYTE_DATA(unicode)); - PyUnicode_1BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; - _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); - _PyUnicode_STATE(unicode).kind = PyUnicode_1BYTE_KIND; - if (maxchar < 128) { - _PyUnicode_STATE(unicode).ascii = 1; - _PyUnicode_UTF8(unicode) = _PyUnicode_DATA_ANY(unicode); - _PyUnicode_UTF8_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); - } - else { - _PyUnicode_STATE(unicode).ascii = 0; - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; - } - PyObject_FREE(_PyUnicode_WSTR(unicode)); - _PyUnicode_WSTR(unicode) = NULL; - _PyUnicode_WSTR_LENGTH(unicode) = 0; - } - /* In this case we might have to convert down from 4-byte native - wchar_t to 2-byte unicode. */ - else if (maxchar < 65536) { - assert(num_surrogates == 0 && - "FindMaxCharAndNumSurrogatePairs() messed up"); - -#if SIZEOF_WCHAR_T == 2 - /* We can share representations and are done. */ - _PyUnicode_DATA_ANY(unicode) = _PyUnicode_WSTR(unicode); - PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; - _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); - _PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND; - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; -#else - /* sizeof(wchar_t) == 4 */ - _PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC( - 2 * (_PyUnicode_WSTR_LENGTH(unicode) + 1)); - if (!_PyUnicode_DATA_ANY(unicode)) { - PyErr_NoMemory(); - return -1; - } - _PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2, - _PyUnicode_WSTR(unicode), end, - PyUnicode_2BYTE_DATA(unicode)); - PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0'; - _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); - _PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND; - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; - PyObject_FREE(_PyUnicode_WSTR(unicode)); - _PyUnicode_WSTR(unicode) = NULL; - _PyUnicode_WSTR_LENGTH(unicode) = 0; -#endif - } - /* maxchar exeeds 16 bit, wee need 4 bytes for unicode characters */ - else { -#if SIZEOF_WCHAR_T == 2 - /* in case the native representation is 2-bytes, we need to allocate a - new normalized 4-byte version. */ - length_wo_surrogates = _PyUnicode_WSTR_LENGTH(unicode) - num_surrogates; - if (length_wo_surrogates > PY_SSIZE_T_MAX / 4 - 1) { - PyErr_NoMemory(); - return -1; - } - _PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(4 * (length_wo_surrogates + 1)); - if (!_PyUnicode_DATA_ANY(unicode)) { - PyErr_NoMemory(); - return -1; - } - _PyUnicode_LENGTH(unicode) = length_wo_surrogates; - _PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND; - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; - /* unicode_convert_wchar_to_ucs4() requires a ready string */ - _PyUnicode_STATE(unicode).ready = 1; - unicode_convert_wchar_to_ucs4(_PyUnicode_WSTR(unicode), end, unicode); - PyObject_FREE(_PyUnicode_WSTR(unicode)); - _PyUnicode_WSTR(unicode) = NULL; - _PyUnicode_WSTR_LENGTH(unicode) = 0; -#else - assert(num_surrogates == 0); - - _PyUnicode_DATA_ANY(unicode) = _PyUnicode_WSTR(unicode); - _PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode); - _PyUnicode_UTF8(unicode) = NULL; - _PyUnicode_UTF8_LENGTH(unicode) = 0; - _PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND; -#endif - PyUnicode_4BYTE_DATA(unicode)[_PyUnicode_LENGTH(unicode)] = '\0'; - } - _PyUnicode_STATE(unicode).ready = 1; - assert(_PyUnicode_CheckConsistency(unicode, 1)); - return 0; -} - -static void -unicode_dealloc(PyObject *unicode) -{ - switch (PyUnicode_CHECK_INTERNED(unicode)) { - case SSTATE_NOT_INTERNED: - break; - - case SSTATE_INTERNED_MORTAL: - /* revive dead object temporarily for DelItem */ - Py_REFCNT(unicode) = 3; - if (PyDict_DelItem(interned, unicode) != 0) - Py_FatalError( - "deletion of interned string failed"); - break; - - case SSTATE_INTERNED_IMMORTAL: - Py_FatalError("Immortal interned string died."); - /* fall through */ - - default: - Py_FatalError("Inconsistent interned string state."); - } - - if (_PyUnicode_HAS_WSTR_MEMORY(unicode)) - PyObject_DEL(_PyUnicode_WSTR(unicode)); - if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) - PyObject_DEL(_PyUnicode_UTF8(unicode)); - if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode)) - PyObject_DEL(_PyUnicode_DATA_ANY(unicode)); - - Py_TYPE(unicode)->tp_free(unicode); -} - -#ifdef Py_DEBUG -static int -unicode_is_singleton(PyObject *unicode) -{ - PyASCIIObject *ascii = (PyASCIIObject *)unicode; - if (unicode == unicode_empty) - return 1; - if (ascii->state.kind != PyUnicode_WCHAR_KIND && ascii->length == 1) - { - Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, 0); - if (ch < 256 && unicode_latin1[ch] == unicode) - return 1; } - return 0; -} -#endif - -static int -unicode_modifiable(PyObject *unicode) -{ - assert(_PyUnicode_CHECK(unicode)); - if (Py_REFCNT(unicode) != 1) - return 0; - if (_PyUnicode_HASH(unicode) != -1) - return 0; - if (PyUnicode_CHECK_INTERNED(unicode)) - return 0; - if (!PyUnicode_CheckExact(unicode)) - return 0; -#ifdef Py_DEBUG - /* singleton refcount is greater than 1 */ - assert(!unicode_is_singleton(unicode)); -#endif - return 1; -} - -static int -unicode_resize(PyObject **p_unicode, Py_ssize_t length) -{ - PyObject *unicode; - Py_ssize_t old_length; - - assert(p_unicode != NULL); - unicode = *p_unicode; - - assert(unicode != NULL); - assert(PyUnicode_Check(unicode)); - assert(0 <= length); - - if (_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND) - old_length = PyUnicode_WSTR_LENGTH(unicode); - else - old_length = PyUnicode_GET_LENGTH(unicode); - if (old_length == length) - return 0; - if (length == 0) { - _Py_INCREF_UNICODE_EMPTY(); - if (!unicode_empty) - return -1; - Py_SETREF(*p_unicode, unicode_empty); - return 0; + *unicode = _PyUnicode_New(0); + Py_DECREF(v); + return (*unicode == NULL) ? -1 : 0; } - if (!unicode_modifiable(unicode)) { - PyObject *copy = resize_copy(unicode, length); - if (copy == NULL) + /* Resizing unicode_empty and single character objects is not + possible since these are being shared. We simply return a fresh + copy with the same Unicode content. */ + if (v->length != length && + (v == unicode_empty || v->length == 1)) { + PyUnicodeObject *w = _PyUnicode_New(length); + if (w == NULL) return -1; - Py_SETREF(*p_unicode, copy); + Py_UNICODE_COPY(w->str, v->str, + length < v->length ? length : v->length); + Py_SETREF(*unicode, w); return 0; } - if (PyUnicode_IS_COMPACT(unicode)) { - PyObject *new_unicode = resize_compact(unicode, length); - if (new_unicode == NULL) - return -1; - *p_unicode = new_unicode; - return 0; - } - return resize_inplace(unicode, length); + /* Note that we don't have to modify *unicode for unshared Unicode + objects, since we can modify them in-place. */ + return unicode_resize(v, length); } -int -PyUnicode_Resize(PyObject **p_unicode, Py_ssize_t length) +int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length) { - PyObject *unicode; - if (p_unicode == NULL) { - PyErr_BadInternalCall(); - return -1; - } - unicode = *p_unicode; - if (unicode == NULL || !PyUnicode_Check(unicode) || length < 0) - { - PyErr_BadInternalCall(); - return -1; - } - return unicode_resize(p_unicode, length); + return _PyUnicode_Resize((PyUnicodeObject **)unicode, length); } -/* Copy an ASCII or latin1 char* string into a Python Unicode string. - - WARNING: The function doesn't copy the terminating null character and - doesn't check the maximum character (may write a latin1 character in an - ASCII string). */ -static void -unicode_write_cstr(PyObject *unicode, Py_ssize_t index, - const char *str, Py_ssize_t len) +PyObject *PyUnicode_FromUnicode(const Py_UNICODE *u, + Py_ssize_t size) { - enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); - void *data = PyUnicode_DATA(unicode); - const char *end = str + len; - - switch (kind) { - case PyUnicode_1BYTE_KIND: { - assert(index + len <= PyUnicode_GET_LENGTH(unicode)); -#ifdef Py_DEBUG - if (PyUnicode_IS_ASCII(unicode)) { - Py_UCS4 maxchar = ucs1lib_find_max_char( - (const Py_UCS1*)str, - (const Py_UCS1*)str + len); - assert(maxchar < 128); - } -#endif - memcpy((char *) data + index, str, len); - break; - } - case PyUnicode_2BYTE_KIND: { - Py_UCS2 *start = (Py_UCS2 *)data + index; - Py_UCS2 *ucs2 = start; - assert(index <= PyUnicode_GET_LENGTH(unicode)); - - for (; str < end; ++ucs2, ++str) - *ucs2 = (Py_UCS2)*str; - - assert((ucs2 - start) <= PyUnicode_GET_LENGTH(unicode)); - break; - } - default: { - Py_UCS4 *start = (Py_UCS4 *)data + index; - Py_UCS4 *ucs4 = start; - assert(kind == PyUnicode_4BYTE_KIND); - assert(index <= PyUnicode_GET_LENGTH(unicode)); + PyUnicodeObject *unicode; - for (; str < end; ++ucs4, ++str) - *ucs4 = (Py_UCS4)*str; + /* If the Unicode data is known at construction time, we can apply + some optimizations which share commonly used objects. */ + if (u != NULL) { - assert((ucs4 - start) <= PyUnicode_GET_LENGTH(unicode)); - } - } -} + /* Optimization for empty strings */ + if (size == 0) + _Py_RETURN_UNICODE_EMPTY(); -static PyObject* -get_latin1_char(unsigned char ch) -{ - PyObject *unicode = unicode_latin1[ch]; - if (!unicode) { - unicode = PyUnicode_New(1, ch); - if (!unicode) - return NULL; - PyUnicode_1BYTE_DATA(unicode)[0] = ch; - assert(_PyUnicode_CheckConsistency(unicode, 1)); - unicode_latin1[ch] = unicode; + /* Single character Unicode objects in the Latin-1 range are + shared when using this constructor */ + if (size == 1 && *u < 256) { + unicode = unicode_latin1[*u]; + if (!unicode) { + unicode = _PyUnicode_New(1); + if (!unicode) + return NULL; + unicode->str[0] = *u; + unicode_latin1[*u] = unicode; + } + Py_INCREF(unicode); + return (PyObject *)unicode; + } } - Py_INCREF(unicode); - return unicode; -} - -static PyObject* -unicode_char(Py_UCS4 ch) -{ - PyObject *unicode; - - assert(ch <= MAX_UNICODE); - if (ch < 256) - return get_latin1_char(ch); - - unicode = PyUnicode_New(1, ch); - if (unicode == NULL) + unicode = _PyUnicode_New(size); + if (!unicode) return NULL; - assert(PyUnicode_KIND(unicode) != PyUnicode_1BYTE_KIND); - if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) { - PyUnicode_2BYTE_DATA(unicode)[0] = (Py_UCS2)ch; - } else { - assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); - PyUnicode_4BYTE_DATA(unicode)[0] = ch; - } - assert(_PyUnicode_CheckConsistency(unicode, 1)); - return unicode; -} - -PyObject * -PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) -{ - if (u == NULL) - return (PyObject*)_PyUnicode_New(size); - - if (size < 0) { - PyErr_BadInternalCall(); - return NULL; - } + /* Copy the Unicode data into the new object */ + if (u != NULL) + Py_UNICODE_COPY(unicode->str, u, size); - return PyUnicode_FromWideChar(u, size); + return (PyObject *)unicode; } -PyObject * -PyUnicode_FromWideChar(const wchar_t *u, Py_ssize_t size) +PyObject *PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) { - PyObject *unicode; - Py_UCS4 maxchar = 0; - Py_ssize_t num_surrogates; + PyUnicodeObject *unicode; - if (u == NULL && size != 0) { - PyErr_BadInternalCall(); + if (size < 0) { + PyErr_SetString(PyExc_SystemError, + "Negative size passed to PyUnicode_FromStringAndSize"); return NULL; } - if (size == -1) { - size = wcslen(u); - } - /* If the Unicode data is known at construction time, we can apply - some optimizations which share commonly used objects. */ + some optimizations which share commonly used objects. + Also, this means the input must be UTF-8, so fall back to the + UTF-8 decoder at the end. */ + if (u != NULL) { - /* Optimization for empty strings */ - if (size == 0) - _Py_RETURN_UNICODE_EMPTY(); + /* Optimization for empty strings */ + if (size == 0) + _Py_RETURN_UNICODE_EMPTY(); - /* Single character Unicode objects in the Latin-1 range are - shared when using this constructor */ - if (size == 1 && (Py_UCS4)*u < 256) - return get_latin1_char((unsigned char)*u); + /* Single characters are shared when using this constructor. + Restrict to ASCII, since the input must be UTF-8. */ + if (size == 1 && Py_CHARMASK(*u) < 128) { + unicode = unicode_latin1[Py_CHARMASK(*u)]; + if (!unicode) { + unicode = _PyUnicode_New(1); + if (!unicode) + return NULL; + unicode->str[0] = Py_CHARMASK(*u); + unicode_latin1[Py_CHARMASK(*u)] = unicode; + } + Py_INCREF(unicode); + return (PyObject *)unicode; + } - /* If not empty and not single character, copy the Unicode data - into the new object */ - if (find_maxchar_surrogates(u, u + size, - &maxchar, &num_surrogates) == -1) - return NULL; + return PyUnicode_DecodeUTF8(u, size, NULL); + } - unicode = PyUnicode_New(size - num_surrogates, maxchar); + unicode = _PyUnicode_New(size); if (!unicode) return NULL; - switch (PyUnicode_KIND(unicode)) { - case PyUnicode_1BYTE_KIND: - _PyUnicode_CONVERT_BYTES(Py_UNICODE, unsigned char, - u, u + size, PyUnicode_1BYTE_DATA(unicode)); - break; - case PyUnicode_2BYTE_KIND: -#if Py_UNICODE_SIZE == 2 - memcpy(PyUnicode_2BYTE_DATA(unicode), u, size * 2); -#else - _PyUnicode_CONVERT_BYTES(Py_UNICODE, Py_UCS2, - u, u + size, PyUnicode_2BYTE_DATA(unicode)); -#endif - break; - case PyUnicode_4BYTE_KIND: -#if SIZEOF_WCHAR_T == 2 - /* This is the only case which has to process surrogates, thus - a simple copy loop is not enough and we need a function. */ - unicode_convert_wchar_to_ucs4(u, u + size, unicode); -#else - assert(num_surrogates == 0); - memcpy(PyUnicode_4BYTE_DATA(unicode), u, size * 4); -#endif - break; - default: - Py_UNREACHABLE(); - } - - return unicode_result(unicode); -} - -PyObject * -PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) -{ - if (size < 0) { - PyErr_SetString(PyExc_SystemError, - "Negative size passed to PyUnicode_FromStringAndSize"); - return NULL; - } - if (u != NULL) - return PyUnicode_DecodeUTF8Stateful(u, size, NULL, NULL); - else - return (PyObject *)_PyUnicode_New(size); + return (PyObject *)unicode; } -PyObject * -PyUnicode_FromString(const char *u) +PyObject *PyUnicode_FromString(const char *u) { size_t size = strlen(u); if (size > PY_SSIZE_T_MAX) { PyErr_SetString(PyExc_OverflowError, "input too long"); return NULL; } - return PyUnicode_DecodeUTF8Stateful(u, (Py_ssize_t)size, NULL, NULL); -} -PyObject * -_PyUnicode_FromId(_Py_Identifier *id) -{ - if (!id->object) { - id->object = PyUnicode_DecodeUTF8Stateful(id->string, - strlen(id->string), - NULL, NULL); - if (!id->object) - return NULL; - PyUnicode_InternInPlace(&id->object); - assert(!id->next); - id->next = static_strings; - static_strings = id; - } - return id->object; + return PyUnicode_FromStringAndSize(u, size); } -void -_PyUnicode_ClearStaticStrings() -{ - _Py_Identifier *tmp, *s = static_strings; - while (s) { - Py_CLEAR(s->object); - tmp = s->next; - s->next = NULL; - s = tmp; - } - static_strings = NULL; -} +/* _Py_UNICODE_NEXT is a private macro used to retrieve the character pointed + * by 'ptr', possibly combining surrogate pairs on narrow builds. + * 'ptr' and 'end' must be Py_UNICODE*, with 'ptr' pointing at the character + * that should be returned and 'end' pointing to the end of the buffer. + * ('end' is used on narrow builds to detect a lone surrogate at the + * end of the buffer that should be returned unchanged.) + * The ptr and end arguments should be side-effect free and ptr must an lvalue. + * The type of the returned char is always Py_UCS4. + * + * Note: the macro advances ptr to next char, so it might have side-effects + * (especially if used with other macros). + */ -/* Internal function, doesn't check maximum character */ +/* helper macros used by _Py_UNICODE_NEXT */ +#define _Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF) +#define _Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF) +/* Join two surrogate characters and return a single Py_UCS4 value. */ +#define _Py_UNICODE_JOIN_SURROGATES(high, low) \ + (((((Py_UCS4)(high) & 0x03FF) << 10) | \ + ((Py_UCS4)(low) & 0x03FF)) + 0x10000) -PyObject* -_PyUnicode_FromASCII(const char *buffer, Py_ssize_t size) -{ - const unsigned char *s = (const unsigned char *)buffer; - PyObject *unicode; - if (size == 1) { -#ifdef Py_DEBUG - assert((unsigned char)s[0] < 128); +#ifdef Py_UNICODE_WIDE +#define _Py_UNICODE_NEXT(ptr, end) *(ptr)++ +#else +#define _Py_UNICODE_NEXT(ptr, end) \ + (((_Py_UNICODE_IS_HIGH_SURROGATE(*(ptr)) && (ptr) < (end)) && \ + _Py_UNICODE_IS_LOW_SURROGATE((ptr)[1])) ? \ + ((ptr) += 2,_Py_UNICODE_JOIN_SURROGATES((ptr)[-2], (ptr)[-1])) : \ + (Py_UCS4)*(ptr)++) #endif - return get_latin1_char(s[0]); - } - unicode = PyUnicode_New(size, 127); - if (!unicode) - return NULL; - memcpy(PyUnicode_1BYTE_DATA(unicode), s, size); - assert(_PyUnicode_CheckConsistency(unicode, 1)); - return unicode; -} -static Py_UCS4 -kind_maxchar_limit(unsigned int kind) -{ - switch (kind) { - case PyUnicode_1BYTE_KIND: - return 0x80; - case PyUnicode_2BYTE_KIND: - return 0x100; - case PyUnicode_4BYTE_KIND: - return 0x10000; - default: - Py_UNREACHABLE(); - } -} +#ifdef HAVE_WCHAR_H -static PyObject* -_PyUnicode_FromUCS1(const Py_UCS1* u, Py_ssize_t size) -{ - PyObject *res; - unsigned char max_char; +#if (Py_UNICODE_SIZE == 2) && defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) +# define CONVERT_WCHAR_TO_SURROGATES +#endif - if (size == 0) - _Py_RETURN_UNICODE_EMPTY(); - assert(size > 0); - if (size == 1) - return get_latin1_char(u[0]); +#ifdef CONVERT_WCHAR_TO_SURROGATES - max_char = ucs1lib_find_max_char(u, u + size); - res = PyUnicode_New(size, max_char); - if (!res) - return NULL; - memcpy(PyUnicode_1BYTE_DATA(res), u, size); - assert(_PyUnicode_CheckConsistency(res, 1)); - return res; -} +/* Here sizeof(wchar_t) is 4 but Py_UNICODE_SIZE == 2, so we need + to convert from UTF32 to UTF16. */ -static PyObject* -_PyUnicode_FromUCS2(const Py_UCS2 *u, Py_ssize_t size) +PyObject *PyUnicode_FromWideChar(register const wchar_t *w, + Py_ssize_t size) { - PyObject *res; - Py_UCS2 max_char; - - if (size == 0) - _Py_RETURN_UNICODE_EMPTY(); - assert(size > 0); - if (size == 1) - return unicode_char(u[0]); + PyUnicodeObject *unicode; + register Py_ssize_t i; + Py_ssize_t alloc; + const wchar_t *orig_w; - max_char = ucs2lib_find_max_char(u, u + size); - res = PyUnicode_New(size, max_char); - if (!res) + if (w == NULL) { + PyErr_BadInternalCall(); return NULL; - if (max_char >= 256) - memcpy(PyUnicode_2BYTE_DATA(res), u, sizeof(Py_UCS2)*size); - else { - _PyUnicode_CONVERT_BYTES( - Py_UCS2, Py_UCS1, u, u + size, PyUnicode_1BYTE_DATA(res)); } - assert(_PyUnicode_CheckConsistency(res, 1)); - return res; -} - -static PyObject* -_PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size) -{ - PyObject *res; - Py_UCS4 max_char; - - if (size == 0) - _Py_RETURN_UNICODE_EMPTY(); - assert(size > 0); - if (size == 1) - return unicode_char(u[0]); - max_char = ucs4lib_find_max_char(u, u + size); - res = PyUnicode_New(size, max_char); - if (!res) - return NULL; - if (max_char < 256) - _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS1, u, u + size, - PyUnicode_1BYTE_DATA(res)); - else if (max_char < 0x10000) - _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS2, u, u + size, - PyUnicode_2BYTE_DATA(res)); - else - memcpy(PyUnicode_4BYTE_DATA(res), u, sizeof(Py_UCS4)*size); - assert(_PyUnicode_CheckConsistency(res, 1)); - return res; -} - -PyObject* -PyUnicode_FromKindAndData(int kind, const void *buffer, Py_ssize_t size) -{ - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "size must be positive"); - return NULL; + alloc = size; + orig_w = w; + for (i = size; i > 0; i--) { + if (*w > 0xFFFF) + alloc++; + w++; } - switch (kind) { - case PyUnicode_1BYTE_KIND: - return _PyUnicode_FromUCS1(buffer, size); - case PyUnicode_2BYTE_KIND: - return _PyUnicode_FromUCS2(buffer, size); - case PyUnicode_4BYTE_KIND: - return _PyUnicode_FromUCS4(buffer, size); - default: - PyErr_SetString(PyExc_SystemError, "invalid kind"); + w = orig_w; + unicode = _PyUnicode_New(alloc); + if (!unicode) return NULL; - } -} - -Py_UCS4 -_PyUnicode_FindMaxChar(PyObject *unicode, Py_ssize_t start, Py_ssize_t end) -{ - enum PyUnicode_Kind kind; - void *startptr, *endptr; - - assert(PyUnicode_IS_READY(unicode)); - assert(0 <= start); - assert(end <= PyUnicode_GET_LENGTH(unicode)); - assert(start <= end); - - if (start == 0 && end == PyUnicode_GET_LENGTH(unicode)) - return PyUnicode_MAX_CHAR_VALUE(unicode); - if (start == end) - return 127; - - if (PyUnicode_IS_ASCII(unicode)) - return 127; - - kind = PyUnicode_KIND(unicode); - startptr = PyUnicode_DATA(unicode); - endptr = (char *)startptr + end * kind; - startptr = (char *)startptr + start * kind; - switch(kind) { - case PyUnicode_1BYTE_KIND: - return ucs1lib_find_max_char(startptr, endptr); - case PyUnicode_2BYTE_KIND: - return ucs2lib_find_max_char(startptr, endptr); - case PyUnicode_4BYTE_KIND: - return ucs4lib_find_max_char(startptr, endptr); - default: - Py_UNREACHABLE(); + /* Copy the wchar_t data into the new object */ + { + register Py_UNICODE *u; + u = PyUnicode_AS_UNICODE(unicode); + for (i = size; i > 0; i--) { + if (*w > 0xFFFF) { + wchar_t ordinal = *w++; + ordinal -= 0x10000; + *u++ = 0xD800 | (ordinal >> 10); + *u++ = 0xDC00 | (ordinal & 0x3FF); + } + else + *u++ = *w++; + } } + return (PyObject *)unicode; } -/* Ensure that a string uses the most efficient storage, if it is not the - case: create a new string with of the right kind. Write NULL into *p_unicode - on error. */ -static void -unicode_adjust_maxchar(PyObject **p_unicode) -{ - PyObject *unicode, *copy; - Py_UCS4 max_char; - Py_ssize_t len; - unsigned int kind; - - assert(p_unicode != NULL); - unicode = *p_unicode; - assert(PyUnicode_IS_READY(unicode)); - if (PyUnicode_IS_ASCII(unicode)) - return; - - len = PyUnicode_GET_LENGTH(unicode); - kind = PyUnicode_KIND(unicode); - if (kind == PyUnicode_1BYTE_KIND) { - const Py_UCS1 *u = PyUnicode_1BYTE_DATA(unicode); - max_char = ucs1lib_find_max_char(u, u + len); - if (max_char >= 128) - return; - } - else if (kind == PyUnicode_2BYTE_KIND) { - const Py_UCS2 *u = PyUnicode_2BYTE_DATA(unicode); - max_char = ucs2lib_find_max_char(u, u + len); - if (max_char >= 256) - return; - } - else { - const Py_UCS4 *u = PyUnicode_4BYTE_DATA(unicode); - assert(kind == PyUnicode_4BYTE_KIND); - max_char = ucs4lib_find_max_char(u, u + len); - if (max_char >= 0x10000) - return; - } - copy = PyUnicode_New(len, max_char); - if (copy != NULL) - _PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, len); - Py_DECREF(unicode); - *p_unicode = copy; -} +#else -PyObject* -_PyUnicode_Copy(PyObject *unicode) +PyObject *PyUnicode_FromWideChar(register const wchar_t *w, + Py_ssize_t size) { - Py_ssize_t length; - PyObject *copy; + PyUnicodeObject *unicode; - if (!PyUnicode_Check(unicode)) { + if (w == NULL) { PyErr_BadInternalCall(); return NULL; } - if (PyUnicode_READY(unicode) == -1) - return NULL; - - length = PyUnicode_GET_LENGTH(unicode); - copy = PyUnicode_New(length, PyUnicode_MAX_CHAR_VALUE(unicode)); - if (!copy) - return NULL; - assert(PyUnicode_KIND(copy) == PyUnicode_KIND(unicode)); - - memcpy(PyUnicode_DATA(copy), PyUnicode_DATA(unicode), - length * PyUnicode_KIND(unicode)); - assert(_PyUnicode_CheckConsistency(copy, 1)); - return copy; -} - - -/* Widen Unicode objects to larger buffers. Don't write terminating null - character. Return NULL on error. */ - -void* -_PyUnicode_AsKind(PyObject *s, unsigned int kind) -{ - Py_ssize_t len; - void *result; - unsigned int skind; - - if (PyUnicode_READY(s) == -1) - return NULL; - - len = PyUnicode_GET_LENGTH(s); - skind = PyUnicode_KIND(s); - if (skind >= kind) { - PyErr_SetString(PyExc_SystemError, "invalid widening attempt"); - return NULL; - } - switch (kind) { - case PyUnicode_2BYTE_KIND: - result = PyMem_New(Py_UCS2, len); - if (!result) - return PyErr_NoMemory(); - assert(skind == PyUnicode_1BYTE_KIND); - _PyUnicode_CONVERT_BYTES( - Py_UCS1, Py_UCS2, - PyUnicode_1BYTE_DATA(s), - PyUnicode_1BYTE_DATA(s) + len, - result); - return result; - case PyUnicode_4BYTE_KIND: - result = PyMem_New(Py_UCS4, len); - if (!result) - return PyErr_NoMemory(); - if (skind == PyUnicode_2BYTE_KIND) { - _PyUnicode_CONVERT_BYTES( - Py_UCS2, Py_UCS4, - PyUnicode_2BYTE_DATA(s), - PyUnicode_2BYTE_DATA(s) + len, - result); - } - else { - assert(skind == PyUnicode_1BYTE_KIND); - _PyUnicode_CONVERT_BYTES( - Py_UCS1, Py_UCS4, - PyUnicode_1BYTE_DATA(s), - PyUnicode_1BYTE_DATA(s) + len, - result); - } - return result; - default: - break; - } - PyErr_SetString(PyExc_SystemError, "invalid kind"); - return NULL; -} -static Py_UCS4* -as_ucs4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize, - int copy_null) -{ - int kind; - void *data; - Py_ssize_t len, targetlen; - if (PyUnicode_READY(string) == -1) + unicode = _PyUnicode_New(size); + if (!unicode) return NULL; - kind = PyUnicode_KIND(string); - data = PyUnicode_DATA(string); - len = PyUnicode_GET_LENGTH(string); - targetlen = len; - if (copy_null) - targetlen++; - if (!target) { - target = PyMem_New(Py_UCS4, targetlen); - if (!target) { - PyErr_NoMemory(); - return NULL; - } - } - else { - if (targetsize < targetlen) { - PyErr_Format(PyExc_SystemError, - "string is longer than the buffer"); - if (copy_null && 0 < targetsize) - target[0] = 0; - return NULL; - } - } - if (kind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *start = (Py_UCS1 *) data; - _PyUnicode_CONVERT_BYTES(Py_UCS1, Py_UCS4, start, start + len, target); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *start = (Py_UCS2 *) data; - _PyUnicode_CONVERT_BYTES(Py_UCS2, Py_UCS4, start, start + len, target); - } - else { - assert(kind == PyUnicode_4BYTE_KIND); - memcpy(target, data, len * sizeof(Py_UCS4)); - } - if (copy_null) - target[len] = 0; - return target; -} -Py_UCS4* -PyUnicode_AsUCS4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize, - int copy_null) -{ - if (target == NULL || targetsize < 0) { - PyErr_BadInternalCall(); - return NULL; + /* Copy the wchar_t data into the new object */ +#ifdef HAVE_USABLE_WCHAR_T + memcpy(unicode->str, w, size * sizeof(wchar_t)); +#else + { + register Py_UNICODE *u; + register Py_ssize_t i; + u = PyUnicode_AS_UNICODE(unicode); + for (i = size; i > 0; i--) + *u++ = *w++; } - return as_ucs4(string, target, targetsize, copy_null); -} +#endif -Py_UCS4* -PyUnicode_AsUCS4Copy(PyObject *string) -{ - return as_ucs4(string, NULL, 0, 1); + return (PyObject *)unicode; } -/* maximum number of characters required for output of %lld or %p. - We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits, - plus 1 for the sign. 53/22 is an upper bound for log10(256). */ -#define MAX_LONG_LONG_CHARS (2 + (SIZEOF_LONG_LONG*53-1) / 22) - -static int -unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, - Py_ssize_t width, Py_ssize_t precision) -{ - Py_ssize_t length, fill, arglen; - Py_UCS4 maxchar; - - if (PyUnicode_READY(str) == -1) - return -1; - - length = PyUnicode_GET_LENGTH(str); - if ((precision == -1 || precision >= length) - && width <= length) - return _PyUnicodeWriter_WriteStr(writer, str); - - if (precision != -1) - length = Py_MIN(precision, length); - - arglen = Py_MAX(length, width); - if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) - maxchar = _PyUnicode_FindMaxChar(str, 0, length); - else - maxchar = writer->maxchar; - - if (_PyUnicodeWriter_Prepare(writer, arglen, maxchar) == -1) - return -1; +#endif /* CONVERT_WCHAR_TO_SURROGATES */ - if (width > length) { - fill = width - length; - if (PyUnicode_Fill(writer->buffer, writer->pos, fill, ' ') == -1) - return -1; - writer->pos += fill; - } +#undef CONVERT_WCHAR_TO_SURROGATES - _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, - str, 0, length); - writer->pos += length; - return 0; -} - -static int -unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, - Py_ssize_t width, Py_ssize_t precision) +static void +makefmt(char *fmt, int longflag, int size_tflag, int zeropad, int width, int precision, char c) { - /* UTF-8 */ - Py_ssize_t length; - PyObject *unicode; - int res; - - if (precision == -1) { - length = strlen(str); + *fmt++ = '%'; + if (width) { + if (zeropad) + *fmt++ = '0'; + fmt += sprintf(fmt, "%d", width); } - else { - length = 0; - while (length < precision && str[length]) { - length++; - } + if (precision) + fmt += sprintf(fmt, ".%d", precision); + if (longflag) + *fmt++ = 'l'; + else if (size_tflag) { + char *f = PY_FORMAT_SIZE_T; + while (*f) + *fmt++ = *f++; } - unicode = PyUnicode_DecodeUTF8Stateful(str, length, "replace", NULL); - if (unicode == NULL) - return -1; - - res = unicode_fromformat_write_str(writer, unicode, width, -1); - Py_DECREF(unicode); - return res; + *fmt++ = c; + *fmt = '\0'; } -static const char* -unicode_fromformat_arg(_PyUnicodeWriter *writer, - const char *f, va_list *vargs) +#define appendstring(string) \ + do { \ + for (copy = string;*copy; copy++) { \ + *s++ = (unsigned char)*copy; \ + } \ + } while (0) + +PyObject * +PyUnicode_FromFormatV(const char *format, va_list vargs) { - const char *p; - Py_ssize_t len; + va_list count; + Py_ssize_t callcount = 0; + PyObject **callresults = NULL; + PyObject **callresult = NULL; + Py_ssize_t n = 0; + int width = 0; + int precision = 0; int zeropad; - Py_ssize_t width; - Py_ssize_t precision; - int longflag; - int longlongflag; - int size_tflag; - Py_ssize_t fill; - - p = f; - f++; - zeropad = 0; - if (*f == '0') { - zeropad = 1; - f++; - } - - /* parse the width.precision part, e.g. "%2.5s" => width=2, precision=5 */ - width = -1; - if (Py_ISDIGIT((unsigned)*f)) { - width = *f - '0'; - f++; - while (Py_ISDIGIT((unsigned)*f)) { - if (width > (PY_SSIZE_T_MAX - ((int)*f - '0')) / 10) { - PyErr_SetString(PyExc_ValueError, - "width too big"); - return NULL; - } - width = (width * 10) + (*f - '0'); - f++; - } - } - precision = -1; - if (*f == '.') { - f++; - if (Py_ISDIGIT((unsigned)*f)) { - precision = (*f - '0'); - f++; - while (Py_ISDIGIT((unsigned)*f)) { - if (precision > (PY_SSIZE_T_MAX - ((int)*f - '0')) / 10) { - PyErr_SetString(PyExc_ValueError, - "precision too big"); - return NULL; - } - precision = (precision * 10) + (*f - '0'); - f++; - } - } - if (*f == '%') { - /* "%.3%s" => f points to "3" */ - f--; - } - } - if (*f == '\0') { - /* bogus format "%.123" => go backward, f points to "3" */ - f--; - } - - /* Handle %ld, %lu, %lld and %llu. */ - longflag = 0; - longlongflag = 0; - size_tflag = 0; - if (*f == 'l') { - if (f[1] == 'd' || f[1] == 'u' || f[1] == 'i') { - longflag = 1; - ++f; - } - else if (f[1] == 'l' && - (f[2] == 'd' || f[2] == 'u' || f[2] == 'i')) { - longlongflag = 1; - f += 2; - } - } - /* handle the size_t flag. */ - else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u' || f[1] == 'i')) { - size_tflag = 1; - ++f; - } - - if (f[1] == '\0') - writer->overallocate = 0; - - switch (*f) { - case 'c': - { - int ordinal = va_arg(*vargs, int); - if (ordinal < 0 || ordinal > MAX_UNICODE) { - PyErr_SetString(PyExc_OverflowError, - "character argument not in range(0x110000)"); - return NULL; - } - if (_PyUnicodeWriter_WriteCharInline(writer, ordinal) < 0) - return NULL; - break; - } - - case 'i': - case 'd': - case 'u': - case 'x': - { - /* used by sprintf */ - char buffer[MAX_LONG_LONG_CHARS]; - Py_ssize_t arglen; - - if (*f == 'u') { - if (longflag) - len = sprintf(buffer, "%lu", - va_arg(*vargs, unsigned long)); - else if (longlongflag) - len = sprintf(buffer, "%llu", - va_arg(*vargs, unsigned long long)); - else if (size_tflag) - len = sprintf(buffer, "%" PY_FORMAT_SIZE_T "u", - va_arg(*vargs, size_t)); - else - len = sprintf(buffer, "%u", - va_arg(*vargs, unsigned int)); - } - else if (*f == 'x') { - len = sprintf(buffer, "%x", va_arg(*vargs, int)); - } - else { - if (longflag) - len = sprintf(buffer, "%li", - va_arg(*vargs, long)); - else if (longlongflag) - len = sprintf(buffer, "%lli", - va_arg(*vargs, long long)); - else if (size_tflag) - len = sprintf(buffer, "%" PY_FORMAT_SIZE_T "i", - va_arg(*vargs, Py_ssize_t)); - else - len = sprintf(buffer, "%i", - va_arg(*vargs, int)); - } - assert(len >= 0); - - if (precision < len) - precision = len; - - arglen = Py_MAX(precision, width); - if (_PyUnicodeWriter_Prepare(writer, arglen, 127) == -1) - return NULL; - - if (width > precision) { - Py_UCS4 fillchar; - fill = width - precision; - fillchar = zeropad?'0':' '; - if (PyUnicode_Fill(writer->buffer, writer->pos, fill, fillchar) == -1) - return NULL; - writer->pos += fill; - } - if (precision > len) { - fill = precision - len; - if (PyUnicode_Fill(writer->buffer, writer->pos, fill, '0') == -1) - return NULL; - writer->pos += fill; - } - - if (_PyUnicodeWriter_WriteASCIIString(writer, buffer, len) < 0) - return NULL; - break; - } - - case 'p': - { - char number[MAX_LONG_LONG_CHARS]; - - len = sprintf(number, "%p", va_arg(*vargs, void*)); - assert(len >= 0); - - /* %p is ill-defined: ensure leading 0x. */ - if (number[1] == 'X') - number[1] = 'x'; - else if (number[1] != 'x') { - memmove(number + 2, number, - strlen(number) + 1); - number[0] = '0'; - number[1] = 'x'; - len += 2; - } - - if (_PyUnicodeWriter_WriteASCIIString(writer, number, len) < 0) - return NULL; - break; - } - - case 's': - { - /* UTF-8 */ - const char *s = va_arg(*vargs, const char*); - if (unicode_fromformat_write_cstr(writer, s, width, precision) < 0) - return NULL; - break; - } - - case 'U': - { - PyObject *obj = va_arg(*vargs, PyObject *); - assert(obj && _PyUnicode_CHECK(obj)); - - if (unicode_fromformat_write_str(writer, obj, width, precision) == -1) + const char* f; + Py_UNICODE *s; + PyObject *string; + /* used by sprintf */ + char buffer[21]; + /* use abuffer instead of buffer, if we need more space + * (which can happen if there's a format specifier with width). */ + char *abuffer = NULL; + char *realbuffer; + Py_ssize_t abuffersize = 0; + char fmt[60]; /* should be enough for %0width.precisionld */ + const char *copy; + +#ifdef VA_LIST_IS_ARRAY + Py_MEMCPY(count, vargs, sizeof(va_list)); +#else +#ifdef __va_copy + __va_copy(count, vargs); +#else + count = vargs; +#endif +#endif + /* step 1: count the number of %S/%R/%s format specifications + * (we call PyObject_Str()/PyObject_Repr()/PyUnicode_DecodeUTF8() for these + * objects once during step 3 and put the result in an array) */ + for (f = format; *f; f++) { + if (*f == '%') { + f++; + while (*f && *f != '%' && !isalpha((unsigned)*f)) + f++; + if (!*f) + break; + if (*f == 's' || *f=='S' || *f=='R') + ++callcount; + } + } + /* step 2: allocate memory for the results of + * PyObject_Str()/PyObject_Repr()/PyUnicode_DecodeUTF8() calls */ + if (callcount) { + callresults = PyObject_Malloc(sizeof(PyObject *)*callcount); + if (!callresults) { + PyErr_NoMemory(); return NULL; - break; - } - - case 'V': - { - PyObject *obj = va_arg(*vargs, PyObject *); - const char *str = va_arg(*vargs, const char *); - if (obj) { - assert(_PyUnicode_CHECK(obj)); - if (unicode_fromformat_write_str(writer, obj, width, precision) == -1) - return NULL; - } - else { - assert(str != NULL); - if (unicode_fromformat_write_cstr(writer, str, width, precision) < 0) - return NULL; } - break; + callresult = callresults; } + /* step 3: figure out how large a buffer we need */ + for (f = format; *f; f++) { + if (*f == '%') { + const char* p = f++; + width = 0; + while (isdigit((unsigned)*f)) + width = (width*10) + *f++ - '0'; + precision = 0; + if (*f == '.') { + f++; + while (isdigit((unsigned)*f)) + precision = (precision*10) + *f++ - '0'; + } - case 'S': - { - PyObject *obj = va_arg(*vargs, PyObject *); - PyObject *str; - assert(obj); - str = PyObject_Str(obj); - if (!str) - return NULL; - if (unicode_fromformat_write_str(writer, str, width, precision) == -1) { - Py_DECREF(str); - return NULL; - } - Py_DECREF(str); - break; - } + /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since + * they don't affect the amount of space we reserve. + */ + if ((*f == 'l' || *f == 'z') && + (f[1] == 'd' || f[1] == 'u')) + ++f; - case 'R': - { - PyObject *obj = va_arg(*vargs, PyObject *); - PyObject *repr; - assert(obj); - repr = PyObject_Repr(obj); - if (!repr) - return NULL; - if (unicode_fromformat_write_str(writer, repr, width, precision) == -1) { - Py_DECREF(repr); - return NULL; - } - Py_DECREF(repr); - break; + switch (*f) { + case 'c': + { + int ordinal = va_arg(count, int); +#ifdef Py_UNICODE_WIDE + if (ordinal < 0 || ordinal > 0x10ffff) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x110000) " + "(wide Python build)"); + goto fail; + } +#else + if (ordinal < 0 || ordinal > 0xffff) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x10000) " + "(narrow Python build)"); + goto fail; + } +#endif + /* fall through... */ + } + case '%': + n++; + break; + case 'd': case 'u': case 'i': case 'x': + (void) va_arg(count, int); + if (width < precision) + width = precision; + /* 20 bytes is enough to hold a 64-bit + integer. Decimal takes the most space. + This isn't enough for octal. + If a width is specified we need more + (which we allocate later). */ + if (width < 20) + width = 20; + n += width; + if (abuffersize < width) + abuffersize = width; + break; + case 's': + { + /* UTF-8 */ + const char *s = va_arg(count, const char*); + PyObject *str = PyUnicode_DecodeUTF8(s, strlen(s), "replace"); + if (!str) + goto fail; + n += PyUnicode_GET_SIZE(str); + /* Remember the str and switch to the next slot */ + *callresult++ = str; + break; + } + case 'U': + { + PyObject *obj = va_arg(count, PyObject *); + assert(obj && PyUnicode_Check(obj)); + n += PyUnicode_GET_SIZE(obj); + break; + } + case 'V': + { + PyObject *obj = va_arg(count, PyObject *); + const char *str = va_arg(count, const char *); + assert(obj || str); + assert(!obj || PyUnicode_Check(obj)); + if (obj) + n += PyUnicode_GET_SIZE(obj); + else + n += strlen(str); + break; + } + case 'S': + { + PyObject *obj = va_arg(count, PyObject *); + PyObject *str; + assert(obj); + str = PyObject_Str(obj); + if (!str) + goto fail; + n += PyString_GET_SIZE(str); + /* Remember the str and switch to the next slot */ + *callresult++ = str; + break; + } + case 'R': + { + PyObject *obj = va_arg(count, PyObject *); + PyObject *repr; + assert(obj); + repr = PyObject_Repr(obj); + if (!repr) + goto fail; + n += PyUnicode_GET_SIZE(repr); + /* Remember the repr and switch to the next slot */ + *callresult++ = repr; + break; + } + case 'p': + (void) va_arg(count, int); + /* maximum 64-bit pointer representation: + * 0xffffffffffffffff + * so 19 characters is enough. + * XXX I count 18 -- what's the extra for? + */ + n += 19; + break; + default: + /* if we stumble upon an unknown + formatting code, copy the rest of + the format string to the output + string. (we cannot just skip the + code, since there's no way to know + what's in the argument list) */ + n += strlen(p); + goto expand; + } + } else + n++; } - - case 'A': - { - PyObject *obj = va_arg(*vargs, PyObject *); - PyObject *ascii; - assert(obj); - ascii = PyObject_ASCII(obj); - if (!ascii) - return NULL; - if (unicode_fromformat_write_str(writer, ascii, width, precision) == -1) { - Py_DECREF(ascii); - return NULL; + expand: + if (abuffersize > 20) { + /* add 1 for sprintf's trailing null byte */ + abuffer = PyObject_Malloc(abuffersize + 1); + if (!abuffer) { + PyErr_NoMemory(); + goto fail; } - Py_DECREF(ascii); - break; - } - - case '%': - if (_PyUnicodeWriter_WriteCharInline(writer, '%') < 0) - return NULL; - break; - - default: - /* if we stumble upon an unknown formatting code, copy the rest - of the format string to the output string. (we cannot just - skip the code, since there's no way to know what's in the - argument list) */ - len = strlen(p); - if (_PyUnicodeWriter_WriteLatin1String(writer, p, len) == -1) - return NULL; - f = p+len; - return f; + realbuffer = abuffer; } - - f++; - return f; -} - -PyObject * -PyUnicode_FromFormatV(const char *format, va_list vargs) -{ - va_list vargs2; - const char *f; - _PyUnicodeWriter writer; - - _PyUnicodeWriter_Init(&writer); - writer.min_length = strlen(format) + 100; - writer.overallocate = 1; - - // Copy varags to be able to pass a reference to a subfunction. - va_copy(vargs2, vargs); - - for (f = format; *f; ) { + else + realbuffer = buffer; + /* step 4: fill the buffer */ + /* Since we've analyzed how much space we need for the worst case, + we don't have to resize the string. + There can be no errors beyond this point. */ + string = PyUnicode_FromUnicode(NULL, n); + if (!string) + goto fail; + + s = PyUnicode_AS_UNICODE(string); + callresult = callresults; + + for (f = format; *f; f++) { if (*f == '%') { - f = unicode_fromformat_arg(&writer, f, &vargs2); - if (f == NULL) - goto fail; - } - else { - const char *p; - Py_ssize_t len; + const char* p = f++; + int longflag = 0; + int size_tflag = 0; + zeropad = (*f == '0'); + /* parse the width.precision part */ + width = 0; + while (isdigit((unsigned)*f)) + width = (width*10) + *f++ - '0'; + precision = 0; + if (*f == '.') { + f++; + while (isdigit((unsigned)*f)) + precision = (precision*10) + *f++ - '0'; + } + /* handle the long flag, but only for %ld and %lu. + others can be added when necessary. */ + if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) { + longflag = 1; + ++f; + } + /* handle the size_t flag. */ + if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { + size_tflag = 1; + ++f; + } - p = f; - do + switch (*f) { + case 'c': + *s++ = va_arg(vargs, int); + break; + case 'd': + makefmt(fmt, longflag, size_tflag, zeropad, width, precision, 'd'); + if (longflag) + sprintf(realbuffer, fmt, va_arg(vargs, long)); + else if (size_tflag) + sprintf(realbuffer, fmt, va_arg(vargs, Py_ssize_t)); + else + sprintf(realbuffer, fmt, va_arg(vargs, int)); + appendstring(realbuffer); + break; + case 'u': + makefmt(fmt, longflag, size_tflag, zeropad, width, precision, 'u'); + if (longflag) + sprintf(realbuffer, fmt, va_arg(vargs, unsigned long)); + else if (size_tflag) + sprintf(realbuffer, fmt, va_arg(vargs, size_t)); + else + sprintf(realbuffer, fmt, va_arg(vargs, unsigned int)); + appendstring(realbuffer); + break; + case 'i': + makefmt(fmt, 0, 0, zeropad, width, precision, 'i'); + sprintf(realbuffer, fmt, va_arg(vargs, int)); + appendstring(realbuffer); + break; + case 'x': + makefmt(fmt, 0, 0, zeropad, width, precision, 'x'); + sprintf(realbuffer, fmt, va_arg(vargs, int)); + appendstring(realbuffer); + break; + case 's': { - if ((unsigned char)*p > 127) { - PyErr_Format(PyExc_ValueError, - "PyUnicode_FromFormatV() expects an ASCII-encoded format " - "string, got a non-ASCII byte: 0x%02x", - (unsigned char)*p); - goto fail; + /* unused, since we already have the result */ + (void) va_arg(vargs, char *); + Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(*callresult), + PyUnicode_GET_SIZE(*callresult)); + s += PyUnicode_GET_SIZE(*callresult); + /* We're done with the unicode()/repr() => forget it */ + Py_DECREF(*callresult); + /* switch to next unicode()/repr() result */ + ++callresult; + break; + } + case 'U': + { + PyObject *obj = va_arg(vargs, PyObject *); + Py_ssize_t size = PyUnicode_GET_SIZE(obj); + Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size); + s += size; + break; + } + case 'V': + { + PyObject *obj = va_arg(vargs, PyObject *); + const char *str = va_arg(vargs, const char *); + if (obj) { + Py_ssize_t size = PyUnicode_GET_SIZE(obj); + Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size); + s += size; + } else { + appendstring(str); } - p++; + break; } - while (*p != '\0' && *p != '%'); - len = p - f; - - if (*p == '\0') - writer.overallocate = 0; - - if (_PyUnicodeWriter_WriteASCIIString(&writer, f, len) < 0) - goto fail; - - f = p; - } + case 'S': + case 'R': + { + const char *str = PyString_AS_STRING(*callresult); + /* unused, since we already have the result */ + (void) va_arg(vargs, PyObject *); + appendstring(str); + /* We're done with the unicode()/repr() => forget it */ + Py_DECREF(*callresult); + /* switch to next unicode()/repr() result */ + ++callresult; + break; + } + case 'p': + sprintf(buffer, "%p", va_arg(vargs, void*)); + /* %p is ill-defined: ensure leading 0x. */ + if (buffer[1] == 'X') + buffer[1] = 'x'; + else if (buffer[1] != 'x') { + memmove(buffer+2, buffer, strlen(buffer)+1); + buffer[0] = '0'; + buffer[1] = 'x'; + } + appendstring(buffer); + break; + case '%': + *s++ = '%'; + break; + default: + appendstring(p); + goto end; + } + } else + *s++ = *f; } - va_end(vargs2); - return _PyUnicodeWriter_Finish(&writer); + end: + if (callresults) + PyObject_Free(callresults); + if (abuffer) + PyObject_Free(abuffer); + PyUnicode_Resize(&string, s - PyUnicode_AS_UNICODE(string)); + return string; fail: - va_end(vargs2); - _PyUnicodeWriter_Dealloc(&writer); + if (callresults) { + PyObject **callresult2 = callresults; + while (callresult2 < callresult) { + Py_DECREF(*callresult2); + ++callresult2; + } + PyObject_Free(callresults); + } + if (abuffer) + PyObject_Free(abuffer); return NULL; } +#undef appendstring + PyObject * PyUnicode_FromFormat(const char *format, ...) { @@ -3056,201 +1103,86 @@ PyUnicode_FromFormat(const char *format, ...) return ret; } -static Py_ssize_t -unicode_get_widechar_size(PyObject *unicode) -{ - Py_ssize_t res; - - assert(unicode != NULL); - assert(_PyUnicode_CHECK(unicode)); - - if (_PyUnicode_WSTR(unicode) != NULL) { - return PyUnicode_WSTR_LENGTH(unicode); - } - assert(PyUnicode_IS_READY(unicode)); - - res = _PyUnicode_LENGTH(unicode); -#if SIZEOF_WCHAR_T == 2 - if (PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND) { - const Py_UCS4 *s = PyUnicode_4BYTE_DATA(unicode); - const Py_UCS4 *end = s + res; - for (; s < end; ++s) { - if (*s > 0xFFFF) { - ++res; - } - } - } -#endif - return res; -} - -static void -unicode_copy_as_widechar(PyObject *unicode, wchar_t *w, Py_ssize_t size) -{ - const wchar_t *wstr; - - assert(unicode != NULL); - assert(_PyUnicode_CHECK(unicode)); - - wstr = _PyUnicode_WSTR(unicode); - if (wstr != NULL) { - memcpy(w, wstr, size * sizeof(wchar_t)); - return; - } - assert(PyUnicode_IS_READY(unicode)); - - if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND) { - const Py_UCS1 *s = PyUnicode_1BYTE_DATA(unicode); - for (; size--; ++s, ++w) { - *w = *s; - } - } - else { -#if SIZEOF_WCHAR_T == 4 - assert(PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND); - const Py_UCS2 *s = PyUnicode_2BYTE_DATA(unicode); - for (; size--; ++s, ++w) { - *w = *s; - } -#else - assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND); - const Py_UCS4 *s = PyUnicode_4BYTE_DATA(unicode); - for (; size--; ++s, ++w) { - Py_UCS4 ch = *s; - if (ch > 0xFFFF) { - assert(ch <= MAX_UNICODE); - /* encode surrogate pair in this case */ - *w++ = Py_UNICODE_HIGH_SURROGATE(ch); - if (!size--) - break; - *w = Py_UNICODE_LOW_SURROGATE(ch); - } - else { - *w = ch; - } - } -#endif - } -} - -#ifdef HAVE_WCHAR_H - -/* Convert a Unicode object to a wide character string. - - - If w is NULL: return the number of wide characters (including the null - character) required to convert the unicode object. Ignore size argument. - - - Otherwise: return the number of wide characters (excluding the null - character) written into w. Write at most size wide characters (including - the null character). */ -Py_ssize_t -PyUnicode_AsWideChar(PyObject *unicode, - wchar_t *w, - Py_ssize_t size) +Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *unicode, + wchar_t *w, + Py_ssize_t size) { - Py_ssize_t res; - if (unicode == NULL) { PyErr_BadInternalCall(); return -1; } - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return -1; - } - res = unicode_get_widechar_size(unicode); - if (w == NULL) { - return res + 1; - } + /* If possible, try to copy the 0-termination as well */ + if (size > PyUnicode_GET_SIZE(unicode)) + size = PyUnicode_GET_SIZE(unicode) + 1; - if (size > res) { - size = res + 1; - } - else { - res = size; +#ifdef HAVE_USABLE_WCHAR_T + memcpy(w, unicode->str, size * sizeof(wchar_t)); +#else + { + register Py_UNICODE *u; + register Py_ssize_t i; + u = PyUnicode_AS_UNICODE(unicode); + for (i = size; i > 0; i--) + *w++ = *u++; } - unicode_copy_as_widechar(unicode, w, size); - return res; +#endif + + if (size > PyUnicode_GET_SIZE(unicode)) + return PyUnicode_GET_SIZE(unicode); + else + return size; } -wchar_t* -PyUnicode_AsWideCharString(PyObject *unicode, - Py_ssize_t *size) -{ - wchar_t *buffer; - Py_ssize_t buflen; +#endif - if (unicode == NULL) { - PyErr_BadInternalCall(); - return NULL; - } - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return NULL; - } +PyObject *PyUnicode_FromOrdinal(int ordinal) +{ + Py_UNICODE s[1]; - buflen = unicode_get_widechar_size(unicode); - buffer = (wchar_t *) PyMem_NEW(wchar_t, (buflen + 1)); - if (buffer == NULL) { - PyErr_NoMemory(); - return NULL; - } - unicode_copy_as_widechar(unicode, buffer, buflen + 1); - if (size != NULL) { - *size = buflen; - } - else if (wcslen(buffer) != (size_t)buflen) { - PyMem_FREE(buffer); +#ifdef Py_UNICODE_WIDE + if (ordinal < 0 || ordinal > 0x10ffff) { PyErr_SetString(PyExc_ValueError, - "embedded null character"); + "unichr() arg not in range(0x110000) " + "(wide Python build)"); return NULL; } - return buffer; -} - -#endif /* HAVE_WCHAR_H */ - -PyObject * -PyUnicode_FromOrdinal(int ordinal) -{ - if (ordinal < 0 || ordinal > MAX_UNICODE) { +#else + if (ordinal < 0 || ordinal > 0xffff) { PyErr_SetString(PyExc_ValueError, - "chr() arg not in range(0x110000)"); + "unichr() arg not in range(0x10000) " + "(narrow Python build)"); return NULL; } +#endif - return unicode_char((Py_UCS4)ordinal); + s[0] = (Py_UNICODE)ordinal; + return PyUnicode_FromUnicode(s, 1); } -PyObject * -PyUnicode_FromObject(PyObject *obj) +PyObject *PyUnicode_FromObject(register PyObject *obj) { /* XXX Perhaps we should make this API an alias of - PyObject_Str() instead ?! */ + PyObject_Unicode() instead ?! */ if (PyUnicode_CheckExact(obj)) { - if (PyUnicode_READY(obj) == -1) - return NULL; Py_INCREF(obj); return obj; } if (PyUnicode_Check(obj)) { /* For a Unicode subtype that's not a Unicode object, return a true Unicode object with the same data. */ - return _PyUnicode_Copy(obj); + return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(obj), + PyUnicode_GET_SIZE(obj)); } - PyErr_Format(PyExc_TypeError, - "Can't convert '%.100s' object to str implicitly", - Py_TYPE(obj)->tp_name); - return NULL; + return PyUnicode_FromEncodedObject(obj, NULL, "strict"); } -PyObject * -PyUnicode_FromEncodedObject(PyObject *obj, - const char *encoding, - const char *errors) +PyObject *PyUnicode_FromEncodedObject(register PyObject *obj, + const char *encoding, + const char *errors) { - Py_buffer buffer; + const char *s = NULL; + Py_ssize_t len; PyObject *v; if (obj == NULL) { @@ -3258,163 +1190,89 @@ PyUnicode_FromEncodedObject(PyObject *obj, return NULL; } - /* Decoding bytes objects is the most common case and should be fast */ - if (PyBytes_Check(obj)) { - if (PyBytes_GET_SIZE(obj) == 0) { - if (unicode_check_encoding_errors(encoding, errors) < 0) { - return NULL; - } - _Py_RETURN_UNICODE_EMPTY(); +#if 0 + /* For b/w compatibility we also accept Unicode objects provided + that no encodings is given and then redirect to + PyObject_Unicode() which then applies the additional logic for + Unicode subclasses. + + NOTE: This API should really only be used for object which + represent *encoded* Unicode ! + + */ + if (PyUnicode_Check(obj)) { + if (encoding) { + PyErr_SetString(PyExc_TypeError, + "decoding Unicode is not supported"); + return NULL; } - return PyUnicode_Decode( - PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj), - encoding, errors); + return PyObject_Unicode(obj); } - +#else if (PyUnicode_Check(obj)) { PyErr_SetString(PyExc_TypeError, - "decoding str is not supported"); + "decoding Unicode is not supported"); return NULL; } +#endif - /* Retrieve a bytes buffer view through the PEP 3118 buffer interface */ - if (PyObject_GetBuffer(obj, &buffer, PyBUF_SIMPLE) < 0) { + /* Coerce object */ + if (PyString_Check(obj)) { + s = PyString_AS_STRING(obj); + len = PyString_GET_SIZE(obj); + } + else if (PyByteArray_Check(obj)) { + /* Python 2.x specific */ PyErr_Format(PyExc_TypeError, - "decoding to str: need a bytes-like object, %.80s found", - Py_TYPE(obj)->tp_name); + "decoding bytearray is not supported"); return NULL; } + else if (PyObject_AsCharBuffer(obj, &s, &len)) { + /* Overwrite the error message with something more useful in + case of a TypeError. */ + if (PyErr_ExceptionMatches(PyExc_TypeError)) + PyErr_Format(PyExc_TypeError, + "coercing to Unicode: need string or buffer, " + "%.80s found", + Py_TYPE(obj)->tp_name); + goto onError; + } - if (buffer.len == 0) { - PyBuffer_Release(&buffer); - if (unicode_check_encoding_errors(encoding, errors) < 0) { - return NULL; - } + /* Convert to Unicode */ + if (len == 0) _Py_RETURN_UNICODE_EMPTY(); - } - v = PyUnicode_Decode((char*) buffer.buf, buffer.len, encoding, errors); - PyBuffer_Release(&buffer); + v = PyUnicode_Decode(s, len, encoding, errors); return v; -} - -/* Normalize an encoding name: similar to encodings.normalize_encoding(), but - also convert to lowercase. Return 1 on success, or 0 on error (encoding is - longer than lower_len-1). */ -int -_Py_normalize_encoding(const char *encoding, - char *lower, - size_t lower_len) -{ - const char *e; - char *l; - char *l_end; - int punct; - - assert(encoding != NULL); - - e = encoding; - l = lower; - l_end = &lower[lower_len - 1]; - punct = 0; - while (1) { - char c = *e; - if (c == 0) { - break; - } - if (Py_ISALNUM(c) || c == '.') { - if (punct && l != lower) { - if (l == l_end) { - return 0; - } - *l++ = '_'; - } - punct = 0; - - if (l == l_end) { - return 0; - } - *l++ = Py_TOLOWER(c); - } - else { - punct = 1; - } - - e++; - } - *l = '\0'; - return 1; + onError: + return NULL; } -PyObject * -PyUnicode_Decode(const char *s, - Py_ssize_t size, - const char *encoding, - const char *errors) +PyObject *PyUnicode_Decode(const char *s, + Py_ssize_t size, + const char *encoding, + const char *errors) { PyObject *buffer = NULL, *unicode; - Py_buffer info; - char buflower[11]; /* strlen("iso-8859-1\0") == 11, longest shortcut */ - - if (unicode_check_encoding_errors(encoding, errors) < 0) { - return NULL; - } - - if (size == 0) { - _Py_RETURN_UNICODE_EMPTY(); - } - if (encoding == NULL) { - return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL); - } + if (encoding == NULL) + encoding = PyUnicode_GetDefaultEncoding(); /* Shortcuts for common default encodings */ - if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) { - char *lower = buflower; - - /* Fast paths */ - if (lower[0] == 'u' && lower[1] == 't' && lower[2] == 'f') { - lower += 3; - if (*lower == '_') { - /* Match "utf8" and "utf_8" */ - lower++; - } - - if (lower[0] == '8' && lower[1] == 0) { - return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL); - } - else if (lower[0] == '1' && lower[1] == '6' && lower[2] == 0) { - return PyUnicode_DecodeUTF16(s, size, errors, 0); - } - else if (lower[0] == '3' && lower[1] == '2' && lower[2] == 0) { - return PyUnicode_DecodeUTF32(s, size, errors, 0); - } - } - else { - if (strcmp(lower, "ascii") == 0 - || strcmp(lower, "us_ascii") == 0) { - return PyUnicode_DecodeASCII(s, size, errors); - } - #ifdef MS_WINDOWS - else if (strcmp(lower, "mbcs") == 0) { - return PyUnicode_DecodeMBCS(s, size, errors); - } - #endif - else if (strcmp(lower, "latin1") == 0 - || strcmp(lower, "latin_1") == 0 - || strcmp(lower, "iso_8859_1") == 0 - || strcmp(lower, "iso8859_1") == 0) { - return PyUnicode_DecodeLatin1(s, size, errors); - } - } - } + if (strcmp(encoding, "utf-8") == 0) + return PyUnicode_DecodeUTF8(s, size, errors); + else if (strcmp(encoding, "latin-1") == 0) + return PyUnicode_DecodeLatin1(s, size, errors); +#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) + else if (strcmp(encoding, "mbcs") == 0) + return PyUnicode_DecodeMBCS(s, size, errors); +#endif + else if (strcmp(encoding, "ascii") == 0) + return PyUnicode_DecodeASCII(s, size, errors); /* Decode via the codec registry */ - buffer = NULL; - if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_FULL_RO) < 0) - goto onError; - buffer = PyMemoryView_FromBuffer(&info); + buffer = PyBuffer_FromMemory((void *)s, size); if (buffer == NULL) goto onError; unicode = _PyCodec_DecodeText(buffer, encoding, errors); @@ -3422,47 +1280,22 @@ PyUnicode_Decode(const char *s, goto onError; if (!PyUnicode_Check(unicode)) { PyErr_Format(PyExc_TypeError, - "'%.400s' decoder returned '%.400s' instead of 'str'; " - "use codecs.decode() to decode to arbitrary types", - encoding, + "decoder did not return an unicode object (type=%.400s)", Py_TYPE(unicode)->tp_name); Py_DECREF(unicode); goto onError; } Py_DECREF(buffer); - return unicode_result(unicode); + return unicode; onError: Py_XDECREF(buffer); return NULL; } -PyObject * -PyUnicode_AsDecodedObject(PyObject *unicode, - const char *encoding, - const char *errors) -{ - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return NULL; - } - - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyUnicode_AsDecodedObject() is deprecated; " - "use PyCodec_Decode() to decode from str", 1) < 0) - return NULL; - - if (encoding == NULL) - encoding = PyUnicode_GetDefaultEncoding(); - - /* Decode via the codec registry */ - return PyCodec_Decode(unicode, encoding, errors); -} - -PyObject * -PyUnicode_AsDecodedUnicode(PyObject *unicode, - const char *encoding, - const char *errors) +PyObject *PyUnicode_AsDecodedObject(PyObject *unicode, + const char *encoding, + const char *errors) { PyObject *v; @@ -3471,42 +1304,30 @@ PyUnicode_AsDecodedUnicode(PyObject *unicode, goto onError; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyUnicode_AsDecodedUnicode() is deprecated; " - "use PyCodec_Decode() to decode from str to str", 1) < 0) - return NULL; + if (PyErr_WarnPy3k("decoding Unicode is not supported in 3.x", 1) < 0) + goto onError; if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); /* Decode via the codec registry */ - v = PyCodec_Decode(unicode, encoding, errors); + v = _PyCodec_DecodeText(unicode, encoding, errors); if (v == NULL) goto onError; - if (!PyUnicode_Check(v)) { - PyErr_Format(PyExc_TypeError, - "'%.400s' decoder returned '%.400s' instead of 'str'; " - "use codecs.decode() to decode to arbitrary types", - encoding, - Py_TYPE(unicode)->tp_name); - Py_DECREF(v); - goto onError; - } - return unicode_result(v); + return v; onError: return NULL; } -PyObject * -PyUnicode_Encode(const Py_UNICODE *s, - Py_ssize_t size, - const char *encoding, - const char *errors) +PyObject *PyUnicode_Encode(const Py_UNICODE *s, + Py_ssize_t size, + const char *encoding, + const char *errors) { PyObject *v, *unicode; - unicode = PyUnicode_FromWideChar(s, size); + unicode = PyUnicode_FromUnicode(s, size); if (unicode == NULL) return NULL; v = PyUnicode_AsEncodedString(unicode, encoding, errors); @@ -3514,10 +1335,9 @@ PyUnicode_Encode(const Py_UNICODE *s, return v; } -PyObject * -PyUnicode_AsEncodedObject(PyObject *unicode, - const char *encoding, - const char *errors) +PyObject *PyUnicode_AsEncodedObject(PyObject *unicode, + const char *encoding, + const char *errors) { PyObject *v; @@ -3526,17 +1346,11 @@ PyUnicode_AsEncodedObject(PyObject *unicode, goto onError; } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyUnicode_AsEncodedObject() is deprecated; " - "use PyUnicode_AsEncodedString() to encode from str to bytes " - "or PyCodec_Encode() for generic encoding", 1) < 0) - return NULL; - if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); /* Encode via the codec registry */ - v = PyCodec_Encode(unicode, encoding, errors); + v = _PyCodec_EncodeText(unicode, encoding, errors); if (v == NULL) goto onError; return v; @@ -3545,232 +1359,41 @@ PyUnicode_AsEncodedObject(PyObject *unicode, return NULL; } - -static PyObject * -unicode_encode_locale(PyObject *unicode, _Py_error_handler error_handler, - int current_locale) -{ - Py_ssize_t wlen; - wchar_t *wstr = PyUnicode_AsWideCharString(unicode, &wlen); - if (wstr == NULL) { - return NULL; - } - - if ((size_t)wlen != wcslen(wstr)) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - PyMem_Free(wstr); - return NULL; - } - - char *str; - size_t error_pos; - const char *reason; - int res = _Py_EncodeLocaleEx(wstr, &str, &error_pos, &reason, - current_locale, error_handler); - PyMem_Free(wstr); - - if (res != 0) { - if (res == -2) { - PyObject *exc; - exc = PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", - "locale", unicode, - (Py_ssize_t)error_pos, - (Py_ssize_t)(error_pos+1), - reason); - if (exc != NULL) { - PyCodec_StrictErrors(exc); - Py_DECREF(exc); - } - } - else if (res == -3) { - PyErr_SetString(PyExc_ValueError, "unsupported error handler"); - } - else { - PyErr_NoMemory(); - } - return NULL; - } - - PyObject *bytes = PyBytes_FromString(str); - PyMem_RawFree(str); - return bytes; -} - -PyObject * -PyUnicode_EncodeLocale(PyObject *unicode, const char *errors) -{ - _Py_error_handler error_handler = _Py_GetErrorHandler(errors); - return unicode_encode_locale(unicode, error_handler, 1); -} - -PyObject * -PyUnicode_EncodeFSDefault(PyObject *unicode) -{ - PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); -#ifdef _Py_FORCE_UTF8_FS_ENCODING - if (interp->fs_codec.encoding) { - return unicode_encode_utf8(unicode, - interp->fs_codec.error_handler, - interp->fs_codec.errors); - } - else { - const wchar_t *filesystem_errors = interp->config.filesystem_errors; - _Py_error_handler errors; - errors = get_error_handler_wide(filesystem_errors); - assert(errors != _Py_ERROR_UNKNOWN); - return unicode_encode_utf8(unicode, errors, NULL); - } -#else - /* Bootstrap check: if the filesystem codec is implemented in Python, we - cannot use it to encode and decode filenames before it is loaded. Load - the Python codec requires to encode at least its own filename. Use the C - implementation of the locale codec until the codec registry is - initialized and the Python codec is loaded. - See _PyUnicode_InitEncodings(). */ - if (interp->fs_codec.encoding) { - return PyUnicode_AsEncodedString(unicode, - interp->fs_codec.encoding, - interp->fs_codec.errors); - } - else { - const wchar_t *filesystem_errors = interp->config.filesystem_errors; - _Py_error_handler errors; - errors = get_error_handler_wide(filesystem_errors); - assert(errors != _Py_ERROR_UNKNOWN); - return unicode_encode_locale(unicode, errors, 0); - } -#endif -} - -PyObject * -PyUnicode_AsEncodedString(PyObject *unicode, - const char *encoding, - const char *errors) +PyObject *PyUnicode_AsEncodedString(PyObject *unicode, + const char *encoding, + const char *errors) { PyObject *v; - char buflower[11]; /* strlen("iso_8859_1\0") == 11, longest shortcut */ if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); - return NULL; - } - - if (unicode_check_encoding_errors(encoding, errors) < 0) { - return NULL; + goto onError; } - if (encoding == NULL) { - return _PyUnicode_AsUTF8String(unicode, errors); - } + if (encoding == NULL) + encoding = PyUnicode_GetDefaultEncoding(); /* Shortcuts for common default encodings */ - if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) { - char *lower = buflower; - - /* Fast paths */ - if (lower[0] == 'u' && lower[1] == 't' && lower[2] == 'f') { - lower += 3; - if (*lower == '_') { - /* Match "utf8" and "utf_8" */ - lower++; - } - - if (lower[0] == '8' && lower[1] == 0) { - return _PyUnicode_AsUTF8String(unicode, errors); - } - else if (lower[0] == '1' && lower[1] == '6' && lower[2] == 0) { - return _PyUnicode_EncodeUTF16(unicode, errors, 0); - } - else if (lower[0] == '3' && lower[1] == '2' && lower[2] == 0) { - return _PyUnicode_EncodeUTF32(unicode, errors, 0); - } - } - else { - if (strcmp(lower, "ascii") == 0 - || strcmp(lower, "us_ascii") == 0) { - return _PyUnicode_AsASCIIString(unicode, errors); - } -#ifdef MS_WINDOWS - else if (strcmp(lower, "mbcs") == 0) { - return PyUnicode_EncodeCodePage(CP_ACP, unicode, errors); - } + if (errors == NULL) { + if (strcmp(encoding, "utf-8") == 0) + return PyUnicode_AsUTF8String(unicode); + else if (strcmp(encoding, "latin-1") == 0) + return PyUnicode_AsLatin1String(unicode); +#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) + else if (strcmp(encoding, "mbcs") == 0) + return PyUnicode_AsMBCSString(unicode); #endif - else if (strcmp(lower, "latin1") == 0 || - strcmp(lower, "latin_1") == 0 || - strcmp(lower, "iso_8859_1") == 0 || - strcmp(lower, "iso8859_1") == 0) { - return _PyUnicode_AsLatin1String(unicode, errors); - } - } + else if (strcmp(encoding, "ascii") == 0) + return PyUnicode_AsASCIIString(unicode); } /* Encode via the codec registry */ v = _PyCodec_EncodeText(unicode, encoding, errors); if (v == NULL) - return NULL; - - /* The normal path */ - if (PyBytes_Check(v)) - return v; - - /* If the codec returns a buffer, raise a warning and convert to bytes */ - if (PyByteArray_Check(v)) { - int error; - PyObject *b; - - error = PyErr_WarnFormat(PyExc_RuntimeWarning, 1, - "encoder %s returned bytearray instead of bytes; " - "use codecs.encode() to encode to arbitrary types", - encoding); - if (error) { - Py_DECREF(v); - return NULL; - } - - b = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), - PyByteArray_GET_SIZE(v)); - Py_DECREF(v); - return b; - } - - PyErr_Format(PyExc_TypeError, - "'%.400s' encoder returned '%.400s' instead of 'bytes'; " - "use codecs.encode() to encode to arbitrary types", - encoding, - Py_TYPE(v)->tp_name); - Py_DECREF(v); - return NULL; -} - -PyObject * -PyUnicode_AsEncodedUnicode(PyObject *unicode, - const char *encoding, - const char *errors) -{ - PyObject *v; - - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - goto onError; - } - - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "PyUnicode_AsEncodedUnicode() is deprecated; " - "use PyCodec_Encode() to encode from str to str", 1) < 0) - return NULL; - - if (encoding == NULL) - encoding = PyUnicode_GetDefaultEncoding(); - - /* Encode via the codec registry */ - v = PyCodec_Encode(unicode, encoding, errors); - if (v == NULL) goto onError; - if (!PyUnicode_Check(v)) { + if (!PyString_Check(v)) { PyErr_Format(PyExc_TypeError, - "'%.400s' encoder returned '%.400s' instead of 'str'; " - "use codecs.encode() to encode to arbitrary types", - encoding, + "encoder did not return a string object (type=%.400s)", Py_TYPE(v)->tp_name); Py_DECREF(v); goto onError; @@ -3781,437 +1404,65 @@ PyUnicode_AsEncodedUnicode(PyObject *unicode, return NULL; } -static PyObject* -unicode_decode_locale(const char *str, Py_ssize_t len, - _Py_error_handler errors, int current_locale) -{ - if (str[len] != '\0' || (size_t)len != strlen(str)) { - PyErr_SetString(PyExc_ValueError, "embedded null byte"); - return NULL; - } - - wchar_t *wstr; - size_t wlen; - const char *reason; - int res = _Py_DecodeLocaleEx(str, &wstr, &wlen, &reason, - current_locale, errors); - if (res != 0) { - if (res == -2) { - PyObject *exc; - exc = PyObject_CallFunction(PyExc_UnicodeDecodeError, "sy#nns", - "locale", str, len, - (Py_ssize_t)wlen, - (Py_ssize_t)(wlen + 1), - reason); - if (exc != NULL) { - PyCodec_StrictErrors(exc); - Py_DECREF(exc); - } - } - else if (res == -3) { - PyErr_SetString(PyExc_ValueError, "unsupported error handler"); - } - else { - PyErr_NoMemory(); - } - return NULL; - } - - PyObject *unicode = PyUnicode_FromWideChar(wstr, wlen); - PyMem_RawFree(wstr); - return unicode; -} - -PyObject* -PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len, - const char *errors) -{ - _Py_error_handler error_handler = _Py_GetErrorHandler(errors); - return unicode_decode_locale(str, len, error_handler, 1); -} - -PyObject* -PyUnicode_DecodeLocale(const char *str, const char *errors) -{ - Py_ssize_t size = (Py_ssize_t)strlen(str); - _Py_error_handler error_handler = _Py_GetErrorHandler(errors); - return unicode_decode_locale(str, size, error_handler, 1); -} - - -PyObject* -PyUnicode_DecodeFSDefault(const char *s) { - Py_ssize_t size = (Py_ssize_t)strlen(s); - return PyUnicode_DecodeFSDefaultAndSize(s, size); -} - -PyObject* -PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) -{ - PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); -#ifdef _Py_FORCE_UTF8_FS_ENCODING - if (interp->fs_codec.encoding) { - return unicode_decode_utf8(s, size, - interp->fs_codec.error_handler, - interp->fs_codec.errors, - NULL); - } - else { - const wchar_t *filesystem_errors = interp->config.filesystem_errors; - _Py_error_handler errors; - errors = get_error_handler_wide(filesystem_errors); - assert(errors != _Py_ERROR_UNKNOWN); - return unicode_decode_utf8(s, size, errors, NULL, NULL); - } -#else - /* Bootstrap check: if the filesystem codec is implemented in Python, we - cannot use it to encode and decode filenames before it is loaded. Load - the Python codec requires to encode at least its own filename. Use the C - implementation of the locale codec until the codec registry is - initialized and the Python codec is loaded. - See _PyUnicode_InitEncodings(). */ - if (interp->fs_codec.encoding) { - return PyUnicode_Decode(s, size, - interp->fs_codec.encoding, - interp->fs_codec.errors); - } - else { - const wchar_t *filesystem_errors = interp->config.filesystem_errors; - _Py_error_handler errors; - errors = get_error_handler_wide(filesystem_errors); - return unicode_decode_locale(s, size, errors, 0); - } -#endif -} - - -int -PyUnicode_FSConverter(PyObject* arg, void* addr) -{ - PyObject *path = NULL; - PyObject *output = NULL; - Py_ssize_t size; - void *data; - if (arg == NULL) { - Py_DECREF(*(PyObject**)addr); - *(PyObject**)addr = NULL; - return 1; - } - path = PyOS_FSPath(arg); - if (path == NULL) { - return 0; - } - if (PyBytes_Check(path)) { - output = path; - } - else { // PyOS_FSPath() guarantees its returned value is bytes or str. - output = PyUnicode_EncodeFSDefault(path); - Py_DECREF(path); - if (!output) { - return 0; - } - assert(PyBytes_Check(output)); - } - - size = PyBytes_GET_SIZE(output); - data = PyBytes_AS_STRING(output); - if ((size_t)size != strlen(data)) { - PyErr_SetString(PyExc_ValueError, "embedded null byte"); - Py_DECREF(output); - return 0; - } - *(PyObject**)addr = output; - return Py_CLEANUP_SUPPORTED; -} - - -int -PyUnicode_FSDecoder(PyObject* arg, void* addr) -{ - int is_buffer = 0; - PyObject *path = NULL; - PyObject *output = NULL; - if (arg == NULL) { - Py_DECREF(*(PyObject**)addr); - *(PyObject**)addr = NULL; - return 1; - } - - is_buffer = PyObject_CheckBuffer(arg); - if (!is_buffer) { - path = PyOS_FSPath(arg); - if (path == NULL) { - return 0; - } - } - else { - path = arg; - Py_INCREF(arg); - } - - if (PyUnicode_Check(path)) { - output = path; - } - else if (PyBytes_Check(path) || is_buffer) { - PyObject *path_bytes = NULL; - - if (!PyBytes_Check(path) && - PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "path should be string, bytes, or os.PathLike, not %.200s", - Py_TYPE(arg)->tp_name)) { - Py_DECREF(path); - return 0; - } - path_bytes = PyBytes_FromObject(path); - Py_DECREF(path); - if (!path_bytes) { - return 0; - } - output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(path_bytes), - PyBytes_GET_SIZE(path_bytes)); - Py_DECREF(path_bytes); - if (!output) { - return 0; - } - } - else { - PyErr_Format(PyExc_TypeError, - "path should be string, bytes, or os.PathLike, not %.200s", - Py_TYPE(arg)->tp_name); - Py_DECREF(path); - return 0; - } - if (PyUnicode_READY(output) == -1) { - Py_DECREF(output); - return 0; - } - if (findchar(PyUnicode_DATA(output), PyUnicode_KIND(output), - PyUnicode_GET_LENGTH(output), 0, 1) >= 0) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - Py_DECREF(output); - return 0; - } - *(PyObject**)addr = output; - return Py_CLEANUP_SUPPORTED; -} - - -const char * -PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) +PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode, + const char *errors) { - PyObject *bytes; - - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return NULL; - } - if (PyUnicode_READY(unicode) == -1) - return NULL; - - if (PyUnicode_UTF8(unicode) == NULL) { - assert(!PyUnicode_IS_COMPACT_ASCII(unicode)); - bytes = _PyUnicode_AsUTF8String(unicode, NULL); - if (bytes == NULL) - return NULL; - _PyUnicode_UTF8(unicode) = PyObject_MALLOC(PyBytes_GET_SIZE(bytes) + 1); - if (_PyUnicode_UTF8(unicode) == NULL) { - PyErr_NoMemory(); - Py_DECREF(bytes); - return NULL; - } - _PyUnicode_UTF8_LENGTH(unicode) = PyBytes_GET_SIZE(bytes); - memcpy(_PyUnicode_UTF8(unicode), - PyBytes_AS_STRING(bytes), - _PyUnicode_UTF8_LENGTH(unicode) + 1); - Py_DECREF(bytes); - } + PyObject *v = ((PyUnicodeObject *)unicode)->defenc; - if (psize) - *psize = PyUnicode_UTF8_LENGTH(unicode); - return PyUnicode_UTF8(unicode); -} - -const char * -PyUnicode_AsUTF8(PyObject *unicode) -{ - return PyUnicode_AsUTF8AndSize(unicode, NULL); + if (v) + return v; + v = PyUnicode_AsEncodedString(unicode, NULL, errors); + if (v && errors == NULL) + ((PyUnicodeObject *)unicode)->defenc = v; + return v; } -Py_UNICODE * -PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) +Py_UNICODE *PyUnicode_AsUnicode(PyObject *unicode) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); - return NULL; + goto onError; } - Py_UNICODE *w = _PyUnicode_WSTR(unicode); - if (w == NULL) { - /* Non-ASCII compact unicode object */ - assert(_PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND); - assert(PyUnicode_IS_READY(unicode)); + return PyUnicode_AS_UNICODE(unicode); - Py_ssize_t wlen = unicode_get_widechar_size(unicode); - if ((size_t)wlen > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { - PyErr_NoMemory(); - return NULL; - } - w = (wchar_t *) PyObject_MALLOC(sizeof(wchar_t) * (wlen + 1)); - if (w == NULL) { - PyErr_NoMemory(); - return NULL; - } - unicode_copy_as_widechar(unicode, w, wlen + 1); - _PyUnicode_WSTR(unicode) = w; - if (!PyUnicode_IS_COMPACT_ASCII(unicode)) { - _PyUnicode_WSTR_LENGTH(unicode) = wlen; - } - } - if (size != NULL) - *size = PyUnicode_WSTR_LENGTH(unicode); - return w; -} - -Py_UNICODE * -PyUnicode_AsUnicode(PyObject *unicode) -{ - return PyUnicode_AsUnicodeAndSize(unicode, NULL); -} - -const Py_UNICODE * -_PyUnicode_AsUnicode(PyObject *unicode) -{ - Py_ssize_t size; - const Py_UNICODE *wstr; - - wstr = PyUnicode_AsUnicodeAndSize(unicode, &size); - if (wstr && wcslen(wstr) != (size_t)size) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - return NULL; - } - return wstr; + onError: + return NULL; } - -Py_ssize_t -PyUnicode_GetSize(PyObject *unicode) +Py_ssize_t PyUnicode_GetSize(PyObject *unicode) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); goto onError; } - if (_PyUnicode_WSTR(unicode) == NULL) { - if (PyUnicode_AsUnicode(unicode) == NULL) - goto onError; - } - return PyUnicode_WSTR_LENGTH(unicode); + return PyUnicode_GET_SIZE(unicode); onError: return -1; } -Py_ssize_t -PyUnicode_GetLength(PyObject *unicode) +const char *PyUnicode_GetDefaultEncoding(void) { - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return -1; - } - if (PyUnicode_READY(unicode) == -1) - return -1; - return PyUnicode_GET_LENGTH(unicode); + return unicode_default_encoding; } -Py_UCS4 -PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index) +int PyUnicode_SetDefaultEncoding(const char *encoding) { - void *data; - int kind; - - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return (Py_UCS4)-1; - } - if (PyUnicode_READY(unicode) == -1) { - return (Py_UCS4)-1; - } - if (index < 0 || index >= PyUnicode_GET_LENGTH(unicode)) { - PyErr_SetString(PyExc_IndexError, "string index out of range"); - return (Py_UCS4)-1; - } - data = PyUnicode_DATA(unicode); - kind = PyUnicode_KIND(unicode); - return PyUnicode_READ(kind, data, index); -} + PyObject *v; -int -PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, Py_UCS4 ch) -{ - if (!PyUnicode_Check(unicode) || !PyUnicode_IS_COMPACT(unicode)) { - PyErr_BadArgument(); - return -1; - } - assert(PyUnicode_IS_READY(unicode)); - if (index < 0 || index >= PyUnicode_GET_LENGTH(unicode)) { - PyErr_SetString(PyExc_IndexError, "string index out of range"); - return -1; - } - if (unicode_check_modifiable(unicode)) - return -1; - if (ch > PyUnicode_MAX_CHAR_VALUE(unicode)) { - PyErr_SetString(PyExc_ValueError, "character out of range"); - return -1; - } - PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode), - index, ch); + /* Make sure the encoding is valid. As side effect, this also + loads the encoding into the codec registry cache. */ + v = _PyCodec_Lookup(encoding); + if (v == NULL) + goto onError; + Py_DECREF(v); + strncpy(unicode_default_encoding, + encoding, + sizeof(unicode_default_encoding) - 1); return 0; -} - -const char * -PyUnicode_GetDefaultEncoding(void) -{ - return "utf-8"; -} - -/* create or adjust a UnicodeDecodeError */ -static void -make_decode_exception(PyObject **exceptionObject, - const char *encoding, - const char *input, Py_ssize_t length, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason) -{ - if (*exceptionObject == NULL) { - *exceptionObject = PyUnicodeDecodeError_Create( - encoding, input, length, startpos, endpos, reason); - } - else { - if (PyUnicodeDecodeError_SetStart(*exceptionObject, startpos)) - goto onError; - if (PyUnicodeDecodeError_SetEnd(*exceptionObject, endpos)) - goto onError; - if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason)) - goto onError; - } - return; - -onError: - Py_CLEAR(*exceptionObject); -} -#ifdef MS_WINDOWS -static int -widechar_resize(wchar_t **buf, Py_ssize_t *size, Py_ssize_t newsize) -{ - if (newsize > *size) { - wchar_t *newbuf = *buf; - if (PyMem_Resize(newbuf, wchar_t, newsize) == NULL) { - PyErr_NoMemory(); - return -1; - } - *buf = newbuf; - } - *size = newsize; - return 0; + onError: + return -1; } /* error handling callback helper: @@ -4221,25 +1472,23 @@ widechar_resize(wchar_t **buf, Py_ssize_t *size, Py_ssize_t newsize) return 0 on success, -1 on error */ -static int -unicode_decode_call_errorhandler_wchar( - const char *errors, PyObject **errorHandler, - const char *encoding, const char *reason, - const char **input, const char **inend, Py_ssize_t *startinpos, - Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, - wchar_t **buf, Py_ssize_t *bufsize, Py_ssize_t *outpos) +static +int unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler, + const char *encoding, const char *reason, + const char *input, Py_ssize_t insize, Py_ssize_t *startinpos, + Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, + PyUnicodeObject **output, Py_ssize_t *outpos, Py_UNICODE **outptr) { - static const char *argparse = "Un;decoding error handler must return (str, int) tuple"; + static char *argparse = "O!n;decoding error handler must return (unicode, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; - Py_ssize_t outsize; - Py_ssize_t insize; + Py_ssize_t outsize = PyUnicode_GET_SIZE(*output); Py_ssize_t requiredsize; Py_ssize_t newpos; - PyObject *inputobj = NULL; - wchar_t *repwstr; - Py_ssize_t repwlen; + Py_UNICODE *repptr; + Py_ssize_t repsize; + int res = -1; if (*errorHandler == NULL) { *errorHandler = PyCodec_LookupError(errors); @@ -4247,36 +1496,30 @@ unicode_decode_call_errorhandler_wchar( goto onError; } - make_decode_exception(exceptionObject, - encoding, - *input, *inend - *input, - *startinpos, *endinpos, - reason); - if (*exceptionObject == NULL) - goto onError; + if (*exceptionObject == NULL) { + *exceptionObject = PyUnicodeDecodeError_Create( + encoding, input, insize, *startinpos, *endinpos, reason); + if (*exceptionObject == NULL) + goto onError; + } + else { + if (PyUnicodeDecodeError_SetStart(*exceptionObject, *startinpos)) + goto onError; + if (PyUnicodeDecodeError_SetEnd(*exceptionObject, *endinpos)) + goto onError; + if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason)) + goto onError; + } - restuple = _PyObject_CallOneArg(*errorHandler, *exceptionObject); + restuple = PyObject_CallFunctionObjArgs(*errorHandler, *exceptionObject, NULL); if (restuple == NULL) goto onError; if (!PyTuple_Check(restuple)) { - PyErr_SetString(PyExc_TypeError, &argparse[3]); + PyErr_SetString(PyExc_TypeError, &argparse[4]); goto onError; } - if (!PyArg_ParseTuple(restuple, argparse, &repunicode, &newpos)) + if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos)) goto onError; - - /* Copy back the bytes variables, which might have been modified by the - callback */ - inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject); - if (!inputobj) - goto onError; - *input = PyBytes_AS_STRING(inputobj); - insize = PyBytes_GET_SIZE(inputobj); - *inend = *input + insize; - /* we can DECREF safely, as the exception has another reference, - so the object won't go away. */ - Py_DECREF(inputobj); - if (newpos<0) newpos = insize+newpos; if (newpos<0 || newpos>insize) { @@ -4284,145 +1527,42 @@ unicode_decode_call_errorhandler_wchar( goto onError; } - repwstr = PyUnicode_AsUnicodeAndSize(repunicode, &repwlen); - if (repwstr == NULL) - goto onError; /* need more space? (at least enough for what we have+the replacement+the rest of the string (starting at the new input position), so we won't have to check space when there are no errors in the rest of the string) */ + repptr = PyUnicode_AS_UNICODE(repunicode); + repsize = PyUnicode_GET_SIZE(repunicode); requiredsize = *outpos; - if (requiredsize > PY_SSIZE_T_MAX - repwlen) + if (requiredsize > PY_SSIZE_T_MAX - repsize) goto overflow; - requiredsize += repwlen; + requiredsize += repsize; if (requiredsize > PY_SSIZE_T_MAX - (insize - newpos)) goto overflow; requiredsize += insize - newpos; - outsize = *bufsize; if (requiredsize > outsize) { if (outsize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*outsize) requiredsize = 2*outsize; - if (widechar_resize(buf, bufsize, requiredsize) < 0) { + if (_PyUnicode_Resize(output, requiredsize) < 0) goto onError; - } + *outptr = PyUnicode_AS_UNICODE(*output) + *outpos; } - wcsncpy(*buf + *outpos, repwstr, repwlen); - *outpos += repwlen; *endinpos = newpos; - *inptr = *input + newpos; - + *inptr = input + newpos; + Py_UNICODE_COPY(*outptr, repptr, repsize); + *outptr += repsize; + *outpos += repsize; /* we made it! */ - Py_DECREF(restuple); - return 0; - - overflow: - PyErr_SetString(PyExc_OverflowError, - "decoded result is too long for a Python string"); + res = 0; onError: Py_XDECREF(restuple); - return -1; -} -#endif /* MS_WINDOWS */ - -static int -unicode_decode_call_errorhandler_writer( - const char *errors, PyObject **errorHandler, - const char *encoding, const char *reason, - const char **input, const char **inend, Py_ssize_t *startinpos, - Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, - _PyUnicodeWriter *writer /* PyObject **output, Py_ssize_t *outpos */) -{ - static const char *argparse = "Un;decoding error handler must return (str, int) tuple"; - - PyObject *restuple = NULL; - PyObject *repunicode = NULL; - Py_ssize_t insize; - Py_ssize_t newpos; - Py_ssize_t replen; - Py_ssize_t remain; - PyObject *inputobj = NULL; - int need_to_grow = 0; - const char *new_inptr; - - if (*errorHandler == NULL) { - *errorHandler = PyCodec_LookupError(errors); - if (*errorHandler == NULL) - goto onError; - } - - make_decode_exception(exceptionObject, - encoding, - *input, *inend - *input, - *startinpos, *endinpos, - reason); - if (*exceptionObject == NULL) - goto onError; - - restuple = _PyObject_CallOneArg(*errorHandler, *exceptionObject); - if (restuple == NULL) - goto onError; - if (!PyTuple_Check(restuple)) { - PyErr_SetString(PyExc_TypeError, &argparse[3]); - goto onError; - } - if (!PyArg_ParseTuple(restuple, argparse, &repunicode, &newpos)) - goto onError; - - /* Copy back the bytes variables, which might have been modified by the - callback */ - inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject); - if (!inputobj) - goto onError; - remain = *inend - *input - *endinpos; - *input = PyBytes_AS_STRING(inputobj); - insize = PyBytes_GET_SIZE(inputobj); - *inend = *input + insize; - /* we can DECREF safely, as the exception has another reference, - so the object won't go away. */ - Py_DECREF(inputobj); - - if (newpos<0) - newpos = insize+newpos; - if (newpos<0 || newpos>insize) { - PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", newpos); - goto onError; - } - - replen = PyUnicode_GET_LENGTH(repunicode); - if (replen > 1) { - writer->min_length += replen - 1; - need_to_grow = 1; - } - new_inptr = *input + newpos; - if (*inend - new_inptr > remain) { - /* We don't know the decoding algorithm here so we make the worst - assumption that one byte decodes to one unicode character. - If unfortunately one byte could decode to more unicode characters, - the decoder may write out-of-bound then. Is it possible for the - algorithms using this function? */ - writer->min_length += *inend - new_inptr - remain; - need_to_grow = 1; - } - if (need_to_grow) { - writer->overallocate = 1; - if (_PyUnicodeWriter_Prepare(writer, writer->min_length - writer->pos, - PyUnicode_MAX_CHAR_VALUE(repunicode)) == -1) - goto onError; - } - if (_PyUnicodeWriter_WriteStr(writer, repunicode) == -1) - goto onError; - - *endinpos = newpos; - *inptr = new_inptr; - - /* we made it! */ - Py_DECREF(restuple); - return 0; + return res; - onError: - Py_XDECREF(restuple); - return -1; + overflow: + PyErr_SetString(PyExc_OverflowError, + "decoded result is too long for a Python string"); + goto onError; } /* --- UTF-7 Codec -------------------------------------------------------- */ @@ -4506,10 +1646,9 @@ char utf7_category[128] = { (directWS && (utf7_category[(c)] == 2)) || \ (directO && (utf7_category[(c)] == 1)))) -PyObject * -PyUnicode_DecodeUTF7(const char *s, - Py_ssize_t size, - const char *errors) +PyObject *PyUnicode_DecodeUTF7(const char *s, + Py_ssize_t size, + const char *errors) { return PyUnicode_DecodeUTF7Stateful(s, size, errors, NULL); } @@ -4521,43 +1660,42 @@ PyUnicode_DecodeUTF7(const char *s, * all the shift state (seen bits, number of bits seen, high * surrogate). */ -PyObject * -PyUnicode_DecodeUTF7Stateful(const char *s, - Py_ssize_t size, - const char *errors, - Py_ssize_t *consumed) +PyObject *PyUnicode_DecodeUTF7Stateful(const char *s, + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) { const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; + Py_ssize_t outpos; const char *e; - _PyUnicodeWriter writer; + PyUnicodeObject *unicode; + Py_UNICODE *p; const char *errmsg = ""; int inShift = 0; - Py_ssize_t shiftOutStart; + Py_UNICODE *shiftOutStart; unsigned int base64bits = 0; unsigned long base64buffer = 0; - Py_UCS4 surrogate = 0; + Py_UNICODE surrogate = 0; PyObject *errorHandler = NULL; PyObject *exc = NULL; + unicode = _PyUnicode_New(size); + if (!unicode) + return NULL; if (size == 0) { if (consumed) *consumed = 0; - _Py_RETURN_UNICODE_EMPTY(); + return (PyObject *)unicode; } - /* Start off assuming it's all ASCII. Widen later as necessary. */ - _PyUnicodeWriter_Init(&writer); - writer.min_length = size; - - shiftOutStart = 0; + p = unicode->str; + shiftOutStart = p; e = s + size; while (s < e) { - Py_UCS4 ch; - restart: - ch = (unsigned char) *s; + Py_UNICODE ch = (unsigned char) *s; if (inShift) { /* in a base-64 section */ if (IS_BASE64(ch)) { /* consume a base-64 character */ @@ -4566,32 +1704,35 @@ PyUnicode_DecodeUTF7Stateful(const char *s, s++; if (base64bits >= 16) { /* we have enough bits for a UTF-16 value */ - Py_UCS4 outCh = (Py_UCS4)(base64buffer >> (base64bits-16)); + Py_UNICODE outCh = (Py_UNICODE) + (base64buffer >> (base64bits-16)); base64bits -= 16; base64buffer &= (1 << base64bits) - 1; /* clear high bits */ assert(outCh <= 0xffff); if (surrogate) { /* expecting a second surrogate */ - if (Py_UNICODE_IS_LOW_SURROGATE(outCh)) { - Py_UCS4 ch2 = Py_UNICODE_JOIN_SURROGATES(surrogate, outCh); - if (_PyUnicodeWriter_WriteCharInline(&writer, ch2) < 0) - goto onError; + if (outCh >= 0xDC00 && outCh <= 0xDFFF) { +#ifdef Py_UNICODE_WIDE + *p++ = (((surrogate & 0x3FF)<<10) + | (outCh & 0x3FF)) + 0x10000; +#else + *p++ = surrogate; + *p++ = outCh; +#endif surrogate = 0; continue; } else { - if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0) - goto onError; + *p++ = surrogate; surrogate = 0; } } - if (Py_UNICODE_IS_HIGH_SURROGATE(outCh)) { + if (outCh >= 0xD800 && outCh <= 0xDBFF) { /* first surrogate */ surrogate = outCh; } else { - if (_PyUnicodeWriter_WriteCharInline(&writer, outCh) < 0) - goto onError; + *p++ = outCh; } } } @@ -4613,10 +1754,8 @@ PyUnicode_DecodeUTF7Stateful(const char *s, } } } - if (surrogate && DECODE_DIRECT(ch)) { - if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0) - goto onError; - } + if (surrogate && DECODE_DIRECT(ch)) + *p++ = surrogate; surrogate = 0; if (ch == '-') { /* '-' is absorbed; other terminating @@ -4630,26 +1769,19 @@ PyUnicode_DecodeUTF7Stateful(const char *s, s++; /* consume '+' */ if (s < e && *s == '-') { /* '+-' encodes '+' */ s++; - if (_PyUnicodeWriter_WriteCharInline(&writer, '+') < 0) - goto onError; - } - else if (s < e && !IS_BASE64(*s)) { - s++; - errmsg = "ill-formed sequence"; - goto utf7Error; + *p++ = '+'; } else { /* begin base64-encoded section */ inShift = 1; surrogate = 0; - shiftOutStart = writer.pos; + shiftOutStart = p; base64bits = 0; base64buffer = 0; } } else if (DECODE_DIRECT(ch)) { /* character decodes as itself */ + *p++ = ch; s++; - if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) - goto onError; } else { startinpos = s-starts; @@ -4659,12 +1791,13 @@ PyUnicode_DecodeUTF7Stateful(const char *s, } continue; utf7Error: + outpos = p-PyUnicode_AS_UNICODE(unicode); endinpos = s-starts; - if (unicode_decode_call_errorhandler_writer( + if (unicode_decode_call_errorhandler( errors, &errorHandler, "utf7", errmsg, - &starts, &e, &startinpos, &endinpos, &exc, &s, - &writer)) + starts, size, &startinpos, &endinpos, &exc, &s, + &unicode, &outpos, &p)) goto onError; } @@ -4676,85 +1809,72 @@ utf7Error: if (surrogate || (base64bits >= 6) || (base64bits > 0 && base64buffer != 0)) { + outpos = p-PyUnicode_AS_UNICODE(unicode); endinpos = size; - if (unicode_decode_call_errorhandler_writer( + if (unicode_decode_call_errorhandler( errors, &errorHandler, "utf7", "unterminated shift sequence", - &starts, &e, &startinpos, &endinpos, &exc, &s, - &writer)) + starts, size, &startinpos, &endinpos, &exc, &s, + &unicode, &outpos, &p)) goto onError; - if (s < e) - goto restart; } } /* return state */ if (consumed) { if (inShift) { + p = shiftOutStart; /* back off output */ *consumed = startinpos; - if (writer.pos != shiftOutStart && writer.maxchar > 127) { - PyObject *result = PyUnicode_FromKindAndData( - writer.kind, writer.data, shiftOutStart); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - _PyUnicodeWriter_Dealloc(&writer); - return result; - } - writer.pos = shiftOutStart; /* back off output */ } else { *consumed = s-starts; } } + if (_PyUnicode_Resize(&unicode, p - PyUnicode_AS_UNICODE(unicode)) < 0) + goto onError; + Py_XDECREF(errorHandler); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return (PyObject *)unicode; onError: Py_XDECREF(errorHandler); Py_XDECREF(exc); - _PyUnicodeWriter_Dealloc(&writer); + Py_DECREF(unicode); return NULL; } -PyObject * -_PyUnicode_EncodeUTF7(PyObject *str, - int base64SetO, - int base64WhiteSpace, - const char *errors) +PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s, + Py_ssize_t size, + int base64SetO, + int base64WhiteSpace, + const char *errors) { - int kind; - void *data; - Py_ssize_t len; PyObject *v; + /* It might be possible to tighten this worst case */ + Py_ssize_t allocated = 8 * size; int inShift = 0; - Py_ssize_t i; + Py_ssize_t i = 0; unsigned int base64bits = 0; unsigned long base64buffer = 0; char * out; char * start; - if (PyUnicode_READY(str) == -1) - return NULL; - kind = PyUnicode_KIND(str); - data = PyUnicode_DATA(str); - len = PyUnicode_GET_LENGTH(str); + if (allocated / 8 != size) + return PyErr_NoMemory(); - if (len == 0) - return PyBytes_FromStringAndSize(NULL, 0); + if (size == 0) + return PyString_FromStringAndSize(NULL, 0); - /* It might be possible to tighten this worst case */ - if (len > PY_SSIZE_T_MAX / 8) - return PyErr_NoMemory(); - v = PyBytes_FromStringAndSize(NULL, len * 8); + v = PyString_FromStringAndSize(NULL, allocated); if (v == NULL) return NULL; - start = out = PyBytes_AS_STRING(v); - for (i = 0; i < len; ++i) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); + start = out = PyString_AS_STRING(v); + for (;i < size; ++i) { + Py_UNICODE ch = s[i]; if (inShift) { if (ENCODE_DIRECT(ch, !base64SetO, !base64WhiteSpace)) { @@ -4792,19 +1912,19 @@ _PyUnicode_EncodeUTF7(PyObject *str, } continue; encode_char: +#ifdef Py_UNICODE_WIDE if (ch >= 0x10000) { - assert(ch <= MAX_UNICODE); - /* code first surrogate */ base64bits += 16; - base64buffer = (base64buffer << 16) | Py_UNICODE_HIGH_SURROGATE(ch); + base64buffer = (base64buffer << 16) | 0xd800 | ((ch-0x10000) >> 10); while (base64bits >= 6) { *out++ = TO_BASE64(base64buffer >> (base64bits-6)); base64bits -= 6; } /* prepare second surrogate */ - ch = Py_UNICODE_LOW_SURROGATE(ch); + ch = 0xDC00 | ((ch-0x10000) & 0x3FF); } +#endif base64bits += 16; base64buffer = (base64buffer << 16) | ch; while (base64bits >= 6) { @@ -4816,26 +1936,11 @@ encode_char: *out++= TO_BASE64(base64buffer << (6-base64bits) ); if (inShift) *out++ = '-'; - if (_PyBytes_Resize(&v, out - start) < 0) + + if (_PyString_Resize(&v, out - start)) return NULL; return v; } -PyObject * -PyUnicode_EncodeUTF7(const Py_UNICODE *s, - Py_ssize_t size, - int base64SetO, - int base64WhiteSpace, - const char *errors) -{ - PyObject *result; - PyObject *tmp = PyUnicode_FromWideChar(s, size); - if (tmp == NULL) - return NULL; - result = _PyUnicode_EncodeUTF7(tmp, base64SetO, - base64WhiteSpace, errors); - Py_DECREF(tmp); - return result; -} #undef IS_BASE64 #undef FROM_BASE64 @@ -4845,502 +1950,290 @@ PyUnicode_EncodeUTF7(const Py_UNICODE *s, /* --- UTF-8 Codec -------------------------------------------------------- */ -PyObject * -PyUnicode_DecodeUTF8(const char *s, - Py_ssize_t size, - const char *errors) +static +char utf8_code_length[256] = { + /* Map UTF-8 encoded prefix byte to sequence length. Zero means + illegal prefix. See RFC 3629 for details */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00-0F */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 70-7F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 80-8F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* B0-BF */ + 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* C0-C1 + C2-CF */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* D0-DF */ + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* E0-EF */ + 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* F0-F4 + F5-FF */ +}; + +PyObject *PyUnicode_DecodeUTF8(const char *s, + Py_ssize_t size, + const char *errors) { return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL); } -#include "stringlib/asciilib.h" -#include "stringlib/codecs.h" -#include "stringlib/undef.h" - -#include "stringlib/ucs1lib.h" -#include "stringlib/codecs.h" -#include "stringlib/undef.h" - -#include "stringlib/ucs2lib.h" -#include "stringlib/codecs.h" -#include "stringlib/undef.h" - -#include "stringlib/ucs4lib.h" -#include "stringlib/codecs.h" -#include "stringlib/undef.h" - -/* Mask to quickly check whether a C 'long' contains a - non-ASCII, UTF8-encoded char. */ -#if (SIZEOF_LONG == 8) -# define ASCII_CHAR_MASK 0x8080808080808080UL -#elif (SIZEOF_LONG == 4) -# define ASCII_CHAR_MASK 0x80808080UL -#else -# error C 'long' size should be either 4 or 8! -#endif - -static Py_ssize_t -ascii_decode(const char *start, const char *end, Py_UCS1 *dest) +PyObject *PyUnicode_DecodeUTF8Stateful(const char *s, + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) { - const char *p = start; - const char *aligned_end = (const char *) _Py_ALIGN_DOWN(end, SIZEOF_LONG); - - /* - * Issue #17237: m68k is a bit different from most architectures in - * that objects do not use "natural alignment" - for example, int and - * long are only aligned at 2-byte boundaries. Therefore the assert() - * won't work; also, tests have shown that skipping the "optimised - * version" will even speed up m68k. - */ -#if !defined(__m68k__) -#if SIZEOF_LONG <= SIZEOF_VOID_P - assert(_Py_IS_ALIGNED(dest, SIZEOF_LONG)); - if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) { - /* Fast path, see in STRINGLIB(utf8_decode) for - an explanation. */ - /* Help allocation */ - const char *_p = p; - Py_UCS1 * q = dest; - while (_p < aligned_end) { - unsigned long value = *(const unsigned long *) _p; - if (value & ASCII_CHAR_MASK) - break; - *((unsigned long *)q) = value; - _p += SIZEOF_LONG; - q += SIZEOF_LONG; - } - p = _p; - while (p < end) { - if ((unsigned char)*p & 0x80) - break; - *q++ = *p++; - } - return p - start; - } -#endif -#endif - while (p < end) { - /* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h - for an explanation. */ - if (_Py_IS_ALIGNED(p, SIZEOF_LONG)) { - /* Help allocation */ - const char *_p = p; - while (_p < aligned_end) { - unsigned long value = *(unsigned long *) _p; - if (value & ASCII_CHAR_MASK) - break; - _p += SIZEOF_LONG; - } - p = _p; - if (_p == end) - break; - } - if ((unsigned char)*p & 0x80) - break; - ++p; - } - memcpy(dest, start, p - start); - return p - start; -} + const char *starts = s; + int n; + int k; + Py_ssize_t startinpos; + Py_ssize_t endinpos; + Py_ssize_t outpos; + const char *e; + PyUnicodeObject *unicode; + Py_UNICODE *p; + const char *errmsg = ""; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; -static PyObject * -unicode_decode_utf8(const char *s, Py_ssize_t size, - _Py_error_handler error_handler, const char *errors, - Py_ssize_t *consumed) -{ + /* Note: size will always be longer than the resulting Unicode + character count */ + unicode = _PyUnicode_New(size); + if (!unicode) + return NULL; if (size == 0) { if (consumed) *consumed = 0; - _Py_RETURN_UNICODE_EMPTY(); - } - - /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 && (unsigned char)s[0] < 128) { - if (consumed) - *consumed = 1; - return get_latin1_char((unsigned char)s[0]); + return (PyObject *)unicode; } - const char *starts = s; - const char *end = s + size; - - // fast path: try ASCII string. - PyObject *u = PyUnicode_New(size, 127); - if (u == NULL) { - return NULL; - } - s += ascii_decode(s, end, PyUnicode_DATA(u)); - if (s == end) { - return u; - } + /* Unpack UTF-8 encoded data */ + p = unicode->str; + e = s + size; - // Use _PyUnicodeWriter after fast path is failed. - _PyUnicodeWriter writer; - _PyUnicodeWriter_InitWithBuffer(&writer, u); - writer.pos = s - starts; + while (s < e) { + Py_UCS4 ch = (unsigned char)*s; - Py_ssize_t startinpos, endinpos; - const char *errmsg = ""; - PyObject *error_handler_obj = NULL; - PyObject *exc = NULL; + if (ch < 0x80) { + *p++ = (Py_UNICODE)ch; + s++; + continue; + } - while (s < end) { - Py_UCS4 ch; - int kind = writer.kind; + n = utf8_code_length[ch]; - if (kind == PyUnicode_1BYTE_KIND) { - if (PyUnicode_IS_ASCII(writer.buffer)) - ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos); - else - ch = ucs1lib_utf8_decode(&s, end, writer.data, &writer.pos); - } else if (kind == PyUnicode_2BYTE_KIND) { - ch = ucs2lib_utf8_decode(&s, end, writer.data, &writer.pos); - } else { - assert(kind == PyUnicode_4BYTE_KIND); - ch = ucs4lib_utf8_decode(&s, end, writer.data, &writer.pos); + if (s + n > e) { + if (consumed) + break; + else { + errmsg = "unexpected end of data"; + startinpos = s-starts; + endinpos = startinpos+1; + for (k=1; (k < size-startinpos) && ((s[k]&0xC0) == 0x80); k++) + endinpos++; + goto utf8Error; + } } - switch (ch) { + switch (n) { + case 0: - if (s == end || consumed) - goto End; - errmsg = "unexpected end of data"; - startinpos = s - starts; - endinpos = end - starts; - break; - case 1: errmsg = "invalid start byte"; - startinpos = s - starts; - endinpos = startinpos + 1; - break; + startinpos = s-starts; + endinpos = startinpos+1; + goto utf8Error; + + case 1: + errmsg = "internal error"; + startinpos = s-starts; + endinpos = startinpos+1; + goto utf8Error; + case 2: - if (consumed && (unsigned char)s[0] == 0xED && end - s == 2 - && (unsigned char)s[1] >= 0xA0 && (unsigned char)s[1] <= 0xBF) - { - /* Truncated surrogate code in range D800-DFFF */ - goto End; - } - /* fall through */ + if ((s[1] & 0xc0) != 0x80) { + errmsg = "invalid continuation byte"; + startinpos = s-starts; + endinpos = startinpos + 1; + goto utf8Error; + } + ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f); + assert ((ch > 0x007F) && (ch <= 0x07FF)); + *p++ = (Py_UNICODE)ch; + break; + case 3: - case 4: - errmsg = "invalid continuation byte"; - startinpos = s - starts; - endinpos = startinpos + ch - 1; + /* XXX: surrogates shouldn't be valid UTF-8! + see http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf + (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt + Uncomment the 2 lines below to make them invalid, + code points: d800-dfff; UTF-8: \xed\xa0\x80-\xed\xbf\xbf. */ + if ((s[1] & 0xc0) != 0x80 || + (s[2] & 0xc0) != 0x80 || + ((unsigned char)s[0] == 0xE0 && + (unsigned char)s[1] < 0xA0)/* || + ((unsigned char)s[0] == 0xED && + (unsigned char)s[1] > 0x9F)*/) { + errmsg = "invalid continuation byte"; + startinpos = s-starts; + endinpos = startinpos + 1; + + /* if s[1] first two bits are 1 and 0, then the invalid + continuation byte is s[2], so increment endinpos by 1, + if not, s[1] is invalid and endinpos doesn't need to + be incremented. */ + if ((s[1] & 0xC0) == 0x80) + endinpos++; + goto utf8Error; + } + ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f); + assert ((ch > 0x07FF) && (ch <= 0xFFFF)); + *p++ = (Py_UNICODE)ch; break; - default: - if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) - goto onError; - continue; - } - if (error_handler == _Py_ERROR_UNKNOWN) - error_handler = _Py_GetErrorHandler(errors); + case 4: + if ((s[1] & 0xc0) != 0x80 || + (s[2] & 0xc0) != 0x80 || + (s[3] & 0xc0) != 0x80 || + ((unsigned char)s[0] == 0xF0 && + (unsigned char)s[1] < 0x90) || + ((unsigned char)s[0] == 0xF4 && + (unsigned char)s[1] > 0x8F)) { + errmsg = "invalid continuation byte"; + startinpos = s-starts; + endinpos = startinpos + 1; + if ((s[1] & 0xC0) == 0x80) { + endinpos++; + if ((s[2] & 0xC0) == 0x80) + endinpos++; + } + goto utf8Error; + } + ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) + + ((s[2] & 0x3f) << 6) + (s[3] & 0x3f); + assert ((ch > 0xFFFF) && (ch <= 0x10ffff)); - switch (error_handler) { - case _Py_ERROR_IGNORE: - s += (endinpos - startinpos); - break; +#ifdef Py_UNICODE_WIDE + *p++ = (Py_UNICODE)ch; +#else + /* compute and append the two surrogates: */ - case _Py_ERROR_REPLACE: - if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0) - goto onError; - s += (endinpos - startinpos); - break; + /* translate from 10000..10FFFF to 0..FFFF */ + ch -= 0x10000; - case _Py_ERROR_SURROGATEESCAPE: - { - Py_ssize_t i; + /* high surrogate = top 10 bits added to D800 */ + *p++ = (Py_UNICODE)(0xD800 + (ch >> 10)); - if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) - goto onError; - for (i=startinpos; i<endinpos; i++) { - ch = (Py_UCS4)(unsigned char)(starts[i]); - PyUnicode_WRITE(writer.kind, writer.data, writer.pos, - ch + 0xdc00); - writer.pos++; - } - s += (endinpos - startinpos); + /* low surrogate = bottom 10 bits added to DC00 */ + *p++ = (Py_UNICODE)(0xDC00 + (ch & 0x03FF)); +#endif break; } + s += n; + continue; - default: - if (unicode_decode_call_errorhandler_writer( - errors, &error_handler_obj, - "utf-8", errmsg, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; - } + utf8Error: + outpos = p-PyUnicode_AS_UNICODE(unicode); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "utf8", errmsg, + starts, size, &startinpos, &endinpos, &exc, &s, + &unicode, &outpos, &p)) + goto onError; } - -End: if (consumed) - *consumed = s - starts; + *consumed = s-starts; + + /* Adjust length */ + if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0) + goto onError; - Py_XDECREF(error_handler_obj); + Py_XDECREF(errorHandler); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return (PyObject *)unicode; -onError: - Py_XDECREF(error_handler_obj); + onError: + Py_XDECREF(errorHandler); Py_XDECREF(exc); - _PyUnicodeWriter_Dealloc(&writer); + Py_DECREF(unicode); return NULL; } - +/* Allocation strategy: if the string is short, convert into a stack buffer + and allocate exactly as much space needed at the end. Else allocate the + maximum possible needed (4 result bytes per Unicode character), and return + the excess memory at the end. +*/ PyObject * -PyUnicode_DecodeUTF8Stateful(const char *s, - Py_ssize_t size, - const char *errors, - Py_ssize_t *consumed) -{ - return unicode_decode_utf8(s, size, _Py_ERROR_UNKNOWN, errors, consumed); -} - - -/* UTF-8 decoder: use surrogateescape error handler if 'surrogateescape' is - non-zero, use strict error handler otherwise. - - On success, write a pointer to a newly allocated wide character string into - *wstr (use PyMem_RawFree() to free the memory) and write the output length - (in number of wchar_t units) into *wlen (if wlen is set). - - On memory allocation failure, return -1. - - On decoding error (if surrogateescape is zero), return -2. If wlen is - non-NULL, write the start of the illegal byte sequence into *wlen. If reason - is not NULL, write the decoding error message into *reason. */ -int -_Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, - const char **reason, _Py_error_handler errors) -{ - const char *orig_s = s; - const char *e; - wchar_t *unicode; - Py_ssize_t outpos; - - int surrogateescape = 0; - int surrogatepass = 0; - switch (errors) - { - case _Py_ERROR_STRICT: - break; - case _Py_ERROR_SURROGATEESCAPE: - surrogateescape = 1; - break; - case _Py_ERROR_SURROGATEPASS: - surrogatepass = 1; - break; - default: - return -3; - } - - /* Note: size will always be longer than the resulting Unicode - character count */ - if (PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(wchar_t) < (size + 1)) { - return -1; - } - - unicode = PyMem_RawMalloc((size + 1) * sizeof(wchar_t)); - if (!unicode) { - return -1; - } - - /* Unpack UTF-8 encoded data */ - e = s + size; - outpos = 0; - while (s < e) { - Py_UCS4 ch; -#if SIZEOF_WCHAR_T == 4 - ch = ucs4lib_utf8_decode(&s, e, (Py_UCS4 *)unicode, &outpos); -#else - ch = ucs2lib_utf8_decode(&s, e, (Py_UCS2 *)unicode, &outpos); -#endif - if (ch > 0xFF) { -#if SIZEOF_WCHAR_T == 4 - Py_UNREACHABLE(); -#else - assert(ch > 0xFFFF && ch <= MAX_UNICODE); - /* write a surrogate pair */ - unicode[outpos++] = (wchar_t)Py_UNICODE_HIGH_SURROGATE(ch); - unicode[outpos++] = (wchar_t)Py_UNICODE_LOW_SURROGATE(ch); -#endif - } - else { - if (!ch && s == e) { - break; - } - - if (surrogateescape) { - unicode[outpos++] = 0xDC00 + (unsigned char)*s++; - } - else { - /* Is it a valid three-byte code? */ - if (surrogatepass - && (e - s) >= 3 - && (s[0] & 0xf0) == 0xe0 - && (s[1] & 0xc0) == 0x80 - && (s[2] & 0xc0) == 0x80) - { - ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f); - s += 3; - unicode[outpos++] = ch; - } - else { - PyMem_RawFree(unicode ); - if (reason != NULL) { - switch (ch) { - case 0: - *reason = "unexpected end of data"; - break; - case 1: - *reason = "invalid start byte"; - break; - /* 2, 3, 4 */ - default: - *reason = "invalid continuation byte"; - break; - } - } - if (wlen != NULL) { - *wlen = s - orig_s; - } - return -2; - } - } - } - } - unicode[outpos] = L'\0'; - if (wlen) { - *wlen = outpos; - } - *wstr = unicode; - return 0; -} - - -wchar_t* -_Py_DecodeUTF8_surrogateescape(const char *arg, Py_ssize_t arglen, - size_t *wlen) -{ - wchar_t *wstr; - int res = _Py_DecodeUTF8Ex(arg, arglen, - &wstr, wlen, - NULL, _Py_ERROR_SURROGATEESCAPE); - if (res != 0) { - /* _Py_DecodeUTF8Ex() must support _Py_ERROR_SURROGATEESCAPE */ - assert(res != -3); - if (wlen) { - *wlen = (size_t)res; - } - return NULL; - } - return wstr; -} - - -/* UTF-8 encoder using the surrogateescape error handler . - - On success, return 0 and write the newly allocated character string (use - PyMem_Free() to free the memory) into *str. - - On encoding failure, return -2 and write the position of the invalid - surrogate character into *error_pos (if error_pos is set) and the decoding - error message into *reason (if reason is set). - - On memory allocation failure, return -1. */ -int -_Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, - const char **reason, int raw_malloc, _Py_error_handler errors) +PyUnicode_EncodeUTF8(const Py_UNICODE *s, + Py_ssize_t size, + const char *errors) { - const Py_ssize_t max_char_size = 4; - Py_ssize_t len = wcslen(text); +#define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */ - assert(len >= 0); + Py_ssize_t i; /* index into s of next input byte */ + PyObject *v; /* result string object */ + char *p; /* next free byte in output buffer */ + Py_ssize_t nallocated; /* number of result bytes allocated */ + Py_ssize_t nneeded; /* number of result bytes needed */ + char stackbuf[MAX_SHORT_UNICHARS * 4]; - int surrogateescape = 0; - int surrogatepass = 0; - switch (errors) - { - case _Py_ERROR_STRICT: - break; - case _Py_ERROR_SURROGATEESCAPE: - surrogateescape = 1; - break; - case _Py_ERROR_SURROGATEPASS: - surrogatepass = 1; - break; - default: - return -3; - } + assert(s != NULL); + assert(size >= 0); - if (len > PY_SSIZE_T_MAX / max_char_size - 1) { - return -1; - } - char *bytes; - if (raw_malloc) { - bytes = PyMem_RawMalloc((len + 1) * max_char_size); + if (size <= MAX_SHORT_UNICHARS) { + /* Write into the stack buffer; nallocated can't overflow. + * At the end, we'll allocate exactly as much heap space as it + * turns out we need. + */ + nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int); + v = NULL; /* will allocate after we're done */ + p = stackbuf; } else { - bytes = PyMem_Malloc((len + 1) * max_char_size); - } - if (bytes == NULL) { - return -1; + /* Overallocate on the heap, and give the excess back at the end. */ + nallocated = size * 4; + if (nallocated / 4 != size) /* overflow! */ + return PyErr_NoMemory(); + v = PyString_FromStringAndSize(NULL, nallocated); + if (v == NULL) + return NULL; + p = PyString_AS_STRING(v); } - char *p = bytes; - Py_ssize_t i; - for (i = 0; i < len; ) { - Py_ssize_t ch_pos = i; - Py_UCS4 ch = text[i]; - i++; -#if Py_UNICODE_SIZE == 2 - if (Py_UNICODE_IS_HIGH_SURROGATE(ch) - && i < len - && Py_UNICODE_IS_LOW_SURROGATE(text[i])) - { - ch = Py_UNICODE_JOIN_SURROGATES(ch, text[i]); - i++; - } -#endif + for (i = 0; i < size;) { + Py_UCS4 ch = s[i++]; - if (ch < 0x80) { + if (ch < 0x80) /* Encode ASCII */ *p++ = (char) ch; - } else if (ch < 0x0800) { /* Encode Latin-1 */ *p++ = (char)(0xc0 | (ch >> 6)); *p++ = (char)(0x80 | (ch & 0x3f)); } - else if (Py_UNICODE_IS_SURROGATE(ch) && !surrogatepass) { - /* surrogateescape error handler */ - if (!surrogateescape || !(0xDC80 <= ch && ch <= 0xDCFF)) { - if (error_pos != NULL) { - *error_pos = (size_t)ch_pos; - } - if (reason != NULL) { - *reason = "encoding error"; - } - if (raw_malloc) { - PyMem_RawFree(bytes); - } - else { - PyMem_Free(bytes); + else { + /* Encode UCS2 Unicode ordinals */ + if (ch < 0x10000) { + /* Special case: check for high surrogate */ + if (0xD800 <= ch && ch <= 0xDBFF && i != size) { + Py_UCS4 ch2 = s[i]; + /* Check for low surrogate and combine the two to + form a UCS4 value */ + if (0xDC00 <= ch2 && ch2 <= 0xDFFF) { + ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000; + i++; + goto encodeUCS4; + } + /* Fall through: handles isolated high surrogates */ } - return -2; + *p++ = (char)(0xe0 | (ch >> 12)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + continue; } - *p++ = (char)(ch & 0xff); - } - else if (ch < 0x10000) { - *p++ = (char)(0xe0 | (ch >> 12)); - *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); - *p++ = (char)(0x80 | (ch & 0x3f)); - } - else { /* ch >= 0x10000 */ - assert(ch <= MAX_UNICODE); + encodeUCS4: /* Encode UCS4 Unicode ordinals */ *p++ = (char)(0xf0 | (ch >> 18)); *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); @@ -5348,104 +2241,34 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, *p++ = (char)(0x80 | (ch & 0x3f)); } } - *p++ = '\0'; - size_t final_size = (p - bytes); - char *bytes2; - if (raw_malloc) { - bytes2 = PyMem_RawRealloc(bytes, final_size); + if (v == NULL) { + /* This was stack allocated. */ + nneeded = p - stackbuf; + assert(nneeded <= nallocated); + v = PyString_FromStringAndSize(stackbuf, nneeded); } else { - bytes2 = PyMem_Realloc(bytes, final_size); - } - if (bytes2 == NULL) { - if (error_pos != NULL) { - *error_pos = (size_t)-1; - } - if (raw_malloc) { - PyMem_RawFree(bytes); - } - else { - PyMem_Free(bytes); - } - return -1; + /* Cut back to size actually needed. */ + nneeded = p - PyString_AS_STRING(v); + assert(nneeded <= nallocated); + if (_PyString_Resize(&v, nneeded)) + return NULL; } - *str = bytes2; - return 0; -} - + return v; -/* Primary internal function which creates utf8 encoded bytes objects. +#undef MAX_SHORT_UNICHARS +} - Allocation strategy: if the string is short, convert into a stack buffer - and allocate exactly as much space needed at the end. Else allocate the - maximum possible needed (4 result bytes per Unicode character), and return - the excess memory at the end. -*/ -static PyObject * -unicode_encode_utf8(PyObject *unicode, _Py_error_handler error_handler, - const char *errors) +PyObject *PyUnicode_AsUTF8String(PyObject *unicode) { - enum PyUnicode_Kind kind; - void *data; - Py_ssize_t size; - if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - - if (PyUnicode_READY(unicode) == -1) - return NULL; - - if (PyUnicode_UTF8(unicode)) - return PyBytes_FromStringAndSize(PyUnicode_UTF8(unicode), - PyUnicode_UTF8_LENGTH(unicode)); - - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - size = PyUnicode_GET_LENGTH(unicode); - - switch (kind) { - default: - Py_UNREACHABLE(); - case PyUnicode_1BYTE_KIND: - /* the string cannot be ASCII, or PyUnicode_UTF8() would be set */ - assert(!PyUnicode_IS_ASCII(unicode)); - return ucs1lib_utf8_encoder(unicode, data, size, error_handler, errors); - case PyUnicode_2BYTE_KIND: - return ucs2lib_utf8_encoder(unicode, data, size, error_handler, errors); - case PyUnicode_4BYTE_KIND: - return ucs4lib_utf8_encoder(unicode, data, size, error_handler, errors); - } -} - -PyObject * -_PyUnicode_AsUTF8String(PyObject *unicode, const char *errors) -{ - return unicode_encode_utf8(unicode, _Py_ERROR_UNKNOWN, errors); -} - - -PyObject * -PyUnicode_EncodeUTF8(const Py_UNICODE *s, - Py_ssize_t size, - const char *errors) -{ - PyObject *v, *unicode; - - unicode = PyUnicode_FromWideChar(s, size); - if (unicode == NULL) - return NULL; - v = _PyUnicode_AsUTF8String(unicode, errors); - Py_DECREF(unicode); - return v; -} - -PyObject * -PyUnicode_AsUTF8String(PyObject *unicode) -{ - return _PyUnicode_AsUTF8String(unicode, NULL); + return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode), + NULL); } /* --- UTF-32 Codec ------------------------------------------------------- */ @@ -5469,11 +2292,24 @@ PyUnicode_DecodeUTF32Stateful(const char *s, const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; - _PyUnicodeWriter writer; + Py_ssize_t outpos; + PyUnicodeObject *unicode; + Py_UNICODE *p; +#ifndef Py_UNICODE_WIDE + int pairs = 0; + const unsigned char *qq; +#else + const int pairs = 0; +#endif const unsigned char *q, *e; - int le, bo = 0; /* assume native ordering by default */ - const char *encoding; + int bo = 0; /* assume native ordering by default */ const char *errmsg = ""; + /* Offsets from q for retrieving bytes in the right order. */ +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + int iorder[] = {0, 1, 2, 3}; +#else + int iorder[] = {3, 2, 1, 0}; +#endif PyObject *errorHandler = NULL; PyObject *exc = NULL; @@ -5487,289 +2323,225 @@ PyUnicode_DecodeUTF32Stateful(const char *s, byte order setting accordingly. In native mode, the leading BOM mark is skipped, in all other modes, it is copied to the output stream as-is (giving a ZWNBSP character). */ - if (bo == 0 && size >= 4) { - Py_UCS4 bom = ((unsigned int)q[3] << 24) | (q[2] << 16) | (q[1] << 8) | q[0]; - if (bom == 0x0000FEFF) { - bo = -1; - q += 4; - } - else if (bom == 0xFFFE0000) { - bo = 1; - q += 4; + if (bo == 0) { + if (size >= 4) { + const Py_UCS4 bom = ((unsigned int)q[iorder[3]] << 24) | (q[iorder[2]] << 16) | + (q[iorder[1]] << 8) | q[iorder[0]]; +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + if (bom == 0x0000FEFF) { + q += 4; + bo = -1; + } + else if (bom == 0xFFFE0000) { + q += 4; + bo = 1; + } +#else + if (bom == 0x0000FEFF) { + q += 4; + bo = 1; + } + else if (bom == 0xFFFE0000) { + q += 4; + bo = -1; + } +#endif } - if (byteorder) - *byteorder = bo; } - if (q == e) { - if (consumed) - *consumed = size; - _Py_RETURN_UNICODE_EMPTY(); + if (bo == -1) { + /* force LE */ + iorder[0] = 0; + iorder[1] = 1; + iorder[2] = 2; + iorder[3] = 3; + } + else if (bo == 1) { + /* force BE */ + iorder[0] = 3; + iorder[1] = 2; + iorder[2] = 1; + iorder[3] = 0; } -#ifdef WORDS_BIGENDIAN - le = bo < 0; -#else - le = bo <= 0; + /* On narrow builds we split characters outside the BMP into two + code points => count how much extra space we need. */ +#ifndef Py_UNICODE_WIDE + for (qq = q; e - qq >= 4; qq += 4) + if (qq[iorder[2]] != 0 || qq[iorder[3]] != 0) + pairs++; #endif - encoding = le ? "utf-32-le" : "utf-32-be"; - - _PyUnicodeWriter_Init(&writer); - writer.min_length = (e - q + 3) / 4; - if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1) - goto onError; - while (1) { - Py_UCS4 ch = 0; - Py_UCS4 maxch = PyUnicode_MAX_CHAR_VALUE(writer.buffer); + /* This might be one to much, because of a BOM */ + unicode = _PyUnicode_New((size+3)/4+pairs); + if (!unicode) + return NULL; + if (size == 0) + return (PyObject *)unicode; - if (e - q >= 4) { - enum PyUnicode_Kind kind = writer.kind; - void *data = writer.data; - const unsigned char *last = e - 4; - Py_ssize_t pos = writer.pos; - if (le) { - do { - ch = ((unsigned int)q[3] << 24) | (q[2] << 16) | (q[1] << 8) | q[0]; - if (ch > maxch) - break; - if (kind != PyUnicode_1BYTE_KIND && - Py_UNICODE_IS_SURROGATE(ch)) - break; - PyUnicode_WRITE(kind, data, pos++, ch); - q += 4; - } while (q <= last); - } - else { - do { - ch = ((unsigned int)q[0] << 24) | (q[1] << 16) | (q[2] << 8) | q[3]; - if (ch > maxch) - break; - if (kind != PyUnicode_1BYTE_KIND && - Py_UNICODE_IS_SURROGATE(ch)) - break; - PyUnicode_WRITE(kind, data, pos++, ch); - q += 4; - } while (q <= last); - } - writer.pos = pos; - } + /* Unpack UTF-32 encoded data */ + p = unicode->str; - if (Py_UNICODE_IS_SURROGATE(ch)) { - errmsg = "code point in surrogate code point range(0xd800, 0xe000)"; - startinpos = ((const char *)q) - starts; - endinpos = startinpos + 4; - } - else if (ch <= maxch) { - if (q == e || consumed) + while (q < e) { + Py_UCS4 ch; + /* remaining bytes at the end? (size should be divisible by 4) */ + if (e-q<4) { + if (consumed) break; - /* remaining bytes at the end? (size should be divisible by 4) */ errmsg = "truncated data"; - startinpos = ((const char *)q) - starts; - endinpos = ((const char *)e) - starts; + startinpos = ((const char *)q)-starts; + endinpos = ((const char *)e)-starts; + goto utf32Error; + /* The remaining input chars are ignored if the callback + chooses to skip the input */ } - else { - if (ch < 0x110000) { - if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) - goto onError; - q += 4; - continue; - } + ch = ((unsigned int)q[iorder[3]] << 24) | (q[iorder[2]] << 16) | + (q[iorder[1]] << 8) | q[iorder[0]]; + + if (ch >= 0x110000) + { errmsg = "code point not in range(0x110000)"; - startinpos = ((const char *)q) - starts; - endinpos = startinpos + 4; + startinpos = ((const char *)q)-starts; + endinpos = startinpos+4; + goto utf32Error; } - - /* The remaining input chars are ignored if the callback - chooses to skip the input */ - if (unicode_decode_call_errorhandler_writer( +#ifndef Py_UNICODE_WIDE + if (ch >= 0x10000) + { + *p++ = 0xD800 | ((ch-0x10000) >> 10); + *p++ = 0xDC00 | ((ch-0x10000) & 0x3FF); + } + else +#endif + *p++ = ch; + q += 4; + continue; + utf32Error: + outpos = p-PyUnicode_AS_UNICODE(unicode); + if (unicode_decode_call_errorhandler( errors, &errorHandler, - encoding, errmsg, - &starts, (const char **)&e, &startinpos, &endinpos, &exc, (const char **)&q, - &writer)) + "utf32", errmsg, + starts, size, &startinpos, &endinpos, &exc, (const char **)&q, + &unicode, &outpos, &p)) goto onError; } + if (byteorder) + *byteorder = bo; + if (consumed) *consumed = (const char *)q-starts; + /* Adjust length */ + if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0) + goto onError; + Py_XDECREF(errorHandler); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return (PyObject *)unicode; onError: - _PyUnicodeWriter_Dealloc(&writer); + Py_DECREF(unicode); Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; } PyObject * -_PyUnicode_EncodeUTF32(PyObject *str, - const char *errors, - int byteorder) +PyUnicode_EncodeUTF32(const Py_UNICODE *s, + Py_ssize_t size, + const char *errors, + int byteorder) { - enum PyUnicode_Kind kind; - const void *data; - Py_ssize_t len; PyObject *v; - uint32_t *out; -#if PY_LITTLE_ENDIAN - int native_ordering = byteorder <= 0; + unsigned char *p; + Py_ssize_t nsize, bytesize; +#ifndef Py_UNICODE_WIDE + Py_ssize_t i, pairs; #else - int native_ordering = byteorder >= 0; + const int pairs = 0; +#endif + /* Offsets from p for storing byte pairs in the right order. */ +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + int iorder[] = {0, 1, 2, 3}; +#else + int iorder[] = {3, 2, 1, 0}; #endif - const char *encoding; - Py_ssize_t nsize, pos; - PyObject *errorHandler = NULL; - PyObject *exc = NULL; - PyObject *rep = NULL; - - if (!PyUnicode_Check(str)) { - PyErr_BadArgument(); - return NULL; - } - if (PyUnicode_READY(str) == -1) - return NULL; - kind = PyUnicode_KIND(str); - data = PyUnicode_DATA(str); - len = PyUnicode_GET_LENGTH(str); - if (len > PY_SSIZE_T_MAX / 4 - (byteorder == 0)) +#define STORECHAR(CH) \ + do { \ + p[iorder[3]] = ((CH) >> 24) & 0xff; \ + p[iorder[2]] = ((CH) >> 16) & 0xff; \ + p[iorder[1]] = ((CH) >> 8) & 0xff; \ + p[iorder[0]] = (CH) & 0xff; \ + p += 4; \ + } while(0) + + /* In narrow builds we can output surrogate pairs as one code point, + so we need less space. */ +#ifndef Py_UNICODE_WIDE + for (i = pairs = 0; i < size-1; i++) + if (0xD800 <= s[i] && s[i] <= 0xDBFF && + 0xDC00 <= s[i+1] && s[i+1] <= 0xDFFF) + pairs++; +#endif + nsize = (size - pairs + (byteorder == 0)); + bytesize = nsize * 4; + if (bytesize / 4 != nsize) return PyErr_NoMemory(); - nsize = len + (byteorder == 0); - v = PyBytes_FromStringAndSize(NULL, nsize * 4); + v = PyString_FromStringAndSize(NULL, bytesize); if (v == NULL) return NULL; - /* output buffer is 4-bytes aligned */ - assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 4)); - out = (uint32_t *)PyBytes_AS_STRING(v); + p = (unsigned char *)PyString_AS_STRING(v); if (byteorder == 0) - *out++ = 0xFEFF; - if (len == 0) - goto done; - - if (byteorder == -1) - encoding = "utf-32-le"; - else if (byteorder == 1) - encoding = "utf-32-be"; - else - encoding = "utf-32"; - - if (kind == PyUnicode_1BYTE_KIND) { - ucs1lib_utf32_encode((const Py_UCS1 *)data, len, &out, native_ordering); - goto done; - } - - pos = 0; - while (pos < len) { - Py_ssize_t repsize, moreunits; - - if (kind == PyUnicode_2BYTE_KIND) { - pos += ucs2lib_utf32_encode((const Py_UCS2 *)data + pos, len - pos, - &out, native_ordering); - } - else { - assert(kind == PyUnicode_4BYTE_KIND); - pos += ucs4lib_utf32_encode((const Py_UCS4 *)data + pos, len - pos, - &out, native_ordering); - } - if (pos == len) - break; - - rep = unicode_encode_call_errorhandler( - errors, &errorHandler, - encoding, "surrogates not allowed", - str, &exc, pos, pos + 1, &pos); - if (!rep) - goto error; - - if (PyBytes_Check(rep)) { - repsize = PyBytes_GET_SIZE(rep); - if (repsize & 3) { - raise_encode_exception(&exc, encoding, - str, pos - 1, pos, - "surrogates not allowed"); - goto error; - } - moreunits = repsize / 4; - } - else { - assert(PyUnicode_Check(rep)); - if (PyUnicode_READY(rep) < 0) - goto error; - moreunits = repsize = PyUnicode_GET_LENGTH(rep); - if (!PyUnicode_IS_ASCII(rep)) { - raise_encode_exception(&exc, encoding, - str, pos - 1, pos, - "surrogates not allowed"); - goto error; - } - } + STORECHAR(0xFEFF); + if (size == 0) + return v; - /* four bytes are reserved for each surrogate */ - if (moreunits > 1) { - Py_ssize_t outpos = out - (uint32_t*) PyBytes_AS_STRING(v); - if (moreunits >= (PY_SSIZE_T_MAX - PyBytes_GET_SIZE(v)) / 4) { - /* integer overflow */ - PyErr_NoMemory(); - goto error; + if (byteorder == -1) { + /* force LE */ + iorder[0] = 0; + iorder[1] = 1; + iorder[2] = 2; + iorder[3] = 3; + } + else if (byteorder == 1) { + /* force BE */ + iorder[0] = 3; + iorder[1] = 2; + iorder[2] = 1; + iorder[3] = 0; + } + + while (size-- > 0) { + Py_UCS4 ch = *s++; +#ifndef Py_UNICODE_WIDE + if (0xD800 <= ch && ch <= 0xDBFF && size > 0) { + Py_UCS4 ch2 = *s; + if (0xDC00 <= ch2 && ch2 <= 0xDFFF) { + ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000; + s++; + size--; } - if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + 4 * (moreunits - 1)) < 0) - goto error; - out = (uint32_t*) PyBytes_AS_STRING(v) + outpos; } - - if (PyBytes_Check(rep)) { - memcpy(out, PyBytes_AS_STRING(rep), repsize); - out += moreunits; - } else /* rep is unicode */ { - assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); - ucs1lib_utf32_encode(PyUnicode_1BYTE_DATA(rep), repsize, - &out, native_ordering); - } - - Py_CLEAR(rep); +#endif + STORECHAR(ch); } - - /* Cut back to size actually needed. This is necessary for, for example, - encoding of a string containing isolated surrogates and the 'ignore' - handler is used. */ - nsize = (unsigned char*) out - (unsigned char*) PyBytes_AS_STRING(v); - if (nsize != PyBytes_GET_SIZE(v)) - _PyBytes_Resize(&v, nsize); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - done: return v; - error: - Py_XDECREF(rep); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - Py_XDECREF(v); - return NULL; +#undef STORECHAR } -PyObject * -PyUnicode_EncodeUTF32(const Py_UNICODE *s, - Py_ssize_t size, - const char *errors, - int byteorder) +PyObject *PyUnicode_AsUTF32String(PyObject *unicode) { - PyObject *result; - PyObject *tmp = PyUnicode_FromWideChar(s, size); - if (tmp == NULL) + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); return NULL; - result = _PyUnicode_EncodeUTF32(tmp, errors, byteorder); - Py_DECREF(tmp); - return result; -} - -PyObject * -PyUnicode_AsUTF32String(PyObject *unicode) -{ - return _PyUnicode_EncodeUTF32(unicode, NULL, 0); + } + return PyUnicode_EncodeUTF32(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode), + NULL, + 0); } /* --- UTF-16 Codec ------------------------------------------------------- */ @@ -5793,15 +2565,31 @@ PyUnicode_DecodeUTF16Stateful(const char *s, const char *starts = s; Py_ssize_t startinpos; Py_ssize_t endinpos; - _PyUnicodeWriter writer; + Py_ssize_t outpos; + PyUnicodeObject *unicode; + Py_UNICODE *p; const unsigned char *q, *e; int bo = 0; /* assume native ordering by default */ - int native_ordering; const char *errmsg = ""; + /* Offsets from q for retrieving byte pairs in the right order. */ +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + int ihi = 1, ilo = 0; +#else + int ihi = 0, ilo = 1; +#endif PyObject *errorHandler = NULL; PyObject *exc = NULL; - const char *encoding; + /* Note: size will always be longer than the resulting Unicode + character count */ + unicode = _PyUnicode_New(size); + if (!unicode) + return NULL; + if (size == 0) + return (PyObject *)unicode; + + /* Unpack UTF-16 encoded data */ + p = unicode->str; q = (unsigned char *)s; e = q + size; @@ -5812,940 +2600,955 @@ PyUnicode_DecodeUTF16Stateful(const char *s, byte order setting accordingly. In native mode, the leading BOM mark is skipped, in all other modes, it is copied to the output stream as-is (giving a ZWNBSP character). */ - if (bo == 0 && size >= 2) { - const Py_UCS4 bom = (q[1] << 8) | q[0]; - if (bom == 0xFEFF) { - q += 2; - bo = -1; - } - else if (bom == 0xFFFE) { - q += 2; - bo = 1; + if (bo == 0) { + if (size >= 2) { + const Py_UNICODE bom = (q[ihi] << 8) | q[ilo]; +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + if (bom == 0xFEFF) { + q += 2; + bo = -1; + } + else if (bom == 0xFFFE) { + q += 2; + bo = 1; + } +#else + if (bom == 0xFEFF) { + q += 2; + bo = 1; + } + else if (bom == 0xFFFE) { + q += 2; + bo = -1; + } +#endif } - if (byteorder) - *byteorder = bo; } - if (q == e) { - if (consumed) - *consumed = size; - _Py_RETURN_UNICODE_EMPTY(); + if (bo == -1) { + /* force LE */ + ihi = 1; + ilo = 0; + } + else if (bo == 1) { + /* force BE */ + ihi = 0; + ilo = 1; } -#if PY_LITTLE_ENDIAN - native_ordering = bo <= 0; - encoding = bo <= 0 ? "utf-16-le" : "utf-16-be"; -#else - native_ordering = bo >= 0; - encoding = bo >= 0 ? "utf-16-be" : "utf-16-le"; -#endif + while (q < e) { + Py_UNICODE ch; + /* remaining bytes at the end? (size should be even) */ + if (e-q<2) { + if (consumed) + break; + errmsg = "truncated data"; + startinpos = ((const char *)q)-starts; + endinpos = ((const char *)e)-starts; + goto utf16Error; + /* The remaining input chars are ignored if the callback + chooses to skip the input */ + } + ch = (q[ihi] << 8) | q[ilo]; - /* Note: size will always be longer than the resulting Unicode - character count normally. Error handler will take care of - resizing when needed. */ - _PyUnicodeWriter_Init(&writer); - writer.min_length = (e - q + 1) / 2; - if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1) - goto onError; + q += 2; - while (1) { - Py_UCS4 ch = 0; - if (e - q >= 2) { - int kind = writer.kind; - if (kind == PyUnicode_1BYTE_KIND) { - if (PyUnicode_IS_ASCII(writer.buffer)) - ch = asciilib_utf16_decode(&q, e, - (Py_UCS1*)writer.data, &writer.pos, - native_ordering); - else - ch = ucs1lib_utf16_decode(&q, e, - (Py_UCS1*)writer.data, &writer.pos, - native_ordering); - } else if (kind == PyUnicode_2BYTE_KIND) { - ch = ucs2lib_utf16_decode(&q, e, - (Py_UCS2*)writer.data, &writer.pos, - native_ordering); - } else { - assert(kind == PyUnicode_4BYTE_KIND); - ch = ucs4lib_utf16_decode(&q, e, - (Py_UCS4*)writer.data, &writer.pos, - native_ordering); - } + if (ch < 0xD800 || ch > 0xDFFF) { + *p++ = ch; + continue; } - switch (ch) - { - case 0: - /* remaining byte at the end? (size should be even) */ - if (q == e || consumed) - goto End; - errmsg = "truncated data"; - startinpos = ((const char *)q) - starts; - endinpos = ((const char *)e) - starts; - break; - /* The remaining input chars are ignored if the callback - chooses to skip the input */ - case 1: + /* UTF-16 code pair: */ + if (e - q < 2) { q -= 2; if (consumed) - goto End; + break; errmsg = "unexpected end of data"; - startinpos = ((const char *)q) - starts; - endinpos = ((const char *)e) - starts; - break; - case 2: - errmsg = "illegal encoding"; - startinpos = ((const char *)q) - 2 - starts; - endinpos = startinpos + 2; - break; - case 3: - errmsg = "illegal UTF-16 surrogate"; - startinpos = ((const char *)q) - 4 - starts; - endinpos = startinpos + 2; - break; - default: - if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) - goto onError; - continue; + startinpos = ((const char *)q)-starts; + endinpos = ((const char *)e)-starts; + goto utf16Error; + } + if (0xD800 <= ch && ch <= 0xDBFF) { + Py_UNICODE ch2 = (q[ihi] << 8) | q[ilo]; + q += 2; + if (0xDC00 <= ch2 && ch2 <= 0xDFFF) { +#ifndef Py_UNICODE_WIDE + *p++ = ch; + *p++ = ch2; +#else + *p++ = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000; +#endif + continue; + } + else { + errmsg = "illegal UTF-16 surrogate"; + startinpos = (((const char *)q)-4)-starts; + endinpos = startinpos+2; + goto utf16Error; + } + } + errmsg = "illegal encoding"; + startinpos = (((const char *)q)-2)-starts; + endinpos = startinpos+2; + /* Fall through to report the error */ - if (unicode_decode_call_errorhandler_writer( - errors, - &errorHandler, - encoding, errmsg, - &starts, - (const char **)&e, - &startinpos, - &endinpos, - &exc, - (const char **)&q, - &writer)) + utf16Error: + outpos = p-PyUnicode_AS_UNICODE(unicode); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "utf16", errmsg, + starts, size, &startinpos, &endinpos, &exc, (const char **)&q, + &unicode, &outpos, &p)) goto onError; } -End: + if (byteorder) + *byteorder = bo; + if (consumed) *consumed = (const char *)q-starts; + /* Adjust length */ + if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0) + goto onError; + Py_XDECREF(errorHandler); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return (PyObject *)unicode; onError: - _PyUnicodeWriter_Dealloc(&writer); + Py_DECREF(unicode); Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; } PyObject * -_PyUnicode_EncodeUTF16(PyObject *str, - const char *errors, - int byteorder) +PyUnicode_EncodeUTF16(const Py_UNICODE *s, + Py_ssize_t size, + const char *errors, + int byteorder) { - enum PyUnicode_Kind kind; - const void *data; - Py_ssize_t len; PyObject *v; - unsigned short *out; - Py_ssize_t pairs; -#if PY_BIG_ENDIAN - int native_ordering = byteorder >= 0; + unsigned char *p; + Py_ssize_t nsize, bytesize; +#ifdef Py_UNICODE_WIDE + Py_ssize_t i, pairs; #else - int native_ordering = byteorder <= 0; + const int pairs = 0; +#endif + /* Offsets from p for storing byte pairs in the right order. */ +#ifdef BYTEORDER_IS_LITTLE_ENDIAN + int ihi = 1, ilo = 0; +#else + int ihi = 0, ilo = 1; #endif - const char *encoding; - Py_ssize_t nsize, pos; - PyObject *errorHandler = NULL; - PyObject *exc = NULL; - PyObject *rep = NULL; - if (!PyUnicode_Check(str)) { - PyErr_BadArgument(); - return NULL; - } - if (PyUnicode_READY(str) == -1) - return NULL; - kind = PyUnicode_KIND(str); - data = PyUnicode_DATA(str); - len = PyUnicode_GET_LENGTH(str); - - pairs = 0; - if (kind == PyUnicode_4BYTE_KIND) { - const Py_UCS4 *in = (const Py_UCS4 *)data; - const Py_UCS4 *end = in + len; - while (in < end) { - if (*in++ >= 0x10000) { - pairs++; - } - } - } - if (len > PY_SSIZE_T_MAX / 2 - pairs - (byteorder == 0)) { +#define STORECHAR(CH) \ + do { \ + p[ihi] = ((CH) >> 8) & 0xff; \ + p[ilo] = (CH) & 0xff; \ + p += 2; \ + } while(0) + +#ifdef Py_UNICODE_WIDE + for (i = pairs = 0; i < size; i++) + if (s[i] >= 0x10000) + pairs++; +#endif + /* 2 * (size + pairs + (byteorder == 0)) */ + if (size > PY_SSIZE_T_MAX || + size > PY_SSIZE_T_MAX - pairs - (byteorder == 0)) return PyErr_NoMemory(); - } - nsize = len + pairs + (byteorder == 0); - v = PyBytes_FromStringAndSize(NULL, nsize * 2); - if (v == NULL) { + nsize = size + pairs + (byteorder == 0); + bytesize = nsize * 2; + if (bytesize / 2 != nsize) + return PyErr_NoMemory(); + v = PyString_FromStringAndSize(NULL, bytesize); + if (v == NULL) return NULL; - } - /* output buffer is 2-bytes aligned */ - assert(_Py_IS_ALIGNED(PyBytes_AS_STRING(v), 2)); - out = (unsigned short *)PyBytes_AS_STRING(v); - if (byteorder == 0) { - *out++ = 0xFEFF; - } - if (len == 0) { - goto done; - } - - if (kind == PyUnicode_1BYTE_KIND) { - ucs1lib_utf16_encode((const Py_UCS1 *)data, len, &out, native_ordering); - goto done; - } + p = (unsigned char *)PyString_AS_STRING(v); + if (byteorder == 0) + STORECHAR(0xFEFF); + if (size == 0) + return v; - if (byteorder < 0) { - encoding = "utf-16-le"; - } - else if (byteorder > 0) { - encoding = "utf-16-be"; + if (byteorder == -1) { + /* force LE */ + ihi = 1; + ilo = 0; } - else { - encoding = "utf-16"; + else if (byteorder == 1) { + /* force BE */ + ihi = 0; + ilo = 1; } - pos = 0; - while (pos < len) { - Py_ssize_t repsize, moreunits; - - if (kind == PyUnicode_2BYTE_KIND) { - pos += ucs2lib_utf16_encode((const Py_UCS2 *)data + pos, len - pos, - &out, native_ordering); - } - else { - assert(kind == PyUnicode_4BYTE_KIND); - pos += ucs4lib_utf16_encode((const Py_UCS4 *)data + pos, len - pos, - &out, native_ordering); - } - if (pos == len) - break; - - rep = unicode_encode_call_errorhandler( - errors, &errorHandler, - encoding, "surrogates not allowed", - str, &exc, pos, pos + 1, &pos); - if (!rep) - goto error; - - if (PyBytes_Check(rep)) { - repsize = PyBytes_GET_SIZE(rep); - if (repsize & 1) { - raise_encode_exception(&exc, encoding, - str, pos - 1, pos, - "surrogates not allowed"); - goto error; - } - moreunits = repsize / 2; - } - else { - assert(PyUnicode_Check(rep)); - if (PyUnicode_READY(rep) < 0) - goto error; - moreunits = repsize = PyUnicode_GET_LENGTH(rep); - if (!PyUnicode_IS_ASCII(rep)) { - raise_encode_exception(&exc, encoding, - str, pos - 1, pos, - "surrogates not allowed"); - goto error; - } - } - - /* two bytes are reserved for each surrogate */ - if (moreunits > 1) { - Py_ssize_t outpos = out - (unsigned short*) PyBytes_AS_STRING(v); - if (moreunits >= (PY_SSIZE_T_MAX - PyBytes_GET_SIZE(v)) / 2) { - /* integer overflow */ - PyErr_NoMemory(); - goto error; - } - if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + 2 * (moreunits - 1)) < 0) - goto error; - out = (unsigned short*) PyBytes_AS_STRING(v) + outpos; - } - - if (PyBytes_Check(rep)) { - memcpy(out, PyBytes_AS_STRING(rep), repsize); - out += moreunits; - } else /* rep is unicode */ { - assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); - ucs1lib_utf16_encode(PyUnicode_1BYTE_DATA(rep), repsize, - &out, native_ordering); + while (size-- > 0) { + Py_UNICODE ch = *s++; + Py_UNICODE ch2 = 0; +#ifdef Py_UNICODE_WIDE + if (ch >= 0x10000) { + ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF); + ch = 0xD800 | ((ch-0x10000) >> 10); } - - Py_CLEAR(rep); +#endif + STORECHAR(ch); + if (ch2) + STORECHAR(ch2); } - - /* Cut back to size actually needed. This is necessary for, for example, - encoding of a string containing isolated surrogates and the 'ignore' handler - is used. */ - nsize = (unsigned char*) out - (unsigned char*) PyBytes_AS_STRING(v); - if (nsize != PyBytes_GET_SIZE(v)) - _PyBytes_Resize(&v, nsize); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - done: return v; - error: - Py_XDECREF(rep); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - Py_XDECREF(v); - return NULL; #undef STORECHAR } -PyObject * -PyUnicode_EncodeUTF16(const Py_UNICODE *s, - Py_ssize_t size, - const char *errors, - int byteorder) +PyObject *PyUnicode_AsUTF16String(PyObject *unicode) { - PyObject *result; - PyObject *tmp = PyUnicode_FromWideChar(s, size); - if (tmp == NULL) + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); return NULL; - result = _PyUnicode_EncodeUTF16(tmp, errors, byteorder); - Py_DECREF(tmp); - return result; -} - -PyObject * -PyUnicode_AsUTF16String(PyObject *unicode) -{ - return _PyUnicode_EncodeUTF16(unicode, NULL, 0); + } + return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode), + NULL, + 0); } /* --- Unicode Escape Codec ----------------------------------------------- */ static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL; -PyObject * -_PyUnicode_DecodeUnicodeEscape(const char *s, - Py_ssize_t size, - const char *errors, - const char **first_invalid_escape) +PyObject *PyUnicode_DecodeUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) { const char *starts = s; - _PyUnicodeWriter writer; + Py_ssize_t startinpos; + Py_ssize_t endinpos; + Py_ssize_t outpos; + PyUnicodeObject *v; + Py_UNICODE *p; const char *end; + char* message; + Py_UCS4 chr = 0xffffffff; /* in case 'getcode' messes up */ PyObject *errorHandler = NULL; PyObject *exc = NULL; - // so we can remember if we've seen an invalid escape char or not - *first_invalid_escape = NULL; - - if (size == 0) { - _Py_RETURN_UNICODE_EMPTY(); - } /* Escaped strings will always be longer than the resulting Unicode string, so we start with size here and then reduce the length after conversion to the true value. (but if the error callback returns a long replacement string we'll have to allocate more space) */ - _PyUnicodeWriter_Init(&writer); - writer.min_length = size; - if (_PyUnicodeWriter_Prepare(&writer, size, 127) < 0) { + v = _PyUnicode_New(size); + if (v == NULL) goto onError; - } + if (size == 0) + return (PyObject *)v; + p = PyUnicode_AS_UNICODE(v); end = s + size; + while (s < end) { - unsigned char c = (unsigned char) *s++; - Py_UCS4 ch; - int count; - Py_ssize_t startinpos; - Py_ssize_t endinpos; - const char *message; - -#define WRITE_ASCII_CHAR(ch) \ - do { \ - assert(ch <= 127); \ - assert(writer.pos < writer.size); \ - PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \ - } while(0) - -#define WRITE_CHAR(ch) \ - do { \ - if (ch <= writer.maxchar) { \ - assert(writer.pos < writer.size); \ - PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \ - } \ - else if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) { \ - goto onError; \ - } \ - } while(0) + unsigned char c; + Py_UNICODE x; + int digits; /* Non-escape characters are interpreted as Unicode ordinals */ - if (c != '\\') { - WRITE_CHAR(c); + if (*s != '\\') { + *p++ = (unsigned char) *s++; continue; } - startinpos = s - starts - 1; + startinpos = s-starts; /* \ - Escapes */ - if (s >= end) { - message = "\\ at end of string"; - goto error; - } - c = (unsigned char) *s++; - - assert(writer.pos < writer.size); + s++; + c = *s++; + if (s > end) + c = '\0'; /* Invalid after \ */ switch (c) { /* \x escapes */ - case '\n': continue; - case '\\': WRITE_ASCII_CHAR('\\'); continue; - case '\'': WRITE_ASCII_CHAR('\''); continue; - case '\"': WRITE_ASCII_CHAR('\"'); continue; - case 'b': WRITE_ASCII_CHAR('\b'); continue; - /* FF */ - case 'f': WRITE_ASCII_CHAR('\014'); continue; - case 't': WRITE_ASCII_CHAR('\t'); continue; - case 'n': WRITE_ASCII_CHAR('\n'); continue; - case 'r': WRITE_ASCII_CHAR('\r'); continue; - /* VT */ - case 'v': WRITE_ASCII_CHAR('\013'); continue; - /* BEL, not classic C */ - case 'a': WRITE_ASCII_CHAR('\007'); continue; + case '\n': break; + case '\\': *p++ = '\\'; break; + case '\'': *p++ = '\''; break; + case '\"': *p++ = '\"'; break; + case 'b': *p++ = '\b'; break; + case 'f': *p++ = '\014'; break; /* FF */ + case 't': *p++ = '\t'; break; + case 'n': *p++ = '\n'; break; + case 'r': *p++ = '\r'; break; + case 'v': *p++ = '\013'; break; /* VT */ + case 'a': *p++ = '\007'; break; /* BEL, not classic C */ /* \OOO (octal) escapes */ case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': - ch = c - '0'; + x = s[-1] - '0'; if (s < end && '0' <= *s && *s <= '7') { - ch = (ch<<3) + *s++ - '0'; - if (s < end && '0' <= *s && *s <= '7') { - ch = (ch<<3) + *s++ - '0'; - } + x = (x<<3) + *s++ - '0'; + if (s < end && '0' <= *s && *s <= '7') + x = (x<<3) + *s++ - '0'; } - WRITE_CHAR(ch); - continue; + *p++ = x; + break; /* hex escapes */ /* \xXX */ case 'x': - count = 2; + digits = 2; message = "truncated \\xXX escape"; goto hexescape; /* \uXXXX */ case 'u': - count = 4; + digits = 4; message = "truncated \\uXXXX escape"; goto hexescape; /* \UXXXXXXXX */ case 'U': - count = 8; + digits = 8; message = "truncated \\UXXXXXXXX escape"; hexescape: - for (ch = 0; count && s < end; ++s, --count) { - c = (unsigned char)*s; - ch <<= 4; - if (c >= '0' && c <= '9') { - ch += c - '0'; - } - else if (c >= 'a' && c <= 'f') { - ch += c - ('a' - 10); - } - else if (c >= 'A' && c <= 'F') { - ch += c - ('A' - 10); + chr = 0; + if (end - s < digits) { + /* count only hex digits */ + for (; s < end; ++s) { + c = (unsigned char)*s; + if (!Py_ISXDIGIT(c)) + goto error; } - else { - break; - } - } - if (count) { goto error; } - - /* when we get here, ch is a 32-bit unicode character */ - if (ch > MAX_UNICODE) { + for (; digits--; ++s) { + c = (unsigned char)*s; + if (!Py_ISXDIGIT(c)) + goto error; + chr = (chr<<4) & ~0xF; + if (c >= '0' && c <= '9') + chr += c - '0'; + else if (c >= 'a' && c <= 'f') + chr += 10 + c - 'a'; + else + chr += 10 + c - 'A'; + } + if (chr == 0xffffffff && PyErr_Occurred()) + /* _decoding_error will have already written into the + target buffer. */ + break; + store: + /* when we get here, chr is a 32-bit unicode character */ + if (chr <= 0xffff) + /* UCS-2 character */ + *p++ = (Py_UNICODE) chr; + else if (chr <= 0x10ffff) { + /* UCS-4 character. Either store directly, or as + surrogate pair. */ +#ifdef Py_UNICODE_WIDE + *p++ = chr; +#else + chr -= 0x10000L; + *p++ = 0xD800 + (Py_UNICODE) (chr >> 10); + *p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF); +#endif + } else { message = "illegal Unicode character"; goto error; } - - WRITE_CHAR(ch); - continue; + break; /* \N{name} */ case 'N': + message = "malformed \\N character escape"; if (ucnhash_CAPI == NULL) { /* load the unicode data module */ - ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import( - PyUnicodeData_CAPSULE_NAME, 1); - if (ucnhash_CAPI == NULL) { - PyErr_SetString( - PyExc_UnicodeError, - "\\N escapes not supported (can't load unicodedata module)" - ); - goto onError; - } + ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCapsule_Import(PyUnicodeData_CAPSULE_NAME, 1); + if (ucnhash_CAPI == NULL) + goto ucnhashError; } - - message = "malformed \\N character escape"; if (s < end && *s == '{') { - const char *start = ++s; - size_t namelen; + const char *start = s+1; /* look for the closing brace */ - while (s < end && *s != '}') + while (*s != '}' && s < end) s++; - namelen = s - start; - if (namelen && s < end) { + if (s > start && s < end && *s == '}') { /* found a name. look it up in the unicode database */ - s++; - ch = 0xffffffff; /* in case 'getcode' messes up */ - if (namelen <= INT_MAX && - ucnhash_CAPI->getcode(NULL, start, (int)namelen, - &ch, 0)) { - assert(ch <= MAX_UNICODE); - WRITE_CHAR(ch); - continue; - } message = "unknown Unicode character name"; + s++; + if (s - start - 1 <= INT_MAX && + ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), &chr)) + goto store; } } goto error; default: - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-1; /* Back up one char, since we've - already incremented s. */ + if (s > end) { + message = "\\ at end of string"; + s--; + goto error; } - WRITE_ASCII_CHAR('\\'); - WRITE_CHAR(c); - continue; + else { + *p++ = '\\'; + *p++ = (unsigned char)s[-1]; + } + break; } + continue; error: endinpos = s-starts; - writer.min_length = end - s + writer.pos; - if (unicode_decode_call_errorhandler_writer( + outpos = p-PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( errors, &errorHandler, "unicodeescape", message, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) { + starts, size, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) goto onError; - } - assert(end - s <= writer.size - writer.pos); - -#undef WRITE_ASCII_CHAR -#undef WRITE_CHAR + continue; } + if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) + goto onError; + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return (PyObject *)v; + ucnhashError: + PyErr_SetString( + PyExc_UnicodeError, + "\\N escapes not supported (can't load unicodedata module)" + ); + Py_XDECREF(v); Py_XDECREF(errorHandler); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return NULL; onError: - _PyUnicodeWriter_Dealloc(&writer); + Py_XDECREF(v); Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; } -PyObject * -PyUnicode_DecodeUnicodeEscape(const char *s, - Py_ssize_t size, - const char *errors) +/* Return a Unicode-Escape string version of the Unicode object. + + If quotes is true, the string is enclosed in u"" or u'' quotes as + appropriate. + +*/ + +Py_LOCAL_INLINE(const Py_UNICODE *) findchar(const Py_UNICODE *s, + Py_ssize_t size, + Py_UNICODE ch) { - const char *first_invalid_escape; - PyObject *result = _PyUnicode_DecodeUnicodeEscape(s, size, errors, - &first_invalid_escape); - if (result == NULL) - return NULL; - if (first_invalid_escape != NULL) { - if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "invalid escape sequence '\\%c'", - (unsigned char)*first_invalid_escape) < 0) { - Py_DECREF(result); - return NULL; - } + /* like wcschr, but doesn't stop at NULL characters */ + + while (size-- > 0) { + if (*s == ch) + return s; + s++; } - return result; -} -/* Return a Unicode-Escape string version of the Unicode object. */ + return NULL; +} -PyObject * -PyUnicode_AsUnicodeEscapeString(PyObject *unicode) +static +PyObject *unicodeescape_string(const Py_UNICODE *s, + Py_ssize_t size, + int quotes) { - Py_ssize_t i, len; PyObject *repr; char *p; - enum PyUnicode_Kind kind; - void *data; - Py_ssize_t expandsize; - /* Initial allocation is based on the longest-possible character - escape. + static const char *hexdigit = "0123456789abcdef"; +#ifdef Py_UNICODE_WIDE + const Py_ssize_t expandsize = 10; +#else + const Py_ssize_t expandsize = 6; +#endif - For UCS1 strings it's '\xxx', 4 bytes per source character. - For UCS2 strings it's '\uxxxx', 6 bytes per source character. - For UCS4 strings it's '\U00xxxxxx', 10 bytes per source character. + /* XXX(nnorwitz): rather than over-allocating, it would be + better to choose a different scheme. Perhaps scan the + first N-chars of the string and allocate based on that size. */ + /* Initial allocation is based on the longest-possible unichr + escape. - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return NULL; - } - if (PyUnicode_READY(unicode) == -1) { - return NULL; - } + In wide (UTF-32) builds '\U00xxxxxx' is 10 chars per source + unichr, so in this case it's the longest unichr escape. In + narrow (UTF-16) builds this is five chars per source unichr + since there are two unichrs in the surrogate pair, so in narrow + (UTF-16) builds it's not the longest unichr escape. - len = PyUnicode_GET_LENGTH(unicode); - if (len == 0) { - return PyBytes_FromStringAndSize(NULL, 0); - } + In wide or narrow builds '\uxxxx' is 6 chars per source unichr, + so in the narrow (UTF-16) build case it's the longest unichr + escape. + */ - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - /* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6 - bytes, and 1 byte characters 4. */ - expandsize = kind * 2 + 2; - if (len > PY_SSIZE_T_MAX / expandsize) { + if (size > (PY_SSIZE_T_MAX - 2 - 1) / expandsize) return PyErr_NoMemory(); - } - repr = PyBytes_FromStringAndSize(NULL, expandsize * len); - if (repr == NULL) { + + repr = PyString_FromStringAndSize(NULL, + 2 + + expandsize*size + + 1); + if (repr == NULL) return NULL; - } - p = PyBytes_AS_STRING(repr); - for (i = 0; i < len; i++) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); + p = PyString_AS_STRING(repr); - /* U+0000-U+00ff range */ - if (ch < 0x100) { - if (ch >= ' ' && ch < 127) { - if (ch != '\\') { - /* Copy printable US ASCII as-is */ - *p++ = (char) ch; - } - /* Escape backslashes */ - else { - *p++ = '\\'; - *p++ = '\\'; - } - } + if (quotes) { + *p++ = 'u'; + *p++ = (findchar(s, size, '\'') && + !findchar(s, size, '"')) ? '"' : '\''; + } + while (size-- > 0) { + Py_UNICODE ch = *s++; - /* Map special whitespace to '\t', \n', '\r' */ - else if (ch == '\t') { - *p++ = '\\'; - *p++ = 't'; - } - else if (ch == '\n') { - *p++ = '\\'; - *p++ = 'n'; - } - else if (ch == '\r') { - *p++ = '\\'; - *p++ = 'r'; - } + /* Escape quotes and backslashes */ + if ((quotes && + ch == (Py_UNICODE) PyString_AS_STRING(repr)[1]) || ch == '\\') { + *p++ = '\\'; + *p++ = (char) ch; + continue; + } - /* Map non-printable US ASCII and 8-bit characters to '\xHH' */ - else { +#ifdef Py_UNICODE_WIDE + /* Map 21-bit characters to '\U00xxxxxx' */ + else if (ch >= 0x10000) { + *p++ = '\\'; + *p++ = 'U'; + *p++ = hexdigit[(ch >> 28) & 0x0000000F]; + *p++ = hexdigit[(ch >> 24) & 0x0000000F]; + *p++ = hexdigit[(ch >> 20) & 0x0000000F]; + *p++ = hexdigit[(ch >> 16) & 0x0000000F]; + *p++ = hexdigit[(ch >> 12) & 0x0000000F]; + *p++ = hexdigit[(ch >> 8) & 0x0000000F]; + *p++ = hexdigit[(ch >> 4) & 0x0000000F]; + *p++ = hexdigit[ch & 0x0000000F]; + continue; + } +#else + /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */ + else if (ch >= 0xD800 && ch < 0xDC00) { + Py_UNICODE ch2; + Py_UCS4 ucs; + + ch2 = *s++; + size--; + if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) { + ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000; *p++ = '\\'; - *p++ = 'x'; - *p++ = Py_hexdigits[(ch >> 4) & 0x000F]; - *p++ = Py_hexdigits[ch & 0x000F]; + *p++ = 'U'; + *p++ = hexdigit[(ucs >> 28) & 0x0000000F]; + *p++ = hexdigit[(ucs >> 24) & 0x0000000F]; + *p++ = hexdigit[(ucs >> 20) & 0x0000000F]; + *p++ = hexdigit[(ucs >> 16) & 0x0000000F]; + *p++ = hexdigit[(ucs >> 12) & 0x0000000F]; + *p++ = hexdigit[(ucs >> 8) & 0x0000000F]; + *p++ = hexdigit[(ucs >> 4) & 0x0000000F]; + *p++ = hexdigit[ucs & 0x0000000F]; + continue; } + /* Fall through: isolated surrogates are copied as-is */ + s--; + size++; } - /* U+0100-U+ffff range: Map 16-bit characters to '\uHHHH' */ - else if (ch < 0x10000) { +#endif + + /* Map 16-bit characters to '\uxxxx' */ + if (ch >= 256) { *p++ = '\\'; *p++ = 'u'; - *p++ = Py_hexdigits[(ch >> 12) & 0x000F]; - *p++ = Py_hexdigits[(ch >> 8) & 0x000F]; - *p++ = Py_hexdigits[(ch >> 4) & 0x000F]; - *p++ = Py_hexdigits[ch & 0x000F]; + *p++ = hexdigit[(ch >> 12) & 0x000F]; + *p++ = hexdigit[(ch >> 8) & 0x000F]; + *p++ = hexdigit[(ch >> 4) & 0x000F]; + *p++ = hexdigit[ch & 0x000F]; } - /* U+010000-U+10ffff range: Map 21-bit characters to '\U00HHHHHH' */ - else { - /* Make sure that the first two digits are zero */ - assert(ch <= MAX_UNICODE && MAX_UNICODE <= 0x10ffff); + /* Map special whitespace to '\t', \n', '\r' */ + else if (ch == '\t') { *p++ = '\\'; - *p++ = 'U'; - *p++ = '0'; - *p++ = '0'; - *p++ = Py_hexdigits[(ch >> 20) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 16) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 12) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 8) & 0x0000000F]; - *p++ = Py_hexdigits[(ch >> 4) & 0x0000000F]; - *p++ = Py_hexdigits[ch & 0x0000000F]; + *p++ = 't'; + } + else if (ch == '\n') { + *p++ = '\\'; + *p++ = 'n'; + } + else if (ch == '\r') { + *p++ = '\\'; + *p++ = 'r'; + } + + /* Map non-printable US ASCII to '\xhh' */ + else if (ch < ' ' || ch >= 0x7F) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = hexdigit[(ch >> 4) & 0x000F]; + *p++ = hexdigit[ch & 0x000F]; } + + /* Copy everything else as-is */ + else + *p++ = (char) ch; } + if (quotes) + *p++ = PyString_AS_STRING(repr)[1]; - assert(p - PyBytes_AS_STRING(repr) > 0); - if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) { + *p = '\0'; + if (_PyString_Resize(&repr, p - PyString_AS_STRING(repr))) return NULL; - } return repr; } -PyObject * -PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, - Py_ssize_t size) +PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s, + Py_ssize_t size) { - PyObject *result; - PyObject *tmp = PyUnicode_FromWideChar(s, size); - if (tmp == NULL) { + return unicodeescape_string(s, size, 0); +} + +PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode) +{ + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); return NULL; } - - result = PyUnicode_AsUnicodeEscapeString(tmp); - Py_DECREF(tmp); - return result; + return PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode)); } /* --- Raw Unicode Escape Codec ------------------------------------------- */ -PyObject * -PyUnicode_DecodeRawUnicodeEscape(const char *s, - Py_ssize_t size, - const char *errors) +PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s, + Py_ssize_t size, + const char *errors) { const char *starts = s; - _PyUnicodeWriter writer; + Py_ssize_t startinpos; + Py_ssize_t endinpos; + Py_ssize_t outpos; + PyUnicodeObject *v; + Py_UNICODE *p; const char *end; + const char *bs; PyObject *errorHandler = NULL; PyObject *exc = NULL; - if (size == 0) { - _Py_RETURN_UNICODE_EMPTY(); - } - /* Escaped strings will always be longer than the resulting Unicode string, so we start with size here and then reduce the length after conversion to the true value. (But decoding error handler might have to resize the string) */ - _PyUnicodeWriter_Init(&writer); - writer.min_length = size; - if (_PyUnicodeWriter_Prepare(&writer, size, 127) < 0) { + v = _PyUnicode_New(size); + if (v == NULL) goto onError; - } - + if (size == 0) + return (PyObject *)v; + p = PyUnicode_AS_UNICODE(v); end = s + size; while (s < end) { - unsigned char c = (unsigned char) *s++; - Py_UCS4 ch; + unsigned char c; + Py_UCS4 x; + int i; int count; - Py_ssize_t startinpos; - Py_ssize_t endinpos; - const char *message; - -#define WRITE_CHAR(ch) \ - do { \ - if (ch <= writer.maxchar) { \ - assert(writer.pos < writer.size); \ - PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, ch); \ - } \ - else if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0) { \ - goto onError; \ - } \ - } while(0) /* Non-escape characters are interpreted as Unicode ordinals */ - if (c != '\\' || s >= end) { - WRITE_CHAR(c); + if (*s != '\\') { + *p++ = (unsigned char)*s++; continue; } + startinpos = s-starts; - c = (unsigned char) *s++; - if (c == 'u') { - count = 4; - message = "truncated \\uXXXX escape"; - } - else if (c == 'U') { - count = 8; - message = "truncated \\UXXXXXXXX escape"; + /* \u-escapes are only interpreted iff the number of leading + backslashes if odd */ + bs = s; + for (;s < end;) { + if (*s != '\\') + break; + *p++ = (unsigned char)*s++; } - else { - assert(writer.pos < writer.size); - PyUnicode_WRITE(writer.kind, writer.data, writer.pos++, '\\'); - WRITE_CHAR(c); + if (((s - bs) & 1) == 0 || + s >= end || + (*s != 'u' && *s != 'U')) { continue; } - startinpos = s - starts - 2; + p--; + count = *s=='u' ? 4 : 8; + s++; - /* \uHHHH with 4 hex digits, \U00HHHHHH with 8 */ - for (ch = 0; count && s < end; ++s, --count) { + /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ + outpos = p-PyUnicode_AS_UNICODE(v); + for (x = 0, i = 0; i < count; ++i, ++s) { c = (unsigned char)*s; - ch <<= 4; - if (c >= '0' && c <= '9') { - ch += c - '0'; - } - else if (c >= 'a' && c <= 'f') { - ch += c - ('a' - 10); - } - else if (c >= 'A' && c <= 'F') { - ch += c - ('A' - 10); - } - else { - break; - } - } - if (!count) { - if (ch <= MAX_UNICODE) { - WRITE_CHAR(ch); - continue; + if (!isxdigit(c)) { + endinpos = s-starts; + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "rawunicodeescape", "truncated \\uXXXX", + starts, size, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) + goto onError; + goto nextByte; } - message = "\\Uxxxxxxxx out of range"; - } - - endinpos = s-starts; - writer.min_length = end - s + writer.pos; - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "rawunicodeescape", message, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) { - goto onError; + x = (x<<4) & ~0xF; + if (c >= '0' && c <= '9') + x += c - '0'; + else if (c >= 'a' && c <= 'f') + x += 10 + c - 'a'; + else + x += 10 + c - 'A'; + } + if (x <= 0xffff) + /* UCS-2 character */ + *p++ = (Py_UNICODE) x; + else if (x <= 0x10ffff) { + /* UCS-4 character. Either store directly, or as + surrogate pair. */ +#ifdef Py_UNICODE_WIDE + *p++ = (Py_UNICODE) x; +#else + x -= 0x10000L; + *p++ = 0xD800 + (Py_UNICODE) (x >> 10); + *p++ = 0xDC00 + (Py_UNICODE) (x & 0x03FF); +#endif + } else { + endinpos = s-starts; + outpos = p-PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "rawunicodeescape", "\\Uxxxxxxxx out of range", + starts, size, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) + goto onError; } - assert(end - s <= writer.size - writer.pos); - -#undef WRITE_CHAR + nextByte: + ; } + if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) + goto onError; Py_XDECREF(errorHandler); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return (PyObject *)v; onError: - _PyUnicodeWriter_Dealloc(&writer); + Py_XDECREF(v); Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; - } - -PyObject * -PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) +PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, + Py_ssize_t size) { PyObject *repr; char *p; - Py_ssize_t expandsize, pos; - int kind; - void *data; - Py_ssize_t len; - - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return NULL; - } - if (PyUnicode_READY(unicode) == -1) { - return NULL; - } - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - len = PyUnicode_GET_LENGTH(unicode); - if (kind == PyUnicode_1BYTE_KIND) { - return PyBytes_FromStringAndSize(data, len); - } + char *q; - /* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6 - bytes, and 1 byte characters 4. */ - expandsize = kind * 2 + 2; + static const char *hexdigit = "0123456789abcdef"; +#ifdef Py_UNICODE_WIDE + const Py_ssize_t expandsize = 10; +#else + const Py_ssize_t expandsize = 6; +#endif - if (len > PY_SSIZE_T_MAX / expandsize) { + if (size > PY_SSIZE_T_MAX / expandsize) return PyErr_NoMemory(); - } - repr = PyBytes_FromStringAndSize(NULL, expandsize * len); - if (repr == NULL) { + + repr = PyString_FromStringAndSize(NULL, expandsize * size); + if (repr == NULL) return NULL; - } - if (len == 0) { + if (size == 0) return repr; - } - - p = PyBytes_AS_STRING(repr); - for (pos = 0; pos < len; pos++) { - Py_UCS4 ch = PyUnicode_READ(kind, data, pos); - /* U+0000-U+00ff range: Copy 8-bit characters as-is */ - if (ch < 0x100) { - *p++ = (char) ch; - } - /* U+0100-U+ffff range: Map 16-bit characters to '\uHHHH' */ - else if (ch < 0x10000) { + p = q = PyString_AS_STRING(repr); + while (size-- > 0) { + Py_UNICODE ch = *s++; +#ifdef Py_UNICODE_WIDE + /* Map 32-bit characters to '\Uxxxxxxxx' */ + if (ch >= 0x10000) { *p++ = '\\'; - *p++ = 'u'; - *p++ = Py_hexdigits[(ch >> 12) & 0xf]; - *p++ = Py_hexdigits[(ch >> 8) & 0xf]; - *p++ = Py_hexdigits[(ch >> 4) & 0xf]; - *p++ = Py_hexdigits[ch & 15]; + *p++ = 'U'; + *p++ = hexdigit[(ch >> 28) & 0xf]; + *p++ = hexdigit[(ch >> 24) & 0xf]; + *p++ = hexdigit[(ch >> 20) & 0xf]; + *p++ = hexdigit[(ch >> 16) & 0xf]; + *p++ = hexdigit[(ch >> 12) & 0xf]; + *p++ = hexdigit[(ch >> 8) & 0xf]; + *p++ = hexdigit[(ch >> 4) & 0xf]; + *p++ = hexdigit[ch & 15]; } - /* U+010000-U+10ffff range: Map 32-bit characters to '\U00HHHHHH' */ - else { - assert(ch <= MAX_UNICODE && MAX_UNICODE <= 0x10ffff); + else +#else + /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */ + if (ch >= 0xD800 && ch < 0xDC00) { + Py_UNICODE ch2; + Py_UCS4 ucs; + + ch2 = *s++; + size--; + if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) { + ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000; + *p++ = '\\'; + *p++ = 'U'; + *p++ = hexdigit[(ucs >> 28) & 0xf]; + *p++ = hexdigit[(ucs >> 24) & 0xf]; + *p++ = hexdigit[(ucs >> 20) & 0xf]; + *p++ = hexdigit[(ucs >> 16) & 0xf]; + *p++ = hexdigit[(ucs >> 12) & 0xf]; + *p++ = hexdigit[(ucs >> 8) & 0xf]; + *p++ = hexdigit[(ucs >> 4) & 0xf]; + *p++ = hexdigit[ucs & 0xf]; + continue; + } + /* Fall through: isolated surrogates are copied as-is */ + s--; + size++; + } +#endif + /* Map 16-bit characters to '\uxxxx' */ + if (ch >= 256) { *p++ = '\\'; - *p++ = 'U'; - *p++ = '0'; - *p++ = '0'; - *p++ = Py_hexdigits[(ch >> 20) & 0xf]; - *p++ = Py_hexdigits[(ch >> 16) & 0xf]; - *p++ = Py_hexdigits[(ch >> 12) & 0xf]; - *p++ = Py_hexdigits[(ch >> 8) & 0xf]; - *p++ = Py_hexdigits[(ch >> 4) & 0xf]; - *p++ = Py_hexdigits[ch & 15]; + *p++ = 'u'; + *p++ = hexdigit[(ch >> 12) & 0xf]; + *p++ = hexdigit[(ch >> 8) & 0xf]; + *p++ = hexdigit[(ch >> 4) & 0xf]; + *p++ = hexdigit[ch & 15]; } + /* Copy everything else as-is */ + else + *p++ = (char) ch; } - - assert(p > PyBytes_AS_STRING(repr)); - if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) { + *p = '\0'; + if (_PyString_Resize(&repr, p - q)) return NULL; - } return repr; } -PyObject * -PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, - Py_ssize_t size) +PyObject *PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) { - PyObject *result; - PyObject *tmp = PyUnicode_FromWideChar(s, size); - if (tmp == NULL) + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); return NULL; - result = PyUnicode_AsRawUnicodeEscapeString(tmp); - Py_DECREF(tmp); - return result; + } + return PyUnicode_EncodeRawUnicodeEscape(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode)); +} + +/* --- Unicode Internal Codec ------------------------------------------- */ + +PyObject *_PyUnicode_DecodeUnicodeInternal(const char *s, + Py_ssize_t size, + const char *errors) +{ + const char *starts = s; + Py_ssize_t startinpos; + Py_ssize_t endinpos; + Py_ssize_t outpos; + PyUnicodeObject *v; + Py_UNICODE *p; + const char *end; + const char *reason; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + +#ifdef Py_UNICODE_WIDE + Py_UNICODE unimax = PyUnicode_GetMax(); +#endif + + /* XXX overflow detection missing */ + v = _PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE); + if (v == NULL) + goto onError; + if (PyUnicode_GetSize((PyObject *)v) == 0) + return (PyObject *)v; + p = PyUnicode_AS_UNICODE(v); + end = s + size; + + while (s < end) { + if (end-s < Py_UNICODE_SIZE) { + endinpos = end-starts; + reason = "truncated input"; + goto error; + } + memcpy(p, s, sizeof(Py_UNICODE)); +#ifdef Py_UNICODE_WIDE + /* We have to sanity check the raw data, otherwise doom looms for + some malformed UCS-4 data. */ + if (*p > unimax || *p < 0) { + endinpos = s - starts + Py_UNICODE_SIZE; + reason = "illegal code point (> 0x10FFFF)"; + goto error; + } +#endif + p++; + s += Py_UNICODE_SIZE; + continue; + + error: + startinpos = s - starts; + outpos = p - PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "unicode_internal", reason, + starts, size, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) { + goto onError; + } + } + + if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) + goto onError; + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return (PyObject *)v; + + onError: + Py_XDECREF(v); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + return NULL; } /* --- Latin-1 Codec ------------------------------------------------------ */ -PyObject * -PyUnicode_DecodeLatin1(const char *s, - Py_ssize_t size, - const char *errors) +PyObject *PyUnicode_DecodeLatin1(const char *s, + Py_ssize_t size, + const char *errors) { + PyUnicodeObject *v; + Py_UNICODE *p; + /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ - return _PyUnicode_FromUCS1((unsigned char*)s, size); + if (size == 1) { + Py_UNICODE r = *(unsigned char*)s; + return PyUnicode_FromUnicode(&r, 1); + } + + v = _PyUnicode_New(size); + if (v == NULL) + goto onError; + if (size == 0) + return (PyObject *)v; + p = PyUnicode_AS_UNICODE(v); + while (size-- > 0) + *p++ = (unsigned char)*s++; + return (PyObject *)v; + + onError: + Py_XDECREF(v); + return NULL; } /* create or adjust a UnicodeEncodeError */ -static void -make_encode_exception(PyObject **exceptionObject, - const char *encoding, - PyObject *unicode, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason) +static void make_encode_exception(PyObject **exceptionObject, + const char *encoding, + const Py_UNICODE *unicode, Py_ssize_t size, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason) { if (*exceptionObject == NULL) { - *exceptionObject = PyObject_CallFunction( - PyExc_UnicodeEncodeError, "sOnns", - encoding, unicode, startpos, endpos, reason); + *exceptionObject = PyUnicodeEncodeError_Create( + encoding, unicode, size, startpos, endpos, reason); } else { if (PyUnicodeEncodeError_SetStart(*exceptionObject, startpos)) @@ -6761,15 +3564,14 @@ make_encode_exception(PyObject **exceptionObject, } /* raises a UnicodeEncodeError */ -static void -raise_encode_exception(PyObject **exceptionObject, - const char *encoding, - PyObject *unicode, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason) +static void raise_encode_exception(PyObject **exceptionObject, + const char *encoding, + const Py_UNICODE *unicode, Py_ssize_t size, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason) { make_encode_exception(exceptionObject, - encoding, unicode, startpos, endpos, reason); + encoding, unicode, size, startpos, endpos, reason); if (*exceptionObject != NULL) PyCodec_StrictErrors(*exceptionObject); } @@ -6778,16 +3580,15 @@ raise_encode_exception(PyObject **exceptionObject, build arguments, call the callback and check the arguments, put the result into newpos and return the replacement string, which has to be freed by the caller */ -static PyObject * -unicode_encode_call_errorhandler(const char *errors, - PyObject **errorHandler, - const char *encoding, const char *reason, - PyObject *unicode, PyObject **exceptionObject, - Py_ssize_t startpos, Py_ssize_t endpos, - Py_ssize_t *newpos) -{ - static const char *argparse = "On;encoding error handler must return (str/bytes, int) tuple"; - Py_ssize_t len; +static PyObject *unicode_encode_call_errorhandler(const char *errors, + PyObject **errorHandler, + const char *encoding, const char *reason, + const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, + Py_ssize_t startpos, Py_ssize_t endpos, + Py_ssize_t *newpos) +{ + static char *argparse = "O!n;encoding error handler must return (unicode, int) tuple"; + PyObject *restuple; PyObject *resunicode; @@ -6797,36 +3598,28 @@ unicode_encode_call_errorhandler(const char *errors, return NULL; } - if (PyUnicode_READY(unicode) == -1) - return NULL; - len = PyUnicode_GET_LENGTH(unicode); - make_encode_exception(exceptionObject, - encoding, unicode, startpos, endpos, reason); + encoding, unicode, size, startpos, endpos, reason); if (*exceptionObject == NULL) return NULL; - restuple = _PyObject_CallOneArg(*errorHandler, *exceptionObject); + restuple = PyObject_CallFunctionObjArgs( + *errorHandler, *exceptionObject, NULL); if (restuple == NULL) return NULL; if (!PyTuple_Check(restuple)) { - PyErr_SetString(PyExc_TypeError, &argparse[3]); + PyErr_SetString(PyExc_TypeError, &argparse[4]); Py_DECREF(restuple); return NULL; } - if (!PyArg_ParseTuple(restuple, argparse, + if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &resunicode, newpos)) { Py_DECREF(restuple); return NULL; } - if (!PyUnicode_Check(resunicode) && !PyBytes_Check(resunicode)) { - PyErr_SetString(PyExc_TypeError, &argparse[3]); - Py_DECREF(restuple); - return NULL; - } if (*newpos<0) - *newpos = len + *newpos; - if (*newpos<0 || *newpos>len) { + *newpos = size+*newpos; + if (*newpos<0 || *newpos>size) { PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos); Py_DECREF(restuple); return NULL; @@ -6836,356 +3629,292 @@ unicode_encode_call_errorhandler(const char *errors, return resunicode; } -static PyObject * -unicode_encode_ucs1(PyObject *unicode, - const char *errors, - const Py_UCS4 limit) -{ - /* input state */ - Py_ssize_t pos=0, size; - int kind; - void *data; +static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, + Py_ssize_t size, + const char *errors, + int limit) +{ + /* output object */ + PyObject *res; + /* pointers to the beginning and end+1 of input */ + const Py_UNICODE *startp = p; + const Py_UNICODE *endp = p + size; + /* pointer to the beginning of the unencodable characters */ + /* const Py_UNICODE *badp = NULL; */ /* pointer into the output */ char *str; + /* current output position */ + Py_ssize_t respos = 0; + Py_ssize_t ressize; const char *encoding = (limit == 256) ? "latin-1" : "ascii"; const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)"; - PyObject *error_handler_obj = NULL; + PyObject *errorHandler = NULL; PyObject *exc = NULL; - _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; - PyObject *rep = NULL; - /* output object */ - _PyBytesWriter writer; + /* the following variable is used for caching string comparisons + * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */ + int known_errorHandler = -1; - if (PyUnicode_READY(unicode) == -1) - return NULL; - size = PyUnicode_GET_LENGTH(unicode); - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); /* allocate enough for a simple encoding without replacements, if we need more, we'll resize */ + res = PyString_FromStringAndSize(NULL, size); + if (res == NULL) + goto onError; if (size == 0) - return PyBytes_FromStringAndSize(NULL, 0); - - _PyBytesWriter_Init(&writer); - str = _PyBytesWriter_Alloc(&writer, size); - if (str == NULL) - return NULL; + return res; + str = PyString_AS_STRING(res); + ressize = size; - while (pos < size) { - Py_UCS4 ch = PyUnicode_READ(kind, data, pos); + while (p<endp) { + Py_UNICODE c = *p; /* can we encode this? */ - if (ch < limit) { + if (c<limit) { /* no overflow check, because we know that the space is enough */ - *str++ = (char)ch; - ++pos; + *str++ = (char)c; + ++p; } else { - Py_ssize_t newpos, i; + Py_ssize_t unicodepos = p-startp; + Py_ssize_t requiredsize; + PyObject *repunicode; + Py_ssize_t repsize; + Py_ssize_t newpos; + Py_ssize_t respos; + Py_UNICODE *uni2; /* startpos for collecting unencodable chars */ - Py_ssize_t collstart = pos; - Py_ssize_t collend = collstart + 1; + const Py_UNICODE *collstart = p; + const Py_UNICODE *collend = p; /* find all unecodable characters */ - - while ((collend < size) && (PyUnicode_READ(kind, data, collend) >= limit)) + while ((collend < endp) && ((*collend) >= limit)) ++collend; - - /* Only overallocate the buffer if it's not the last write */ - writer.overallocate = (collend < size); - /* cache callback name lookup (if not done yet, i.e. it's the first error) */ - if (error_handler == _Py_ERROR_UNKNOWN) - error_handler = _Py_GetErrorHandler(errors); - - switch (error_handler) { - case _Py_ERROR_STRICT: - raise_encode_exception(&exc, encoding, unicode, collstart, collend, reason); + if (known_errorHandler==-1) { + if ((errors==NULL) || (!strcmp(errors, "strict"))) + known_errorHandler = 1; + else if (!strcmp(errors, "replace")) + known_errorHandler = 2; + else if (!strcmp(errors, "ignore")) + known_errorHandler = 3; + else if (!strcmp(errors, "xmlcharrefreplace")) + known_errorHandler = 4; + else + known_errorHandler = 0; + } + switch (known_errorHandler) { + case 1: /* strict */ + raise_encode_exception(&exc, encoding, startp, size, collstart-startp, collend-startp, reason); goto onError; - - case _Py_ERROR_REPLACE: - memset(str, '?', collend - collstart); - str += (collend - collstart); - /* fall through */ - case _Py_ERROR_IGNORE: - pos = collend; - break; - - case _Py_ERROR_BACKSLASHREPLACE: - /* subtract preallocated bytes */ - writer.min_size -= (collend - collstart); - str = backslashreplace(&writer, str, - unicode, collstart, collend); - if (str == NULL) - goto onError; - pos = collend; - break; - - case _Py_ERROR_XMLCHARREFREPLACE: - /* subtract preallocated bytes */ - writer.min_size -= (collend - collstart); - str = xmlcharrefreplace(&writer, str, - unicode, collstart, collend); - if (str == NULL) - goto onError; - pos = collend; + case 2: /* replace */ + while (collstart++ < collend) + *str++ = '?'; /* fall through */ + case 3: /* ignore */ + p = collend; break; - - case _Py_ERROR_SURROGATEESCAPE: - for (i = collstart; i < collend; ++i) { - ch = PyUnicode_READ(kind, data, i); - if (ch < 0xdc80 || 0xdcff < ch) { - /* Not a UTF-8b surrogate */ - break; - } - *str++ = (char)(ch - 0xdc00); - ++pos; + case 4: /* xmlcharrefreplace */ + respos = str - PyString_AS_STRING(res); + /* determine replacement size (temporarily (mis)uses p) */ + requiredsize = respos; + for (p = collstart; p < collend;) { + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + Py_ssize_t incr; + if (ch < 10) + incr = 2+1+1; + else if (ch < 100) + incr = 2+2+1; + else if (ch < 1000) + incr = 2+3+1; + else if (ch < 10000) + incr = 2+4+1; + else if (ch < 100000) + incr = 2+5+1; + else if (ch < 1000000) + incr = 2+6+1; + else + incr = 2+7+1; + if (requiredsize > PY_SSIZE_T_MAX - incr) + goto overflow; + requiredsize += incr; } - if (i >= collend) - break; - collstart = pos; - assert(collstart != collend); - /* fall through */ - + if (requiredsize > PY_SSIZE_T_MAX - (endp - collend)) + goto overflow; + requiredsize += endp - collend; + if (requiredsize > ressize) { + if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize) + requiredsize = 2*ressize; + if (_PyString_Resize(&res, requiredsize)) + goto onError; + str = PyString_AS_STRING(res) + respos; + ressize = requiredsize; + } + /* generate replacement (temporarily (mis)uses p) */ + for (p = collstart; p < collend;) { + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + str += sprintf(str, "&#%d;", (int)ch); + } + p = collend; + break; default: - rep = unicode_encode_call_errorhandler(errors, &error_handler_obj, - encoding, reason, unicode, &exc, - collstart, collend, &newpos); - if (rep == NULL) + repunicode = unicode_encode_call_errorhandler(errors, &errorHandler, + encoding, reason, startp, size, &exc, + collstart-startp, collend-startp, &newpos); + if (repunicode == NULL) goto onError; - - /* subtract preallocated bytes */ - writer.min_size -= newpos - collstart; - - if (PyBytes_Check(rep)) { - /* Directly copy bytes result to output. */ - str = _PyBytesWriter_WriteBytes(&writer, str, - PyBytes_AS_STRING(rep), - PyBytes_GET_SIZE(rep)); - } - else { - assert(PyUnicode_Check(rep)); - - if (PyUnicode_READY(rep) < 0) + /* need more space? (at least enough for what we have+the + replacement+the rest of the string, so we won't have to + check space for encodable characters) */ + respos = str - PyString_AS_STRING(res); + repsize = PyUnicode_GET_SIZE(repunicode); + if (respos > PY_SSIZE_T_MAX - repsize) + goto overflow; + requiredsize = respos + repsize; + if (requiredsize > PY_SSIZE_T_MAX - (endp - collend)) + goto overflow; + requiredsize += endp - collend; + if (requiredsize > ressize) { + if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize) + requiredsize = 2*ressize; + if (_PyString_Resize(&res, requiredsize)) { + Py_DECREF(repunicode); goto onError; - - if (limit == 256 ? - PyUnicode_KIND(rep) != PyUnicode_1BYTE_KIND : - !PyUnicode_IS_ASCII(rep)) - { - /* Not all characters are smaller than limit */ - raise_encode_exception(&exc, encoding, unicode, - collstart, collend, reason); + } + str = PyString_AS_STRING(res) + respos; + ressize = requiredsize; + } + /* check if there is anything unencodable in the replacement + and copy it to the output */ + for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2, ++str) { + c = *uni2; + if (c >= limit) { + raise_encode_exception(&exc, encoding, startp, size, + unicodepos, unicodepos+1, reason); + Py_DECREF(repunicode); goto onError; } - assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); - str = _PyBytesWriter_WriteBytes(&writer, str, - PyUnicode_DATA(rep), - PyUnicode_GET_LENGTH(rep)); + *str = (char)c; } - if (str == NULL) - goto onError; - - pos = newpos; - Py_CLEAR(rep); + p = startp + newpos; + Py_DECREF(repunicode); } - - /* If overallocation was disabled, ensure that it was the last - write. Otherwise, we missed an optimization */ - assert(writer.overallocate || pos == size); } } - - Py_XDECREF(error_handler_obj); + /* Resize if we allocated to much */ + respos = str - PyString_AS_STRING(res); + if (respos < ressize) + /* If this falls res will be NULL */ + _PyString_Resize(&res, respos); + Py_XDECREF(errorHandler); Py_XDECREF(exc); - return _PyBytesWriter_Finish(&writer, str); + return res; + + overflow: + PyErr_SetString(PyExc_OverflowError, + "encoded result is too long for a Python string"); onError: - Py_XDECREF(rep); - _PyBytesWriter_Dealloc(&writer); - Py_XDECREF(error_handler_obj); + Py_XDECREF(res); + Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; } -/* Deprecated */ -PyObject * -PyUnicode_EncodeLatin1(const Py_UNICODE *p, - Py_ssize_t size, - const char *errors) +PyObject *PyUnicode_EncodeLatin1(const Py_UNICODE *p, + Py_ssize_t size, + const char *errors) { - PyObject *result; - PyObject *unicode = PyUnicode_FromWideChar(p, size); - if (unicode == NULL) - return NULL; - result = unicode_encode_ucs1(unicode, errors, 256); - Py_DECREF(unicode); - return result; + return unicode_encode_ucs1(p, size, errors, 256); } -PyObject * -_PyUnicode_AsLatin1String(PyObject *unicode, const char *errors) +PyObject *PyUnicode_AsLatin1String(PyObject *unicode) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - if (PyUnicode_READY(unicode) == -1) - return NULL; - /* Fast path: if it is a one-byte string, construct - bytes object directly. */ - if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND) - return PyBytes_FromStringAndSize(PyUnicode_DATA(unicode), - PyUnicode_GET_LENGTH(unicode)); - /* Non-Latin-1 characters present. Defer to above function to - raise the exception. */ - return unicode_encode_ucs1(unicode, errors, 256); -} - -PyObject* -PyUnicode_AsLatin1String(PyObject *unicode) -{ - return _PyUnicode_AsLatin1String(unicode, NULL); + return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode), + NULL); } /* --- 7-bit ASCII Codec -------------------------------------------------- */ -PyObject * -PyUnicode_DecodeASCII(const char *s, - Py_ssize_t size, - const char *errors) +PyObject *PyUnicode_DecodeASCII(const char *s, + Py_ssize_t size, + const char *errors) { const char *starts = s; - const char *e = s + size; - PyObject *error_handler_obj = NULL; + PyUnicodeObject *v; + Py_UNICODE *p; + Py_ssize_t startinpos; + Py_ssize_t endinpos; + Py_ssize_t outpos; + const char *e; + PyObject *errorHandler = NULL; PyObject *exc = NULL; - _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; - - if (size == 0) - _Py_RETURN_UNICODE_EMPTY(); /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 && (unsigned char)s[0] < 128) - return get_latin1_char((unsigned char)s[0]); - - // Shortcut for simple case - PyObject *u = PyUnicode_New(size, 127); - if (u == NULL) { - return NULL; + if (size == 1 && *(unsigned char*)s < 128) { + Py_UNICODE r = *(unsigned char*)s; + return PyUnicode_FromUnicode(&r, 1); } - Py_ssize_t outpos = ascii_decode(s, e, PyUnicode_DATA(u)); - if (outpos == size) { - return u; - } - - _PyUnicodeWriter writer; - _PyUnicodeWriter_InitWithBuffer(&writer, u); - writer.pos = outpos; - - s += outpos; - int kind = writer.kind; - void *data = writer.data; - Py_ssize_t startinpos, endinpos; + v = _PyUnicode_New(size); + if (v == NULL) + goto onError; + if (size == 0) + return (PyObject *)v; + p = PyUnicode_AS_UNICODE(v); + e = s + size; while (s < e) { - unsigned char c = (unsigned char)*s; + register unsigned char c = (unsigned char)*s; if (c < 128) { - PyUnicode_WRITE(kind, data, writer.pos, c); - writer.pos++; + *p++ = c; ++s; - continue; } - - /* byte outsize range 0x00..0x7f: call the error handler */ - - if (error_handler == _Py_ERROR_UNKNOWN) - error_handler = _Py_GetErrorHandler(errors); - - switch (error_handler) - { - case _Py_ERROR_REPLACE: - case _Py_ERROR_SURROGATEESCAPE: - /* Fast-path: the error handler only writes one character, - but we may switch to UCS2 at the first write */ - if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) - goto onError; - kind = writer.kind; - data = writer.data; - - if (error_handler == _Py_ERROR_REPLACE) - PyUnicode_WRITE(kind, data, writer.pos, 0xfffd); - else - PyUnicode_WRITE(kind, data, writer.pos, c + 0xdc00); - writer.pos++; - ++s; - break; - - case _Py_ERROR_IGNORE: - ++s; - break; - - default: + else { startinpos = s-starts; endinpos = startinpos + 1; - if (unicode_decode_call_errorhandler_writer( - errors, &error_handler_obj, + outpos = p - (Py_UNICODE *)PyUnicode_AS_UNICODE(v); + if (unicode_decode_call_errorhandler( + errors, &errorHandler, "ascii", "ordinal not in range(128)", - &starts, &e, &startinpos, &endinpos, &exc, &s, - &writer)) + starts, size, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) goto onError; - kind = writer.kind; - data = writer.data; } } - Py_XDECREF(error_handler_obj); + if (p - PyUnicode_AS_UNICODE(v) < PyString_GET_SIZE(v)) + if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) + goto onError; + Py_XDECREF(errorHandler); Py_XDECREF(exc); - return _PyUnicodeWriter_Finish(&writer); + return (PyObject *)v; onError: - _PyUnicodeWriter_Dealloc(&writer); - Py_XDECREF(error_handler_obj); + Py_XDECREF(v); + Py_XDECREF(errorHandler); Py_XDECREF(exc); return NULL; } -/* Deprecated */ -PyObject * -PyUnicode_EncodeASCII(const Py_UNICODE *p, - Py_ssize_t size, - const char *errors) +PyObject *PyUnicode_EncodeASCII(const Py_UNICODE *p, + Py_ssize_t size, + const char *errors) { - PyObject *result; - PyObject *unicode = PyUnicode_FromWideChar(p, size); - if (unicode == NULL) - return NULL; - result = unicode_encode_ucs1(unicode, errors, 128); - Py_DECREF(unicode); - return result; + return unicode_encode_ucs1(p, size, errors, 128); } -PyObject * -_PyUnicode_AsASCIIString(PyObject *unicode, const char *errors) +PyObject *PyUnicode_AsASCIIString(PyObject *unicode) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); return NULL; } - if (PyUnicode_READY(unicode) == -1) - return NULL; - /* Fast path: if it is an ASCII-only string, construct bytes object - directly. Else defer to above function to raise the exception. */ - if (PyUnicode_IS_ASCII(unicode)) - return PyBytes_FromStringAndSize(PyUnicode_DATA(unicode), - PyUnicode_GET_LENGTH(unicode)); - return unicode_encode_ucs1(unicode, errors, 128); -} - -PyObject * -PyUnicode_AsASCIIString(PyObject *unicode) -{ - return _PyUnicode_AsASCIIString(unicode, NULL); + return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode), + NULL); } -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) /* --- MBCS codecs for Windows -------------------------------------------- */ @@ -7193,709 +3922,207 @@ PyUnicode_AsASCIIString(PyObject *unicode) #define NEED_RETRY #endif -/* INT_MAX is the theoretical largest chunk (or INT_MAX / 2 when - transcoding from UTF-16), but INT_MAX / 4 perfoms better in - both cases also and avoids partial characters overrunning the - length limit in MultiByteToWideChar on Windows */ -#define DECODING_CHUNK_SIZE (INT_MAX/4) - -#ifndef WC_ERR_INVALID_CHARS -# define WC_ERR_INVALID_CHARS 0x0080 -#endif +/* XXX This code is limited to "true" double-byte encodings, as + a) it assumes an incomplete character consists of a single byte, and + b) IsDBCSLeadByte (probably) does not work for non-DBCS multi-byte + encodings, see IsDBCSLeadByteEx documentation. */ -static const char* -code_page_name(UINT code_page, PyObject **obj) +static int is_dbcs_lead_byte(const char *s, int offset) { - *obj = NULL; - if (code_page == CP_ACP) - return "mbcs"; - if (code_page == CP_UTF7) - return "CP_UTF7"; - if (code_page == CP_UTF8) - return "CP_UTF8"; + const char *curr = s + offset; - *obj = PyBytes_FromFormat("cp%u", code_page); - if (*obj == NULL) - return NULL; - return PyBytes_AS_STRING(*obj); -} - -static DWORD -decode_code_page_flags(UINT code_page) -{ - if (code_page == CP_UTF7) { - /* The CP_UTF7 decoder only supports flags=0 */ - return 0; + if (IsDBCSLeadByte(*curr)) { + const char *prev = CharPrev(s, curr); + return (prev == curr) || !IsDBCSLeadByte(*prev) || (curr - prev == 2); } - else - return MB_ERR_INVALID_CHARS; + return 0; } /* - * Decode a byte string from a Windows code page into unicode object in strict - * mode. - * - * Returns consumed size if succeed, returns -2 on decode error, or raise an - * OSError and returns -1 on other error. + * Decode MBCS string into unicode object. If 'final' is set, converts + * trailing lead-byte too. Returns consumed size if succeed, -1 otherwise. */ -static int -decode_code_page_strict(UINT code_page, - wchar_t **buf, - Py_ssize_t *bufsize, - const char *in, - int insize) +static int decode_mbcs(PyUnicodeObject **v, + const char *s, /* MBCS string */ + int size, /* sizeof MBCS string */ + int final) { - DWORD flags = MB_ERR_INVALID_CHARS; - wchar_t *out; - DWORD outsize; - - /* First get the size of the result */ - assert(insize > 0); - while ((outsize = MultiByteToWideChar(code_page, flags, - in, insize, NULL, 0)) <= 0) - { - if (!flags || GetLastError() != ERROR_INVALID_FLAGS) { - goto error; - } - /* For some code pages (e.g. UTF-7) flags must be set to 0. */ - flags = 0; - } - - /* Extend a wchar_t* buffer */ - Py_ssize_t n = *bufsize; /* Get the current length */ - if (widechar_resize(buf, bufsize, n + outsize) < 0) { - return -1; - } - out = *buf + n; - - /* Do the conversion */ - outsize = MultiByteToWideChar(code_page, flags, in, insize, out, outsize); - if (outsize <= 0) - goto error; - return insize; - -error: - if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION) - return -2; - PyErr_SetFromWindowsErr(0); - return -1; -} - -/* - * Decode a byte string from a code page into unicode object with an error - * handler. - * - * Returns consumed size if succeed, or raise an OSError or - * UnicodeDecodeError exception and returns -1 on error. - */ -static int -decode_code_page_errors(UINT code_page, - wchar_t **buf, - Py_ssize_t *bufsize, - const char *in, const int size, - const char *errors, int final) -{ - const char *startin = in; - const char *endin = in + size; - DWORD flags = MB_ERR_INVALID_CHARS; - /* Ideally, we should get reason from FormatMessage. This is the Windows - 2000 English version of the message. */ - const char *reason = "No mapping for the Unicode character exists " - "in the target code page."; - /* each step cannot decode more than 1 character, but a character can be - represented as a surrogate pair */ - wchar_t buffer[2], *out; - int insize; - Py_ssize_t outsize; - PyObject *errorHandler = NULL; - PyObject *exc = NULL; - PyObject *encoding_obj = NULL; - const char *encoding; - DWORD err; - int ret = -1; + Py_UNICODE *p; + Py_ssize_t n = 0; + int usize = 0; - assert(size > 0); + assert(size >= 0); - encoding = code_page_name(code_page, &encoding_obj); - if (encoding == NULL) - return -1; + /* Skip trailing lead-byte unless 'final' is set */ + if (!final && size >= 1 && is_dbcs_lead_byte(s, size - 1)) + --size; - if ((errors == NULL || strcmp(errors, "strict") == 0) && final) { - /* The last error was ERROR_NO_UNICODE_TRANSLATION, then we raise a - UnicodeDecodeError. */ - make_decode_exception(&exc, encoding, in, size, 0, 0, reason); - if (exc != NULL) { - PyCodec_StrictErrors(exc); - Py_CLEAR(exc); + /* First get the size of the result */ + if (size > 0) { + usize = MultiByteToWideChar(CP_ACP, 0, s, size, NULL, 0); + if (usize == 0) { + PyErr_SetFromWindowsErrWithFilename(0, NULL); + return -1; } - goto error; } - /* Extend a wchar_t* buffer */ - Py_ssize_t n = *bufsize; /* Get the current length */ - if (size > (PY_SSIZE_T_MAX - n) / (Py_ssize_t)Py_ARRAY_LENGTH(buffer)) { - PyErr_NoMemory(); - goto error; + if (*v == NULL) { + /* Create unicode object */ + *v = _PyUnicode_New(usize); + if (*v == NULL) + return -1; } - if (widechar_resize(buf, bufsize, n + size * Py_ARRAY_LENGTH(buffer)) < 0) { - goto error; + else { + /* Extend unicode object */ + n = PyUnicode_GET_SIZE(*v); + if (_PyUnicode_Resize(v, n + usize) < 0) + return -1; } - out = *buf + n; - - /* Decode the byte string character per character */ - while (in < endin) - { - /* Decode a character */ - insize = 1; - do - { - outsize = MultiByteToWideChar(code_page, flags, - in, insize, - buffer, Py_ARRAY_LENGTH(buffer)); - if (outsize > 0) - break; - err = GetLastError(); - if (err == ERROR_INVALID_FLAGS && flags) { - /* For some code pages (e.g. UTF-7) flags must be set to 0. */ - flags = 0; - continue; - } - if (err != ERROR_NO_UNICODE_TRANSLATION - && err != ERROR_INSUFFICIENT_BUFFER) - { - PyErr_SetFromWindowsErr(0); - goto error; - } - insize++; - } - /* 4=maximum length of a UTF-8 sequence */ - while (insize <= 4 && (in + insize) <= endin); - if (outsize <= 0) { - Py_ssize_t startinpos, endinpos, outpos; - - /* last character in partial decode? */ - if (in + insize >= endin && !final) - break; - - startinpos = in - startin; - endinpos = startinpos + 1; - outpos = out - *buf; - if (unicode_decode_call_errorhandler_wchar( - errors, &errorHandler, - encoding, reason, - &startin, &endin, &startinpos, &endinpos, &exc, &in, - buf, bufsize, &outpos)) - { - goto error; - } - out = *buf + outpos; - } - else { - in += insize; - memcpy(out, buffer, outsize * sizeof(wchar_t)); - out += outsize; + /* Do the conversion */ + if (size > 0) { + p = PyUnicode_AS_UNICODE(*v) + n; + if (0 == MultiByteToWideChar(CP_ACP, 0, s, size, p, usize)) { + PyErr_SetFromWindowsErrWithFilename(0, NULL); + return -1; } } - /* Shrink the buffer */ - assert(out - *buf <= *bufsize); - *bufsize = out - *buf; - /* (in - startin) <= size and size is an int */ - ret = Py_SAFE_DOWNCAST(in - startin, Py_ssize_t, int); - -error: - Py_XDECREF(encoding_obj); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - return ret; + return size; } -static PyObject * -decode_code_page_stateful(int code_page, - const char *s, Py_ssize_t size, - const char *errors, Py_ssize_t *consumed) +PyObject *PyUnicode_DecodeMBCSStateful(const char *s, + Py_ssize_t size, + const char *errors, + Py_ssize_t *consumed) { - wchar_t *buf = NULL; - Py_ssize_t bufsize = 0; - int chunk_size, final, converted, done; - - if (code_page < 0) { - PyErr_SetString(PyExc_ValueError, "invalid code page number"); - return NULL; - } - if (size < 0) { - PyErr_BadInternalCall(); - return NULL; - } + PyUnicodeObject *v = NULL; + int done; if (consumed) *consumed = 0; - do - { #ifdef NEED_RETRY - if (size > DECODING_CHUNK_SIZE) { - chunk_size = DECODING_CHUNK_SIZE; - final = 0; - done = 0; - } - else + retry: + if (size > INT_MAX) + done = decode_mbcs(&v, s, INT_MAX, 0); + else #endif - { - chunk_size = (int)size; - final = (consumed == NULL); - done = 1; - } - - if (chunk_size == 0 && done) { - if (buf != NULL) - break; - _Py_RETURN_UNICODE_EMPTY(); - } - - converted = decode_code_page_strict(code_page, &buf, &bufsize, - s, chunk_size); - if (converted == -2) - converted = decode_code_page_errors(code_page, &buf, &bufsize, - s, chunk_size, - errors, final); - assert(converted != 0 || done); - - if (converted < 0) { - PyMem_Free(buf); - return NULL; - } + done = decode_mbcs(&v, s, (int)size, !consumed); - if (consumed) - *consumed += converted; - - s += converted; - size -= converted; - } while (!done); + if (done < 0) { + Py_XDECREF(v); + return NULL; + } - PyObject *v = PyUnicode_FromWideChar(buf, bufsize); - PyMem_Free(buf); - return v; -} + if (consumed) + *consumed += done; -PyObject * -PyUnicode_DecodeCodePageStateful(int code_page, - const char *s, - Py_ssize_t size, - const char *errors, - Py_ssize_t *consumed) -{ - return decode_code_page_stateful(code_page, s, size, errors, consumed); -} +#ifdef NEED_RETRY + if (size > INT_MAX) { + s += done; + size -= done; + goto retry; + } +#endif -PyObject * -PyUnicode_DecodeMBCSStateful(const char *s, - Py_ssize_t size, - const char *errors, - Py_ssize_t *consumed) -{ - return decode_code_page_stateful(CP_ACP, s, size, errors, consumed); + return (PyObject *)v; } -PyObject * -PyUnicode_DecodeMBCS(const char *s, - Py_ssize_t size, - const char *errors) +PyObject *PyUnicode_DecodeMBCS(const char *s, + Py_ssize_t size, + const char *errors) { return PyUnicode_DecodeMBCSStateful(s, size, errors, NULL); } -static DWORD -encode_code_page_flags(UINT code_page, const char *errors) -{ - if (code_page == CP_UTF8) { - return WC_ERR_INVALID_CHARS; - } - else if (code_page == CP_UTF7) { - /* CP_UTF7 only supports flags=0 */ - return 0; - } - else { - if (errors != NULL && strcmp(errors, "replace") == 0) - return 0; - else - return WC_NO_BEST_FIT_CHARS; - } -} - /* - * Encode a Unicode string to a Windows code page into a byte string in strict - * mode. - * - * Returns consumed characters if succeed, returns -2 on encode error, or raise - * an OSError and returns -1 on other error. + * Convert unicode into string object (MBCS). + * Returns 0 if succeed, -1 otherwise. */ -static int -encode_code_page_strict(UINT code_page, PyObject **outbytes, - PyObject *unicode, Py_ssize_t offset, int len, - const char* errors) -{ - BOOL usedDefaultChar = FALSE; - BOOL *pusedDefaultChar = &usedDefaultChar; - int outsize; - wchar_t *p; - Py_ssize_t size; - const DWORD flags = encode_code_page_flags(code_page, NULL); - char *out; - /* Create a substring so that we can get the UTF-16 representation - of just the slice under consideration. */ - PyObject *substring; - - assert(len > 0); - - if (code_page != CP_UTF8 && code_page != CP_UTF7) - pusedDefaultChar = &usedDefaultChar; - else - pusedDefaultChar = NULL; +static int encode_mbcs(PyObject **repr, + const Py_UNICODE *p, /* unicode */ + int size) /* size of unicode */ +{ + int mbcssize = 0; + Py_ssize_t n = 0; - substring = PyUnicode_Substring(unicode, offset, offset+len); - if (substring == NULL) - return -1; - p = PyUnicode_AsUnicodeAndSize(substring, &size); - if (p == NULL) { - Py_DECREF(substring); - return -1; - } - assert(size <= INT_MAX); + assert(size >= 0); /* First get the size of the result */ - outsize = WideCharToMultiByte(code_page, flags, - p, (int)size, - NULL, 0, - NULL, pusedDefaultChar); - if (outsize <= 0) - goto error; - /* If we used a default char, then we failed! */ - if (pusedDefaultChar && *pusedDefaultChar) { - Py_DECREF(substring); - return -2; + if (size > 0) { + mbcssize = WideCharToMultiByte(CP_ACP, 0, p, size, NULL, 0, NULL, NULL); + if (mbcssize == 0) { + PyErr_SetFromWindowsErrWithFilename(0, NULL); + return -1; + } } - if (*outbytes == NULL) { + if (*repr == NULL) { /* Create string object */ - *outbytes = PyBytes_FromStringAndSize(NULL, outsize); - if (*outbytes == NULL) { - Py_DECREF(substring); + *repr = PyString_FromStringAndSize(NULL, mbcssize); + if (*repr == NULL) return -1; - } - out = PyBytes_AS_STRING(*outbytes); } else { /* Extend string object */ - const Py_ssize_t n = PyBytes_Size(*outbytes); - if (outsize > PY_SSIZE_T_MAX - n) { - PyErr_NoMemory(); - Py_DECREF(substring); + n = PyString_Size(*repr); + if (_PyString_Resize(repr, n + mbcssize) < 0) return -1; - } - if (_PyBytes_Resize(outbytes, n + outsize) < 0) { - Py_DECREF(substring); - return -1; - } - out = PyBytes_AS_STRING(*outbytes) + n; } /* Do the conversion */ - outsize = WideCharToMultiByte(code_page, flags, - p, (int)size, - out, outsize, - NULL, pusedDefaultChar); - Py_CLEAR(substring); - if (outsize <= 0) - goto error; - if (pusedDefaultChar && *pusedDefaultChar) - return -2; - return 0; - -error: - Py_XDECREF(substring); - if (GetLastError() == ERROR_NO_UNICODE_TRANSLATION) - return -2; - PyErr_SetFromWindowsErr(0); - return -1; -} - -/* - * Encode a Unicode string to a Windows code page into a byte string using an - * error handler. - * - * Returns consumed characters if succeed, or raise an OSError and returns - * -1 on other error. - */ -static int -encode_code_page_errors(UINT code_page, PyObject **outbytes, - PyObject *unicode, Py_ssize_t unicode_offset, - Py_ssize_t insize, const char* errors) -{ - const DWORD flags = encode_code_page_flags(code_page, errors); - Py_ssize_t pos = unicode_offset; - Py_ssize_t endin = unicode_offset + insize; - /* Ideally, we should get reason from FormatMessage. This is the Windows - 2000 English version of the message. */ - const char *reason = "invalid character"; - /* 4=maximum length of a UTF-8 sequence */ - char buffer[4]; - BOOL usedDefaultChar = FALSE, *pusedDefaultChar; - Py_ssize_t outsize; - char *out; - PyObject *errorHandler = NULL; - PyObject *exc = NULL; - PyObject *encoding_obj = NULL; - const char *encoding; - Py_ssize_t newpos, newoutsize; - PyObject *rep; - int ret = -1; - - assert(insize > 0); - - encoding = code_page_name(code_page, &encoding_obj); - if (encoding == NULL) - return -1; - - if (errors == NULL || strcmp(errors, "strict") == 0) { - /* The last error was ERROR_NO_UNICODE_TRANSLATION, - then we raise a UnicodeEncodeError. */ - make_encode_exception(&exc, encoding, unicode, 0, 0, reason); - if (exc != NULL) { - PyCodec_StrictErrors(exc); - Py_DECREF(exc); - } - Py_XDECREF(encoding_obj); - return -1; - } - - if (code_page != CP_UTF8 && code_page != CP_UTF7) - pusedDefaultChar = &usedDefaultChar; - else - pusedDefaultChar = NULL; - - if (Py_ARRAY_LENGTH(buffer) > PY_SSIZE_T_MAX / insize) { - PyErr_NoMemory(); - goto error; - } - outsize = insize * Py_ARRAY_LENGTH(buffer); - - if (*outbytes == NULL) { - /* Create string object */ - *outbytes = PyBytes_FromStringAndSize(NULL, outsize); - if (*outbytes == NULL) - goto error; - out = PyBytes_AS_STRING(*outbytes); - } - else { - /* Extend string object */ - Py_ssize_t n = PyBytes_Size(*outbytes); - if (n > PY_SSIZE_T_MAX - outsize) { - PyErr_NoMemory(); - goto error; - } - if (_PyBytes_Resize(outbytes, n + outsize) < 0) - goto error; - out = PyBytes_AS_STRING(*outbytes) + n; - } - - /* Encode the string character per character */ - while (pos < endin) - { - Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, pos); - wchar_t chars[2]; - int charsize; - if (ch < 0x10000) { - chars[0] = (wchar_t)ch; - charsize = 1; - } - else { - chars[0] = Py_UNICODE_HIGH_SURROGATE(ch); - chars[1] = Py_UNICODE_LOW_SURROGATE(ch); - charsize = 2; - } - - outsize = WideCharToMultiByte(code_page, flags, - chars, charsize, - buffer, Py_ARRAY_LENGTH(buffer), - NULL, pusedDefaultChar); - if (outsize > 0) { - if (pusedDefaultChar == NULL || !(*pusedDefaultChar)) - { - pos++; - memcpy(out, buffer, outsize); - out += outsize; - continue; - } - } - else if (GetLastError() != ERROR_NO_UNICODE_TRANSLATION) { - PyErr_SetFromWindowsErr(0); - goto error; - } - - rep = unicode_encode_call_errorhandler( - errors, &errorHandler, encoding, reason, - unicode, &exc, - pos, pos + 1, &newpos); - if (rep == NULL) - goto error; - pos = newpos; - - if (PyBytes_Check(rep)) { - outsize = PyBytes_GET_SIZE(rep); - if (outsize != 1) { - Py_ssize_t offset = out - PyBytes_AS_STRING(*outbytes); - newoutsize = PyBytes_GET_SIZE(*outbytes) + (outsize - 1); - if (_PyBytes_Resize(outbytes, newoutsize) < 0) { - Py_DECREF(rep); - goto error; - } - out = PyBytes_AS_STRING(*outbytes) + offset; - } - memcpy(out, PyBytes_AS_STRING(rep), outsize); - out += outsize; - } - else { - Py_ssize_t i; - enum PyUnicode_Kind kind; - void *data; - - if (PyUnicode_READY(rep) == -1) { - Py_DECREF(rep); - goto error; - } - - outsize = PyUnicode_GET_LENGTH(rep); - if (outsize != 1) { - Py_ssize_t offset = out - PyBytes_AS_STRING(*outbytes); - newoutsize = PyBytes_GET_SIZE(*outbytes) + (outsize - 1); - if (_PyBytes_Resize(outbytes, newoutsize) < 0) { - Py_DECREF(rep); - goto error; - } - out = PyBytes_AS_STRING(*outbytes) + offset; - } - kind = PyUnicode_KIND(rep); - data = PyUnicode_DATA(rep); - for (i=0; i < outsize; i++) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); - if (ch > 127) { - raise_encode_exception(&exc, - encoding, unicode, - pos, pos + 1, - "unable to encode error handler result to ASCII"); - Py_DECREF(rep); - goto error; - } - *out = (unsigned char)ch; - out++; - } + if (size > 0) { + char *s = PyString_AS_STRING(*repr) + n; + if (0 == WideCharToMultiByte(CP_ACP, 0, p, size, s, mbcssize, NULL, NULL)) { + PyErr_SetFromWindowsErrWithFilename(0, NULL); + return -1; } - Py_DECREF(rep); } - /* write a NUL byte */ - *out = 0; - outsize = out - PyBytes_AS_STRING(*outbytes); - assert(outsize <= PyBytes_GET_SIZE(*outbytes)); - if (_PyBytes_Resize(outbytes, outsize) < 0) - goto error; - ret = 0; -error: - Py_XDECREF(encoding_obj); - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - return ret; + return 0; } -static PyObject * -encode_code_page(int code_page, - PyObject *unicode, - const char *errors) +PyObject *PyUnicode_EncodeMBCS(const Py_UNICODE *p, + Py_ssize_t size, + const char *errors) { - Py_ssize_t len; - PyObject *outbytes = NULL; - Py_ssize_t offset; - int chunk_len, ret, done; - - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return NULL; - } + PyObject *repr = NULL; + int ret; - if (PyUnicode_READY(unicode) == -1) - return NULL; - len = PyUnicode_GET_LENGTH(unicode); +#ifdef NEED_RETRY + retry: + if (size > INT_MAX) + ret = encode_mbcs(&repr, p, INT_MAX); + else +#endif + ret = encode_mbcs(&repr, p, (int)size); - if (code_page < 0) { - PyErr_SetString(PyExc_ValueError, "invalid code page number"); + if (ret < 0) { + Py_XDECREF(repr); return NULL; } - if (len == 0) - return PyBytes_FromStringAndSize(NULL, 0); - - offset = 0; - do - { #ifdef NEED_RETRY - if (len > DECODING_CHUNK_SIZE) { - chunk_len = DECODING_CHUNK_SIZE; - done = 0; - } - else + if (size > INT_MAX) { + p += INT_MAX; + size -= INT_MAX; + goto retry; + } #endif - { - chunk_len = (int)len; - done = 1; - } - - ret = encode_code_page_strict(code_page, &outbytes, - unicode, offset, chunk_len, - errors); - if (ret == -2) - ret = encode_code_page_errors(code_page, &outbytes, - unicode, offset, - chunk_len, errors); - if (ret < 0) { - Py_XDECREF(outbytes); - return NULL; - } - - offset += chunk_len; - len -= chunk_len; - } while (!done); - return outbytes; + return repr; } -PyObject * -PyUnicode_EncodeMBCS(const Py_UNICODE *p, - Py_ssize_t size, - const char *errors) +PyObject *PyUnicode_AsMBCSString(PyObject *unicode) { - PyObject *unicode, *res; - unicode = PyUnicode_FromWideChar(p, size); - if (unicode == NULL) + if (!PyUnicode_Check(unicode)) { + PyErr_BadArgument(); return NULL; - res = encode_code_page(CP_ACP, unicode, errors); - Py_DECREF(unicode); - return res; -} - -PyObject * -PyUnicode_EncodeCodePage(int code_page, - PyObject *unicode, - const char *errors) -{ - return encode_code_page(code_page, unicode, errors); -} - -PyObject * -PyUnicode_AsMBCSString(PyObject *unicode) -{ - return PyUnicode_EncodeCodePage(CP_ACP, unicode, NULL); + } + return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode), + NULL); } #undef NEED_RETRY @@ -7904,264 +4131,198 @@ PyUnicode_AsMBCSString(PyObject *unicode) /* --- Character Mapping Codec -------------------------------------------- */ -static int -charmap_decode_string(const char *s, - Py_ssize_t size, - PyObject *mapping, - const char *errors, - _PyUnicodeWriter *writer) +PyObject *PyUnicode_DecodeCharmap(const char *s, + Py_ssize_t size, + PyObject *mapping, + const char *errors) { const char *starts = s; + Py_ssize_t startinpos; + Py_ssize_t endinpos; + Py_ssize_t outpos; const char *e; - Py_ssize_t startinpos, endinpos; - PyObject *errorHandler = NULL, *exc = NULL; - Py_ssize_t maplen; - enum PyUnicode_Kind mapkind; - void *mapdata; - Py_UCS4 x; - unsigned char ch; - - if (PyUnicode_READY(mapping) == -1) - return -1; + PyUnicodeObject *v; + Py_UNICODE *p; + Py_ssize_t extrachars = 0; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + Py_UNICODE *mapstring = NULL; + Py_ssize_t maplen = 0; - maplen = PyUnicode_GET_LENGTH(mapping); - mapdata = PyUnicode_DATA(mapping); - mapkind = PyUnicode_KIND(mapping); + /* Default to Latin-1 */ + if (mapping == NULL) + return PyUnicode_DecodeLatin1(s, size, errors); + v = _PyUnicode_New(size); + if (v == NULL) + goto onError; + if (size == 0) + return (PyObject *)v; + p = PyUnicode_AS_UNICODE(v); e = s + size; - - if (mapkind == PyUnicode_1BYTE_KIND && maplen >= 256) { - /* fast-path for cp037, cp500 and iso8859_1 encodings. iso8859_1 - * is disabled in encoding aliases, latin1 is preferred because - * its implementation is faster. */ - Py_UCS1 *mapdata_ucs1 = (Py_UCS1 *)mapdata; - Py_UCS1 *outdata = (Py_UCS1 *)writer->data; - Py_UCS4 maxchar = writer->maxchar; - - assert (writer->kind == PyUnicode_1BYTE_KIND); + if (PyUnicode_CheckExact(mapping)) { + mapstring = PyUnicode_AS_UNICODE(mapping); + maplen = PyUnicode_GET_SIZE(mapping); while (s < e) { - ch = *s; - x = mapdata_ucs1[ch]; - if (x > maxchar) { - if (_PyUnicodeWriter_Prepare(writer, 1, 0xff) == -1) + unsigned char ch = *s; + Py_UNICODE x = 0xfffe; /* illegal value */ + + if (ch < maplen) + x = mapstring[ch]; + + if (x == 0xfffe) { + /* undefined mapping */ + outpos = p-PyUnicode_AS_UNICODE(v); + startinpos = s-starts; + endinpos = startinpos+1; + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "charmap", "character maps to <undefined>", + starts, size, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) { goto onError; - maxchar = writer->maxchar; - outdata = (Py_UCS1 *)writer->data; + } + continue; } - outdata[writer->pos] = x; - writer->pos++; + *p++ = x; ++s; } - return 0; } + else { + while (s < e) { + unsigned char ch = *s; + PyObject *w, *x; - while (s < e) { - if (mapkind == PyUnicode_2BYTE_KIND && maplen >= 256) { - enum PyUnicode_Kind outkind = writer->kind; - Py_UCS2 *mapdata_ucs2 = (Py_UCS2 *)mapdata; - if (outkind == PyUnicode_1BYTE_KIND) { - Py_UCS1 *outdata = (Py_UCS1 *)writer->data; - Py_UCS4 maxchar = writer->maxchar; - while (s < e) { - ch = *s; - x = mapdata_ucs2[ch]; - if (x > maxchar) - goto Error; - outdata[writer->pos] = x; - writer->pos++; - ++s; - } - break; + /* Get mapping (char ordinal -> integer, Unicode char or None) */ + w = PyInt_FromLong((long)ch); + if (w == NULL) + goto onError; + x = PyObject_GetItem(mapping, w); + Py_DECREF(w); + if (x == NULL) { + if (PyErr_ExceptionMatches(PyExc_LookupError)) { + /* No mapping found means: mapping is undefined. */ + PyErr_Clear(); + goto Undefined; + } else + goto onError; } - else if (outkind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *outdata = (Py_UCS2 *)writer->data; - while (s < e) { - ch = *s; - x = mapdata_ucs2[ch]; - if (x == 0xFFFE) - goto Error; - outdata[writer->pos] = x; - writer->pos++; - ++s; + + /* Apply mapping */ + if (x == Py_None) + goto Undefined; + if (PyInt_Check(x)) { + long value = PyInt_AS_LONG(x); + if (value == 0xFFFE) + goto Undefined; + if (value < 0 || value > 0x10FFFF) { + PyErr_SetString(PyExc_TypeError, + "character mapping must be in range(0x110000)"); + Py_DECREF(x); + goto onError; } - break; + +#ifndef Py_UNICODE_WIDE + if (value > 0xFFFF) { + /* see the code for 1-n mapping below */ + if (extrachars < 2) { + /* resize first */ + Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v); + Py_ssize_t needed = 10 - extrachars; + extrachars += needed; + /* XXX overflow detection missing */ + if (_PyUnicode_Resize(&v, + PyUnicode_GET_SIZE(v) + needed) < 0) { + Py_DECREF(x); + goto onError; + } + p = PyUnicode_AS_UNICODE(v) + oldpos; + } + value -= 0x10000; + *p++ = 0xD800 | (Py_UNICODE)(value >> 10); + *p++ = 0xDC00 | (value & 0x3FF); + extrachars -= 2; + } + else +#endif + *p++ = (Py_UNICODE)value; } - } - ch = *s; + else if (PyUnicode_Check(x)) { + Py_ssize_t targetsize = PyUnicode_GET_SIZE(x); - if (ch < maplen) - x = PyUnicode_READ(mapkind, mapdata, ch); - else - x = 0xfffe; /* invalid value */ -Error: - if (x == 0xfffe) - { + if (targetsize == 1) { + /* 1-1 mapping */ + Py_UNICODE value = *PyUnicode_AS_UNICODE(x); + if (value == 0xFFFE) + goto Undefined; + *p++ = value; + } + else if (targetsize > 1) { + /* 1-n mapping */ + if (targetsize > extrachars) { + /* resize first */ + Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v); + Py_ssize_t needed = (targetsize - extrachars) + \ + (targetsize << 2); + extrachars += needed; + /* XXX overflow detection missing */ + if (_PyUnicode_Resize(&v, + PyUnicode_GET_SIZE(v) + needed) < 0) { + Py_DECREF(x); + goto onError; + } + p = PyUnicode_AS_UNICODE(v) + oldpos; + } + Py_UNICODE_COPY(p, + PyUnicode_AS_UNICODE(x), + targetsize); + p += targetsize; + extrachars -= targetsize; + } + /* 1-0 mapping: skip the character */ + } + else { + /* wrong return value */ + PyErr_SetString(PyExc_TypeError, + "character mapping must return integer, None or unicode"); + Py_DECREF(x); + goto onError; + } + Py_DECREF(x); + ++s; + continue; +Undefined: /* undefined mapping */ + Py_XDECREF(x); + outpos = p-PyUnicode_AS_UNICODE(v); startinpos = s-starts; endinpos = startinpos+1; - if (unicode_decode_call_errorhandler_writer( + if (unicode_decode_call_errorhandler( errors, &errorHandler, "charmap", "character maps to <undefined>", - &starts, &e, &startinpos, &endinpos, &exc, &s, - writer)) { + starts, size, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) { goto onError; } - continue; } - - if (_PyUnicodeWriter_WriteCharInline(writer, x) < 0) - goto onError; - ++s; } - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - return 0; - -onError: - Py_XDECREF(errorHandler); - Py_XDECREF(exc); - return -1; -} - -static int -charmap_decode_mapping(const char *s, - Py_ssize_t size, - PyObject *mapping, - const char *errors, - _PyUnicodeWriter *writer) -{ - const char *starts = s; - const char *e; - Py_ssize_t startinpos, endinpos; - PyObject *errorHandler = NULL, *exc = NULL; - unsigned char ch; - PyObject *key, *item = NULL; - - e = s + size; - - while (s < e) { - ch = *s; - - /* Get mapping (char ordinal -> integer, Unicode char or None) */ - key = PyLong_FromLong((long)ch); - if (key == NULL) - goto onError; - - item = PyObject_GetItem(mapping, key); - Py_DECREF(key); - if (item == NULL) { - if (PyErr_ExceptionMatches(PyExc_LookupError)) { - /* No mapping found means: mapping is undefined. */ - PyErr_Clear(); - goto Undefined; - } else - goto onError; - } - - /* Apply mapping */ - if (item == Py_None) - goto Undefined; - if (PyLong_Check(item)) { - long value = PyLong_AS_LONG(item); - if (value == 0xFFFE) - goto Undefined; - if (value < 0 || value > MAX_UNICODE) { - PyErr_Format(PyExc_TypeError, - "character mapping must be in range(0x%lx)", - (unsigned long)MAX_UNICODE + 1); - goto onError; - } - - if (_PyUnicodeWriter_WriteCharInline(writer, value) < 0) - goto onError; - } - else if (PyUnicode_Check(item)) { - if (PyUnicode_READY(item) == -1) - goto onError; - if (PyUnicode_GET_LENGTH(item) == 1) { - Py_UCS4 value = PyUnicode_READ_CHAR(item, 0); - if (value == 0xFFFE) - goto Undefined; - if (_PyUnicodeWriter_WriteCharInline(writer, value) < 0) - goto onError; - } - else { - writer->overallocate = 1; - if (_PyUnicodeWriter_WriteStr(writer, item) == -1) - goto onError; - } - } - else { - /* wrong return value */ - PyErr_SetString(PyExc_TypeError, - "character mapping must return integer, None or str"); - goto onError; - } - Py_CLEAR(item); - ++s; - continue; - -Undefined: - /* undefined mapping */ - Py_CLEAR(item); - startinpos = s-starts; - endinpos = startinpos+1; - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "charmap", "character maps to <undefined>", - &starts, &e, &startinpos, &endinpos, &exc, &s, - writer)) { + if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v)) + if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0) goto onError; - } - } Py_XDECREF(errorHandler); Py_XDECREF(exc); - return 0; + return (PyObject *)v; -onError: - Py_XDECREF(item); + onError: Py_XDECREF(errorHandler); Py_XDECREF(exc); - return -1; -} - -PyObject * -PyUnicode_DecodeCharmap(const char *s, - Py_ssize_t size, - PyObject *mapping, - const char *errors) -{ - _PyUnicodeWriter writer; - - /* Default to Latin-1 */ - if (mapping == NULL) - return PyUnicode_DecodeLatin1(s, size, errors); - - if (size == 0) - _Py_RETURN_UNICODE_EMPTY(); - _PyUnicodeWriter_Init(&writer); - writer.min_length = size; - if (_PyUnicodeWriter_Prepare(&writer, writer.min_length, 127) == -1) - goto onError; - - if (PyUnicode_CheckExact(mapping)) { - if (charmap_decode_string(s, size, mapping, errors, &writer) < 0) - goto onError; - } - else { - if (charmap_decode_mapping(s, size, mapping, errors, &writer) < 0) - goto onError; - } - return _PyUnicodeWriter_Finish(&writer); - - onError: - _PyUnicodeWriter_Dealloc(&writer); + Py_XDECREF(v); return NULL; } /* Charmap encoding: the lookup table */ -struct encoding_map { +struct encoding_map{ PyObject_HEAD unsigned char level1[32]; int count2, count3; @@ -8172,8 +4333,8 @@ static PyObject* encoding_map_size(PyObject *obj, PyObject* args) { struct encoding_map *map = (struct encoding_map*)obj; - return PyLong_FromLong(sizeof(*map) - 1 + 16*map->count2 + - 128*map->count3); + return PyInt_FromLong(sizeof(*map) - 1 + 16*map->count2 + + 128*map->count3); } static PyMethodDef encoding_map_methods[] = { @@ -8182,17 +4343,23 @@ static PyMethodDef encoding_map_methods[] = { { 0 } }; +static void +encoding_map_dealloc(PyObject* o) +{ + PyObject_FREE(o); +} + static PyTypeObject EncodingMapType = { PyVarObject_HEAD_INIT(NULL, 0) "EncodingMap", /*tp_name*/ sizeof(struct encoding_map), /*tp_basicsize*/ 0, /*tp_itemsize*/ /* methods */ - 0, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ + encoding_map_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ - 0, /*tp_as_async*/ + 0, /*tp_compare*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ @@ -8229,6 +4396,7 @@ static PyTypeObject EncodingMapType = { PyObject* PyUnicode_BuildEncodingMap(PyObject* string) { + Py_UNICODE *decode; PyObject *result; struct encoding_map *mresult; int i; @@ -8237,39 +4405,35 @@ PyUnicode_BuildEncodingMap(PyObject* string) unsigned char level2[512]; unsigned char *mlevel1, *mlevel2, *mlevel3; int count2 = 0, count3 = 0; - int kind; - void *data; - Py_ssize_t length; - Py_UCS4 ch; - if (!PyUnicode_Check(string) || !PyUnicode_GET_LENGTH(string)) { + if (!PyUnicode_Check(string) || PyUnicode_GetSize(string) != 256) { PyErr_BadArgument(); return NULL; } - kind = PyUnicode_KIND(string); - data = PyUnicode_DATA(string); - length = PyUnicode_GET_LENGTH(string); - length = Py_MIN(length, 256); + decode = PyUnicode_AS_UNICODE(string); memset(level1, 0xFF, sizeof level1); memset(level2, 0xFF, sizeof level2); /* If there isn't a one-to-one mapping of NULL to \0, or if there are non-BMP characters, we need to use a mapping dictionary. */ - if (PyUnicode_READ(kind, data, 0) != 0) + if (decode[0] != 0) need_dict = 1; - for (i = 1; i < length; i++) { + for (i = 1; i < 256; i++) { int l1, l2; - ch = PyUnicode_READ(kind, data, i); - if (ch == 0 || ch > 0xFFFF) { + if (decode[i] == 0 +#ifdef Py_UNICODE_WIDE + || decode[i] > 0xFFFF +#endif + ) { need_dict = 1; break; } - if (ch == 0xFFFE) + if (decode[i] == 0xFFFE) /* unmapped character */ continue; - l1 = ch >> 11; - l2 = ch >> 7; + l1 = decode[i] >> 11; + l2 = decode[i] >> 7; if (level1[l1] == 0xFF) level1[l1] = count2++; if (level2[l2] == 0xFF) @@ -8284,9 +4448,10 @@ PyUnicode_BuildEncodingMap(PyObject* string) PyObject *key, *value; if (!result) return NULL; - for (i = 0; i < length; i++) { - key = PyLong_FromLong(PyUnicode_READ(kind, data, i)); - value = PyLong_FromLong(i); + for (i = 0; i < 256; i++) { + value = NULL; + key = PyInt_FromLong(decode[i]); + value = PyInt_FromLong(i); if (!key || !value) goto failed1; if (PyDict_SetItem(result, key, value) == -1) @@ -8318,18 +4483,17 @@ PyUnicode_BuildEncodingMap(PyObject* string) memset(mlevel2, 0xFF, 16*count2); memset(mlevel3, 0, 128*count3); count3 = 0; - for (i = 1; i < length; i++) { + for (i = 1; i < 256; i++) { int o1, o2, o3, i2, i3; - Py_UCS4 ch = PyUnicode_READ(kind, data, i); - if (ch == 0xFFFE) + if (decode[i] == 0xFFFE) /* unmapped character */ continue; - o1 = ch>>11; - o2 = (ch>>7) & 0xF; + o1 = decode[i]>>11; + o2 = (decode[i]>>7) & 0xF; i2 = 16*mlevel1[o1] + o2; if (mlevel2[i2] == 0xFF) mlevel2[i2] = count3++; - o3 = ch & 0x7F; + o3 = decode[i] & 0x7F; i3 = 128*mlevel2[i2] + o3; mlevel3[i3] = i; } @@ -8337,7 +4501,7 @@ PyUnicode_BuildEncodingMap(PyObject* string) } static int -encoding_map_lookup(Py_UCS4 c, PyObject *mapping) +encoding_map_lookup(Py_UNICODE c, PyObject *mapping) { struct encoding_map *map = (struct encoding_map*)mapping; int l1 = c>>11; @@ -8345,8 +4509,11 @@ encoding_map_lookup(Py_UCS4 c, PyObject *mapping) int l3 = c & 0x7F; int i; - if (c > 0xFFFF) +#ifdef Py_UNICODE_WIDE + if (c > 0xFFFF) { return -1; + } +#endif if (c == 0) return 0; /* level 1*/ @@ -8370,10 +4537,9 @@ encoding_map_lookup(Py_UCS4 c, PyObject *mapping) /* Lookup the character ch in the mapping. If the character can't be found, Py_None is returned (or NULL, if another error occurred). */ -static PyObject * -charmapencode_lookup(Py_UCS4 c, PyObject *mapping) +static PyObject *charmapencode_lookup(Py_UNICODE c, PyObject *mapping) { - PyObject *w = PyLong_FromLong((long)c); + PyObject *w = PyInt_FromLong((long)c); PyObject *x; if (w == NULL) @@ -8384,14 +4550,16 @@ charmapencode_lookup(Py_UCS4 c, PyObject *mapping) if (PyErr_ExceptionMatches(PyExc_LookupError)) { /* No mapping found means: mapping is undefined. */ PyErr_Clear(); - Py_RETURN_NONE; + x = Py_None; + Py_INCREF(x); + return x; } else return NULL; } else if (x == Py_None) return x; - else if (PyLong_Check(x)) { - long value = PyLong_AS_LONG(x); + else if (PyInt_Check(x)) { + long value = PyInt_AS_LONG(x); if (value < 0 || value > 255) { PyErr_SetString(PyExc_TypeError, "character mapping must be in range(256)"); @@ -8400,13 +4568,12 @@ charmapencode_lookup(Py_UCS4 c, PyObject *mapping) } return x; } - else if (PyBytes_Check(x)) + else if (PyString_Check(x)) return x; else { /* wrong return value */ - PyErr_Format(PyExc_TypeError, - "character mapping must return integer, bytes or None, not %.400s", - x->ob_type->tp_name); + PyErr_SetString(PyExc_TypeError, + "character mapping must return integer, None or str"); Py_DECREF(x); return NULL; } @@ -8415,31 +4582,32 @@ charmapencode_lookup(Py_UCS4 c, PyObject *mapping) static int charmapencode_resize(PyObject **outobj, Py_ssize_t *outpos, Py_ssize_t requiredsize) { - Py_ssize_t outsize = PyBytes_GET_SIZE(*outobj); + Py_ssize_t outsize = PyString_GET_SIZE(*outobj); /* exponentially overallocate to minimize reallocations */ if (requiredsize < 2*outsize) requiredsize = 2*outsize; - if (_PyBytes_Resize(outobj, requiredsize)) - return -1; - return 0; + if (_PyString_Resize(outobj, requiredsize)) { + return 0; + } + return 1; } typedef enum charmapencode_result { enc_SUCCESS, enc_FAILED, enc_EXCEPTION -} charmapencode_result; +}charmapencode_result; /* lookup the character, put the result in the output string and adjust - various state variables. Resize the output bytes object if not enough + various state variables. Reallocate the output string if not enough space is available. Return a new reference to the object that was put in the output buffer, or Py_None, if the mapping was undefined (in which case no character was written) or NULL, if a reallocation error occurred. The caller must decref the result */ -static charmapencode_result -charmapencode_output(Py_UCS4 c, PyObject *mapping, - PyObject **outobj, Py_ssize_t *outpos) +static +charmapencode_result charmapencode_output(Py_UNICODE c, PyObject *mapping, + PyObject **outobj, Py_ssize_t *outpos) { PyObject *rep; char *outstart; - Py_ssize_t outsize = PyBytes_GET_SIZE(*outobj); + Py_ssize_t outsize = PyString_GET_SIZE(*outobj); if (Py_TYPE(mapping) == &EncodingMapType) { int res = encoding_map_lookup(c, mapping); @@ -8447,9 +4615,9 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, if (res == -1) return enc_FAILED; if (outsize<requiredsize) - if (charmapencode_resize(outobj, outpos, requiredsize)) + if (!charmapencode_resize(outobj, outpos, requiredsize)) return enc_EXCEPTION; - outstart = PyBytes_AS_STRING(*outobj); + outstart = PyString_AS_STRING(*outobj); outstart[(*outpos)++] = (char)res; return enc_SUCCESS; } @@ -8461,26 +4629,26 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, Py_DECREF(rep); return enc_FAILED; } else { - if (PyLong_Check(rep)) { + if (PyInt_Check(rep)) { Py_ssize_t requiredsize = *outpos+1; if (outsize<requiredsize) - if (charmapencode_resize(outobj, outpos, requiredsize)) { + if (!charmapencode_resize(outobj, outpos, requiredsize)) { Py_DECREF(rep); return enc_EXCEPTION; } - outstart = PyBytes_AS_STRING(*outobj); - outstart[(*outpos)++] = (char)PyLong_AS_LONG(rep); + outstart = PyString_AS_STRING(*outobj); + outstart[(*outpos)++] = (char)PyInt_AS_LONG(rep); } else { - const char *repchars = PyBytes_AS_STRING(rep); - Py_ssize_t repsize = PyBytes_GET_SIZE(rep); + const char *repchars = PyString_AS_STRING(rep); + Py_ssize_t repsize = PyString_GET_SIZE(rep); Py_ssize_t requiredsize = *outpos+repsize; if (outsize<requiredsize) - if (charmapencode_resize(outobj, outpos, requiredsize)) { + if (!charmapencode_resize(outobj, outpos, requiredsize)) { Py_DECREF(rep); return enc_EXCEPTION; } - outstart = PyBytes_AS_STRING(*outobj); + outstart = PyString_AS_STRING(*outobj); memcpy(outstart + *outpos, repchars, repsize); *outpos += repsize; } @@ -8491,46 +4659,37 @@ charmapencode_output(Py_UCS4 c, PyObject *mapping, /* handle an error in PyUnicode_EncodeCharmap Return 0 on success, -1 on error */ -static int -charmap_encoding_error( - PyObject *unicode, Py_ssize_t *inpos, PyObject *mapping, +static +int charmap_encoding_error( + const Py_UNICODE *p, Py_ssize_t size, Py_ssize_t *inpos, PyObject *mapping, PyObject **exceptionObject, - _Py_error_handler *error_handler, PyObject **error_handler_obj, const char *errors, + int *known_errorHandler, PyObject **errorHandler, const char *errors, PyObject **res, Py_ssize_t *respos) { PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ - Py_ssize_t size, repsize; + Py_ssize_t repsize; Py_ssize_t newpos; - enum PyUnicode_Kind kind; - void *data; - Py_ssize_t index; + Py_UNICODE *uni2; /* startpos for collecting unencodable chars */ Py_ssize_t collstartpos = *inpos; Py_ssize_t collendpos = *inpos+1; Py_ssize_t collpos; - const char *encoding = "charmap"; - const char *reason = "character maps to <undefined>"; + char *encoding = "charmap"; + char *reason = "character maps to <undefined>"; charmapencode_result x; - Py_UCS4 ch; - int val; - if (PyUnicode_READY(unicode) == -1) - return -1; - size = PyUnicode_GET_LENGTH(unicode); /* find all unencodable characters */ while (collendpos < size) { PyObject *rep; if (Py_TYPE(mapping) == &EncodingMapType) { - ch = PyUnicode_READ_CHAR(unicode, collendpos); - val = encoding_map_lookup(ch, mapping); - if (val != -1) + int res = encoding_map_lookup(p[collendpos], mapping); + if (res != -1) break; ++collendpos; continue; } - ch = PyUnicode_READ_CHAR(unicode, collendpos); - rep = charmapencode_lookup(ch, mapping); + rep = charmapencode_lookup(p[collendpos], mapping); if (rep==NULL) return -1; else if (rep!=Py_None) { @@ -8542,92 +4701,80 @@ charmap_encoding_error( } /* cache callback name lookup * (if not done yet, i.e. it's the first error) */ - if (*error_handler == _Py_ERROR_UNKNOWN) - *error_handler = _Py_GetErrorHandler(errors); - - switch (*error_handler) { - case _Py_ERROR_STRICT: - raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); + if (*known_errorHandler==-1) { + if ((errors==NULL) || (!strcmp(errors, "strict"))) + *known_errorHandler = 1; + else if (!strcmp(errors, "replace")) + *known_errorHandler = 2; + else if (!strcmp(errors, "ignore")) + *known_errorHandler = 3; + else if (!strcmp(errors, "xmlcharrefreplace")) + *known_errorHandler = 4; + else + *known_errorHandler = 0; + } + switch (*known_errorHandler) { + case 1: /* strict */ + raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); return -1; - - case _Py_ERROR_REPLACE: + case 2: /* replace */ for (collpos = collstartpos; collpos<collendpos; ++collpos) { x = charmapencode_output('?', mapping, res, respos); if (x==enc_EXCEPTION) { return -1; } else if (x==enc_FAILED) { - raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); + raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); return -1; } } /* fall through */ - case _Py_ERROR_IGNORE: + case 3: /* ignore */ *inpos = collendpos; break; - - case _Py_ERROR_XMLCHARREFREPLACE: - /* generate replacement (temporarily (mis)uses p) */ - for (collpos = collstartpos; collpos < collendpos; ++collpos) { + case 4: /* xmlcharrefreplace */ + /* generate replacement */ + for (collpos = collstartpos; collpos < collendpos;) { char buffer[2+29+1+1]; char *cp; - sprintf(buffer, "&#%d;", (int)PyUnicode_READ_CHAR(unicode, collpos)); + Py_UCS4 ch = p[collpos++]; +#ifndef Py_UNICODE_WIDE + if ((0xD800 <= ch && ch <= 0xDBFF) && + (collpos < collendpos) && + (0xDC00 <= p[collpos] && p[collpos] <= 0xDFFF)) { + ch = ((((ch & 0x03FF) << 10) | + ((Py_UCS4)p[collpos++] & 0x03FF)) + 0x10000); + } +#endif + sprintf(buffer, "&#%d;", (int)ch); for (cp = buffer; *cp; ++cp) { x = charmapencode_output(*cp, mapping, res, respos); if (x==enc_EXCEPTION) return -1; else if (x==enc_FAILED) { - raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); + raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); return -1; } } } *inpos = collendpos; break; - default: - repunicode = unicode_encode_call_errorhandler(errors, error_handler_obj, - encoding, reason, unicode, exceptionObject, + repunicode = unicode_encode_call_errorhandler(errors, errorHandler, + encoding, reason, p, size, exceptionObject, collstartpos, collendpos, &newpos); if (repunicode == NULL) return -1; - if (PyBytes_Check(repunicode)) { - /* Directly copy bytes result to output. */ - Py_ssize_t outsize = PyBytes_Size(*res); - Py_ssize_t requiredsize; - repsize = PyBytes_Size(repunicode); - requiredsize = *respos + repsize; - if (requiredsize > outsize) - /* Make room for all additional bytes. */ - if (charmapencode_resize(res, respos, requiredsize)) { - Py_DECREF(repunicode); - return -1; - } - memcpy(PyBytes_AsString(*res) + *respos, - PyBytes_AsString(repunicode), repsize); - *respos += repsize; - *inpos = newpos; - Py_DECREF(repunicode); - break; - } /* generate replacement */ - if (PyUnicode_READY(repunicode) == -1) { - Py_DECREF(repunicode); - return -1; - } - repsize = PyUnicode_GET_LENGTH(repunicode); - data = PyUnicode_DATA(repunicode); - kind = PyUnicode_KIND(repunicode); - for (index = 0; index < repsize; index++) { - Py_UCS4 repch = PyUnicode_READ(kind, data, index); - x = charmapencode_output(repch, mapping, res, respos); + repsize = PyUnicode_GET_SIZE(repunicode); + for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) { + x = charmapencode_output(*uni2, mapping, res, respos); if (x==enc_EXCEPTION) { - Py_DECREF(repunicode); return -1; } else if (x==enc_FAILED) { Py_DECREF(repunicode); - raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); + raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); return -1; } } @@ -8637,52 +4784,45 @@ charmap_encoding_error( return 0; } -PyObject * -_PyUnicode_EncodeCharmap(PyObject *unicode, - PyObject *mapping, - const char *errors) +PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p, + Py_ssize_t size, + PyObject *mapping, + const char *errors) { /* output object */ PyObject *res = NULL; /* current input position */ Py_ssize_t inpos = 0; - Py_ssize_t size; /* current output position */ Py_ssize_t respos = 0; - PyObject *error_handler_obj = NULL; + PyObject *errorHandler = NULL; PyObject *exc = NULL; - _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; - void *data; - int kind; - - if (PyUnicode_READY(unicode) == -1) - return NULL; - size = PyUnicode_GET_LENGTH(unicode); - data = PyUnicode_DATA(unicode); - kind = PyUnicode_KIND(unicode); + /* the following variable is used for caching string comparisons + * -1=not initialized, 0=unknown, 1=strict, 2=replace, + * 3=ignore, 4=xmlcharrefreplace */ + int known_errorHandler = -1; /* Default to Latin-1 */ if (mapping == NULL) - return unicode_encode_ucs1(unicode, errors, 256); + return PyUnicode_EncodeLatin1(p, size, errors); /* allocate enough for a simple encoding without replacements, if we need more, we'll resize */ - res = PyBytes_FromStringAndSize(NULL, size); + res = PyString_FromStringAndSize(NULL, size); if (res == NULL) goto onError; if (size == 0) return res; while (inpos<size) { - Py_UCS4 ch = PyUnicode_READ(kind, data, inpos); /* try to encode it */ - charmapencode_result x = charmapencode_output(ch, mapping, &res, &respos); + charmapencode_result x = charmapencode_output(p[inpos], mapping, &res, &respos); if (x==enc_EXCEPTION) /* error */ goto onError; if (x==enc_FAILED) { /* unencodable character */ - if (charmap_encoding_error(unicode, &inpos, mapping, + if (charmap_encoding_error(p, size, &inpos, mapping, &exc, - &error_handler, &error_handler_obj, errors, + &known_errorHandler, &errorHandler, errors, &res, &respos)) { goto onError; } @@ -8693,58 +4833,43 @@ _PyUnicode_EncodeCharmap(PyObject *unicode, } /* Resize if we allocated to much */ - if (respos<PyBytes_GET_SIZE(res)) - if (_PyBytes_Resize(&res, respos) < 0) + if (respos<PyString_GET_SIZE(res)) { + if (_PyString_Resize(&res, respos)) goto onError; - + } Py_XDECREF(exc); - Py_XDECREF(error_handler_obj); + Py_XDECREF(errorHandler); return res; onError: Py_XDECREF(res); Py_XDECREF(exc); - Py_XDECREF(error_handler_obj); + Py_XDECREF(errorHandler); return NULL; } -/* Deprecated */ -PyObject * -PyUnicode_EncodeCharmap(const Py_UNICODE *p, - Py_ssize_t size, - PyObject *mapping, - const char *errors) -{ - PyObject *result; - PyObject *unicode = PyUnicode_FromWideChar(p, size); - if (unicode == NULL) - return NULL; - result = _PyUnicode_EncodeCharmap(unicode, mapping, errors); - Py_DECREF(unicode); - return result; -} - -PyObject * -PyUnicode_AsCharmapString(PyObject *unicode, - PyObject *mapping) +PyObject *PyUnicode_AsCharmapString(PyObject *unicode, + PyObject *mapping) { if (!PyUnicode_Check(unicode) || mapping == NULL) { PyErr_BadArgument(); return NULL; } - return _PyUnicode_EncodeCharmap(unicode, mapping, NULL); + return PyUnicode_EncodeCharmap(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode), + mapping, + NULL); } /* create or adjust a UnicodeTranslateError */ -static void -make_translate_exception(PyObject **exceptionObject, - PyObject *unicode, - Py_ssize_t startpos, Py_ssize_t endpos, - const char *reason) +static void make_translate_exception(PyObject **exceptionObject, + const Py_UNICODE *unicode, Py_ssize_t size, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason) { if (*exceptionObject == NULL) { - *exceptionObject = _PyUnicodeTranslateError_Create( - unicode, startpos, endpos, reason); + *exceptionObject = PyUnicodeTranslateError_Create( + unicode, size, startpos, endpos, reason); } else { if (PyUnicodeTranslateError_SetStart(*exceptionObject, startpos)) @@ -8759,19 +4884,30 @@ make_translate_exception(PyObject **exceptionObject, } } +/* raises a UnicodeTranslateError */ +static void raise_translate_exception(PyObject **exceptionObject, + const Py_UNICODE *unicode, Py_ssize_t size, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason) +{ + make_translate_exception(exceptionObject, + unicode, size, startpos, endpos, reason); + if (*exceptionObject != NULL) + PyCodec_StrictErrors(*exceptionObject); +} + /* error handling callback helper: build arguments, call the callback and check the arguments, put the result into newpos and return the replacement string, which has to be freed by the caller */ -static PyObject * -unicode_translate_call_errorhandler(const char *errors, - PyObject **errorHandler, - const char *reason, - PyObject *unicode, PyObject **exceptionObject, - Py_ssize_t startpos, Py_ssize_t endpos, - Py_ssize_t *newpos) +static PyObject *unicode_translate_call_errorhandler(const char *errors, + PyObject **errorHandler, + const char *reason, + const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, + Py_ssize_t startpos, Py_ssize_t endpos, + Py_ssize_t *newpos) { - static const char *argparse = "Un;translating error handler must return (str, int) tuple"; + static char *argparse = "O!n;translating error handler must return (unicode, int) tuple"; Py_ssize_t i_newpos; PyObject *restuple; @@ -8784,28 +4920,29 @@ unicode_translate_call_errorhandler(const char *errors, } make_translate_exception(exceptionObject, - unicode, startpos, endpos, reason); + unicode, size, startpos, endpos, reason); if (*exceptionObject == NULL) return NULL; - restuple = _PyObject_CallOneArg(*errorHandler, *exceptionObject); + restuple = PyObject_CallFunctionObjArgs( + *errorHandler, *exceptionObject, NULL); if (restuple == NULL) return NULL; if (!PyTuple_Check(restuple)) { - PyErr_SetString(PyExc_TypeError, &argparse[3]); + PyErr_SetString(PyExc_TypeError, &argparse[4]); Py_DECREF(restuple); return NULL; } - if (!PyArg_ParseTuple(restuple, argparse, + if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &resunicode, &i_newpos)) { Py_DECREF(restuple); return NULL; } if (i_newpos<0) - *newpos = PyUnicode_GET_LENGTH(unicode)+i_newpos; + *newpos = size+i_newpos; else *newpos = i_newpos; - if (*newpos<0 || *newpos>PyUnicode_GET_LENGTH(unicode)) { + if (*newpos<0 || *newpos>size) { PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos); Py_DECREF(restuple); return NULL; @@ -8818,10 +4955,10 @@ unicode_translate_call_errorhandler(const char *errors, /* Lookup the character ch in the mapping and put the result in result, which must be decrefed by the caller. Return 0 on success, -1 on error */ -static int -charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) +static +int charmaptranslate_lookup(Py_UNICODE c, PyObject *mapping, PyObject **result) { - PyObject *w = PyLong_FromLong((long)c); + PyObject *w = PyInt_FromLong((long)c); PyObject *x; if (w == NULL) @@ -8841,12 +4978,12 @@ charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) *result = x; return 0; } - else if (PyLong_Check(x)) { - long value = PyLong_AS_LONG(x); - if (value < 0 || value > MAX_UNICODE) { - PyErr_Format(PyExc_ValueError, - "character mapping must be in range(0x%x)", - MAX_UNICODE+1); + else if (PyInt_Check(x)) { + long value = PyInt_AS_LONG(x); + long max = PyUnicode_GetMax(); + if (value < 0 || value > max) { + PyErr_Format(PyExc_TypeError, + "character mapping must be in range(0x%lx)", max+1); Py_DECREF(x); return -1; } @@ -8860,474 +4997,382 @@ charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result) else { /* wrong return value */ PyErr_SetString(PyExc_TypeError, - "character mapping must return integer, None or str"); + "character mapping must return integer, None or unicode"); Py_DECREF(x); return -1; } } - -/* lookup the character, write the result into the writer. - Return 1 if the result was written into the writer, return 0 if the mapping - was undefined, raise an exception return -1 on error. */ -static int -charmaptranslate_output(Py_UCS4 ch, PyObject *mapping, - _PyUnicodeWriter *writer) -{ - PyObject *item; - - if (charmaptranslate_lookup(ch, mapping, &item)) - return -1; - - if (item == NULL) { - /* not found => default to 1:1 mapping */ - if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) { - return -1; - } - return 1; - } - - if (item == Py_None) { - Py_DECREF(item); - return 0; - } - - if (PyLong_Check(item)) { - long ch = (Py_UCS4)PyLong_AS_LONG(item); - /* PyLong_AS_LONG() cannot fail, charmaptranslate_lookup() already - used it */ - if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) { - Py_DECREF(item); +/* ensure that *outobj is at least requiredsize characters long, + if not reallocate and adjust various state variables. + Return 0 on success, -1 on error */ +static +int charmaptranslate_makespace(PyObject **outobj, Py_UNICODE **outp, + Py_ssize_t requiredsize) +{ + Py_ssize_t oldsize = PyUnicode_GET_SIZE(*outobj); + if (requiredsize > oldsize) { + /* remember old output position */ + Py_ssize_t outpos = *outp-PyUnicode_AS_UNICODE(*outobj); + /* exponentially overallocate to minimize reallocations */ + if (requiredsize < 2 * oldsize) + requiredsize = 2 * oldsize; + if (PyUnicode_Resize(outobj, requiredsize) < 0) return -1; - } - Py_DECREF(item); - return 1; - } - - if (!PyUnicode_Check(item)) { - Py_DECREF(item); - return -1; - } - - if (_PyUnicodeWriter_WriteStr(writer, item) < 0) { - Py_DECREF(item); - return -1; + *outp = PyUnicode_AS_UNICODE(*outobj) + outpos; } - - Py_DECREF(item); - return 1; + return 0; } - -static int -unicode_fast_translate_lookup(PyObject *mapping, Py_UCS1 ch, - Py_UCS1 *translate) +/* lookup the character, put the result in the output string and adjust + various state variables. Return a new reference to the object that + was put in the output buffer in *result, or Py_None, if the mapping was + undefined (in which case no character was written). + The called must decref result. + Return 0 on success, -1 on error. */ +static +int charmaptranslate_output(const Py_UNICODE *startinp, const Py_UNICODE *curinp, + Py_ssize_t insize, PyObject *mapping, PyObject **outobj, Py_UNICODE **outp, + PyObject **res) { - PyObject *item = NULL; - int ret = 0; - - if (charmaptranslate_lookup(ch, mapping, &item)) { + if (charmaptranslate_lookup(*curinp, mapping, res)) return -1; - } - - if (item == Py_None) { - /* deletion */ - translate[ch] = 0xfe; - } - else if (item == NULL) { + if (*res==NULL) { /* not found => default to 1:1 mapping */ - translate[ch] = ch; - return 1; - } - else if (PyLong_Check(item)) { - long replace = PyLong_AS_LONG(item); - /* PyLong_AS_LONG() cannot fail, charmaptranslate_lookup() already - used it */ - if (127 < replace) { - /* invalid character or character outside ASCII: - skip the fast translate */ - goto exit; - } - translate[ch] = (Py_UCS1)replace; - } - else if (PyUnicode_Check(item)) { - Py_UCS4 replace; - - if (PyUnicode_READY(item) == -1) { - Py_DECREF(item); - return -1; - } - if (PyUnicode_GET_LENGTH(item) != 1) - goto exit; - - replace = PyUnicode_READ_CHAR(item, 0); - if (replace > 127) - goto exit; - translate[ch] = (Py_UCS1)replace; + *(*outp)++ = *curinp; } - else { - /* not None, NULL, long or unicode */ - goto exit; + else if (*res==Py_None) + ; + else if (PyInt_Check(*res)) { + /* no overflow check, because we know that the space is enough */ + *(*outp)++ = (Py_UNICODE)PyInt_AS_LONG(*res); } - ret = 1; - - exit: - Py_DECREF(item); - return ret; -} - -/* Fast path for ascii => ascii translation. Return 1 if the whole string - was translated into writer, return 0 if the input string was partially - translated into writer, raise an exception and return -1 on error. */ -static int -unicode_fast_translate(PyObject *input, PyObject *mapping, - _PyUnicodeWriter *writer, int ignore, - Py_ssize_t *input_pos) -{ - Py_UCS1 ascii_table[128], ch, ch2; - Py_ssize_t len; - Py_UCS1 *in, *end, *out; - int res = 0; - - len = PyUnicode_GET_LENGTH(input); - - memset(ascii_table, 0xff, 128); - - in = PyUnicode_1BYTE_DATA(input); - end = in + len; - - assert(PyUnicode_IS_ASCII(writer->buffer)); - assert(PyUnicode_GET_LENGTH(writer->buffer) == len); - out = PyUnicode_1BYTE_DATA(writer->buffer); - - for (; in < end; in++) { - ch = *in; - ch2 = ascii_table[ch]; - if (ch2 == 0xff) { - int translate = unicode_fast_translate_lookup(mapping, ch, - ascii_table); - if (translate < 0) + else if (PyUnicode_Check(*res)) { + Py_ssize_t repsize = PyUnicode_GET_SIZE(*res); + if (repsize==1) { + /* no overflow check, because we know that the space is enough */ + *(*outp)++ = *PyUnicode_AS_UNICODE(*res); + } + else if (repsize!=0) { + /* more than one character */ + Py_ssize_t requiredsize = (*outp-PyUnicode_AS_UNICODE(*outobj)) + + (insize - (curinp-startinp)) + + repsize - 1; + if (charmaptranslate_makespace(outobj, outp, requiredsize)) return -1; - if (translate == 0) - goto exit; - ch2 = ascii_table[ch]; + memcpy(*outp, PyUnicode_AS_UNICODE(*res), sizeof(Py_UNICODE)*repsize); + *outp += repsize; } - if (ch2 == 0xfe) { - if (ignore) - continue; - goto exit; - } - assert(ch2 < 128); - *out = ch2; - out++; } - res = 1; - -exit: - writer->pos = out - PyUnicode_1BYTE_DATA(writer->buffer); - *input_pos = in - PyUnicode_1BYTE_DATA(input); - return res; + else + return -1; + return 0; } -static PyObject * -_PyUnicode_TranslateCharmap(PyObject *input, - PyObject *mapping, - const char *errors) +PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p, + Py_ssize_t size, + PyObject *mapping, + const char *errors) { - /* input object */ - char *data; - Py_ssize_t size, i; - int kind; - /* output buffer */ - _PyUnicodeWriter writer; - /* error handler */ - const char *reason = "character maps to <undefined>"; + /* output object */ + PyObject *res = NULL; + /* pointers to the beginning and end+1 of input */ + const Py_UNICODE *startp = p; + const Py_UNICODE *endp = p + size; + /* pointer into the output */ + Py_UNICODE *str; + /* current output position */ + Py_ssize_t respos = 0; + char *reason = "character maps to <undefined>"; PyObject *errorHandler = NULL; PyObject *exc = NULL; - int ignore; - int res; + /* the following variable is used for caching string comparisons + * -1=not initialized, 0=unknown, 1=strict, 2=replace, + * 3=ignore, 4=xmlcharrefreplace */ + int known_errorHandler = -1; if (mapping == NULL) { PyErr_BadArgument(); return NULL; } - if (PyUnicode_READY(input) == -1) - return NULL; - data = (char*)PyUnicode_DATA(input); - kind = PyUnicode_KIND(input); - size = PyUnicode_GET_LENGTH(input); - - if (size == 0) - return PyUnicode_FromObject(input); - /* allocate enough for a simple 1:1 translation without replacements, if we need more, we'll resize */ - _PyUnicodeWriter_Init(&writer); - if (_PyUnicodeWriter_Prepare(&writer, size, 127) == -1) + res = PyUnicode_FromUnicode(NULL, size); + if (res == NULL) goto onError; + if (size == 0) + return res; + str = PyUnicode_AS_UNICODE(res); - ignore = (errors != NULL && strcmp(errors, "ignore") == 0); - - if (PyUnicode_READY(input) == -1) - return NULL; - if (PyUnicode_IS_ASCII(input)) { - res = unicode_fast_translate(input, mapping, &writer, ignore, &i); - if (res < 0) { - _PyUnicodeWriter_Dealloc(&writer); - return NULL; - } - if (res == 1) - return _PyUnicodeWriter_Finish(&writer); - } - else { - i = 0; - } - - while (i<size) { + while (p<endp) { /* try to encode it */ - int translate; - PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ - Py_ssize_t newpos; - /* startpos for collecting untranslatable chars */ - Py_ssize_t collstart; - Py_ssize_t collend; - Py_UCS4 ch; - - ch = PyUnicode_READ(kind, data, i); - translate = charmaptranslate_output(ch, mapping, &writer); - if (translate < 0) + PyObject *x = NULL; + if (charmaptranslate_output(startp, p, size, mapping, &res, &str, &x)) { + Py_XDECREF(x); goto onError; - - if (translate != 0) { - /* it worked => adjust input pointer */ - ++i; - continue; } - - /* untranslatable character */ - collstart = i; - collend = i+1; - - /* find all untranslatable characters */ - while (collend < size) { - PyObject *x; - ch = PyUnicode_READ(kind, data, collend); - if (charmaptranslate_lookup(ch, mapping, &x)) + Py_XDECREF(x); + if (x!=Py_None) /* it worked => adjust input pointer */ + ++p; + else { /* untranslatable character */ + PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ + Py_ssize_t repsize; + Py_ssize_t newpos; + Py_UNICODE *uni2; + /* startpos for collecting untranslatable chars */ + const Py_UNICODE *collstart = p; + const Py_UNICODE *collend = p+1; + const Py_UNICODE *coll; + + /* find all untranslatable characters */ + while (collend < endp) { + if (charmaptranslate_lookup(*collend, mapping, &x)) + goto onError; + Py_XDECREF(x); + if (x!=Py_None) + break; + ++collend; + } + /* cache callback name lookup + * (if not done yet, i.e. it's the first error) */ + if (known_errorHandler==-1) { + if ((errors==NULL) || (!strcmp(errors, "strict"))) + known_errorHandler = 1; + else if (!strcmp(errors, "replace")) + known_errorHandler = 2; + else if (!strcmp(errors, "ignore")) + known_errorHandler = 3; + else if (!strcmp(errors, "xmlcharrefreplace")) + known_errorHandler = 4; + else + known_errorHandler = 0; + } + switch (known_errorHandler) { + case 1: /* strict */ + raise_translate_exception(&exc, startp, size, collstart-startp, collend-startp, reason); goto onError; - Py_XDECREF(x); - if (x != Py_None) + case 2: /* replace */ + /* No need to check for space, this is a 1:1 replacement */ + for (coll = collstart; coll<collend; ++coll) + *str++ = '?'; + /* fall through */ + case 3: /* ignore */ + p = collend; break; - ++collend; - } - - if (ignore) { - i = collend; - } - else { - repunicode = unicode_translate_call_errorhandler(errors, &errorHandler, - reason, input, &exc, - collstart, collend, &newpos); - if (repunicode == NULL) - goto onError; - if (_PyUnicodeWriter_WriteStr(&writer, repunicode) < 0) { + case 4: /* xmlcharrefreplace */ + /* generate replacement (temporarily (mis)uses p) */ + for (p = collstart; p < collend;) { + char buffer[2+29+1+1]; + char *cp; + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + sprintf(buffer, "&#%d;", (int)ch); + if (charmaptranslate_makespace(&res, &str, + (str-PyUnicode_AS_UNICODE(res))+strlen(buffer)+(endp-collend))) + goto onError; + for (cp = buffer; *cp; ++cp) + *str++ = *cp; + } + p = collend; + break; + default: + repunicode = unicode_translate_call_errorhandler(errors, &errorHandler, + reason, startp, size, &exc, + collstart-startp, collend-startp, &newpos); + if (repunicode == NULL) + goto onError; + /* generate replacement */ + repsize = PyUnicode_GET_SIZE(repunicode); + if (charmaptranslate_makespace(&res, &str, + (str-PyUnicode_AS_UNICODE(res))+repsize+(endp-collend))) { + Py_DECREF(repunicode); + goto onError; + } + for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) + *str++ = *uni2; + p = startp + newpos; Py_DECREF(repunicode); - goto onError; } - Py_DECREF(repunicode); - i = newpos; } } + /* Resize if we allocated to much */ + respos = str-PyUnicode_AS_UNICODE(res); + if (respos<PyUnicode_GET_SIZE(res)) { + if (PyUnicode_Resize(&res, respos) < 0) + goto onError; + } Py_XDECREF(exc); Py_XDECREF(errorHandler); - return _PyUnicodeWriter_Finish(&writer); + return res; onError: - _PyUnicodeWriter_Dealloc(&writer); + Py_XDECREF(res); Py_XDECREF(exc); Py_XDECREF(errorHandler); return NULL; } -/* Deprecated. Use PyUnicode_Translate instead. */ -PyObject * -PyUnicode_TranslateCharmap(const Py_UNICODE *p, - Py_ssize_t size, - PyObject *mapping, - const char *errors) +PyObject *PyUnicode_Translate(PyObject *str, + PyObject *mapping, + const char *errors) { PyObject *result; - PyObject *unicode = PyUnicode_FromWideChar(p, size); - if (!unicode) - return NULL; - result = _PyUnicode_TranslateCharmap(unicode, mapping, errors); - Py_DECREF(unicode); - return result; -} - -PyObject * -PyUnicode_Translate(PyObject *str, - PyObject *mapping, - const char *errors) -{ - if (ensure_unicode(str) < 0) - return NULL; - return _PyUnicode_TranslateCharmap(str, mapping, errors); -} - -PyObject * -_PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode) -{ - if (!PyUnicode_Check(unicode)) { - PyErr_BadInternalCall(); - return NULL; - } - if (PyUnicode_READY(unicode) == -1) - return NULL; - if (PyUnicode_IS_ASCII(unicode)) { - /* If the string is already ASCII, just return the same string */ - Py_INCREF(unicode); - return unicode; - } - - Py_ssize_t len = PyUnicode_GET_LENGTH(unicode); - PyObject *result = PyUnicode_New(len, 127); - if (result == NULL) { - return NULL; - } - - Py_UCS1 *out = PyUnicode_1BYTE_DATA(result); - int kind = PyUnicode_KIND(unicode); - const void *data = PyUnicode_DATA(unicode); - Py_ssize_t i; - for (i = 0; i < len; ++i) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); - if (ch < 127) { - out[i] = ch; - } - else if (Py_UNICODE_ISSPACE(ch)) { - out[i] = ' '; - } - else { - int decimal = Py_UNICODE_TODECIMAL(ch); - if (decimal < 0) { - out[i] = '?'; - out[i+1] = '\0'; - _PyUnicode_LENGTH(result) = i + 1; - break; - } - out[i] = '0' + decimal; - } - } - assert(_PyUnicode_CheckConsistency(result, 1)); + str = PyUnicode_FromObject(str); + if (str == NULL) + goto onError; + result = PyUnicode_TranslateCharmap(PyUnicode_AS_UNICODE(str), + PyUnicode_GET_SIZE(str), + mapping, + errors); + Py_DECREF(str); return result; -} -PyObject * -PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, - Py_ssize_t length) -{ - PyObject *decimal; - Py_ssize_t i; - Py_UCS4 maxchar; - enum PyUnicode_Kind kind; - void *data; - - maxchar = 127; - for (i = 0; i < length; i++) { - Py_UCS4 ch = s[i]; - if (ch > 127) { - int decimal = Py_UNICODE_TODECIMAL(ch); - if (decimal >= 0) - ch = '0' + decimal; - maxchar = Py_MAX(maxchar, ch); - } - } - - /* Copy to a new string */ - decimal = PyUnicode_New(length, maxchar); - if (decimal == NULL) - return decimal; - kind = PyUnicode_KIND(decimal); - data = PyUnicode_DATA(decimal); - /* Iterate over code points */ - for (i = 0; i < length; i++) { - Py_UCS4 ch = s[i]; - if (ch > 127) { - int decimal = Py_UNICODE_TODECIMAL(ch); - if (decimal >= 0) - ch = '0' + decimal; - } - PyUnicode_WRITE(kind, data, i, ch); - } - return unicode_result(decimal); + onError: + Py_XDECREF(str); + return NULL; } + /* --- Decimal Encoder ---------------------------------------------------- */ -int -PyUnicode_EncodeDecimal(Py_UNICODE *s, - Py_ssize_t length, - char *output, - const char *errors) +int PyUnicode_EncodeDecimal(Py_UNICODE *s, + Py_ssize_t length, + char *output, + const char *errors) { - PyObject *unicode; - Py_ssize_t i; - enum PyUnicode_Kind kind; - void *data; + Py_UNICODE *p, *end; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + const char *encoding = "decimal"; + const char *reason = "invalid decimal Unicode string"; + /* the following variable is used for caching string comparisons + * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */ + int known_errorHandler = -1; if (output == NULL) { PyErr_BadArgument(); return -1; } - unicode = PyUnicode_FromWideChar(s, length); - if (unicode == NULL) - return -1; - - kind = PyUnicode_KIND(unicode); - data = PyUnicode_DATA(unicode); - - for (i=0; i < length; ) { - PyObject *exc; - Py_UCS4 ch; + p = s; + end = s + length; + while (p < end) { + register Py_UNICODE ch = *p; int decimal; - Py_ssize_t startpos; - - ch = PyUnicode_READ(kind, data, i); + PyObject *repunicode; + Py_ssize_t repsize; + Py_ssize_t newpos; + Py_UNICODE *uni2; + Py_UNICODE *collstart; + Py_UNICODE *collend; if (Py_UNICODE_ISSPACE(ch)) { *output++ = ' '; - i++; + ++p; continue; } decimal = Py_UNICODE_TODECIMAL(ch); if (decimal >= 0) { *output++ = '0' + decimal; - i++; + ++p; continue; } if (0 < ch && ch < 256) { *output++ = (char)ch; - i++; + ++p; continue; } - - startpos = i; - exc = NULL; - raise_encode_exception(&exc, "decimal", unicode, - startpos, startpos+1, - "invalid decimal Unicode string"); - Py_XDECREF(exc); - Py_DECREF(unicode); - return -1; + /* All other characters are considered unencodable */ + collstart = p; + for (collend = p+1; collend < end; collend++) { + if ((0 < *collend && *collend < 256) || + Py_UNICODE_ISSPACE(*collend) || + 0 <= Py_UNICODE_TODECIMAL(*collend)) + break; + } + /* cache callback name lookup + * (if not done yet, i.e. it's the first error) */ + if (known_errorHandler==-1) { + if ((errors==NULL) || (!strcmp(errors, "strict"))) + known_errorHandler = 1; + else if (!strcmp(errors, "replace")) + known_errorHandler = 2; + else if (!strcmp(errors, "ignore")) + known_errorHandler = 3; + else if (!strcmp(errors, "xmlcharrefreplace")) + known_errorHandler = 4; + else + known_errorHandler = 0; + } + switch (known_errorHandler) { + case 1: /* strict */ + raise_encode_exception(&exc, encoding, s, length, collstart-s, collend-s, reason); + goto onError; + case 2: /* replace */ + for (p = collstart; p < collend; ++p) + *output++ = '?'; + /* fall through */ + case 3: /* ignore */ + p = collend; + break; + case 4: /* xmlcharrefreplace */ + /* generate replacement (temporarily (mis)uses p) */ + for (p = collstart; p < collend;) { + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + output += sprintf(output, "&#%d;", ch); + } + p = collend; + break; + default: + repunicode = unicode_encode_call_errorhandler(errors, &errorHandler, + encoding, reason, s, length, &exc, + collstart-s, collend-s, &newpos); + if (repunicode == NULL) + goto onError; + /* generate replacement */ + repsize = PyUnicode_GET_SIZE(repunicode); + for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) { + Py_UNICODE ch = *uni2; + if (Py_UNICODE_ISSPACE(ch)) + *output++ = ' '; + else { + decimal = Py_UNICODE_TODECIMAL(ch); + if (decimal >= 0) + *output++ = '0' + decimal; + else if (0 < ch && ch < 256) + *output++ = (char)ch; + else { + Py_DECREF(repunicode); + raise_encode_exception(&exc, encoding, + s, length, collstart-s, collend-s, reason); + goto onError; + } + } + } + p = s + newpos; + Py_DECREF(repunicode); + } } /* 0-terminate the output string */ *output++ = '\0'; - Py_DECREF(unicode); + Py_XDECREF(exc); + Py_XDECREF(errorHandler); return 0; + + onError: + Py_XDECREF(exc); + Py_XDECREF(errorHandler); + return -1; } /* --- Helpers ------------------------------------------------------------ */ +#include "stringlib/unicodedefs.h" +#include "stringlib/fastsearch.h" + +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/partition.h" +#include "stringlib/split.h" + /* helper macro to fixup start/end slice values */ #define ADJUST_INDICES(start, end, len) \ if (end > len) \ @@ -9343,2345 +5388,1227 @@ PyUnicode_EncodeDecimal(Py_UNICODE *s, start = 0; \ } -static Py_ssize_t -any_find_slice(PyObject* s1, PyObject* s2, - Py_ssize_t start, - Py_ssize_t end, - int direction) -{ - int kind1, kind2; - void *buf1, *buf2; - Py_ssize_t len1, len2, result; - - kind1 = PyUnicode_KIND(s1); - kind2 = PyUnicode_KIND(s2); - if (kind1 < kind2) - return -1; +Py_ssize_t PyUnicode_Count(PyObject *str, + PyObject *substr, + Py_ssize_t start, + Py_ssize_t end) +{ + Py_ssize_t result; + PyUnicodeObject* str_obj; + PyUnicodeObject* sub_obj; - len1 = PyUnicode_GET_LENGTH(s1); - len2 = PyUnicode_GET_LENGTH(s2); - ADJUST_INDICES(start, end, len1); - if (end - start < len2) + str_obj = (PyUnicodeObject*) PyUnicode_FromObject(str); + if (!str_obj) return -1; - - buf1 = PyUnicode_DATA(s1); - buf2 = PyUnicode_DATA(s2); - if (len2 == 1) { - Py_UCS4 ch = PyUnicode_READ(kind2, buf2, 0); - result = findchar((const char *)buf1 + kind1*start, - kind1, end - start, ch, direction); - if (result == -1) - return -1; - else - return start + result; - } - - if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(s2, kind1); - if (!buf2) - return -2; - } - - if (direction > 0) { - switch (kind1) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(s1) && PyUnicode_IS_ASCII(s2)) - result = asciilib_find_slice(buf1, len1, buf2, len2, start, end); - else - result = ucs1lib_find_slice(buf1, len1, buf2, len2, start, end); - break; - case PyUnicode_2BYTE_KIND: - result = ucs2lib_find_slice(buf1, len1, buf2, len2, start, end); - break; - case PyUnicode_4BYTE_KIND: - result = ucs4lib_find_slice(buf1, len1, buf2, len2, start, end); - break; - default: - Py_UNREACHABLE(); - } - } - else { - switch (kind1) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(s1) && PyUnicode_IS_ASCII(s2)) - result = asciilib_rfind_slice(buf1, len1, buf2, len2, start, end); - else - result = ucs1lib_rfind_slice(buf1, len1, buf2, len2, start, end); - break; - case PyUnicode_2BYTE_KIND: - result = ucs2lib_rfind_slice(buf1, len1, buf2, len2, start, end); - break; - case PyUnicode_4BYTE_KIND: - result = ucs4lib_rfind_slice(buf1, len1, buf2, len2, start, end); - break; - default: - Py_UNREACHABLE(); - } - } - - if (kind2 != kind1) - PyMem_Free(buf2); - - return result; -} - -/* _PyUnicode_InsertThousandsGrouping() helper functions */ -#include "stringlib/localeutil.h" - -/** - * InsertThousandsGrouping: - * @writer: Unicode writer. - * @n_buffer: Number of characters in @buffer. - * @digits: Digits we're reading from. If count is non-NULL, this is unused. - * @d_pos: Start of digits string. - * @n_digits: The number of digits in the string, in which we want - * to put the grouping chars. - * @min_width: The minimum width of the digits in the output string. - * Output will be zero-padded on the left to fill. - * @grouping: see definition in localeconv(). - * @thousands_sep: see definition in localeconv(). - * - * There are 2 modes: counting and filling. If @writer is NULL, - * we are in counting mode, else filling mode. - * If counting, the required buffer size is returned. - * If filling, we know the buffer will be large enough, so we don't - * need to pass in the buffer size. - * Inserts thousand grouping characters (as defined by grouping and - * thousands_sep) into @writer. - * - * Return value: -1 on error, number of characters otherwise. - **/ -Py_ssize_t -_PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar) -{ - min_width = Py_MAX(0, min_width); - if (writer) { - assert(digits != NULL); - assert(maxchar == NULL); - } - else { - assert(digits == NULL); - assert(maxchar != NULL); - } - assert(0 <= d_pos); - assert(0 <= n_digits); - assert(grouping != NULL); - - if (digits != NULL) { - if (PyUnicode_READY(digits) == -1) { - return -1; - } - } - if (PyUnicode_READY(thousands_sep) == -1) { + sub_obj = (PyUnicodeObject*) PyUnicode_FromObject(substr); + if (!sub_obj) { + Py_DECREF(str_obj); return -1; } - Py_ssize_t count = 0; - Py_ssize_t n_zeros; - int loop_broken = 0; - int use_separator = 0; /* First time through, don't append the - separator. They only go between - groups. */ - Py_ssize_t buffer_pos; - Py_ssize_t digits_pos; - Py_ssize_t len; - Py_ssize_t n_chars; - Py_ssize_t remaining = n_digits; /* Number of chars remaining to - be looked at */ - /* A generator that returns all of the grouping widths, until it - returns 0. */ - GroupGenerator groupgen; - GroupGenerator_init(&groupgen, grouping); - const Py_ssize_t thousands_sep_len = PyUnicode_GET_LENGTH(thousands_sep); - - /* if digits are not grouped, thousands separator - should be an empty string */ - assert(!(grouping[0] == CHAR_MAX && thousands_sep_len != 0)); - - digits_pos = d_pos + n_digits; - if (writer) { - buffer_pos = writer->pos + n_buffer; - assert(buffer_pos <= PyUnicode_GET_LENGTH(writer->buffer)); - assert(digits_pos <= PyUnicode_GET_LENGTH(digits)); - } - else { - buffer_pos = n_buffer; - } - - if (!writer) { - *maxchar = 127; - } - - while ((len = GroupGenerator_next(&groupgen)) > 0) { - len = Py_MIN(len, Py_MAX(Py_MAX(remaining, min_width), 1)); - n_zeros = Py_MAX(0, len - remaining); - n_chars = Py_MAX(0, Py_MIN(remaining, len)); - - /* Use n_zero zero's and n_chars chars */ - - /* Count only, don't do anything. */ - count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; - - /* Copy into the writer. */ - InsertThousandsGrouping_fill(writer, &buffer_pos, - digits, &digits_pos, - n_chars, n_zeros, - use_separator ? thousands_sep : NULL, - thousands_sep_len, maxchar); - - /* Use a separator next time. */ - use_separator = 1; - - remaining -= n_chars; - min_width -= len; - - if (remaining <= 0 && min_width <= 0) { - loop_broken = 1; - break; - } - min_width -= thousands_sep_len; - } - if (!loop_broken) { - /* We left the loop without using a break statement. */ - - len = Py_MAX(Py_MAX(remaining, min_width), 1); - n_zeros = Py_MAX(0, len - remaining); - n_chars = Py_MAX(0, Py_MIN(remaining, len)); + ADJUST_INDICES(start, end, str_obj->length); + result = stringlib_count( + str_obj->str + start, end - start, sub_obj->str, sub_obj->length, + PY_SSIZE_T_MAX + ); - /* Use n_zero zero's and n_chars chars */ - count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; + Py_DECREF(sub_obj); + Py_DECREF(str_obj); - /* Copy into the writer. */ - InsertThousandsGrouping_fill(writer, &buffer_pos, - digits, &digits_pos, - n_chars, n_zeros, - use_separator ? thousands_sep : NULL, - thousands_sep_len, maxchar); - } - return count; + return result; } - -Py_ssize_t -PyUnicode_Count(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end) +Py_ssize_t PyUnicode_Find(PyObject *str, + PyObject *sub, + Py_ssize_t start, + Py_ssize_t end, + int direction) { Py_ssize_t result; - int kind1, kind2; - void *buf1 = NULL, *buf2 = NULL; - Py_ssize_t len1, len2; - - if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) - return -1; - - kind1 = PyUnicode_KIND(str); - kind2 = PyUnicode_KIND(substr); - if (kind1 < kind2) - return 0; - len1 = PyUnicode_GET_LENGTH(str); - len2 = PyUnicode_GET_LENGTH(substr); - ADJUST_INDICES(start, end, len1); - if (end - start < len2) - return 0; - - buf1 = PyUnicode_DATA(str); - buf2 = PyUnicode_DATA(substr); - if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(substr, kind1); - if (!buf2) - goto onError; + str = PyUnicode_FromObject(str); + if (!str) + return -2; + sub = PyUnicode_FromObject(sub); + if (!sub) { + Py_DECREF(str); + return -2; } - switch (kind1) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(str) && PyUnicode_IS_ASCII(substr)) - result = asciilib_count( - ((Py_UCS1*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - else - result = ucs1lib_count( - ((Py_UCS1*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_2BYTE_KIND: - result = ucs2lib_count( - ((Py_UCS2*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX + if (direction > 0) + result = stringlib_find_slice( + PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str), + PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub), + start, end ); - break; - case PyUnicode_4BYTE_KIND: - result = ucs4lib_count( - ((Py_UCS4*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX + else + result = stringlib_rfind_slice( + PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str), + PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub), + start, end ); - break; - default: - Py_UNREACHABLE(); - } - if (kind2 != kind1) - PyMem_Free(buf2); + Py_DECREF(str); + Py_DECREF(sub); return result; - onError: - if (kind2 != kind1 && buf2) - PyMem_Free(buf2); - return -1; } -Py_ssize_t -PyUnicode_Find(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end, - int direction) -{ - if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) - return -2; - - return any_find_slice(str, substr, start, end, direction); -} - -Py_ssize_t -PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, - Py_ssize_t start, Py_ssize_t end, - int direction) +static +int tailmatch(PyUnicodeObject *self, + PyUnicodeObject *substring, + Py_ssize_t start, + Py_ssize_t end, + int direction) { - int kind; - Py_ssize_t len, result; - if (PyUnicode_READY(str) == -1) - return -2; - len = PyUnicode_GET_LENGTH(str); - ADJUST_INDICES(start, end, len); - if (end - start < 1) - return -1; - kind = PyUnicode_KIND(str); - result = findchar(PyUnicode_1BYTE_DATA(str) + kind*start, - kind, end-start, ch, direction); - if (result == -1) - return -1; - else - return start + result; -} - -static int -tailmatch(PyObject *self, - PyObject *substring, - Py_ssize_t start, - Py_ssize_t end, - int direction) -{ - int kind_self; - int kind_sub; - void *data_self; - void *data_sub; - Py_ssize_t offset; - Py_ssize_t i; - Py_ssize_t end_sub; - - if (PyUnicode_READY(self) == -1 || - PyUnicode_READY(substring) == -1) - return -1; + if (substring->length == 0) + return 1; - ADJUST_INDICES(start, end, PyUnicode_GET_LENGTH(self)); - end -= PyUnicode_GET_LENGTH(substring); + ADJUST_INDICES(start, end, self->length); + end -= substring->length; if (end < start) return 0; - if (PyUnicode_GET_LENGTH(substring) == 0) - return 1; - - kind_self = PyUnicode_KIND(self); - data_self = PyUnicode_DATA(self); - kind_sub = PyUnicode_KIND(substring); - data_sub = PyUnicode_DATA(substring); - end_sub = PyUnicode_GET_LENGTH(substring) - 1; - - if (direction > 0) - offset = end; - else - offset = start; - - if (PyUnicode_READ(kind_self, data_self, offset) == - PyUnicode_READ(kind_sub, data_sub, 0) && - PyUnicode_READ(kind_self, data_self, offset + end_sub) == - PyUnicode_READ(kind_sub, data_sub, end_sub)) { - /* If both are of the same kind, memcmp is sufficient */ - if (kind_self == kind_sub) { - return ! memcmp((char *)data_self + - (offset * PyUnicode_KIND(substring)), - data_sub, - PyUnicode_GET_LENGTH(substring) * - PyUnicode_KIND(substring)); - } - /* otherwise we have to compare each character by first accessing it */ - else { - /* We do not need to compare 0 and len(substring)-1 because - the if statement above ensured already that they are equal - when we end up here. */ - for (i = 1; i < end_sub; ++i) { - if (PyUnicode_READ(kind_self, data_self, offset + i) != - PyUnicode_READ(kind_sub, data_sub, i)) - return 0; - } + if (direction > 0) { + if (Py_UNICODE_MATCH(self, end, substring)) + return 1; + } else { + if (Py_UNICODE_MATCH(self, start, substring)) return 1; - } } return 0; } -Py_ssize_t -PyUnicode_Tailmatch(PyObject *str, - PyObject *substr, - Py_ssize_t start, - Py_ssize_t end, - int direction) +Py_ssize_t PyUnicode_Tailmatch(PyObject *str, + PyObject *substr, + Py_ssize_t start, + Py_ssize_t end, + int direction) { - if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) + Py_ssize_t result; + + str = PyUnicode_FromObject(str); + if (str == NULL) return -1; + substr = PyUnicode_FromObject(substr); + if (substr == NULL) { + Py_DECREF(str); + return -1; + } - return tailmatch(str, substr, start, end, direction); + result = tailmatch((PyUnicodeObject *)str, + (PyUnicodeObject *)substr, + start, end, direction); + Py_DECREF(str); + Py_DECREF(substr); + return result; } -static PyObject * -ascii_upper_or_lower(PyObject *self, int lower) +/* Apply fixfct filter to the Unicode object self and return a + reference to the modified object */ + +static +PyObject *fixup(PyUnicodeObject *self, + int (*fixfct)(PyUnicodeObject *s)) { - Py_ssize_t len = PyUnicode_GET_LENGTH(self); - char *resdata, *data = PyUnicode_DATA(self); - PyObject *res; - res = PyUnicode_New(len, 127); - if (res == NULL) - return NULL; - resdata = PyUnicode_DATA(res); - if (lower) - _Py_bytes_lower(resdata, data, len); - else - _Py_bytes_upper(resdata, data, len); - return res; -} + PyUnicodeObject *u; -static Py_UCS4 -handle_capital_sigma(int kind, void *data, Py_ssize_t length, Py_ssize_t i) -{ - Py_ssize_t j; - int final_sigma; - Py_UCS4 c = 0; /* initialize to prevent gcc warning */ - /* U+03A3 is in the Final_Sigma context when, it is found like this: + u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length); + if (u == NULL) + return NULL; - \p{cased}\p{case-ignorable}*U+03A3!(\p{case-ignorable}*\p{cased}) + Py_UNICODE_COPY(u->str, self->str, self->length); - where ! is a negation and \p{xxx} is a character with property xxx. - */ - for (j = i - 1; j >= 0; j--) { - c = PyUnicode_READ(kind, data, j); - if (!_PyUnicode_IsCaseIgnorable(c)) - break; - } - final_sigma = j >= 0 && _PyUnicode_IsCased(c); - if (final_sigma) { - for (j = i + 1; j < length; j++) { - c = PyUnicode_READ(kind, data, j); - if (!_PyUnicode_IsCaseIgnorable(c)) - break; - } - final_sigma = j == length || !_PyUnicode_IsCased(c); + if (!fixfct(u) && PyUnicode_CheckExact(self)) { + /* fixfct should return TRUE if it modified the buffer. If + FALSE, return a reference to the original buffer instead + (to save space, not time) */ + Py_INCREF(self); + Py_DECREF(u); + return (PyObject*) self; } - return (final_sigma) ? 0x3C2 : 0x3C3; + return (PyObject*) u; } -static int -lower_ucs4(int kind, void *data, Py_ssize_t length, Py_ssize_t i, - Py_UCS4 c, Py_UCS4 *mapped) +static +int fixupper(PyUnicodeObject *self) { - /* Obscure special case. */ - if (c == 0x3A3) { - mapped[0] = handle_capital_sigma(kind, data, length, i); - return 1; - } - return _PyUnicode_ToLowerFull(c, mapped); -} + Py_ssize_t len = self->length; + Py_UNICODE *s = self->str; + int status = 0; -static Py_ssize_t -do_capitalize(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) -{ - Py_ssize_t i, k = 0; - int n_res, j; - Py_UCS4 c, mapped[3]; + while (len-- > 0) { + register Py_UNICODE ch; - c = PyUnicode_READ(kind, data, 0); - n_res = _PyUnicode_ToTitleFull(c, mapped); - for (j = 0; j < n_res; j++) { - *maxchar = Py_MAX(*maxchar, mapped[j]); - res[k++] = mapped[j]; - } - for (i = 1; i < length; i++) { - c = PyUnicode_READ(kind, data, i); - n_res = lower_ucs4(kind, data, length, i, c, mapped); - for (j = 0; j < n_res; j++) { - *maxchar = Py_MAX(*maxchar, mapped[j]); - res[k++] = mapped[j]; + ch = Py_UNICODE_TOUPPER(*s); + if (ch != *s) { + status = 1; + *s = ch; } + s++; } - return k; + + return status; } -static Py_ssize_t -do_swapcase(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) { - Py_ssize_t i, k = 0; +static +int fixlower(PyUnicodeObject *self) +{ + Py_ssize_t len = self->length; + Py_UNICODE *s = self->str; + int status = 0; - for (i = 0; i < length; i++) { - Py_UCS4 c = PyUnicode_READ(kind, data, i), mapped[3]; - int n_res, j; - if (Py_UNICODE_ISUPPER(c)) { - n_res = lower_ucs4(kind, data, length, i, c, mapped); - } - else if (Py_UNICODE_ISLOWER(c)) { - n_res = _PyUnicode_ToUpperFull(c, mapped); - } - else { - n_res = 1; - mapped[0] = c; - } - for (j = 0; j < n_res; j++) { - *maxchar = Py_MAX(*maxchar, mapped[j]); - res[k++] = mapped[j]; + while (len-- > 0) { + register Py_UNICODE ch; + + ch = Py_UNICODE_TOLOWER(*s); + if (ch != *s) { + status = 1; + *s = ch; } + s++; } - return k; + + return status; } -static Py_ssize_t -do_upper_or_lower(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, - Py_UCS4 *maxchar, int lower) +static +int fixswapcase(PyUnicodeObject *self) { - Py_ssize_t i, k = 0; + Py_ssize_t len = self->length; + Py_UNICODE *s = self->str; + int status = 0; - for (i = 0; i < length; i++) { - Py_UCS4 c = PyUnicode_READ(kind, data, i), mapped[3]; - int n_res, j; - if (lower) - n_res = lower_ucs4(kind, data, length, i, c, mapped); - else - n_res = _PyUnicode_ToUpperFull(c, mapped); - for (j = 0; j < n_res; j++) { - *maxchar = Py_MAX(*maxchar, mapped[j]); - res[k++] = mapped[j]; + while (len-- > 0) { + if (Py_UNICODE_ISUPPER(*s)) { + *s = Py_UNICODE_TOLOWER(*s); + status = 1; + } else if (Py_UNICODE_ISLOWER(*s)) { + *s = Py_UNICODE_TOUPPER(*s); + status = 1; } + s++; } - return k; -} -static Py_ssize_t -do_upper(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) -{ - return do_upper_or_lower(kind, data, length, res, maxchar, 0); -} - -static Py_ssize_t -do_lower(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) -{ - return do_upper_or_lower(kind, data, length, res, maxchar, 1); + return status; } -static Py_ssize_t -do_casefold(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) +static +int fixcapitalize(PyUnicodeObject *self) { - Py_ssize_t i, k = 0; + Py_ssize_t len = self->length; + Py_UNICODE *s = self->str; + int status = 0; - for (i = 0; i < length; i++) { - Py_UCS4 c = PyUnicode_READ(kind, data, i); - Py_UCS4 mapped[3]; - int j, n_res = _PyUnicode_ToFoldedFull(c, mapped); - for (j = 0; j < n_res; j++) { - *maxchar = Py_MAX(*maxchar, mapped[j]); - res[k++] = mapped[j]; + if (len == 0) + return 0; + if (!Py_UNICODE_ISUPPER(*s)) { + *s = Py_UNICODE_TOUPPER(*s); + status = 1; + } + s++; + while (--len > 0) { + if (!Py_UNICODE_ISLOWER(*s)) { + *s = Py_UNICODE_TOLOWER(*s); + status = 1; } + s++; } - return k; + return status; } -static Py_ssize_t -do_title(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar) +static +int fixtitle(PyUnicodeObject *self) { - Py_ssize_t i, k = 0; + register Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register Py_UNICODE *e; int previous_is_cased; + /* Shortcut for single character strings */ + if (PyUnicode_GET_SIZE(self) == 1) { + Py_UNICODE ch = Py_UNICODE_TOTITLE(*p); + if (*p != ch) { + *p = ch; + return 1; + } + else + return 0; + } + + e = p + PyUnicode_GET_SIZE(self); previous_is_cased = 0; - for (i = 0; i < length; i++) { - const Py_UCS4 c = PyUnicode_READ(kind, data, i); - Py_UCS4 mapped[3]; - int n_res, j; + for (; p < e; p++) { + register const Py_UNICODE ch = *p; if (previous_is_cased) - n_res = lower_ucs4(kind, data, length, i, c, mapped); + *p = Py_UNICODE_TOLOWER(ch); else - n_res = _PyUnicode_ToTitleFull(c, mapped); - - for (j = 0; j < n_res; j++) { - *maxchar = Py_MAX(*maxchar, mapped[j]); - res[k++] = mapped[j]; - } + *p = Py_UNICODE_TOTITLE(ch); - previous_is_cased = _PyUnicode_IsCased(c); - } - return k; -} - -static PyObject * -case_operation(PyObject *self, - Py_ssize_t (*perform)(int, void *, Py_ssize_t, Py_UCS4 *, Py_UCS4 *)) -{ - PyObject *res = NULL; - Py_ssize_t length, newlength = 0; - int kind, outkind; - void *data, *outdata; - Py_UCS4 maxchar = 0, *tmp, *tmpend; - - assert(PyUnicode_IS_READY(self)); - - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); - length = PyUnicode_GET_LENGTH(self); - if ((size_t) length > PY_SSIZE_T_MAX / (3 * sizeof(Py_UCS4))) { - PyErr_SetString(PyExc_OverflowError, "string is too long"); - return NULL; - } - tmp = PyMem_MALLOC(sizeof(Py_UCS4) * 3 * length); - if (tmp == NULL) - return PyErr_NoMemory(); - newlength = perform(kind, data, length, tmp, &maxchar); - res = PyUnicode_New(newlength, maxchar); - if (res == NULL) - goto leave; - tmpend = tmp + newlength; - outdata = PyUnicode_DATA(res); - outkind = PyUnicode_KIND(res); - switch (outkind) { - case PyUnicode_1BYTE_KIND: - _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS1, tmp, tmpend, outdata); - break; - case PyUnicode_2BYTE_KIND: - _PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS2, tmp, tmpend, outdata); - break; - case PyUnicode_4BYTE_KIND: - memcpy(outdata, tmp, sizeof(Py_UCS4) * newlength); - break; - default: - Py_UNREACHABLE(); + if (Py_UNICODE_ISLOWER(ch) || + Py_UNICODE_ISUPPER(ch) || + Py_UNICODE_ISTITLE(ch)) + previous_is_cased = 1; + else + previous_is_cased = 0; } - leave: - PyMem_FREE(tmp); - return res; + return 1; } PyObject * PyUnicode_Join(PyObject *separator, PyObject *seq) { - PyObject *res; - PyObject *fseq; - Py_ssize_t seqlen; - PyObject **items; + PyObject *internal_separator = NULL; + const Py_UNICODE blank = ' '; + const Py_UNICODE *sep = ␣ + Py_ssize_t seplen = 1; + PyUnicodeObject *res = NULL; /* the result */ + Py_ssize_t res_alloc = 100; /* # allocated bytes for string in res */ + Py_ssize_t res_used; /* # used bytes */ + Py_UNICODE *res_p; /* pointer to free byte in res's string area */ + PyObject *fseq; /* PySequence_Fast(seq) */ + Py_ssize_t seqlen; /* len(fseq) -- number of items in sequence */ + PyObject *item; + Py_ssize_t i; fseq = PySequence_Fast(seq, "can only join an iterable"); if (fseq == NULL) { return NULL; } - /* NOTE: the following code can't call back into Python code, - * so we are sure that fseq won't be mutated. + /* Grrrr. A codec may be invoked to convert str objects to + * Unicode, and so it's possible to call back into Python code + * during PyUnicode_FromObject(), and so it's possible for a sick + * codec to change the size of fseq (if seq is a list). Therefore + * we have to keep refetching the size -- can't assume seqlen + * is invariant. */ - - items = PySequence_Fast_ITEMS(fseq); seqlen = PySequence_Fast_GET_SIZE(fseq); - res = _PyUnicode_JoinArray(separator, items, seqlen); - Py_DECREF(fseq); - return res; -} - -PyObject * -_PyUnicode_JoinArray(PyObject *separator, PyObject *const *items, Py_ssize_t seqlen) -{ - PyObject *res = NULL; /* the result */ - PyObject *sep = NULL; - Py_ssize_t seplen; - PyObject *item; - Py_ssize_t sz, i, res_offset; - Py_UCS4 maxchar; - Py_UCS4 item_maxchar; - int use_memcpy; - unsigned char *res_data = NULL, *sep_data = NULL; - PyObject *last_obj; - unsigned int kind = 0; - /* If empty sequence, return u"". */ if (seqlen == 0) { - _Py_RETURN_UNICODE_EMPTY(); + res = _PyUnicode_New(0); /* empty sequence; return u"" */ + goto Done; } - /* If singleton sequence with an exact Unicode, return that. */ - last_obj = NULL; if (seqlen == 1) { - if (PyUnicode_CheckExact(items[0])) { - res = items[0]; - Py_INCREF(res); - return res; + item = PySequence_Fast_GET_ITEM(fseq, 0); + if (PyUnicode_CheckExact(item)) { + Py_INCREF(item); + res = (PyUnicodeObject *)item; + goto Done; } - seplen = 0; - maxchar = 0; } - else { - /* Set up sep and seplen */ + + /* At least two items to join, or one that isn't exact Unicode. */ + if (seqlen > 1) { + /* Set up sep and seplen -- they're needed. */ if (separator == NULL) { - /* fall back to a blank space separator */ - sep = PyUnicode_FromOrdinal(' '); - if (!sep) - goto onError; + sep = ␣ seplen = 1; - maxchar = 32; } else { - if (!PyUnicode_Check(separator)) { - PyErr_Format(PyExc_TypeError, - "separator: expected str instance," - " %.80s found", - Py_TYPE(separator)->tp_name); + internal_separator = PyUnicode_FromObject(separator); + if (internal_separator == NULL) goto onError; - } - if (PyUnicode_READY(separator)) - goto onError; - sep = separator; - seplen = PyUnicode_GET_LENGTH(separator); - maxchar = PyUnicode_MAX_CHAR_VALUE(separator); - /* inc refcount to keep this code path symmetric with the - above case of a blank separator */ - Py_INCREF(sep); + sep = PyUnicode_AS_UNICODE(internal_separator); + seplen = PyUnicode_GET_SIZE(internal_separator); + /* In case PyUnicode_FromObject() mutated seq. */ + seqlen = PySequence_Fast_GET_SIZE(fseq); } - last_obj = sep; } - /* There are at least two things to join, or else we have a subclass - * of str in the sequence. - * Do a pre-pass to figure out the total amount of space we'll - * need (sz), and see whether all argument are strings. - */ - sz = 0; -#ifdef Py_DEBUG - use_memcpy = 0; -#else - use_memcpy = 1; -#endif - for (i = 0; i < seqlen; i++) { - size_t add_sz; - item = items[i]; - if (!PyUnicode_Check(item)) { + /* Get space. */ + res = _PyUnicode_New(res_alloc); + if (res == NULL) + goto onError; + res_p = PyUnicode_AS_UNICODE(res); + res_used = 0; + + for (i = 0; i < seqlen; ++i) { + Py_ssize_t itemlen; + Py_ssize_t new_res_used; + + item = PySequence_Fast_GET_ITEM(fseq, i); + /* Convert item to Unicode. */ + if (! PyUnicode_Check(item) && ! PyString_Check(item)) { PyErr_Format(PyExc_TypeError, - "sequence item %zd: expected str instance," + "sequence item %zd: expected string or Unicode," " %.80s found", i, Py_TYPE(item)->tp_name); goto onError; } - if (PyUnicode_READY(item) == -1) - goto onError; - add_sz = PyUnicode_GET_LENGTH(item); - item_maxchar = PyUnicode_MAX_CHAR_VALUE(item); - maxchar = Py_MAX(maxchar, item_maxchar); - if (i != 0) { - add_sz += seplen; - } - if (add_sz > (size_t)(PY_SSIZE_T_MAX - sz)) { - PyErr_SetString(PyExc_OverflowError, - "join() result is too long for a Python string"); + item = PyUnicode_FromObject(item); + if (item == NULL) goto onError; + /* We own a reference to item from here on. */ + + /* In case PyUnicode_FromObject() mutated seq. */ + seqlen = PySequence_Fast_GET_SIZE(fseq); + + /* Make sure we have enough space for the separator and the item. */ + itemlen = PyUnicode_GET_SIZE(item); + if (res_used > PY_SSIZE_T_MAX - itemlen) + goto Overflow; + new_res_used = res_used + itemlen; + if (i < seqlen - 1) { + if (new_res_used > PY_SSIZE_T_MAX - seplen) + goto Overflow; + new_res_used += seplen; + } + if (new_res_used > res_alloc) { + /* double allocated size until it's big enough */ + do { + if (res_alloc > PY_SSIZE_T_MAX / 2) + goto Overflow; + res_alloc += res_alloc; + } while (new_res_used > res_alloc); + if (_PyUnicode_Resize(&res, res_alloc) < 0) { + Py_DECREF(item); + goto onError; + } + res_p = PyUnicode_AS_UNICODE(res) + res_used; } - sz += add_sz; - if (use_memcpy && last_obj != NULL) { - if (PyUnicode_KIND(last_obj) != PyUnicode_KIND(item)) - use_memcpy = 0; + + /* Copy item, and maybe the separator. */ + Py_UNICODE_COPY(res_p, PyUnicode_AS_UNICODE(item), itemlen); + res_p += itemlen; + if (i < seqlen - 1) { + Py_UNICODE_COPY(res_p, sep, seplen); + res_p += seplen; } - last_obj = item; + Py_DECREF(item); + res_used = new_res_used; } - res = PyUnicode_New(sz, maxchar); - if (res == NULL) + /* Shrink res to match the used area; this probably can't fail, + * but it's cheap to check. + */ + if (_PyUnicode_Resize(&res, res_used) < 0) goto onError; - /* Catenate everything. */ -#ifdef Py_DEBUG - use_memcpy = 0; -#else - if (use_memcpy) { - res_data = PyUnicode_1BYTE_DATA(res); - kind = PyUnicode_KIND(res); - if (seplen != 0) - sep_data = PyUnicode_1BYTE_DATA(sep); - } -#endif - if (use_memcpy) { - for (i = 0; i < seqlen; ++i) { - Py_ssize_t itemlen; - item = items[i]; - - /* Copy item, and maybe the separator. */ - if (i && seplen != 0) { - memcpy(res_data, - sep_data, - kind * seplen); - res_data += kind * seplen; - } - - itemlen = PyUnicode_GET_LENGTH(item); - if (itemlen != 0) { - memcpy(res_data, - PyUnicode_DATA(item), - kind * itemlen); - res_data += kind * itemlen; - } - } - assert(res_data == PyUnicode_1BYTE_DATA(res) - + kind * PyUnicode_GET_LENGTH(res)); - } - else { - for (i = 0, res_offset = 0; i < seqlen; ++i) { - Py_ssize_t itemlen; - item = items[i]; - - /* Copy item, and maybe the separator. */ - if (i && seplen != 0) { - _PyUnicode_FastCopyCharacters(res, res_offset, sep, 0, seplen); - res_offset += seplen; - } - - itemlen = PyUnicode_GET_LENGTH(item); - if (itemlen != 0) { - _PyUnicode_FastCopyCharacters(res, res_offset, item, 0, itemlen); - res_offset += itemlen; - } - } - assert(res_offset == PyUnicode_GET_LENGTH(res)); - } + Done: + Py_XDECREF(internal_separator); + Py_DECREF(fseq); + return (PyObject *)res; - Py_XDECREF(sep); - assert(_PyUnicode_CheckConsistency(res, 1)); - return res; + Overflow: + PyErr_SetString(PyExc_OverflowError, + "join() result is too long for a Python string"); + Py_DECREF(item); + /* fall through */ onError: - Py_XDECREF(sep); + Py_XDECREF(internal_separator); + Py_DECREF(fseq); Py_XDECREF(res); return NULL; } -void -_PyUnicode_FastFill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, - Py_UCS4 fill_char) -{ - const enum PyUnicode_Kind kind = PyUnicode_KIND(unicode); - void *data = PyUnicode_DATA(unicode); - assert(PyUnicode_IS_READY(unicode)); - assert(unicode_modifiable(unicode)); - assert(fill_char <= PyUnicode_MAX_CHAR_VALUE(unicode)); - assert(start >= 0); - assert(start + length <= PyUnicode_GET_LENGTH(unicode)); - unicode_fill(kind, data, fill_char, start, length); -} - -Py_ssize_t -PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, - Py_UCS4 fill_char) -{ - Py_ssize_t maxlen; - - if (!PyUnicode_Check(unicode)) { - PyErr_BadInternalCall(); - return -1; - } - if (PyUnicode_READY(unicode) == -1) - return -1; - if (unicode_check_modifiable(unicode)) - return -1; - - if (start < 0) { - PyErr_SetString(PyExc_IndexError, "string index out of range"); - return -1; - } - if (fill_char > PyUnicode_MAX_CHAR_VALUE(unicode)) { - PyErr_SetString(PyExc_ValueError, - "fill character is bigger than " - "the string maximum character"); - return -1; - } - - maxlen = PyUnicode_GET_LENGTH(unicode) - start; - length = Py_MIN(maxlen, length); - if (length <= 0) - return 0; - - _PyUnicode_FastFill(unicode, start, length, fill_char); - return length; -} - -static PyObject * -pad(PyObject *self, - Py_ssize_t left, - Py_ssize_t right, - Py_UCS4 fill) +static +PyUnicodeObject *pad(PyUnicodeObject *self, + Py_ssize_t left, + Py_ssize_t right, + Py_UNICODE fill) { - PyObject *u; - Py_UCS4 maxchar; - int kind; - void *data; + PyUnicodeObject *u; if (left < 0) left = 0; if (right < 0) right = 0; - if (left == 0 && right == 0) - return unicode_result_unchanged(self); + if (left == 0 && right == 0 && PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return self; + } - if (left > PY_SSIZE_T_MAX - _PyUnicode_LENGTH(self) || - right > PY_SSIZE_T_MAX - (left + _PyUnicode_LENGTH(self))) { + if (left > PY_SSIZE_T_MAX - self->length || + right > PY_SSIZE_T_MAX - (left + self->length)) { PyErr_SetString(PyExc_OverflowError, "padded string is too long"); return NULL; } - maxchar = PyUnicode_MAX_CHAR_VALUE(self); - maxchar = Py_MAX(maxchar, fill); - u = PyUnicode_New(left + _PyUnicode_LENGTH(self) + right, maxchar); - if (!u) - return NULL; + u = _PyUnicode_New(left + self->length + right); + if (u) { + if (left) + Py_UNICODE_FILL(u->str, fill, left); + Py_UNICODE_COPY(u->str + left, self->str, self->length); + if (right) + Py_UNICODE_FILL(u->str + left + self->length, fill, right); + } - kind = PyUnicode_KIND(u); - data = PyUnicode_DATA(u); - if (left) - unicode_fill(kind, data, fill, 0, left); - if (right) - unicode_fill(kind, data, fill, left + _PyUnicode_LENGTH(self), right); - _PyUnicode_FastCopyCharacters(u, left, self, 0, _PyUnicode_LENGTH(self)); - assert(_PyUnicode_CheckConsistency(u, 1)); return u; } -PyObject * -PyUnicode_Splitlines(PyObject *string, int keepends) +PyObject *PyUnicode_Splitlines(PyObject *string, int keepends) { PyObject *list; - if (ensure_unicode(string) < 0) + string = PyUnicode_FromObject(string); + if (string == NULL) return NULL; - switch (PyUnicode_KIND(string)) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(string)) - list = asciilib_splitlines( - string, PyUnicode_1BYTE_DATA(string), - PyUnicode_GET_LENGTH(string), keepends); - else - list = ucs1lib_splitlines( - string, PyUnicode_1BYTE_DATA(string), - PyUnicode_GET_LENGTH(string), keepends); - break; - case PyUnicode_2BYTE_KIND: - list = ucs2lib_splitlines( - string, PyUnicode_2BYTE_DATA(string), - PyUnicode_GET_LENGTH(string), keepends); - break; - case PyUnicode_4BYTE_KIND: - list = ucs4lib_splitlines( - string, PyUnicode_4BYTE_DATA(string), - PyUnicode_GET_LENGTH(string), keepends); - break; - default: - Py_UNREACHABLE(); - } + list = stringlib_splitlines( + (PyObject*) string, PyUnicode_AS_UNICODE(string), + PyUnicode_GET_SIZE(string), keepends); + + Py_DECREF(string); return list; } -static PyObject * -split(PyObject *self, - PyObject *substring, - Py_ssize_t maxcount) +static +PyObject *split(PyUnicodeObject *self, + PyUnicodeObject *substring, + Py_ssize_t maxcount) { - int kind1, kind2; - void *buf1, *buf2; - Py_ssize_t len1, len2; - PyObject* out; - if (maxcount < 0) maxcount = PY_SSIZE_T_MAX; - if (PyUnicode_READY(self) == -1) - return NULL; - if (substring == NULL) - switch (PyUnicode_KIND(self)) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(self)) - return asciilib_split_whitespace( - self, PyUnicode_1BYTE_DATA(self), - PyUnicode_GET_LENGTH(self), maxcount - ); - else - return ucs1lib_split_whitespace( - self, PyUnicode_1BYTE_DATA(self), - PyUnicode_GET_LENGTH(self), maxcount - ); - case PyUnicode_2BYTE_KIND: - return ucs2lib_split_whitespace( - self, PyUnicode_2BYTE_DATA(self), - PyUnicode_GET_LENGTH(self), maxcount - ); - case PyUnicode_4BYTE_KIND: - return ucs4lib_split_whitespace( - self, PyUnicode_4BYTE_DATA(self), - PyUnicode_GET_LENGTH(self), maxcount - ); - default: - Py_UNREACHABLE(); - } - - if (PyUnicode_READY(substring) == -1) - return NULL; - - kind1 = PyUnicode_KIND(self); - kind2 = PyUnicode_KIND(substring); - len1 = PyUnicode_GET_LENGTH(self); - len2 = PyUnicode_GET_LENGTH(substring); - if (kind1 < kind2 || len1 < len2) { - out = PyList_New(1); - if (out == NULL) - return NULL; - Py_INCREF(self); - PyList_SET_ITEM(out, 0, self); - return out; - } - buf1 = PyUnicode_DATA(self); - buf2 = PyUnicode_DATA(substring); - if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(substring, kind1); - if (!buf2) - return NULL; - } + return stringlib_split_whitespace( + (PyObject*) self, self->str, self->length, maxcount + ); - switch (kind1) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(self) && PyUnicode_IS_ASCII(substring)) - out = asciilib_split( - self, buf1, len1, buf2, len2, maxcount); - else - out = ucs1lib_split( - self, buf1, len1, buf2, len2, maxcount); - break; - case PyUnicode_2BYTE_KIND: - out = ucs2lib_split( - self, buf1, len1, buf2, len2, maxcount); - break; - case PyUnicode_4BYTE_KIND: - out = ucs4lib_split( - self, buf1, len1, buf2, len2, maxcount); - break; - default: - out = NULL; - } - if (kind2 != kind1) - PyMem_Free(buf2); - return out; + return stringlib_split( + (PyObject*) self, self->str, self->length, + substring->str, substring->length, + maxcount + ); } -static PyObject * -rsplit(PyObject *self, - PyObject *substring, - Py_ssize_t maxcount) +static +PyObject *rsplit(PyUnicodeObject *self, + PyUnicodeObject *substring, + Py_ssize_t maxcount) { - int kind1, kind2; - void *buf1, *buf2; - Py_ssize_t len1, len2; - PyObject* out; - if (maxcount < 0) maxcount = PY_SSIZE_T_MAX; - if (PyUnicode_READY(self) == -1) - return NULL; - if (substring == NULL) - switch (PyUnicode_KIND(self)) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(self)) - return asciilib_rsplit_whitespace( - self, PyUnicode_1BYTE_DATA(self), - PyUnicode_GET_LENGTH(self), maxcount - ); - else - return ucs1lib_rsplit_whitespace( - self, PyUnicode_1BYTE_DATA(self), - PyUnicode_GET_LENGTH(self), maxcount - ); - case PyUnicode_2BYTE_KIND: - return ucs2lib_rsplit_whitespace( - self, PyUnicode_2BYTE_DATA(self), - PyUnicode_GET_LENGTH(self), maxcount - ); - case PyUnicode_4BYTE_KIND: - return ucs4lib_rsplit_whitespace( - self, PyUnicode_4BYTE_DATA(self), - PyUnicode_GET_LENGTH(self), maxcount - ); - default: - Py_UNREACHABLE(); - } - - if (PyUnicode_READY(substring) == -1) - return NULL; - - kind1 = PyUnicode_KIND(self); - kind2 = PyUnicode_KIND(substring); - len1 = PyUnicode_GET_LENGTH(self); - len2 = PyUnicode_GET_LENGTH(substring); - if (kind1 < kind2 || len1 < len2) { - out = PyList_New(1); - if (out == NULL) - return NULL; - Py_INCREF(self); - PyList_SET_ITEM(out, 0, self); - return out; - } - buf1 = PyUnicode_DATA(self); - buf2 = PyUnicode_DATA(substring); - if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(substring, kind1); - if (!buf2) - return NULL; - } - - switch (kind1) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(self) && PyUnicode_IS_ASCII(substring)) - out = asciilib_rsplit( - self, buf1, len1, buf2, len2, maxcount); - else - out = ucs1lib_rsplit( - self, buf1, len1, buf2, len2, maxcount); - break; - case PyUnicode_2BYTE_KIND: - out = ucs2lib_rsplit( - self, buf1, len1, buf2, len2, maxcount); - break; - case PyUnicode_4BYTE_KIND: - out = ucs4lib_rsplit( - self, buf1, len1, buf2, len2, maxcount); - break; - default: - out = NULL; - } - if (kind2 != kind1) - PyMem_Free(buf2); - return out; -} + return stringlib_rsplit_whitespace( + (PyObject*) self, self->str, self->length, maxcount + ); -static Py_ssize_t -anylib_find(int kind, PyObject *str1, void *buf1, Py_ssize_t len1, - PyObject *str2, void *buf2, Py_ssize_t len2, Py_ssize_t offset) -{ - switch (kind) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(str1) && PyUnicode_IS_ASCII(str2)) - return asciilib_find(buf1, len1, buf2, len2, offset); - else - return ucs1lib_find(buf1, len1, buf2, len2, offset); - case PyUnicode_2BYTE_KIND: - return ucs2lib_find(buf1, len1, buf2, len2, offset); - case PyUnicode_4BYTE_KIND: - return ucs4lib_find(buf1, len1, buf2, len2, offset); - } - Py_UNREACHABLE(); + return stringlib_rsplit( + (PyObject*) self, self->str, self->length, + substring->str, substring->length, + maxcount + ); } -static Py_ssize_t -anylib_count(int kind, PyObject *sstr, void* sbuf, Py_ssize_t slen, - PyObject *str1, void *buf1, Py_ssize_t len1, Py_ssize_t maxcount) +static +PyObject *replace(PyUnicodeObject *self, + PyUnicodeObject *str1, + PyUnicodeObject *str2, + Py_ssize_t maxcount) { - switch (kind) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(sstr) && PyUnicode_IS_ASCII(str1)) - return asciilib_count(sbuf, slen, buf1, len1, maxcount); - else - return ucs1lib_count(sbuf, slen, buf1, len1, maxcount); - case PyUnicode_2BYTE_KIND: - return ucs2lib_count(sbuf, slen, buf1, len1, maxcount); - case PyUnicode_4BYTE_KIND: - return ucs4lib_count(sbuf, slen, buf1, len1, maxcount); - } - Py_UNREACHABLE(); -} - -static void -replace_1char_inplace(PyObject *u, Py_ssize_t pos, - Py_UCS4 u1, Py_UCS4 u2, Py_ssize_t maxcount) -{ - int kind = PyUnicode_KIND(u); - void *data = PyUnicode_DATA(u); - Py_ssize_t len = PyUnicode_GET_LENGTH(u); - if (kind == PyUnicode_1BYTE_KIND) { - ucs1lib_replace_1char_inplace((Py_UCS1 *)data + pos, - (Py_UCS1 *)data + len, - u1, u2, maxcount); - } - else if (kind == PyUnicode_2BYTE_KIND) { - ucs2lib_replace_1char_inplace((Py_UCS2 *)data + pos, - (Py_UCS2 *)data + len, - u1, u2, maxcount); - } - else { - assert(kind == PyUnicode_4BYTE_KIND); - ucs4lib_replace_1char_inplace((Py_UCS4 *)data + pos, - (Py_UCS4 *)data + len, - u1, u2, maxcount); - } -} - -static PyObject * -replace(PyObject *self, PyObject *str1, - PyObject *str2, Py_ssize_t maxcount) -{ - PyObject *u; - char *sbuf = PyUnicode_DATA(self); - char *buf1 = PyUnicode_DATA(str1); - char *buf2 = PyUnicode_DATA(str2); - int srelease = 0, release1 = 0, release2 = 0; - int skind = PyUnicode_KIND(self); - int kind1 = PyUnicode_KIND(str1); - int kind2 = PyUnicode_KIND(str2); - Py_ssize_t slen = PyUnicode_GET_LENGTH(self); - Py_ssize_t len1 = PyUnicode_GET_LENGTH(str1); - Py_ssize_t len2 = PyUnicode_GET_LENGTH(str2); - int mayshrink; - Py_UCS4 maxchar, maxchar_str1, maxchar_str2; - - if (slen < len1) - goto nothing; + PyUnicodeObject *u; if (maxcount < 0) maxcount = PY_SSIZE_T_MAX; - else if (maxcount == 0) - goto nothing; - - if (str1 == str2) - goto nothing; - - maxchar = PyUnicode_MAX_CHAR_VALUE(self); - maxchar_str1 = PyUnicode_MAX_CHAR_VALUE(str1); - if (maxchar < maxchar_str1) - /* substring too wide to be present */ + else if (maxcount == 0 || self->length == 0) goto nothing; - maxchar_str2 = PyUnicode_MAX_CHAR_VALUE(str2); - /* Replacing str1 with str2 may cause a maxchar reduction in the - result string. */ - mayshrink = (maxchar_str2 < maxchar_str1) && (maxchar == maxchar_str1); - maxchar = Py_MAX(maxchar, maxchar_str2); - if (len1 == len2) { + if (str1->length == str2->length) { + Py_ssize_t i; /* same length */ - if (len1 == 0) + if (str1->length == 0) goto nothing; - if (len1 == 1) { + if (str1->length == 1) { /* replace characters */ - Py_UCS4 u1, u2; - Py_ssize_t pos; - - u1 = PyUnicode_READ(kind1, buf1, 0); - pos = findchar(sbuf, skind, slen, u1, 1); - if (pos < 0) + Py_UNICODE u1, u2; + if (!findchar(self->str, self->length, str1->str[0])) goto nothing; - u2 = PyUnicode_READ(kind2, buf2, 0); - u = PyUnicode_New(slen, maxchar); + u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length); if (!u) - goto error; - - _PyUnicode_FastCopyCharacters(u, 0, self, 0, slen); - replace_1char_inplace(u, pos, u1, u2, maxcount); - } - else { - int rkind = skind; - char *res; - Py_ssize_t i; - - if (kind1 < rkind) { - /* widen substring */ - buf1 = _PyUnicode_AsKind(str1, rkind); - if (!buf1) goto error; - release1 = 1; - } - i = anylib_find(rkind, self, sbuf, slen, str1, buf1, len1, 0); + return NULL; + Py_UNICODE_COPY(u->str, self->str, self->length); + u1 = str1->str[0]; + u2 = str2->str[0]; + for (i = 0; i < u->length; i++) + if (u->str[i] == u1) { + if (--maxcount < 0) + break; + u->str[i] = u2; + } + } else { + i = stringlib_find( + self->str, self->length, str1->str, str1->length, 0 + ); if (i < 0) goto nothing; - if (rkind > kind2) { - /* widen replacement */ - buf2 = _PyUnicode_AsKind(str2, rkind); - if (!buf2) goto error; - release2 = 1; - } - else if (rkind < kind2) { - /* widen self and buf1 */ - rkind = kind2; - if (release1) PyMem_Free(buf1); - release1 = 0; - sbuf = _PyUnicode_AsKind(self, rkind); - if (!sbuf) goto error; - srelease = 1; - buf1 = _PyUnicode_AsKind(str1, rkind); - if (!buf1) goto error; - release1 = 1; - } - u = PyUnicode_New(slen, maxchar); + u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length); if (!u) - goto error; - assert(PyUnicode_KIND(u) == rkind); - res = PyUnicode_DATA(u); + return NULL; + Py_UNICODE_COPY(u->str, self->str, self->length); - memcpy(res, sbuf, rkind * slen); /* change everything in-place, starting with this one */ - memcpy(res + rkind * i, - buf2, - rkind * len2); - i += len1; + Py_UNICODE_COPY(u->str+i, str2->str, str2->length); + i += str1->length; while ( --maxcount > 0) { - i = anylib_find(rkind, self, - sbuf+rkind*i, slen-i, - str1, buf1, len1, i); + i = stringlib_find(self->str+i, self->length-i, + str1->str, str1->length, + i); if (i == -1) break; - memcpy(res + rkind * i, - buf2, - rkind * len2); - i += len1; + Py_UNICODE_COPY(u->str+i, str2->str, str2->length); + i += str1->length; } } - } - else { - Py_ssize_t n, i, j, ires; - Py_ssize_t new_size; - int rkind = skind; - char *res; - - if (kind1 < rkind) { - /* widen substring */ - buf1 = _PyUnicode_AsKind(str1, rkind); - if (!buf1) goto error; - release1 = 1; - } - n = anylib_count(rkind, self, sbuf, slen, str1, buf1, len1, maxcount); + } else { + + Py_ssize_t n, i, j; + Py_ssize_t new_size, delta; + Py_UNICODE *p; + + /* replace strings */ + n = stringlib_count(self->str, self->length, str1->str, str1->length, + maxcount); if (n == 0) goto nothing; - if (kind2 < rkind) { - /* widen replacement */ - buf2 = _PyUnicode_AsKind(str2, rkind); - if (!buf2) goto error; - release2 = 1; - } - else if (kind2 > rkind) { - /* widen self and buf1 */ - rkind = kind2; - sbuf = _PyUnicode_AsKind(self, rkind); - if (!sbuf) goto error; - srelease = 1; - if (release1) PyMem_Free(buf1); - release1 = 0; - buf1 = _PyUnicode_AsKind(str1, rkind); - if (!buf1) goto error; - release1 = 1; - } - /* new_size = PyUnicode_GET_LENGTH(self) + n * (PyUnicode_GET_LENGTH(str2) - - PyUnicode_GET_LENGTH(str1))); */ - if (len1 < len2 && len2 - len1 > (PY_SSIZE_T_MAX - slen) / n) { + /* new_size = self->length + n * (str2->length - str1->length)); */ + delta = (str2->length - str1->length); + if (delta == 0) { + new_size = self->length; + } else { + assert(n > 0); + if (delta > (PY_SSIZE_T_MAX - self->length) / n) { PyErr_SetString(PyExc_OverflowError, "replace string is too long"); - goto error; - } - new_size = slen + n * (len2 - len1); - if (new_size == 0) { - _Py_INCREF_UNICODE_EMPTY(); - if (!unicode_empty) - goto error; - u = unicode_empty; - goto done; - } - if (new_size > (PY_SSIZE_T_MAX / rkind)) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - goto error; + return NULL; + } + new_size = self->length + delta * n; } - u = PyUnicode_New(new_size, maxchar); + u = _PyUnicode_New(new_size); if (!u) - goto error; - assert(PyUnicode_KIND(u) == rkind); - res = PyUnicode_DATA(u); - ires = i = 0; - if (len1 > 0) { + return NULL; + i = 0; + p = u->str; + if (str1->length > 0) { while (n-- > 0) { /* look for next match */ - j = anylib_find(rkind, self, - sbuf + rkind * i, slen-i, - str1, buf1, len1, i); + j = stringlib_find(self->str+i, self->length-i, + str1->str, str1->length, + i); if (j == -1) break; else if (j > i) { /* copy unchanged part [i:j] */ - memcpy(res + rkind * ires, - sbuf + rkind * i, - rkind * (j-i)); - ires += j - i; + Py_UNICODE_COPY(p, self->str+i, j-i); + p += j - i; } /* copy substitution string */ - if (len2 > 0) { - memcpy(res + rkind * ires, - buf2, - rkind * len2); - ires += len2; + if (str2->length > 0) { + Py_UNICODE_COPY(p, str2->str, str2->length); + p += str2->length; } - i = j + len1; + i = j + str1->length; } - if (i < slen) + if (i < self->length) /* copy tail [i:] */ - memcpy(res + rkind * ires, - sbuf + rkind * i, - rkind * (slen-i)); - } - else { + Py_UNICODE_COPY(p, self->str+i, self->length-i); + } else { /* interleave */ while (n > 0) { - memcpy(res + rkind * ires, - buf2, - rkind * len2); - ires += len2; + Py_UNICODE_COPY(p, str2->str, str2->length); + p += str2->length; if (--n <= 0) break; - memcpy(res + rkind * ires, - sbuf + rkind * i, - rkind); - ires++; - i++; + *p++ = self->str[i++]; } - memcpy(res + rkind * ires, - sbuf + rkind * i, - rkind * (slen-i)); + Py_UNICODE_COPY(p, self->str+i, self->length-i); } } - - if (mayshrink) { - unicode_adjust_maxchar(&u); - if (u == NULL) - goto error; - } - - done: - if (srelease) - PyMem_FREE(sbuf); - if (release1) - PyMem_FREE(buf1); - if (release2) - PyMem_FREE(buf2); - assert(_PyUnicode_CheckConsistency(u, 1)); - return u; + return (PyObject *) u; nothing: /* nothing to replace; return original string (when possible) */ - if (srelease) - PyMem_FREE(sbuf); - if (release1) - PyMem_FREE(buf1); - if (release2) - PyMem_FREE(buf2); - return unicode_result_unchanged(self); - - error: - if (srelease && sbuf) - PyMem_FREE(sbuf); - if (release1 && buf1) - PyMem_FREE(buf1); - if (release2 && buf2) - PyMem_FREE(buf2); - return NULL; + if (PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return (PyObject *) self; + } + return PyUnicode_FromUnicode(self->str, self->length); } /* --- Unicode Object Methods --------------------------------------------- */ -/*[clinic input] -str.title as unicode_title - -Return a version of the string where each word is titlecased. - -More specifically, words start with uppercased characters and all remaining -cased characters have lower case. -[clinic start generated code]*/ +PyDoc_STRVAR(title__doc__, + "S.title() -> unicode\n\ +\n\ +Return a titlecased version of S, i.e. words start with title case\n\ +characters, all remaining cased characters have lower case."); -static PyObject * -unicode_title_impl(PyObject *self) -/*[clinic end generated code: output=c75ae03809574902 input=fa945d669b26e683]*/ +static PyObject* +unicode_title(PyUnicodeObject *self) { - if (PyUnicode_READY(self) == -1) - return NULL; - return case_operation(self, do_title); + return fixup(self, fixtitle); } -/*[clinic input] -str.capitalize as unicode_capitalize +PyDoc_STRVAR(capitalize__doc__, + "S.capitalize() -> unicode\n\ +\n\ +Return a capitalized version of S, i.e. make the first character\n\ +have upper case and the rest lower case."); -Return a capitalized version of the string. +static PyObject* +unicode_capitalize(PyUnicodeObject *self) +{ + return fixup(self, fixcapitalize); +} -More specifically, make the first character have upper case and the rest lower -case. -[clinic start generated code]*/ +#if 0 +PyDoc_STRVAR(capwords__doc__, + "S.capwords() -> unicode\n\ +\n\ +Apply .capitalize() to all words in S and return the result with\n\ +normalized whitespace (all whitespace strings are replaced by ' ')."); -static PyObject * -unicode_capitalize_impl(PyObject *self) -/*[clinic end generated code: output=e49a4c333cdb7667 input=f4cbf1016938da6d]*/ +static PyObject* +unicode_capwords(PyUnicodeObject *self) { - if (PyUnicode_READY(self) == -1) + PyObject *list; + PyObject *item; + Py_ssize_t i; + + /* Split into words */ + list = split(self, NULL, -1); + if (!list) return NULL; - if (PyUnicode_GET_LENGTH(self) == 0) - return unicode_result_unchanged(self); - return case_operation(self, do_capitalize); -} -/*[clinic input] -str.casefold as unicode_casefold + /* Capitalize each word */ + for (i = 0; i < PyList_GET_SIZE(list); i++) { + item = fixup((PyUnicodeObject *)PyList_GET_ITEM(list, i), + fixcapitalize); + if (item == NULL) + goto onError; + Py_DECREF(PyList_GET_ITEM(list, i)); + PyList_SET_ITEM(list, i, item); + } -Return a version of the string suitable for caseless comparisons. -[clinic start generated code]*/ + /* Join the words to form a new string */ + item = PyUnicode_Join(NULL, list); -static PyObject * -unicode_casefold_impl(PyObject *self) -/*[clinic end generated code: output=0120daf657ca40af input=384d66cc2ae30daf]*/ -{ - if (PyUnicode_READY(self) == -1) - return NULL; - if (PyUnicode_IS_ASCII(self)) - return ascii_upper_or_lower(self, 1); - return case_operation(self, do_casefold); + onError: + Py_DECREF(list); + return (PyObject *)item; } +#endif - -/* Argument converter. Accepts a single Unicode character. */ +/* Argument converter. Coerces to a single unicode character */ static int convert_uc(PyObject *obj, void *addr) { - Py_UCS4 *fillcharloc = (Py_UCS4 *)addr; + Py_UNICODE *fillcharloc = (Py_UNICODE *)addr; + PyObject *uniobj; + Py_UNICODE *unistr; - if (!PyUnicode_Check(obj)) { - PyErr_Format(PyExc_TypeError, - "The fill character must be a unicode character, " - "not %.100s", Py_TYPE(obj)->tp_name); + uniobj = PyUnicode_FromObject(obj); + if (uniobj == NULL) { + PyErr_SetString(PyExc_TypeError, + "The fill character cannot be converted to Unicode"); return 0; } - if (PyUnicode_READY(obj) < 0) - return 0; - if (PyUnicode_GET_LENGTH(obj) != 1) { + if (PyUnicode_GET_SIZE(uniobj) != 1) { PyErr_SetString(PyExc_TypeError, "The fill character must be exactly one character long"); + Py_DECREF(uniobj); return 0; } - *fillcharloc = PyUnicode_READ_CHAR(obj, 0); + unistr = PyUnicode_AS_UNICODE(uniobj); + *fillcharloc = unistr[0]; + Py_DECREF(uniobj); return 1; } -/*[clinic input] -str.center as unicode_center - - width: Py_ssize_t - fillchar: Py_UCS4 = ' ' - / - -Return a centered string of length width. - -Padding is done using the specified fill character (default is a space). -[clinic start generated code]*/ +PyDoc_STRVAR(center__doc__, + "S.center(width[, fillchar]) -> unicode\n\ +\n\ +Return S centered in a Unicode string of length width. Padding is\n\ +done using the specified fill character (default is a space)"); static PyObject * -unicode_center_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=420c8859effc7c0c input=b42b247eb26e6519]*/ +unicode_center(PyUnicodeObject *self, PyObject *args) { Py_ssize_t marg, left; + Py_ssize_t width; + Py_UNICODE fillchar = ' '; - if (PyUnicode_READY(self) == -1) + if (!PyArg_ParseTuple(args, "n|O&:center", &width, convert_uc, &fillchar)) return NULL; - if (PyUnicode_GET_LENGTH(self) >= width) - return unicode_result_unchanged(self); + if (self->length >= width && PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return (PyObject*) self; + } - marg = width - PyUnicode_GET_LENGTH(self); + marg = width - self->length; left = marg / 2 + (marg & width & 1); - return pad(self, left, marg - left, fillchar); + return (PyObject*) pad(self, left, marg - left, fillchar); } -/* This function assumes that str1 and str2 are readied by the caller. */ +#if 0 -static int -unicode_compare(PyObject *str1, PyObject *str2) -{ -#define COMPARE(TYPE1, TYPE2) \ - do { \ - TYPE1* p1 = (TYPE1 *)data1; \ - TYPE2* p2 = (TYPE2 *)data2; \ - TYPE1* end = p1 + len; \ - Py_UCS4 c1, c2; \ - for (; p1 != end; p1++, p2++) { \ - c1 = *p1; \ - c2 = *p2; \ - if (c1 != c2) \ - return (c1 < c2) ? -1 : 1; \ - } \ - } \ - while (0) - - int kind1, kind2; - void *data1, *data2; - Py_ssize_t len1, len2, len; - - kind1 = PyUnicode_KIND(str1); - kind2 = PyUnicode_KIND(str2); - data1 = PyUnicode_DATA(str1); - data2 = PyUnicode_DATA(str2); - len1 = PyUnicode_GET_LENGTH(str1); - len2 = PyUnicode_GET_LENGTH(str2); - len = Py_MIN(len1, len2); - - switch(kind1) { - case PyUnicode_1BYTE_KIND: - { - switch(kind2) { - case PyUnicode_1BYTE_KIND: - { - int cmp = memcmp(data1, data2, len); - /* normalize result of memcmp() into the range [-1; 1] */ - if (cmp < 0) - return -1; - if (cmp > 0) - return 1; - break; - } - case PyUnicode_2BYTE_KIND: - COMPARE(Py_UCS1, Py_UCS2); - break; - case PyUnicode_4BYTE_KIND: - COMPARE(Py_UCS1, Py_UCS4); - break; - default: - Py_UNREACHABLE(); - } - break; - } - case PyUnicode_2BYTE_KIND: - { - switch(kind2) { - case PyUnicode_1BYTE_KIND: - COMPARE(Py_UCS2, Py_UCS1); - break; - case PyUnicode_2BYTE_KIND: - { - COMPARE(Py_UCS2, Py_UCS2); - break; - } - case PyUnicode_4BYTE_KIND: - COMPARE(Py_UCS2, Py_UCS4); - break; - default: - Py_UNREACHABLE(); - } - break; - } - case PyUnicode_4BYTE_KIND: - { - switch(kind2) { - case PyUnicode_1BYTE_KIND: - COMPARE(Py_UCS4, Py_UCS1); - break; - case PyUnicode_2BYTE_KIND: - COMPARE(Py_UCS4, Py_UCS2); - break; - case PyUnicode_4BYTE_KIND: - { -#if defined(HAVE_WMEMCMP) && SIZEOF_WCHAR_T == 4 - int cmp = wmemcmp((wchar_t *)data1, (wchar_t *)data2, len); - /* normalize result of wmemcmp() into the range [-1; 1] */ - if (cmp < 0) - return -1; - if (cmp > 0) - return 1; -#else - COMPARE(Py_UCS4, Py_UCS4); -#endif - break; - } - default: - Py_UNREACHABLE(); - } - break; - } - default: - Py_UNREACHABLE(); - } +/* This code should go into some future Unicode collation support + module. The basic comparison should compare ordinals on a naive + basis (this is what Java does and thus Jython too). */ - if (len1 == len2) - return 0; - if (len1 < len2) - return -1; - else - return 1; +/* speedy UTF-16 code point order comparison */ +/* gleaned from: */ +/* http://www-4.ibm.com/software/developer/library/utf16.html?dwzone=unicode */ -#undef COMPARE -} +static short utf16Fixup[32] = +{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0x2000, -0x800, -0x800, -0x800, -0x800 +}; static int -unicode_compare_eq(PyObject *str1, PyObject *str2) +unicode_compare(PyUnicodeObject *str1, PyUnicodeObject *str2) { - int kind; - void *data1, *data2; - Py_ssize_t len; - int cmp; + Py_ssize_t len1, len2; - len = PyUnicode_GET_LENGTH(str1); - if (PyUnicode_GET_LENGTH(str2) != len) - return 0; - kind = PyUnicode_KIND(str1); - if (PyUnicode_KIND(str2) != kind) - return 0; - data1 = PyUnicode_DATA(str1); - data2 = PyUnicode_DATA(str2); + Py_UNICODE *s1 = str1->str; + Py_UNICODE *s2 = str2->str; - cmp = memcmp(data1, data2, len * kind); - return (cmp == 0); -} + len1 = str1->length; + len2 = str2->length; + while (len1 > 0 && len2 > 0) { + Py_UNICODE c1, c2; -int -PyUnicode_Compare(PyObject *left, PyObject *right) -{ - if (PyUnicode_Check(left) && PyUnicode_Check(right)) { - if (PyUnicode_READY(left) == -1 || - PyUnicode_READY(right) == -1) - return -1; + c1 = *s1++; + c2 = *s2++; - /* a string is equal to itself */ - if (left == right) - return 0; + if (c1 > (1<<11) * 26) + c1 += utf16Fixup[c1>>11]; + if (c2 > (1<<11) * 26) + c2 += utf16Fixup[c2>>11]; + /* now c1 and c2 are in UTF-32-compatible order */ - return unicode_compare(left, right); - } - PyErr_Format(PyExc_TypeError, - "Can't compare %.100s and %.100s", - left->ob_type->tp_name, - right->ob_type->tp_name); - return -1; -} + if (c1 != c2) + return (c1 < c2) ? -1 : 1; -int -PyUnicode_CompareWithASCIIString(PyObject* uni, const char* str) -{ - Py_ssize_t i; - int kind; - Py_UCS4 chr; - const unsigned char *ustr = (const unsigned char *)str; - - assert(_PyUnicode_CHECK(uni)); - if (!PyUnicode_IS_READY(uni)) { - const wchar_t *ws = _PyUnicode_WSTR(uni); - /* Compare Unicode string and source character set string */ - for (i = 0; (chr = ws[i]) && ustr[i]; i++) { - if (chr != ustr[i]) - return (chr < ustr[i]) ? -1 : 1; - } - /* This check keeps Python strings that end in '\0' from comparing equal - to C strings identical up to that point. */ - if (_PyUnicode_WSTR_LENGTH(uni) != i || chr) - return 1; /* uni is longer */ - if (ustr[i]) - return -1; /* str is longer */ - return 0; - } - kind = PyUnicode_KIND(uni); - if (kind == PyUnicode_1BYTE_KIND) { - const void *data = PyUnicode_1BYTE_DATA(uni); - size_t len1 = (size_t)PyUnicode_GET_LENGTH(uni); - size_t len, len2 = strlen(str); - int cmp; - - len = Py_MIN(len1, len2); - cmp = memcmp(data, str, len); - if (cmp != 0) { - if (cmp < 0) - return -1; - else - return 1; - } - if (len1 > len2) - return 1; /* uni is longer */ - if (len1 < len2) - return -1; /* str is longer */ - return 0; - } - else { - void *data = PyUnicode_DATA(uni); - /* Compare Unicode string and source character set string */ - for (i = 0; (chr = PyUnicode_READ(kind, data, i)) && str[i]; i++) - if (chr != (unsigned char)str[i]) - return (chr < (unsigned char)(str[i])) ? -1 : 1; - /* This check keeps Python strings that end in '\0' from comparing equal - to C strings identical up to that point. */ - if (PyUnicode_GET_LENGTH(uni) != i || chr) - return 1; /* uni is longer */ - if (str[i]) - return -1; /* str is longer */ - return 0; + len1--; len2--; } -} -static int -non_ready_unicode_equal_to_ascii_string(PyObject *unicode, const char *str) -{ - size_t i, len; - const wchar_t *p; - len = (size_t)_PyUnicode_WSTR_LENGTH(unicode); - if (strlen(str) != len) - return 0; - p = _PyUnicode_WSTR(unicode); - assert(p); - for (i = 0; i < len; i++) { - unsigned char c = (unsigned char)str[i]; - if (c >= 128 || p[i] != (wchar_t)c) - return 0; - } - return 1; + return (len1 < len2) ? -1 : (len1 != len2); } -int -_PyUnicode_EqualToASCIIString(PyObject *unicode, const char *str) -{ - size_t len; - assert(_PyUnicode_CHECK(unicode)); - assert(str); -#ifndef NDEBUG - for (const char *p = str; *p; p++) { - assert((unsigned char)*p < 128); - } -#endif - if (PyUnicode_READY(unicode) == -1) { - /* Memory error or bad data */ - PyErr_Clear(); - return non_ready_unicode_equal_to_ascii_string(unicode, str); - } - if (!PyUnicode_IS_ASCII(unicode)) - return 0; - len = (size_t)PyUnicode_GET_LENGTH(unicode); - return strlen(str) == len && - memcmp(PyUnicode_1BYTE_DATA(unicode), str, len) == 0; -} +#else -int -_PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) +static int +unicode_compare(PyUnicodeObject *str1, PyUnicodeObject *str2) { - PyObject *right_uni; - Py_hash_t hash; + register Py_ssize_t len1, len2; - assert(_PyUnicode_CHECK(left)); - assert(right->string); -#ifndef NDEBUG - for (const char *p = right->string; *p; p++) { - assert((unsigned char)*p < 128); - } -#endif + Py_UNICODE *s1 = str1->str; + Py_UNICODE *s2 = str2->str; - if (PyUnicode_READY(left) == -1) { - /* memory error or bad data */ - PyErr_Clear(); - return non_ready_unicode_equal_to_ascii_string(left, right->string); - } + len1 = str1->length; + len2 = str2->length; - if (!PyUnicode_IS_ASCII(left)) - return 0; + while (len1 > 0 && len2 > 0) { + Py_UNICODE c1, c2; - right_uni = _PyUnicode_FromId(right); /* borrowed */ - if (right_uni == NULL) { - /* memory error or bad data */ - PyErr_Clear(); - return _PyUnicode_EqualToASCIIString(left, right->string); - } + c1 = *s1++; + c2 = *s2++; - if (left == right_uni) - return 1; - - if (PyUnicode_CHECK_INTERNED(left)) - return 0; + if (c1 != c2) + return (c1 < c2) ? -1 : 1; - assert(_PyUnicode_HASH(right_uni) != -1); - hash = _PyUnicode_HASH(left); - if (hash != -1 && hash != _PyUnicode_HASH(right_uni)) - return 0; + len1--; len2--; + } - return unicode_compare_eq(left, right_uni); + return (len1 < len2) ? -1 : (len1 != len2); } -PyObject * -PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) +#endif + +int PyUnicode_Compare(PyObject *left, + PyObject *right) { + PyUnicodeObject *u = NULL, *v = NULL; int result; - if (!PyUnicode_Check(left) || !PyUnicode_Check(right)) - Py_RETURN_NOTIMPLEMENTED; - - if (PyUnicode_READY(left) == -1 || - PyUnicode_READY(right) == -1) - return NULL; + /* Coerce the two arguments */ + u = (PyUnicodeObject *)PyUnicode_FromObject(left); + if (u == NULL) + goto onError; + v = (PyUnicodeObject *)PyUnicode_FromObject(right); + if (v == NULL) + goto onError; - if (left == right) { - switch (op) { - case Py_EQ: - case Py_LE: - case Py_GE: - /* a string is equal to itself */ - Py_RETURN_TRUE; - case Py_NE: - case Py_LT: - case Py_GT: - Py_RETURN_FALSE; - default: - PyErr_BadArgument(); - return NULL; - } - } - else if (op == Py_EQ || op == Py_NE) { - result = unicode_compare_eq(left, right); - result ^= (op == Py_NE); - return PyBool_FromLong(result); - } - else { - result = unicode_compare(left, right); - Py_RETURN_RICHCOMPARE(result, 0, op); + /* Shortcut for empty or interned objects */ + if (v == u) { + Py_DECREF(u); + Py_DECREF(v); + return 0; } -} -int -_PyUnicode_EQ(PyObject *aa, PyObject *bb) -{ - return unicode_eq(aa, bb); + result = unicode_compare(u, v); + + Py_DECREF(u); + Py_DECREF(v); + return result; + + onError: + Py_XDECREF(u); + Py_XDECREF(v); + return -1; } -int -PyUnicode_Contains(PyObject *str, PyObject *substr) +PyObject *PyUnicode_RichCompare(PyObject *left, + PyObject *right, + int op) { - int kind1, kind2; - void *buf1, *buf2; - Py_ssize_t len1, len2; int result; - if (!PyUnicode_Check(substr)) { - PyErr_Format(PyExc_TypeError, - "'in <string>' requires string as left operand, not %.100s", - Py_TYPE(substr)->tp_name); - return -1; - } - if (PyUnicode_READY(substr) == -1) - return -1; - if (ensure_unicode(str) < 0) - return -1; - - kind1 = PyUnicode_KIND(str); - kind2 = PyUnicode_KIND(substr); - if (kind1 < kind2) - return 0; - len1 = PyUnicode_GET_LENGTH(str); - len2 = PyUnicode_GET_LENGTH(substr); - if (len1 < len2) - return 0; - buf1 = PyUnicode_DATA(str); - buf2 = PyUnicode_DATA(substr); - if (len2 == 1) { - Py_UCS4 ch = PyUnicode_READ(kind2, buf2, 0); - result = findchar((const char *)buf1, kind1, len1, ch, 1) != -1; - return result; - } - if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(substr, kind1); - if (!buf2) - return -1; - } + result = PyUnicode_Compare(left, right); + if (result == -1 && PyErr_Occurred()) + goto onError; - switch (kind1) { - case PyUnicode_1BYTE_KIND: - result = ucs1lib_find(buf1, len1, buf2, len2, 0) != -1; + /* Convert the return value to a Boolean */ + switch (op) { + case Py_EQ: + result = (result == 0); break; - case PyUnicode_2BYTE_KIND: - result = ucs2lib_find(buf1, len1, buf2, len2, 0) != -1; + case Py_NE: + result = (result != 0); break; - case PyUnicode_4BYTE_KIND: - result = ucs4lib_find(buf1, len1, buf2, len2, 0) != -1; + case Py_LE: + result = (result <= 0); + break; + case Py_GE: + result = (result >= 0); + break; + case Py_LT: + result = (result == -1); + break; + case Py_GT: + result = (result == 1); break; - default: - Py_UNREACHABLE(); } + return PyBool_FromLong(result); - if (kind2 != kind1) - PyMem_Free(buf2); - - return result; -} - -/* Concat to string or Unicode object giving a new Unicode object. */ + onError: -PyObject * -PyUnicode_Concat(PyObject *left, PyObject *right) -{ - PyObject *result; - Py_UCS4 maxchar, maxchar2; - Py_ssize_t left_len, right_len, new_len; + /* Standard case - if (ensure_unicode(left) < 0) - return NULL; + Type errors mean that PyUnicode_FromObject() could not convert + one of the arguments (usually the right hand side) to Unicode, + ie. we can't handle the comparison request. However, it is + possible that the other object knows a comparison method, which + is why we return Py_NotImplemented to give the other object a + chance. - if (!PyUnicode_Check(right)) { - PyErr_Format(PyExc_TypeError, - "can only concatenate str (not \"%.200s\") to str", - right->ob_type->tp_name); - return NULL; + */ + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; } - if (PyUnicode_READY(right) < 0) + if (op != Py_EQ && op != Py_NE) return NULL; - /* Shortcuts */ - if (left == unicode_empty) - return PyUnicode_FromObject(right); - if (right == unicode_empty) - return PyUnicode_FromObject(left); - - left_len = PyUnicode_GET_LENGTH(left); - right_len = PyUnicode_GET_LENGTH(right); - if (left_len > PY_SSIZE_T_MAX - right_len) { - PyErr_SetString(PyExc_OverflowError, - "strings are too large to concat"); - return NULL; - } - new_len = left_len + right_len; + /* Equality comparison. - maxchar = PyUnicode_MAX_CHAR_VALUE(left); - maxchar2 = PyUnicode_MAX_CHAR_VALUE(right); - maxchar = Py_MAX(maxchar, maxchar2); + This is a special case: we silence any PyExc_UnicodeDecodeError + and instead turn it into a PyErr_UnicodeWarning. - /* Concat the two Unicode strings */ - result = PyUnicode_New(new_len, maxchar); - if (result == NULL) - return NULL; - _PyUnicode_FastCopyCharacters(result, 0, left, 0, left_len); - _PyUnicode_FastCopyCharacters(result, left_len, right, 0, right_len); - assert(_PyUnicode_CheckConsistency(result, 1)); - return result; + */ + if (!PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) + return NULL; + PyErr_Clear(); + if (PyErr_Warn(PyExc_UnicodeWarning, + (op == Py_EQ) ? + "Unicode equal comparison " + "failed to convert both arguments to Unicode - " + "interpreting them as being unequal" : + "Unicode unequal comparison " + "failed to convert both arguments to Unicode - " + "interpreting them as being unequal" + ) < 0) + return NULL; + result = (op == Py_NE); + return PyBool_FromLong(result); } -void -PyUnicode_Append(PyObject **p_left, PyObject *right) +int PyUnicode_Contains(PyObject *container, + PyObject *element) { - PyObject *left, *res; - Py_UCS4 maxchar, maxchar2; - Py_ssize_t left_len, right_len, new_len; + PyObject *str, *sub; + int result; - if (p_left == NULL) { - if (!PyErr_Occurred()) - PyErr_BadInternalCall(); - return; + /* Coerce the two arguments */ + sub = PyUnicode_FromObject(element); + if (!sub) { + return -1; } - left = *p_left; - if (right == NULL || left == NULL - || !PyUnicode_Check(left) || !PyUnicode_Check(right)) { - if (!PyErr_Occurred()) - PyErr_BadInternalCall(); - goto error; + + str = PyUnicode_FromObject(container); + if (!str) { + Py_DECREF(sub); + return -1; } - if (PyUnicode_READY(left) == -1) - goto error; - if (PyUnicode_READY(right) == -1) - goto error; + result = stringlib_contains_obj(str, sub); + + Py_DECREF(str); + Py_DECREF(sub); + + return result; +} + +/* Concat to string or Unicode object giving a new Unicode object. */ + +PyObject *PyUnicode_Concat(PyObject *left, + PyObject *right) +{ + PyUnicodeObject *u = NULL, *v = NULL, *w; + + /* Coerce the two arguments */ + u = (PyUnicodeObject *)PyUnicode_FromObject(left); + if (u == NULL) + goto onError; + v = (PyUnicodeObject *)PyUnicode_FromObject(right); + if (v == NULL) + goto onError; /* Shortcuts */ - if (left == unicode_empty) { - Py_DECREF(left); - Py_INCREF(right); - *p_left = right; - return; + if (v == unicode_empty) { + Py_DECREF(v); + return (PyObject *)u; + } + if (u == unicode_empty) { + Py_DECREF(u); + return (PyObject *)v; } - if (right == unicode_empty) - return; - left_len = PyUnicode_GET_LENGTH(left); - right_len = PyUnicode_GET_LENGTH(right); - if (left_len > PY_SSIZE_T_MAX - right_len) { + if (u->length > PY_SSIZE_T_MAX - v->length) { PyErr_SetString(PyExc_OverflowError, "strings are too large to concat"); - goto error; - } - new_len = left_len + right_len; - - if (unicode_modifiable(left) - && PyUnicode_CheckExact(right) - && PyUnicode_KIND(right) <= PyUnicode_KIND(left) - /* Don't resize for ascii += latin1. Convert ascii to latin1 requires - to change the structure size, but characters are stored just after - the structure, and so it requires to move all characters which is - not so different than duplicating the string. */ - && !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) - { - /* append inplace */ - if (unicode_resize(p_left, new_len) != 0) - goto error; - - /* copy 'right' into the newly allocated area of 'left' */ - _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len); - } - else { - maxchar = PyUnicode_MAX_CHAR_VALUE(left); - maxchar2 = PyUnicode_MAX_CHAR_VALUE(right); - maxchar = Py_MAX(maxchar, maxchar2); - - /* Concat the two Unicode strings */ - res = PyUnicode_New(new_len, maxchar); - if (res == NULL) - goto error; - _PyUnicode_FastCopyCharacters(res, 0, left, 0, left_len); - _PyUnicode_FastCopyCharacters(res, left_len, right, 0, right_len); - Py_DECREF(left); - *p_left = res; + goto onError; } - assert(_PyUnicode_CheckConsistency(*p_left, 1)); - return; -error: - Py_CLEAR(*p_left); -} - -void -PyUnicode_AppendAndDel(PyObject **pleft, PyObject *right) -{ - PyUnicode_Append(pleft, right); - Py_XDECREF(right); -} + /* Concat the two Unicode strings */ + w = _PyUnicode_New(u->length + v->length); + if (w == NULL) + goto onError; + Py_UNICODE_COPY(w->str, u->str, u->length); + Py_UNICODE_COPY(w->str + u->length, v->str, v->length); -/* -Wraps stringlib_parse_args_finds() and additionally ensures that the -first argument is a unicode object. -*/ + Py_DECREF(u); + Py_DECREF(v); + return (PyObject *)w; -static inline int -parse_args_finds_unicode(const char * function_name, PyObject *args, - PyObject **substring, - Py_ssize_t *start, Py_ssize_t *end) -{ - if(stringlib_parse_args_finds(function_name, args, substring, - start, end)) { - if (ensure_unicode(*substring) < 0) - return 0; - return 1; - } - return 0; + onError: + Py_XDECREF(u); + Py_XDECREF(v); + return NULL; } PyDoc_STRVAR(count__doc__, "S.count(sub[, start[, end]]) -> int\n\ \n\ Return the number of non-overlapping occurrences of substring sub in\n\ -string S[start:end]. Optional arguments start and end are\n\ +Unicode string S[start:end]. Optional arguments start and end are\n\ interpreted as in slice notation."); static PyObject * -unicode_count(PyObject *self, PyObject *args) +unicode_count(PyUnicodeObject *self, PyObject *args) { - PyObject *substring = NULL; /* initialize to fix a compiler warning */ + PyUnicodeObject *substring; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; PyObject *result; - int kind1, kind2; - void *buf1, *buf2; - Py_ssize_t len1, len2, iresult; - if (!parse_args_finds_unicode("count", args, &substring, &start, &end)) + if (!stringlib_parse_args_finds_unicode("count", args, &substring, + &start, &end)) return NULL; - kind1 = PyUnicode_KIND(self); - kind2 = PyUnicode_KIND(substring); - if (kind1 < kind2) - return PyLong_FromLong(0); - - len1 = PyUnicode_GET_LENGTH(self); - len2 = PyUnicode_GET_LENGTH(substring); - ADJUST_INDICES(start, end, len1); - if (end - start < len2) - return PyLong_FromLong(0); - - buf1 = PyUnicode_DATA(self); - buf2 = PyUnicode_DATA(substring); - if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(substring, kind1); - if (!buf2) - return NULL; - } - switch (kind1) { - case PyUnicode_1BYTE_KIND: - iresult = ucs1lib_count( - ((Py_UCS1*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_2BYTE_KIND: - iresult = ucs2lib_count( - ((Py_UCS2*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - case PyUnicode_4BYTE_KIND: - iresult = ucs4lib_count( - ((Py_UCS4*)buf1) + start, end - start, - buf2, len2, PY_SSIZE_T_MAX - ); - break; - default: - Py_UNREACHABLE(); - } - - result = PyLong_FromSsize_t(iresult); + ADJUST_INDICES(start, end, self->length); + result = PyInt_FromSsize_t( + stringlib_count(self->str + start, end - start, + substring->str, substring->length, + PY_SSIZE_T_MAX) + ); - if (kind2 != kind1) - PyMem_Free(buf2); + Py_DECREF(substring); return result; } -/*[clinic input] -str.encode as unicode_encode - - encoding: str(c_default="NULL") = 'utf-8' - The encoding in which to encode the string. - errors: str(c_default="NULL") = 'strict' - The error handling scheme to use for encoding errors. - The default is 'strict' meaning that encoding errors raise a - UnicodeEncodeError. Other possible values are 'ignore', 'replace' and - 'xmlcharrefreplace' as well as any other name registered with - codecs.register_error that can handle UnicodeEncodeErrors. - -Encode the string using the codec registered for encoding. -[clinic start generated code]*/ +PyDoc_STRVAR(encode__doc__, + "S.encode([encoding[,errors]]) -> string or unicode\n\ +\n\ +Encodes S using the codec registered for encoding. encoding defaults\n\ +to the default encoding. errors may be given to set a different error\n\ +handling scheme. Default is 'strict' meaning that encoding errors raise\n\ +a UnicodeEncodeError. Other possible values are 'ignore', 'replace' and\n\ +'xmlcharrefreplace' as well as any other name registered with\n\ +codecs.register_error that can handle UnicodeEncodeErrors."); static PyObject * -unicode_encode_impl(PyObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=bf78b6e2a9470e3c input=f0a9eb293d08fe02]*/ +unicode_encode(PyUnicodeObject *self, PyObject *args, PyObject *kwargs) { - return PyUnicode_AsEncodedString(self, encoding, errors); + static char *kwlist[] = {"encoding", "errors", 0}; + char *encoding = NULL; + char *errors = NULL; + PyObject *v; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ss:encode", + kwlist, &encoding, &errors)) + return NULL; + v = PyUnicode_AsEncodedObject((PyObject *)self, encoding, errors); + if (v == NULL) + goto onError; + if (!PyString_Check(v) && !PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "encoder did not return a string/unicode object " + "(type=%.400s)", + Py_TYPE(v)->tp_name); + Py_DECREF(v); + return NULL; + } + return v; + + onError: + return NULL; } -/*[clinic input] -str.expandtabs as unicode_expandtabs +PyDoc_STRVAR(decode__doc__, + "S.decode([encoding[,errors]]) -> string or unicode\n\ +\n\ +Decodes S using the codec registered for encoding. encoding defaults\n\ +to the default encoding. errors may be given to set a different error\n\ +handling scheme. Default is 'strict' meaning that encoding errors raise\n\ +a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n\ +as well as any other name registered with codecs.register_error that is\n\ +able to handle UnicodeDecodeErrors."); - tabsize: int = 8 +static PyObject * +unicode_decode(PyUnicodeObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"encoding", "errors", 0}; + char *encoding = NULL; + char *errors = NULL; + PyObject *v; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|ss:decode", + kwlist, &encoding, &errors)) + return NULL; + v = PyUnicode_AsDecodedObject((PyObject *)self, encoding, errors); + if (v == NULL) + goto onError; + if (!PyString_Check(v) && !PyUnicode_Check(v)) { + PyErr_Format(PyExc_TypeError, + "decoder did not return a string/unicode object " + "(type=%.400s)", + Py_TYPE(v)->tp_name); + Py_DECREF(v); + return NULL; + } + return v; -Return a copy where all tab characters are expanded using spaces. + onError: + return NULL; +} -If tabsize is not given, a tab size of 8 characters is assumed. -[clinic start generated code]*/ +PyDoc_STRVAR(expandtabs__doc__, + "S.expandtabs([tabsize]) -> unicode\n\ +\n\ +Return a copy of S where all tab characters are expanded using spaces.\n\ +If tabsize is not given, a tab size of 8 characters is assumed."); -static PyObject * -unicode_expandtabs_impl(PyObject *self, int tabsize) -/*[clinic end generated code: output=3457c5dcee26928f input=8a01914034af4c85]*/ +static PyObject* +unicode_expandtabs(PyUnicodeObject *self, PyObject *args) { - Py_ssize_t i, j, line_pos, src_len, incr; - Py_UCS4 ch; - PyObject *u; - void *src_data, *dest_data; - int kind; - int found; + Py_UNICODE *e; + Py_UNICODE *p; + Py_UNICODE *q; + Py_UNICODE *qe; + Py_ssize_t i, j, incr; + PyUnicodeObject *u; + int tabsize = 8; - if (PyUnicode_READY(self) == -1) + if (!PyArg_ParseTuple(args, "|i:expandtabs", &tabsize)) return NULL; /* First pass: determine size of output string */ - src_len = PyUnicode_GET_LENGTH(self); - i = j = line_pos = 0; - kind = PyUnicode_KIND(self); - src_data = PyUnicode_DATA(self); - found = 0; - for (; i < src_len; i++) { - ch = PyUnicode_READ(kind, src_data, i); - if (ch == '\t') { - found = 1; + i = 0; /* chars up to and including most recent \n or \r */ + j = 0; /* chars since most recent \n or \r (use in tab calculations) */ + e = self->str + self->length; /* end of input */ + for (p = self->str; p < e; p++) + if (*p == '\t') { if (tabsize > 0) { - incr = tabsize - (line_pos % tabsize); /* cannot overflow */ + incr = tabsize - (j % tabsize); /* cannot overflow */ if (j > PY_SSIZE_T_MAX - incr) - goto overflow; - line_pos += incr; + goto overflow1; j += incr; } } else { if (j > PY_SSIZE_T_MAX - 1) - goto overflow; - line_pos++; + goto overflow1; j++; - if (ch == '\n' || ch == '\r') - line_pos = 0; + if (*p == '\n' || *p == '\r') { + if (i > PY_SSIZE_T_MAX - j) + goto overflow1; + i += j; + j = 0; + } } - } - if (!found) - return unicode_result_unchanged(self); + + if (i > PY_SSIZE_T_MAX - j) + goto overflow1; /* Second pass: create output string and fill it */ - u = PyUnicode_New(j, PyUnicode_MAX_CHAR_VALUE(self)); + u = _PyUnicode_New(i + j); if (!u) return NULL; - dest_data = PyUnicode_DATA(u); - i = j = line_pos = 0; + j = 0; /* same as in first pass */ + q = u->str; /* next output char */ + qe = u->str + u->length; /* end of output */ - for (; i < src_len; i++) { - ch = PyUnicode_READ(kind, src_data, i); - if (ch == '\t') { + for (p = self->str; p < e; p++) + if (*p == '\t') { if (tabsize > 0) { - incr = tabsize - (line_pos % tabsize); - line_pos += incr; - unicode_fill(kind, dest_data, ' ', j, incr); - j += incr; + i = tabsize - (j % tabsize); + j += i; + while (i--) { + if (q >= qe) + goto overflow2; + *q++ = ' '; + } } } else { - line_pos++; - PyUnicode_WRITE(kind, dest_data, j, ch); + if (q >= qe) + goto overflow2; + *q++ = *p; j++; - if (ch == '\n' || ch == '\r') - line_pos = 0; + if (*p == '\n' || *p == '\r') + j = 0; } - } - assert (j == PyUnicode_GET_LENGTH(u)); - return unicode_result(u); - overflow: + return (PyObject*) u; + + overflow2: + Py_DECREF(u); + overflow1: PyErr_SetString(PyExc_OverflowError, "new string is too long"); return NULL; } PyDoc_STRVAR(find__doc__, - "S.find(sub[, start[, end]]) -> int\n\ + "S.find(sub [,start [,end]]) -> int\n\ \n\ Return the lowest index in S where substring sub is found,\n\ such that sub is contained within S[start:end]. Optional\n\ @@ -11690,268 +6617,219 @@ arguments start and end are interpreted as in slice notation.\n\ Return -1 on failure."); static PyObject * -unicode_find(PyObject *self, PyObject *args) +unicode_find(PyUnicodeObject *self, PyObject *args) { - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; + PyUnicodeObject *substring; + Py_ssize_t start; + Py_ssize_t end; Py_ssize_t result; - if (!parse_args_finds_unicode("find", args, &substring, &start, &end)) - return NULL; - - if (PyUnicode_READY(self) == -1) + if (!stringlib_parse_args_finds_unicode("find", args, &substring, + &start, &end)) return NULL; - result = any_find_slice(self, substring, start, end, 1); + result = stringlib_find_slice( + PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), + PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), + start, end + ); - if (result == -2) - return NULL; + Py_DECREF(substring); - return PyLong_FromSsize_t(result); + return PyInt_FromSsize_t(result); } static PyObject * -unicode_getitem(PyObject *self, Py_ssize_t index) +unicode_getitem(PyUnicodeObject *self, Py_ssize_t index) { - void *data; - enum PyUnicode_Kind kind; - Py_UCS4 ch; - - if (!PyUnicode_Check(self)) { - PyErr_BadArgument(); - return NULL; - } - if (PyUnicode_READY(self) == -1) { - return NULL; - } - if (index < 0 || index >= PyUnicode_GET_LENGTH(self)) { + if (index < 0 || index >= self->length) { PyErr_SetString(PyExc_IndexError, "string index out of range"); return NULL; } - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); - ch = PyUnicode_READ(kind, data, index); - return unicode_char(ch); + + return (PyObject*) PyUnicode_FromUnicode(&self->str[index], 1); } -/* Believe it or not, this produces the same value for ASCII strings - as bytes_hash(). */ -static Py_hash_t -unicode_hash(PyObject *self) +static long +unicode_hash(PyUnicodeObject *self) { - Py_uhash_t x; /* Unsigned for defined overflow behavior. */ + /* Since Unicode objects compare equal to their ASCII string + counterparts, they should use the individual character values + as basis for their hash value. This is needed to assure that + strings and Unicode objects behave in the same way as + dictionary keys. */ + + register Py_ssize_t len; + register Py_UNICODE *p; + register long x; #ifdef Py_DEBUG assert(_Py_HashSecret_Initialized); #endif - if (_PyUnicode_HASH(self) != -1) - return _PyUnicode_HASH(self); - if (PyUnicode_READY(self) == -1) - return -1; - - x = _Py_HashBytes(PyUnicode_DATA(self), - PyUnicode_GET_LENGTH(self) * PyUnicode_KIND(self)); - _PyUnicode_HASH(self) = x; + if (self->hash != -1) + return self->hash; + len = PyUnicode_GET_SIZE(self); + /* + We make the hash of the empty string be 0, rather than using + (prefix ^ suffix), since this slightly obfuscates the hash secret + */ + if (len == 0) { + self->hash = 0; + return 0; + } + p = PyUnicode_AS_UNICODE(self); + x = _Py_HashSecret.prefix; + x ^= *p << 7; + while (--len >= 0) + x = (1000003*x) ^ *p++; + x ^= PyUnicode_GET_SIZE(self); + x ^= _Py_HashSecret.suffix; + if (x == -1) + x = -2; + self->hash = x; return x; } PyDoc_STRVAR(index__doc__, - "S.index(sub[, start[, end]]) -> int\n\ + "S.index(sub [,start [,end]]) -> int\n\ \n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Raises ValueError when the substring is not found."); +Like S.find() but raise ValueError when the substring is not found."); static PyObject * -unicode_index(PyObject *self, PyObject *args) +unicode_index(PyUnicodeObject *self, PyObject *args) { - /* initialize variables to prevent gcc warning */ Py_ssize_t result; - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; + PyUnicodeObject *substring; + Py_ssize_t start; + Py_ssize_t end; - if (!parse_args_finds_unicode("index", args, &substring, &start, &end)) + if (!stringlib_parse_args_finds_unicode("index", args, &substring, + &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) - return NULL; - - result = any_find_slice(self, substring, start, end, 1); + result = stringlib_find_slice( + PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), + PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), + start, end + ); - if (result == -2) - return NULL; + Py_DECREF(substring); if (result < 0) { PyErr_SetString(PyExc_ValueError, "substring not found"); return NULL; } - return PyLong_FromSsize_t(result); -} - -/*[clinic input] -str.isascii as unicode_isascii - -Return True if all characters in the string are ASCII, False otherwise. - -ASCII characters have code points in the range U+0000-U+007F. -Empty string is ASCII too. -[clinic start generated code]*/ - -static PyObject * -unicode_isascii_impl(PyObject *self) -/*[clinic end generated code: output=c5910d64b5a8003f input=5a43cbc6399621d5]*/ -{ - if (PyUnicode_READY(self) == -1) { - return NULL; - } - return PyBool_FromLong(PyUnicode_IS_ASCII(self)); + return PyInt_FromSsize_t(result); } -/*[clinic input] -str.islower as unicode_islower - -Return True if the string is a lowercase string, False otherwise. - -A string is lowercase if all cased characters in the string are lowercase and -there is at least one cased character in the string. -[clinic start generated code]*/ +PyDoc_STRVAR(islower__doc__, + "S.islower() -> bool\n\ +\n\ +Return True if all cased characters in S are lowercase and there is\n\ +at least one cased character in S, False otherwise."); -static PyObject * -unicode_islower_impl(PyObject *self) -/*[clinic end generated code: output=dbd41995bd005b81 input=acec65ac6821ae47]*/ +static PyObject* +unicode_islower(PyUnicodeObject *self) { - Py_ssize_t i, length; - int kind; - void *data; + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; int cased; - if (PyUnicode_READY(self) == -1) - return NULL; - length = PyUnicode_GET_LENGTH(self); - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); - /* Shortcut for single character strings */ - if (length == 1) - return PyBool_FromLong( - Py_UNICODE_ISLOWER(PyUnicode_READ(kind, data, 0))); + if (PyUnicode_GET_SIZE(self) == 1) + return PyBool_FromLong(Py_UNICODE_ISLOWER(*p)); /* Special case for empty strings */ - if (length == 0) - Py_RETURN_FALSE; + if (PyUnicode_GET_SIZE(self) == 0) + return PyBool_FromLong(0); + e = p + PyUnicode_GET_SIZE(self); cased = 0; - for (i = 0; i < length; i++) { - const Py_UCS4 ch = PyUnicode_READ(kind, data, i); + for (; p < e; p++) { + register const Py_UNICODE ch = *p; if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch)) - Py_RETURN_FALSE; + return PyBool_FromLong(0); else if (!cased && Py_UNICODE_ISLOWER(ch)) cased = 1; } return PyBool_FromLong(cased); } -/*[clinic input] -str.isupper as unicode_isupper - -Return True if the string is an uppercase string, False otherwise. - -A string is uppercase if all cased characters in the string are uppercase and -there is at least one cased character in the string. -[clinic start generated code]*/ +PyDoc_STRVAR(isupper__doc__, + "S.isupper() -> bool\n\ +\n\ +Return True if all cased characters in S are uppercase and there is\n\ +at least one cased character in S, False otherwise."); -static PyObject * -unicode_isupper_impl(PyObject *self) -/*[clinic end generated code: output=049209c8e7f15f59 input=e9b1feda5d17f2d3]*/ +static PyObject* +unicode_isupper(PyUnicodeObject *self) { - Py_ssize_t i, length; - int kind; - void *data; + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; int cased; - if (PyUnicode_READY(self) == -1) - return NULL; - length = PyUnicode_GET_LENGTH(self); - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); - /* Shortcut for single character strings */ - if (length == 1) - return PyBool_FromLong( - Py_UNICODE_ISUPPER(PyUnicode_READ(kind, data, 0)) != 0); + if (PyUnicode_GET_SIZE(self) == 1) + return PyBool_FromLong(Py_UNICODE_ISUPPER(*p) != 0); /* Special case for empty strings */ - if (length == 0) - Py_RETURN_FALSE; + if (PyUnicode_GET_SIZE(self) == 0) + return PyBool_FromLong(0); + e = p + PyUnicode_GET_SIZE(self); cased = 0; - for (i = 0; i < length; i++) { - const Py_UCS4 ch = PyUnicode_READ(kind, data, i); + for (; p < e; p++) { + register const Py_UNICODE ch = *p; if (Py_UNICODE_ISLOWER(ch) || Py_UNICODE_ISTITLE(ch)) - Py_RETURN_FALSE; + return PyBool_FromLong(0); else if (!cased && Py_UNICODE_ISUPPER(ch)) cased = 1; } return PyBool_FromLong(cased); } -/*[clinic input] -str.istitle as unicode_istitle - -Return True if the string is a title-cased string, False otherwise. - -In a title-cased string, upper- and title-case characters may only -follow uncased characters and lowercase characters only cased ones. -[clinic start generated code]*/ +PyDoc_STRVAR(istitle__doc__, + "S.istitle() -> bool\n\ +\n\ +Return True if S is a titlecased string and there is at least one\n\ +character in S, i.e. upper- and titlecase characters may only\n\ +follow uncased characters and lowercase characters only cased ones.\n\ +Return False otherwise."); -static PyObject * -unicode_istitle_impl(PyObject *self) -/*[clinic end generated code: output=e9bf6eb91f5d3f0e input=98d32bd2e1f06f8c]*/ +static PyObject* +unicode_istitle(PyUnicodeObject *self) { - Py_ssize_t i, length; - int kind; - void *data; + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; int cased, previous_is_cased; - if (PyUnicode_READY(self) == -1) - return NULL; - length = PyUnicode_GET_LENGTH(self); - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); - /* Shortcut for single character strings */ - if (length == 1) { - Py_UCS4 ch = PyUnicode_READ(kind, data, 0); - return PyBool_FromLong((Py_UNICODE_ISTITLE(ch) != 0) || - (Py_UNICODE_ISUPPER(ch) != 0)); - } + if (PyUnicode_GET_SIZE(self) == 1) + return PyBool_FromLong((Py_UNICODE_ISTITLE(*p) != 0) || + (Py_UNICODE_ISUPPER(*p) != 0)); /* Special case for empty strings */ - if (length == 0) - Py_RETURN_FALSE; + if (PyUnicode_GET_SIZE(self) == 0) + return PyBool_FromLong(0); + e = p + PyUnicode_GET_SIZE(self); cased = 0; previous_is_cased = 0; - for (i = 0; i < length; i++) { - const Py_UCS4 ch = PyUnicode_READ(kind, data, i); + for (; p < e; p++) { + register const Py_UNICODE ch = *p; if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch)) { if (previous_is_cased) - Py_RETURN_FALSE; + return PyBool_FromLong(0); previous_is_cased = 1; cased = 1; } else if (Py_UNICODE_ISLOWER(ch)) { if (!previous_is_cased) - Py_RETURN_FALSE; + return PyBool_FromLong(0); previous_is_cased = 1; cased = 1; } @@ -11961,403 +6839,230 @@ unicode_istitle_impl(PyObject *self) return PyBool_FromLong(cased); } -/*[clinic input] -str.isspace as unicode_isspace - -Return True if the string is a whitespace string, False otherwise. - -A string is whitespace if all characters in the string are whitespace and there -is at least one character in the string. -[clinic start generated code]*/ +PyDoc_STRVAR(isspace__doc__, + "S.isspace() -> bool\n\ +\n\ +Return True if all characters in S are whitespace\n\ +and there is at least one character in S, False otherwise."); -static PyObject * -unicode_isspace_impl(PyObject *self) -/*[clinic end generated code: output=163a63bfa08ac2b9 input=fe462cb74f8437d8]*/ +static PyObject* +unicode_isspace(PyUnicodeObject *self) { - Py_ssize_t i, length; - int kind; - void *data; - - if (PyUnicode_READY(self) == -1) - return NULL; - length = PyUnicode_GET_LENGTH(self); - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; /* Shortcut for single character strings */ - if (length == 1) - return PyBool_FromLong( - Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, 0))); + if (PyUnicode_GET_SIZE(self) == 1 && + Py_UNICODE_ISSPACE(*p)) + return PyBool_FromLong(1); /* Special case for empty strings */ - if (length == 0) - Py_RETURN_FALSE; + if (PyUnicode_GET_SIZE(self) == 0) + return PyBool_FromLong(0); - for (i = 0; i < length; i++) { - const Py_UCS4 ch = PyUnicode_READ(kind, data, i); - if (!Py_UNICODE_ISSPACE(ch)) - Py_RETURN_FALSE; + e = p + PyUnicode_GET_SIZE(self); + for (; p < e; p++) { + if (!Py_UNICODE_ISSPACE(*p)) + return PyBool_FromLong(0); } - Py_RETURN_TRUE; + return PyBool_FromLong(1); } -/*[clinic input] -str.isalpha as unicode_isalpha - -Return True if the string is an alphabetic string, False otherwise. - -A string is alphabetic if all characters in the string are alphabetic and there -is at least one character in the string. -[clinic start generated code]*/ +PyDoc_STRVAR(isalpha__doc__, + "S.isalpha() -> bool\n\ +\n\ +Return True if all characters in S are alphabetic\n\ +and there is at least one character in S, False otherwise."); -static PyObject * -unicode_isalpha_impl(PyObject *self) -/*[clinic end generated code: output=cc81b9ac3883ec4f input=d0fd18a96cbca5eb]*/ +static PyObject* +unicode_isalpha(PyUnicodeObject *self) { - Py_ssize_t i, length; - int kind; - void *data; - - if (PyUnicode_READY(self) == -1) - return NULL; - length = PyUnicode_GET_LENGTH(self); - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; /* Shortcut for single character strings */ - if (length == 1) - return PyBool_FromLong( - Py_UNICODE_ISALPHA(PyUnicode_READ(kind, data, 0))); + if (PyUnicode_GET_SIZE(self) == 1 && + Py_UNICODE_ISALPHA(*p)) + return PyBool_FromLong(1); /* Special case for empty strings */ - if (length == 0) - Py_RETURN_FALSE; + if (PyUnicode_GET_SIZE(self) == 0) + return PyBool_FromLong(0); - for (i = 0; i < length; i++) { - if (!Py_UNICODE_ISALPHA(PyUnicode_READ(kind, data, i))) - Py_RETURN_FALSE; + e = p + PyUnicode_GET_SIZE(self); + for (; p < e; p++) { + if (!Py_UNICODE_ISALPHA(*p)) + return PyBool_FromLong(0); } - Py_RETURN_TRUE; + return PyBool_FromLong(1); } -/*[clinic input] -str.isalnum as unicode_isalnum - -Return True if the string is an alpha-numeric string, False otherwise. - -A string is alpha-numeric if all characters in the string are alpha-numeric and -there is at least one character in the string. -[clinic start generated code]*/ +PyDoc_STRVAR(isalnum__doc__, + "S.isalnum() -> bool\n\ +\n\ +Return True if all characters in S are alphanumeric\n\ +and there is at least one character in S, False otherwise."); -static PyObject * -unicode_isalnum_impl(PyObject *self) -/*[clinic end generated code: output=a5a23490ffc3660c input=5c6579bf2e04758c]*/ +static PyObject* +unicode_isalnum(PyUnicodeObject *self) { - int kind; - void *data; - Py_ssize_t len, i; - - if (PyUnicode_READY(self) == -1) - return NULL; - - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); - len = PyUnicode_GET_LENGTH(self); + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; /* Shortcut for single character strings */ - if (len == 1) { - const Py_UCS4 ch = PyUnicode_READ(kind, data, 0); - return PyBool_FromLong(Py_UNICODE_ISALNUM(ch)); - } + if (PyUnicode_GET_SIZE(self) == 1 && + Py_UNICODE_ISALNUM(*p)) + return PyBool_FromLong(1); /* Special case for empty strings */ - if (len == 0) - Py_RETURN_FALSE; + if (PyUnicode_GET_SIZE(self) == 0) + return PyBool_FromLong(0); - for (i = 0; i < len; i++) { - const Py_UCS4 ch = PyUnicode_READ(kind, data, i); - if (!Py_UNICODE_ISALNUM(ch)) - Py_RETURN_FALSE; + e = p + PyUnicode_GET_SIZE(self); + for (; p < e; p++) { + if (!Py_UNICODE_ISALNUM(*p)) + return PyBool_FromLong(0); } - Py_RETURN_TRUE; + return PyBool_FromLong(1); } -/*[clinic input] -str.isdecimal as unicode_isdecimal - -Return True if the string is a decimal string, False otherwise. - -A string is a decimal string if all characters in the string are decimal and -there is at least one character in the string. -[clinic start generated code]*/ +PyDoc_STRVAR(isdecimal__doc__, + "S.isdecimal() -> bool\n\ +\n\ +Return True if there are only decimal characters in S,\n\ +False otherwise."); -static PyObject * -unicode_isdecimal_impl(PyObject *self) -/*[clinic end generated code: output=fb2dcdb62d3fc548 input=336bc97ab4c8268f]*/ +static PyObject* +unicode_isdecimal(PyUnicodeObject *self) { - Py_ssize_t i, length; - int kind; - void *data; - - if (PyUnicode_READY(self) == -1) - return NULL; - length = PyUnicode_GET_LENGTH(self); - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; /* Shortcut for single character strings */ - if (length == 1) - return PyBool_FromLong( - Py_UNICODE_ISDECIMAL(PyUnicode_READ(kind, data, 0))); + if (PyUnicode_GET_SIZE(self) == 1 && + Py_UNICODE_ISDECIMAL(*p)) + return PyBool_FromLong(1); /* Special case for empty strings */ - if (length == 0) - Py_RETURN_FALSE; + if (PyUnicode_GET_SIZE(self) == 0) + return PyBool_FromLong(0); - for (i = 0; i < length; i++) { - if (!Py_UNICODE_ISDECIMAL(PyUnicode_READ(kind, data, i))) - Py_RETURN_FALSE; + e = p + PyUnicode_GET_SIZE(self); + for (; p < e; p++) { + if (!Py_UNICODE_ISDECIMAL(*p)) + return PyBool_FromLong(0); } - Py_RETURN_TRUE; + return PyBool_FromLong(1); } -/*[clinic input] -str.isdigit as unicode_isdigit - -Return True if the string is a digit string, False otherwise. - -A string is a digit string if all characters in the string are digits and there -is at least one character in the string. -[clinic start generated code]*/ +PyDoc_STRVAR(isdigit__doc__, + "S.isdigit() -> bool\n\ +\n\ +Return True if all characters in S are digits\n\ +and there is at least one character in S, False otherwise."); -static PyObject * -unicode_isdigit_impl(PyObject *self) -/*[clinic end generated code: output=10a6985311da6858 input=901116c31deeea4c]*/ +static PyObject* +unicode_isdigit(PyUnicodeObject *self) { - Py_ssize_t i, length; - int kind; - void *data; - - if (PyUnicode_READY(self) == -1) - return NULL; - length = PyUnicode_GET_LENGTH(self); - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; /* Shortcut for single character strings */ - if (length == 1) { - const Py_UCS4 ch = PyUnicode_READ(kind, data, 0); - return PyBool_FromLong(Py_UNICODE_ISDIGIT(ch)); - } + if (PyUnicode_GET_SIZE(self) == 1 && + Py_UNICODE_ISDIGIT(*p)) + return PyBool_FromLong(1); /* Special case for empty strings */ - if (length == 0) - Py_RETURN_FALSE; + if (PyUnicode_GET_SIZE(self) == 0) + return PyBool_FromLong(0); - for (i = 0; i < length; i++) { - if (!Py_UNICODE_ISDIGIT(PyUnicode_READ(kind, data, i))) - Py_RETURN_FALSE; + e = p + PyUnicode_GET_SIZE(self); + for (; p < e; p++) { + if (!Py_UNICODE_ISDIGIT(*p)) + return PyBool_FromLong(0); } - Py_RETURN_TRUE; + return PyBool_FromLong(1); } -/*[clinic input] -str.isnumeric as unicode_isnumeric - -Return True if the string is a numeric string, False otherwise. - -A string is numeric if all characters in the string are numeric and there is at -least one character in the string. -[clinic start generated code]*/ +PyDoc_STRVAR(isnumeric__doc__, + "S.isnumeric() -> bool\n\ +\n\ +Return True if there are only numeric characters in S,\n\ +False otherwise."); -static PyObject * -unicode_isnumeric_impl(PyObject *self) -/*[clinic end generated code: output=9172a32d9013051a input=722507db976f826c]*/ +static PyObject* +unicode_isnumeric(PyUnicodeObject *self) { - Py_ssize_t i, length; - int kind; - void *data; - - if (PyUnicode_READY(self) == -1) - return NULL; - length = PyUnicode_GET_LENGTH(self); - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); + register const Py_UNICODE *p = PyUnicode_AS_UNICODE(self); + register const Py_UNICODE *e; /* Shortcut for single character strings */ - if (length == 1) - return PyBool_FromLong( - Py_UNICODE_ISNUMERIC(PyUnicode_READ(kind, data, 0))); + if (PyUnicode_GET_SIZE(self) == 1 && + Py_UNICODE_ISNUMERIC(*p)) + return PyBool_FromLong(1); /* Special case for empty strings */ - if (length == 0) - Py_RETURN_FALSE; + if (PyUnicode_GET_SIZE(self) == 0) + return PyBool_FromLong(0); - for (i = 0; i < length; i++) { - if (!Py_UNICODE_ISNUMERIC(PyUnicode_READ(kind, data, i))) - Py_RETURN_FALSE; + e = p + PyUnicode_GET_SIZE(self); + for (; p < e; p++) { + if (!Py_UNICODE_ISNUMERIC(*p)) + return PyBool_FromLong(0); } - Py_RETURN_TRUE; -} - -int -PyUnicode_IsIdentifier(PyObject *self) -{ - int kind; - void *data; - Py_ssize_t i; - Py_UCS4 first; - - if (PyUnicode_READY(self) == -1) { - Py_FatalError("identifier not ready"); - return 0; - } - - /* Special case for empty strings */ - if (PyUnicode_GET_LENGTH(self) == 0) - return 0; - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); - - /* PEP 3131 says that the first character must be in - XID_Start and subsequent characters in XID_Continue, - and for the ASCII range, the 2.x rules apply (i.e - start with letters and underscore, continue with - letters, digits, underscore). However, given the current - definition of XID_Start and XID_Continue, it is sufficient - to check just for these, except that _ must be allowed - as starting an identifier. */ - first = PyUnicode_READ(kind, data, 0); - if (!_PyUnicode_IsXidStart(first) && first != 0x5F /* LOW LINE */) - return 0; - - for (i = 1; i < PyUnicode_GET_LENGTH(self); i++) - if (!_PyUnicode_IsXidContinue(PyUnicode_READ(kind, data, i))) - return 0; - return 1; -} - -/*[clinic input] -str.isidentifier as unicode_isidentifier - -Return True if the string is a valid Python identifier, False otherwise. - -Call keyword.iskeyword(s) to test whether string s is a reserved identifier, -such as "def" or "class". -[clinic start generated code]*/ - -static PyObject * -unicode_isidentifier_impl(PyObject *self) -/*[clinic end generated code: output=fe585a9666572905 input=2d807a104f21c0c5]*/ -{ - return PyBool_FromLong(PyUnicode_IsIdentifier(self)); + return PyBool_FromLong(1); } -/*[clinic input] -str.isprintable as unicode_isprintable - -Return True if the string is printable, False otherwise. - -A string is printable if all of its characters are considered printable in -repr() or if it is empty. -[clinic start generated code]*/ - -static PyObject * -unicode_isprintable_impl(PyObject *self) -/*[clinic end generated code: output=3ab9626cd32dd1a0 input=98a0e1c2c1813209]*/ -{ - Py_ssize_t i, length; - int kind; - void *data; - - if (PyUnicode_READY(self) == -1) - return NULL; - length = PyUnicode_GET_LENGTH(self); - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); - - /* Shortcut for single character strings */ - if (length == 1) - return PyBool_FromLong( - Py_UNICODE_ISPRINTABLE(PyUnicode_READ(kind, data, 0))); - - for (i = 0; i < length; i++) { - if (!Py_UNICODE_ISPRINTABLE(PyUnicode_READ(kind, data, i))) { - Py_RETURN_FALSE; - } - } - Py_RETURN_TRUE; -} - -/*[clinic input] -str.join as unicode_join - - iterable: object - / - -Concatenate any number of strings. - -The string whose method is called is inserted in between each given string. -The result is returned as a new string. - -Example: '.'.join(['ab', 'pq', 'rs']) -> 'ab.pq.rs' -[clinic start generated code]*/ +PyDoc_STRVAR(join__doc__, + "S.join(iterable) -> unicode\n\ +\n\ +Return a string which is the concatenation of the strings in the\n\ +iterable. The separator between elements is S."); -static PyObject * -unicode_join(PyObject *self, PyObject *iterable) -/*[clinic end generated code: output=6857e7cecfe7bf98 input=2f70422bfb8fa189]*/ +static PyObject* +unicode_join(PyObject *self, PyObject *data) { - return PyUnicode_Join(self, iterable); + return PyUnicode_Join(self, data); } static Py_ssize_t -unicode_length(PyObject *self) +unicode_length(PyUnicodeObject *self) { - if (PyUnicode_READY(self) == -1) - return -1; - return PyUnicode_GET_LENGTH(self); + return self->length; } -/*[clinic input] -str.ljust as unicode_ljust - - width: Py_ssize_t - fillchar: Py_UCS4 = ' ' - / - -Return a left-justified string of length width. - -Padding is done using the specified fill character (default is a space). -[clinic start generated code]*/ +PyDoc_STRVAR(ljust__doc__, + "S.ljust(width[, fillchar]) -> int\n\ +\n\ +Return S left-justified in a Unicode string of length width. Padding is\n\ +done using the specified fill character (default is a space)."); static PyObject * -unicode_ljust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=3ab599e335e60a32]*/ +unicode_ljust(PyUnicodeObject *self, PyObject *args) { - if (PyUnicode_READY(self) == -1) + Py_ssize_t width; + Py_UNICODE fillchar = ' '; + + if (!PyArg_ParseTuple(args, "n|O&:ljust", &width, convert_uc, &fillchar)) return NULL; - if (PyUnicode_GET_LENGTH(self) >= width) - return unicode_result_unchanged(self); + if (self->length >= width && PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return (PyObject*) self; + } - return pad(self, 0, width - PyUnicode_GET_LENGTH(self), fillchar); + return (PyObject*) pad(self, 0, width - self->length, fillchar); } -/*[clinic input] -str.lower as unicode_lower - -Return a copy of the string converted to lowercase. -[clinic start generated code]*/ +PyDoc_STRVAR(lower__doc__, + "S.lower() -> unicode\n\ +\n\ +Return a copy of the string S converted to lowercase."); -static PyObject * -unicode_lower_impl(PyObject *self) -/*[clinic end generated code: output=84ef9ed42efad663 input=60a2984b8beff23a]*/ +static PyObject* +unicode_lower(PyUnicodeObject *self) { - if (PyUnicode_READY(self) == -1) - return NULL; - if (PyUnicode_IS_ASCII(self)) - return ascii_upper_or_lower(self, 1); - return case_operation(self, do_lower); + return fixup(self, fixlower); } #define LEFTSTRIP 0 @@ -12365,173 +7070,99 @@ unicode_lower_impl(PyObject *self) #define BOTHSTRIP 2 /* Arrays indexed by above */ -static const char *stripfuncnames[] = {"lstrip", "rstrip", "strip"}; +static const char *stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; -#define STRIPNAME(i) (stripfuncnames[i]) +#define STRIPNAME(i) (stripformat[i]+3) /* externally visible for str.strip(unicode) */ PyObject * -_PyUnicode_XStrip(PyObject *self, int striptype, PyObject *sepobj) +_PyUnicode_XStrip(PyUnicodeObject *self, int striptype, PyObject *sepobj) { - void *data; - int kind; - Py_ssize_t i, j, len; - BLOOM_MASK sepmask; - Py_ssize_t seplen; - - if (PyUnicode_READY(self) == -1 || PyUnicode_READY(sepobj) == -1) - return NULL; + Py_UNICODE *s = PyUnicode_AS_UNICODE(self); + Py_ssize_t len = PyUnicode_GET_SIZE(self); + Py_UNICODE *sep = PyUnicode_AS_UNICODE(sepobj); + Py_ssize_t seplen = PyUnicode_GET_SIZE(sepobj); + Py_ssize_t i, j; - kind = PyUnicode_KIND(self); - data = PyUnicode_DATA(self); - len = PyUnicode_GET_LENGTH(self); - seplen = PyUnicode_GET_LENGTH(sepobj); - sepmask = make_bloom_mask(PyUnicode_KIND(sepobj), - PyUnicode_DATA(sepobj), - seplen); + BLOOM_MASK sepmask = make_bloom_mask(sep, seplen); i = 0; if (striptype != RIGHTSTRIP) { - while (i < len) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); - if (!BLOOM(sepmask, ch)) - break; - if (PyUnicode_FindChar(sepobj, ch, 0, seplen, 1) < 0) - break; + while (i < len && BLOOM_MEMBER(sepmask, s[i], sep, seplen)) { i++; } } j = len; if (striptype != LEFTSTRIP) { - j--; - while (j >= i) { - Py_UCS4 ch = PyUnicode_READ(kind, data, j); - if (!BLOOM(sepmask, ch)) - break; - if (PyUnicode_FindChar(sepobj, ch, 0, seplen, 1) < 0) - break; + do { j--; - } - + } while (j >= i && BLOOM_MEMBER(sepmask, s[j], sep, seplen)); j++; } - return PyUnicode_Substring(self, i, j); -} - -PyObject* -PyUnicode_Substring(PyObject *self, Py_ssize_t start, Py_ssize_t end) -{ - unsigned char *data; - int kind; - Py_ssize_t length; - - if (PyUnicode_READY(self) == -1) - return NULL; - - length = PyUnicode_GET_LENGTH(self); - end = Py_MIN(end, length); - - if (start == 0 && end == length) - return unicode_result_unchanged(self); - - if (start < 0 || end < 0) { - PyErr_SetString(PyExc_IndexError, "string index out of range"); - return NULL; - } - if (start >= length || end < start) - _Py_RETURN_UNICODE_EMPTY(); - - length = end - start; - if (PyUnicode_IS_ASCII(self)) { - data = PyUnicode_1BYTE_DATA(self); - return _PyUnicode_FromASCII((char*)(data + start), length); - } - else { - kind = PyUnicode_KIND(self); - data = PyUnicode_1BYTE_DATA(self); - return PyUnicode_FromKindAndData(kind, - data + kind * start, - length); + if (i == 0 && j == len && PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return (PyObject*)self; } + else + return PyUnicode_FromUnicode(s+i, j-i); } + static PyObject * -do_strip(PyObject *self, int striptype) +do_strip(PyUnicodeObject *self, int striptype) { - Py_ssize_t len, i, j; - - if (PyUnicode_READY(self) == -1) - return NULL; - - len = PyUnicode_GET_LENGTH(self); - - if (PyUnicode_IS_ASCII(self)) { - Py_UCS1 *data = PyUnicode_1BYTE_DATA(self); + Py_UNICODE *s = PyUnicode_AS_UNICODE(self); + Py_ssize_t len = PyUnicode_GET_SIZE(self), i, j; - i = 0; - if (striptype != RIGHTSTRIP) { - while (i < len) { - Py_UCS1 ch = data[i]; - if (!_Py_ascii_whitespace[ch]) - break; - i++; - } - } - - j = len; - if (striptype != LEFTSTRIP) { - j--; - while (j >= i) { - Py_UCS1 ch = data[j]; - if (!_Py_ascii_whitespace[ch]) - break; - j--; - } - j++; + i = 0; + if (striptype != RIGHTSTRIP) { + while (i < len && Py_UNICODE_ISSPACE(s[i])) { + i++; } } - else { - int kind = PyUnicode_KIND(self); - void *data = PyUnicode_DATA(self); - i = 0; - if (striptype != RIGHTSTRIP) { - while (i < len) { - Py_UCS4 ch = PyUnicode_READ(kind, data, i); - if (!Py_UNICODE_ISSPACE(ch)) - break; - i++; - } - } - - j = len; - if (striptype != LEFTSTRIP) { + j = len; + if (striptype != LEFTSTRIP) { + do { j--; - while (j >= i) { - Py_UCS4 ch = PyUnicode_READ(kind, data, j); - if (!Py_UNICODE_ISSPACE(ch)) - break; - j--; - } - j++; - } + } while (j >= i && Py_UNICODE_ISSPACE(s[j])); + j++; } - return PyUnicode_Substring(self, i, j); + if (i == 0 && j == len && PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return (PyObject*)self; + } + else + return PyUnicode_FromUnicode(s+i, j-i); } static PyObject * -do_argstrip(PyObject *self, int striptype, PyObject *sep) +do_argstrip(PyUnicodeObject *self, int striptype, PyObject *args) { - if (sep != Py_None) { + PyObject *sep = NULL; + + if (!PyArg_ParseTuple(args, (char *)stripformat[striptype], &sep)) + return NULL; + + if (sep != NULL && sep != Py_None) { if (PyUnicode_Check(sep)) return _PyUnicode_XStrip(self, striptype, sep); + else if (PyString_Check(sep)) { + PyObject *res; + sep = PyUnicode_FromObject(sep); + if (sep==NULL) + return NULL; + res = _PyUnicode_XStrip(self, striptype, sep); + Py_DECREF(sep); + return res; + } else { PyErr_Format(PyExc_TypeError, - "%s arg must be None or str", + "%s arg must be None, unicode or str", STRIPNAME(striptype)); return NULL; } @@ -12541,331 +7172,191 @@ do_argstrip(PyObject *self, int striptype, PyObject *sep) } -/*[clinic input] -str.strip as unicode_strip - - chars: object = None - / - -Return a copy of the string with leading and trailing whitespace removed. - -If chars is given and not None, remove characters in chars instead. -[clinic start generated code]*/ +PyDoc_STRVAR(strip__doc__, + "S.strip([chars]) -> unicode\n\ +\n\ +Return a copy of the string S with leading and trailing\n\ +whitespace removed.\n\ +If chars is given and not None, remove characters in chars instead.\n\ +If chars is a str, it will be converted to unicode before stripping"); static PyObject * -unicode_strip_impl(PyObject *self, PyObject *chars) -/*[clinic end generated code: output=ca19018454345d57 input=385289c6f423b954]*/ +unicode_strip(PyUnicodeObject *self, PyObject *args) { - return do_argstrip(self, BOTHSTRIP, chars); + if (PyTuple_GET_SIZE(args) == 0) + return do_strip(self, BOTHSTRIP); /* Common case */ + else + return do_argstrip(self, BOTHSTRIP, args); } -/*[clinic input] -str.lstrip as unicode_lstrip - - chars: object = None - / - -Return a copy of the string with leading whitespace removed. - -If chars is given and not None, remove characters in chars instead. -[clinic start generated code]*/ +PyDoc_STRVAR(lstrip__doc__, + "S.lstrip([chars]) -> unicode\n\ +\n\ +Return a copy of the string S with leading whitespace removed.\n\ +If chars is given and not None, remove characters in chars instead.\n\ +If chars is a str, it will be converted to unicode before stripping"); static PyObject * -unicode_lstrip_impl(PyObject *self, PyObject *chars) -/*[clinic end generated code: output=3b43683251f79ca7 input=529f9f3834448671]*/ +unicode_lstrip(PyUnicodeObject *self, PyObject *args) { - return do_argstrip(self, LEFTSTRIP, chars); + if (PyTuple_GET_SIZE(args) == 0) + return do_strip(self, LEFTSTRIP); /* Common case */ + else + return do_argstrip(self, LEFTSTRIP, args); } -/*[clinic input] -str.rstrip as unicode_rstrip - - chars: object = None - / - -Return a copy of the string with trailing whitespace removed. - -If chars is given and not None, remove characters in chars instead. -[clinic start generated code]*/ +PyDoc_STRVAR(rstrip__doc__, + "S.rstrip([chars]) -> unicode\n\ +\n\ +Return a copy of the string S with trailing whitespace removed.\n\ +If chars is given and not None, remove characters in chars instead.\n\ +If chars is a str, it will be converted to unicode before stripping"); static PyObject * -unicode_rstrip_impl(PyObject *self, PyObject *chars) -/*[clinic end generated code: output=4a59230017cc3b7a input=62566c627916557f]*/ +unicode_rstrip(PyUnicodeObject *self, PyObject *args) { - return do_argstrip(self, RIGHTSTRIP, chars); + if (PyTuple_GET_SIZE(args) == 0) + return do_strip(self, RIGHTSTRIP); /* Common case */ + else + return do_argstrip(self, RIGHTSTRIP, args); } static PyObject* -unicode_repeat(PyObject *str, Py_ssize_t len) +unicode_repeat(PyUnicodeObject *str, Py_ssize_t len) { - PyObject *u; - Py_ssize_t nchars, n; + PyUnicodeObject *u; + Py_UNICODE *p; + Py_ssize_t nchars; + size_t nbytes; - if (len < 1) - _Py_RETURN_UNICODE_EMPTY(); + if (len < 0) + len = 0; - /* no repeat, return original string */ - if (len == 1) - return unicode_result_unchanged(str); + if (len == 1 && PyUnicode_CheckExact(str)) { + /* no repeat, return original string */ + Py_INCREF(str); + return (PyObject*) str; + } - if (PyUnicode_READY(str) == -1) + /* ensure # of chars needed doesn't overflow Py_ssize_t and # of bytes + * needed doesn't overflow size_t + */ + if (len && str->length > PY_SSIZE_T_MAX / len) { + PyErr_SetString(PyExc_OverflowError, + "repeated string is too long"); return NULL; - - if (PyUnicode_GET_LENGTH(str) > PY_SSIZE_T_MAX / len) { + } + nchars = len * str->length; + nbytes = ((size_t)nchars + 1u) * sizeof(Py_UNICODE); + if (nbytes / sizeof(Py_UNICODE) != ((size_t)nchars + 1u)) { PyErr_SetString(PyExc_OverflowError, "repeated string is too long"); return NULL; } - nchars = len * PyUnicode_GET_LENGTH(str); - - u = PyUnicode_New(nchars, PyUnicode_MAX_CHAR_VALUE(str)); + u = _PyUnicode_New(nchars); if (!u) return NULL; - assert(PyUnicode_KIND(u) == PyUnicode_KIND(str)); - - if (PyUnicode_GET_LENGTH(str) == 1) { - const int kind = PyUnicode_KIND(str); - const Py_UCS4 fill_char = PyUnicode_READ(kind, PyUnicode_DATA(str), 0); - if (kind == PyUnicode_1BYTE_KIND) { - void *to = PyUnicode_DATA(u); - memset(to, (unsigned char)fill_char, len); - } - else if (kind == PyUnicode_2BYTE_KIND) { - Py_UCS2 *ucs2 = PyUnicode_2BYTE_DATA(u); - for (n = 0; n < len; ++n) - ucs2[n] = fill_char; - } else { - Py_UCS4 *ucs4 = PyUnicode_4BYTE_DATA(u); - assert(kind == PyUnicode_4BYTE_KIND); - for (n = 0; n < len; ++n) - ucs4[n] = fill_char; + + p = u->str; + + if (str->length == 1 && len > 0) { + Py_UNICODE_FILL(p, str->str[0], len); + } else { + Py_ssize_t done = 0; /* number of characters copied this far */ + if (done < nchars) { + Py_UNICODE_COPY(p, str->str, str->length); + done = str->length; } - } - else { - /* number of characters copied this far */ - Py_ssize_t done = PyUnicode_GET_LENGTH(str); - const Py_ssize_t char_size = PyUnicode_KIND(str); - char *to = (char *) PyUnicode_DATA(u); - memcpy(to, PyUnicode_DATA(str), - PyUnicode_GET_LENGTH(str) * char_size); while (done < nchars) { - n = (done <= nchars-done) ? done : nchars-done; - memcpy(to + (done * char_size), to, n * char_size); + Py_ssize_t n = (done <= nchars-done) ? done : nchars-done; + Py_UNICODE_COPY(p+done, p, n); done += n; } } - assert(_PyUnicode_CheckConsistency(u, 1)); - return u; + return (PyObject*) u; } -PyObject * -PyUnicode_Replace(PyObject *str, - PyObject *substr, - PyObject *replstr, - Py_ssize_t maxcount) +PyObject *PyUnicode_Replace(PyObject *obj, + PyObject *subobj, + PyObject *replobj, + Py_ssize_t maxcount) { - if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0 || - ensure_unicode(replstr) < 0) - return NULL; - return replace(str, substr, replstr, maxcount); -} - -/*[clinic input] -str.replace as unicode_replace - - old: unicode - new: unicode - count: Py_ssize_t = -1 - Maximum number of occurrences to replace. - -1 (the default value) means replace all occurrences. - / - -Return a copy with all occurrences of substring old replaced by new. - -If the optional argument count is given, only the first count occurrences are -replaced. -[clinic start generated code]*/ + PyObject *self; + PyObject *str1; + PyObject *str2; + PyObject *result; -static PyObject * -unicode_replace_impl(PyObject *self, PyObject *old, PyObject *new, - Py_ssize_t count) -/*[clinic end generated code: output=b63f1a8b5eebf448 input=147d12206276ebeb]*/ -{ - if (PyUnicode_READY(self) == -1) + self = PyUnicode_FromObject(obj); + if (self == NULL) + return NULL; + str1 = PyUnicode_FromObject(subobj); + if (str1 == NULL) { + Py_DECREF(self); return NULL; - return replace(self, old, new, count); + } + str2 = PyUnicode_FromObject(replobj); + if (str2 == NULL) { + Py_DECREF(self); + Py_DECREF(str1); + return NULL; + } + result = replace((PyUnicodeObject *)self, + (PyUnicodeObject *)str1, + (PyUnicodeObject *)str2, + maxcount); + Py_DECREF(self); + Py_DECREF(str1); + Py_DECREF(str2); + return result; } -static PyObject * -unicode_repr(PyObject *unicode) +PyDoc_STRVAR(replace__doc__, + "S.replace(old, new[, count]) -> unicode\n\ +\n\ +Return a copy of S with all occurrences of substring\n\ +old replaced by new. If the optional argument count is\n\ +given, only the first count occurrences are replaced."); + +static PyObject* +unicode_replace(PyUnicodeObject *self, PyObject *args) { - PyObject *repr; - Py_ssize_t isize; - Py_ssize_t osize, squote, dquote, i, o; - Py_UCS4 max, quote; - int ikind, okind, unchanged; - void *idata, *odata; + PyUnicodeObject *str1; + PyUnicodeObject *str2; + Py_ssize_t maxcount = -1; + PyObject *result; - if (PyUnicode_READY(unicode) == -1) + if (!PyArg_ParseTuple(args, "OO|n:replace", &str1, &str2, &maxcount)) return NULL; - - isize = PyUnicode_GET_LENGTH(unicode); - idata = PyUnicode_DATA(unicode); - - /* Compute length of output, quote characters, and - maximum character */ - osize = 0; - max = 127; - squote = dquote = 0; - ikind = PyUnicode_KIND(unicode); - for (i = 0; i < isize; i++) { - Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); - Py_ssize_t incr = 1; - switch (ch) { - case '\'': squote++; break; - case '"': dquote++; break; - case '\\': case '\t': case '\r': case '\n': - incr = 2; - break; - default: - /* Fast-path ASCII */ - if (ch < ' ' || ch == 0x7f) - incr = 4; /* \xHH */ - else if (ch < 0x7f) - ; - else if (Py_UNICODE_ISPRINTABLE(ch)) - max = ch > max ? ch : max; - else if (ch < 0x100) - incr = 4; /* \xHH */ - else if (ch < 0x10000) - incr = 6; /* \uHHHH */ - else - incr = 10; /* \uHHHHHHHH */ - } - if (osize > PY_SSIZE_T_MAX - incr) { - PyErr_SetString(PyExc_OverflowError, - "string is too long to generate repr"); - return NULL; - } - osize += incr; - } - - quote = '\''; - unchanged = (osize == isize); - if (squote) { - unchanged = 0; - if (dquote) - /* Both squote and dquote present. Use squote, - and escape them */ - osize += squote; - else - quote = '"'; - } - osize += 2; /* quotes */ - - repr = PyUnicode_New(osize, max); - if (repr == NULL) + str1 = (PyUnicodeObject *)PyUnicode_FromObject((PyObject *)str1); + if (str1 == NULL) + return NULL; + str2 = (PyUnicodeObject *)PyUnicode_FromObject((PyObject *)str2); + if (str2 == NULL) { + Py_DECREF(str1); return NULL; - okind = PyUnicode_KIND(repr); - odata = PyUnicode_DATA(repr); - - PyUnicode_WRITE(okind, odata, 0, quote); - PyUnicode_WRITE(okind, odata, osize-1, quote); - if (unchanged) { - _PyUnicode_FastCopyCharacters(repr, 1, - unicode, 0, - isize); } - else { - for (i = 0, o = 1; i < isize; i++) { - Py_UCS4 ch = PyUnicode_READ(ikind, idata, i); - - /* Escape quotes and backslashes */ - if ((ch == quote) || (ch == '\\')) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, ch); - continue; - } - - /* Map special whitespace to '\t', \n', '\r' */ - if (ch == '\t') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 't'); - } - else if (ch == '\n') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'n'); - } - else if (ch == '\r') { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'r'); - } - /* Map non-printable US ASCII to '\xhh' */ - else if (ch < ' ' || ch == 0x7F) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } + result = replace(self, str1, str2, maxcount); - /* Copy ASCII characters as-is */ - else if (ch < 0x7F) { - PyUnicode_WRITE(okind, odata, o++, ch); - } + Py_DECREF(str1); + Py_DECREF(str2); + return result; +} - /* Non-ASCII characters */ - else { - /* Map Unicode whitespace and control characters - (categories Z* and C* except ASCII space) - */ - if (!Py_UNICODE_ISPRINTABLE(ch)) { - PyUnicode_WRITE(okind, odata, o++, '\\'); - /* Map 8-bit characters to '\xhh' */ - if (ch <= 0xff) { - PyUnicode_WRITE(okind, odata, o++, 'x'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0x000F]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]); - } - /* Map 16-bit characters to '\uxxxx' */ - else if (ch <= 0xffff) { - PyUnicode_WRITE(okind, odata, o++, 'u'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - /* Map 21-bit characters to '\U00xxxxxx' */ - else { - PyUnicode_WRITE(okind, odata, o++, 'U'); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 28) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 24) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 20) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 16) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]); - PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]); - } - } - /* Copy characters as-is */ - else { - PyUnicode_WRITE(okind, odata, o++, ch); - } - } - } - } - /* Closing quote already added at the beginning */ - assert(_PyUnicode_CheckConsistency(repr, 1)); - return repr; +static +PyObject *unicode_repr(PyObject *unicode) +{ + return unicodeescape_string(PyUnicode_AS_UNICODE(unicode), + PyUnicode_GET_SIZE(unicode), + 1); } PyDoc_STRVAR(rfind__doc__, - "S.rfind(sub[, start[, end]]) -> int\n\ + "S.rfind(sub [,start [,end]]) -> int\n\ \n\ Return the highest index in S where substring sub is found,\n\ such that sub is contained within S[start:end]. Optional\n\ @@ -12874,577 +7365,400 @@ arguments start and end are interpreted as in slice notation.\n\ Return -1 on failure."); static PyObject * -unicode_rfind(PyObject *self, PyObject *args) +unicode_rfind(PyUnicodeObject *self, PyObject *args) { - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; + PyUnicodeObject *substring; + Py_ssize_t start; + Py_ssize_t end; Py_ssize_t result; - if (!parse_args_finds_unicode("rfind", args, &substring, &start, &end)) - return NULL; - - if (PyUnicode_READY(self) == -1) + if (!stringlib_parse_args_finds_unicode("rfind", args, &substring, + &start, &end)) return NULL; - result = any_find_slice(self, substring, start, end, -1); + result = stringlib_rfind_slice( + PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), + PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), + start, end + ); - if (result == -2) - return NULL; + Py_DECREF(substring); - return PyLong_FromSsize_t(result); + return PyInt_FromSsize_t(result); } PyDoc_STRVAR(rindex__doc__, - "S.rindex(sub[, start[, end]]) -> int\n\ -\n\ -Return the highest index in S where substring sub is found,\n\ -such that sub is contained within S[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ + "S.rindex(sub [,start [,end]]) -> int\n\ \n\ -Raises ValueError when the substring is not found."); +Like S.rfind() but raise ValueError when the substring is not found."); static PyObject * -unicode_rindex(PyObject *self, PyObject *args) +unicode_rindex(PyUnicodeObject *self, PyObject *args) { - /* initialize variables to prevent gcc warning */ - PyObject *substring = NULL; - Py_ssize_t start = 0; - Py_ssize_t end = 0; + PyUnicodeObject *substring; + Py_ssize_t start; + Py_ssize_t end; Py_ssize_t result; - if (!parse_args_finds_unicode("rindex", args, &substring, &start, &end)) + if (!stringlib_parse_args_finds_unicode("rindex", args, &substring, + &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) - return NULL; + result = stringlib_rfind_slice( + PyUnicode_AS_UNICODE(self), PyUnicode_GET_SIZE(self), + PyUnicode_AS_UNICODE(substring), PyUnicode_GET_SIZE(substring), + start, end + ); - result = any_find_slice(self, substring, start, end, -1); - - if (result == -2) - return NULL; + Py_DECREF(substring); if (result < 0) { PyErr_SetString(PyExc_ValueError, "substring not found"); return NULL; } - - return PyLong_FromSsize_t(result); + return PyInt_FromSsize_t(result); } -/*[clinic input] -str.rjust as unicode_rjust - - width: Py_ssize_t - fillchar: Py_UCS4 = ' ' - / - -Return a right-justified string of length width. - -Padding is done using the specified fill character (default is a space). -[clinic start generated code]*/ +PyDoc_STRVAR(rjust__doc__, + "S.rjust(width[, fillchar]) -> unicode\n\ +\n\ +Return S right-justified in a Unicode string of length width. Padding is\n\ +done using the specified fill character (default is a space)."); static PyObject * -unicode_rjust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=804a1a57fbe8d5cf input=d05f550b5beb1f72]*/ +unicode_rjust(PyUnicodeObject *self, PyObject *args) { - if (PyUnicode_READY(self) == -1) + Py_ssize_t width; + Py_UNICODE fillchar = ' '; + + if (!PyArg_ParseTuple(args, "n|O&:rjust", &width, convert_uc, &fillchar)) return NULL; - if (PyUnicode_GET_LENGTH(self) >= width) - return unicode_result_unchanged(self); + if (self->length >= width && PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return (PyObject*) self; + } - return pad(self, width - PyUnicode_GET_LENGTH(self), 0, fillchar); + return (PyObject*) pad(self, width - self->length, 0, fillchar); } -PyObject * -PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) +static PyObject* +unicode_slice(PyUnicodeObject *self, Py_ssize_t start, Py_ssize_t end) +{ + /* standard clamping */ + if (start < 0) + start = 0; + if (end < 0) + end = 0; + if (end > self->length) + end = self->length; + if (start == 0 && end == self->length && PyUnicode_CheckExact(self)) { + /* full slice, return original string */ + Py_INCREF(self); + return (PyObject*) self; + } + if (start > end) + start = end; + /* copy slice */ + return (PyObject*) PyUnicode_FromUnicode(self->str + start, + end - start); +} + +PyObject *PyUnicode_Split(PyObject *s, + PyObject *sep, + Py_ssize_t maxsplit) { - if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0)) - return NULL; + PyObject *result; - return split(s, sep, maxsplit); -} + s = PyUnicode_FromObject(s); + if (s == NULL) + return NULL; + if (sep != NULL) { + sep = PyUnicode_FromObject(sep); + if (sep == NULL) { + Py_DECREF(s); + return NULL; + } + } -/*[clinic input] -str.split as unicode_split + result = split((PyUnicodeObject *)s, (PyUnicodeObject *)sep, maxsplit); - sep: object = None - The delimiter according which to split the string. - None (the default value) means split according to any whitespace, - and discard empty strings from the result. - maxsplit: Py_ssize_t = -1 - Maximum number of splits to do. - -1 (the default value) means no limit. + Py_DECREF(s); + Py_XDECREF(sep); + return result; +} -Return a list of the words in the string, using sep as the delimiter string. -[clinic start generated code]*/ +PyDoc_STRVAR(split__doc__, + "S.split([sep [,maxsplit]]) -> list of strings\n\ +\n\ +Return a list of the words in S, using sep as the\n\ +delimiter string. If maxsplit is given, at most maxsplit\n\ +splits are done. If sep is not specified or is None, any\n\ +whitespace string is a separator and empty strings are\n\ +removed from the result."); -static PyObject * -unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=3a65b1db356948dc input=606e750488a82359]*/ +static PyObject* +unicode_split(PyUnicodeObject *self, PyObject *args) { - if (sep == Py_None) - return split(self, NULL, maxsplit); - if (PyUnicode_Check(sep)) - return split(self, sep, maxsplit); + PyObject *substring = Py_None; + Py_ssize_t maxcount = -1; - PyErr_Format(PyExc_TypeError, - "must be str or None, not %.100s", - Py_TYPE(sep)->tp_name); - return NULL; + if (!PyArg_ParseTuple(args, "|On:split", &substring, &maxcount)) + return NULL; + + if (substring == Py_None) + return split(self, NULL, maxcount); + else if (PyUnicode_Check(substring)) + return split(self, (PyUnicodeObject *)substring, maxcount); + else + return PyUnicode_Split((PyObject *)self, substring, maxcount); } PyObject * -PyUnicode_Partition(PyObject *str_obj, PyObject *sep_obj) +PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) { + PyObject* str_obj; + PyObject* sep_obj; PyObject* out; - int kind1, kind2; - void *buf1, *buf2; - Py_ssize_t len1, len2; - if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0) + str_obj = PyUnicode_FromObject(str_in); + if (!str_obj) + return NULL; + sep_obj = PyUnicode_FromObject(sep_in); + if (!sep_obj) { + Py_DECREF(str_obj); return NULL; - - kind1 = PyUnicode_KIND(str_obj); - kind2 = PyUnicode_KIND(sep_obj); - len1 = PyUnicode_GET_LENGTH(str_obj); - len2 = PyUnicode_GET_LENGTH(sep_obj); - if (kind1 < kind2 || len1 < len2) { - _Py_INCREF_UNICODE_EMPTY(); - if (!unicode_empty) - out = NULL; - else { - out = PyTuple_Pack(3, str_obj, unicode_empty, unicode_empty); - Py_DECREF(unicode_empty); - } - return out; - } - buf1 = PyUnicode_DATA(str_obj); - buf2 = PyUnicode_DATA(sep_obj); - if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(sep_obj, kind1); - if (!buf2) - return NULL; } - switch (kind1) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sep_obj)) - out = asciilib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); - else - out = ucs1lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); - break; - case PyUnicode_2BYTE_KIND: - out = ucs2lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); - break; - case PyUnicode_4BYTE_KIND: - out = ucs4lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2); - break; - default: - Py_UNREACHABLE(); - } + out = stringlib_partition( + str_obj, PyUnicode_AS_UNICODE(str_obj), PyUnicode_GET_SIZE(str_obj), + sep_obj, PyUnicode_AS_UNICODE(sep_obj), PyUnicode_GET_SIZE(sep_obj) + ); - if (kind2 != kind1) - PyMem_Free(buf2); + Py_DECREF(sep_obj); + Py_DECREF(str_obj); return out; } PyObject * -PyUnicode_RPartition(PyObject *str_obj, PyObject *sep_obj) +PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) { + PyObject* str_obj; + PyObject* sep_obj; PyObject* out; - int kind1, kind2; - void *buf1, *buf2; - Py_ssize_t len1, len2; - if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0) + str_obj = PyUnicode_FromObject(str_in); + if (!str_obj) + return NULL; + sep_obj = PyUnicode_FromObject(sep_in); + if (!sep_obj) { + Py_DECREF(str_obj); return NULL; - - kind1 = PyUnicode_KIND(str_obj); - kind2 = PyUnicode_KIND(sep_obj); - len1 = PyUnicode_GET_LENGTH(str_obj); - len2 = PyUnicode_GET_LENGTH(sep_obj); - if (kind1 < kind2 || len1 < len2) { - _Py_INCREF_UNICODE_EMPTY(); - if (!unicode_empty) - out = NULL; - else { - out = PyTuple_Pack(3, unicode_empty, unicode_empty, str_obj); - Py_DECREF(unicode_empty); - } - return out; - } - buf1 = PyUnicode_DATA(str_obj); - buf2 = PyUnicode_DATA(sep_obj); - if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(sep_obj, kind1); - if (!buf2) - return NULL; } - switch (kind1) { - case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sep_obj)) - out = asciilib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); - else - out = ucs1lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); - break; - case PyUnicode_2BYTE_KIND: - out = ucs2lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); - break; - case PyUnicode_4BYTE_KIND: - out = ucs4lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2); - break; - default: - Py_UNREACHABLE(); - } + out = stringlib_rpartition( + str_obj, PyUnicode_AS_UNICODE(str_obj), PyUnicode_GET_SIZE(str_obj), + sep_obj, PyUnicode_AS_UNICODE(sep_obj), PyUnicode_GET_SIZE(sep_obj) + ); - if (kind2 != kind1) - PyMem_Free(buf2); + Py_DECREF(sep_obj); + Py_DECREF(str_obj); return out; } -/*[clinic input] -str.partition as unicode_partition - - sep: object - / - -Partition the string into three parts using the given separator. - -This will search for the separator in the string. If the separator is found, -returns a 3-tuple containing the part before the separator, the separator -itself, and the part after it. - -If the separator is not found, returns a 3-tuple containing the original string -and two empty strings. -[clinic start generated code]*/ +PyDoc_STRVAR(partition__doc__, + "S.partition(sep) -> (head, sep, tail)\n\ +\n\ +Search for the separator sep in S, and return the part before it,\n\ +the separator itself, and the part after it. If the separator is not\n\ +found, return S and two empty strings."); -static PyObject * -unicode_partition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=e4ced7bd253ca3c4 input=f29b8d06c63e50be]*/ +static PyObject* +unicode_partition(PyUnicodeObject *self, PyObject *separator) { - return PyUnicode_Partition(self, sep); + return PyUnicode_Partition((PyObject *)self, separator); } -/*[clinic input] -str.rpartition as unicode_rpartition = str.partition - -Partition the string into three parts using the given separator. - -This will search for the separator in the string, starting at the end. If -the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it. - -If the separator is not found, returns a 3-tuple containing two empty strings -and the original string. -[clinic start generated code]*/ +PyDoc_STRVAR(rpartition__doc__, + "S.rpartition(sep) -> (head, sep, tail)\n\ +\n\ +Search for the separator sep in S, starting at the end of S, and return\n\ +the part before it, the separator itself, and the part after it. If the\n\ +separator is not found, return two empty strings and S."); -static PyObject * -unicode_rpartition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=1aa13cf1156572aa input=c4b7db3ef5cf336a]*/ +static PyObject* +unicode_rpartition(PyUnicodeObject *self, PyObject *separator) { - return PyUnicode_RPartition(self, sep); + return PyUnicode_RPartition((PyObject *)self, separator); } -PyObject * -PyUnicode_RSplit(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) +PyObject *PyUnicode_RSplit(PyObject *s, + PyObject *sep, + Py_ssize_t maxsplit) { - if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0)) - return NULL; + PyObject *result; - return rsplit(s, sep, maxsplit); -} + s = PyUnicode_FromObject(s); + if (s == NULL) + return NULL; + if (sep != NULL) { + sep = PyUnicode_FromObject(sep); + if (sep == NULL) { + Py_DECREF(s); + return NULL; + } + } -/*[clinic input] -str.rsplit as unicode_rsplit = str.split + result = rsplit((PyUnicodeObject *)s, (PyUnicodeObject *)sep, maxsplit); -Return a list of the words in the string, using sep as the delimiter string. + Py_DECREF(s); + Py_XDECREF(sep); + return result; +} -Splits are done starting at the end of the string and working to the front. -[clinic start generated code]*/ +PyDoc_STRVAR(rsplit__doc__, + "S.rsplit([sep [,maxsplit]]) -> list of strings\n\ +\n\ +Return a list of the words in S, using sep as the\n\ +delimiter string, starting at the end of the string and\n\ +working to the front. If maxsplit is given, at most maxsplit\n\ +splits are done. If sep is not specified, any whitespace string\n\ +is a separator."); -static PyObject * -unicode_rsplit_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=c2b815c63bcabffc input=12ad4bf57dd35f15]*/ +static PyObject* +unicode_rsplit(PyUnicodeObject *self, PyObject *args) { - if (sep == Py_None) - return rsplit(self, NULL, maxsplit); - if (PyUnicode_Check(sep)) - return rsplit(self, sep, maxsplit); + PyObject *substring = Py_None; + Py_ssize_t maxcount = -1; - PyErr_Format(PyExc_TypeError, - "must be str or None, not %.100s", - Py_TYPE(sep)->tp_name); - return NULL; -} + if (!PyArg_ParseTuple(args, "|On:rsplit", &substring, &maxcount)) + return NULL; -/*[clinic input] -str.splitlines as unicode_splitlines + if (substring == Py_None) + return rsplit(self, NULL, maxcount); + else if (PyUnicode_Check(substring)) + return rsplit(self, (PyUnicodeObject *)substring, maxcount); + else + return PyUnicode_RSplit((PyObject *)self, substring, maxcount); +} - keepends: bool(accept={int}) = False +PyDoc_STRVAR(splitlines__doc__, + "S.splitlines(keepends=False) -> list of strings\n\ +\n\ +Return a list of the lines in S, breaking at line boundaries.\n\ +Line breaks are not included in the resulting list unless keepends\n\ +is given and true."); -Return a list of the lines in the string, breaking at line boundaries. +static PyObject* +unicode_splitlines(PyUnicodeObject *self, PyObject *args) +{ + int keepends = 0; -Line breaks are not included in the resulting list unless keepends is given and -true. -[clinic start generated code]*/ + if (!PyArg_ParseTuple(args, "|i:splitlines", &keepends)) + return NULL; -static PyObject * -unicode_splitlines_impl(PyObject *self, int keepends) -/*[clinic end generated code: output=f664dcdad153ec40 input=b508e180459bdd8b]*/ -{ - return PyUnicode_Splitlines(self, keepends); + return PyUnicode_Splitlines((PyObject *)self, keepends); } static -PyObject *unicode_str(PyObject *self) +PyObject *unicode_str(PyUnicodeObject *self) { - return unicode_result_unchanged(self); + return PyUnicode_AsEncodedString((PyObject *)self, NULL, NULL); } -/*[clinic input] -str.swapcase as unicode_swapcase - -Convert uppercase characters to lowercase and lowercase characters to uppercase. -[clinic start generated code]*/ - -static PyObject * -unicode_swapcase_impl(PyObject *self) -/*[clinic end generated code: output=5d28966bf6d7b2af input=3f3ef96d5798a7bb]*/ -{ - if (PyUnicode_READY(self) == -1) - return NULL; - return case_operation(self, do_swapcase); -} - -/*[clinic input] - -@staticmethod -str.maketrans as unicode_maketrans - - x: object - - y: unicode=NULL - - z: unicode=NULL - - / - -Return a translation table usable for str.translate(). - -If there is only one argument, it must be a dictionary mapping Unicode -ordinals (integers) or characters to Unicode ordinals, strings or None. -Character keys will be then converted to ordinals. -If there are two arguments, they must be strings of equal length, and -in the resulting dictionary, each character in x will be mapped to the -character at the same position in y. If there is a third argument, it -must be a string, whose characters will be mapped to None in the result. -[clinic start generated code]*/ +PyDoc_STRVAR(swapcase__doc__, + "S.swapcase() -> unicode\n\ +\n\ +Return a copy of S with uppercase characters converted to lowercase\n\ +and vice versa."); -static PyObject * -unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) -/*[clinic end generated code: output=a925c89452bd5881 input=7bfbf529a293c6c5]*/ +static PyObject* +unicode_swapcase(PyUnicodeObject *self) { - PyObject *new = NULL, *key, *value; - Py_ssize_t i = 0; - int res; - - new = PyDict_New(); - if (!new) - return NULL; - if (y != NULL) { - int x_kind, y_kind, z_kind; - void *x_data, *y_data, *z_data; - - /* x must be a string too, of equal length */ - if (!PyUnicode_Check(x)) { - PyErr_SetString(PyExc_TypeError, "first maketrans argument must " - "be a string if there is a second argument"); - goto err; - } - if (PyUnicode_GET_LENGTH(x) != PyUnicode_GET_LENGTH(y)) { - PyErr_SetString(PyExc_ValueError, "the first two maketrans " - "arguments must have equal length"); - goto err; - } - /* create entries for translating chars in x to those in y */ - x_kind = PyUnicode_KIND(x); - y_kind = PyUnicode_KIND(y); - x_data = PyUnicode_DATA(x); - y_data = PyUnicode_DATA(y); - for (i = 0; i < PyUnicode_GET_LENGTH(x); i++) { - key = PyLong_FromLong(PyUnicode_READ(x_kind, x_data, i)); - if (!key) - goto err; - value = PyLong_FromLong(PyUnicode_READ(y_kind, y_data, i)); - if (!value) { - Py_DECREF(key); - goto err; - } - res = PyDict_SetItem(new, key, value); - Py_DECREF(key); - Py_DECREF(value); - if (res < 0) - goto err; - } - /* create entries for deleting chars in z */ - if (z != NULL) { - z_kind = PyUnicode_KIND(z); - z_data = PyUnicode_DATA(z); - for (i = 0; i < PyUnicode_GET_LENGTH(z); i++) { - key = PyLong_FromLong(PyUnicode_READ(z_kind, z_data, i)); - if (!key) - goto err; - res = PyDict_SetItem(new, key, Py_None); - Py_DECREF(key); - if (res < 0) - goto err; - } - } - } else { - int kind; - void *data; - - /* x must be a dict */ - if (!PyDict_CheckExact(x)) { - PyErr_SetString(PyExc_TypeError, "if you give only one argument " - "to maketrans it must be a dict"); - goto err; - } - /* copy entries into the new dict, converting string keys to int keys */ - while (PyDict_Next(x, &i, &key, &value)) { - if (PyUnicode_Check(key)) { - /* convert string keys to integer keys */ - PyObject *newkey; - if (PyUnicode_GET_LENGTH(key) != 1) { - PyErr_SetString(PyExc_ValueError, "string keys in translate " - "table must be of length 1"); - goto err; - } - kind = PyUnicode_KIND(key); - data = PyUnicode_DATA(key); - newkey = PyLong_FromLong(PyUnicode_READ(kind, data, 0)); - if (!newkey) - goto err; - res = PyDict_SetItem(new, newkey, value); - Py_DECREF(newkey); - if (res < 0) - goto err; - } else if (PyLong_Check(key)) { - /* just keep integer keys */ - if (PyDict_SetItem(new, key, value) < 0) - goto err; - } else { - PyErr_SetString(PyExc_TypeError, "keys in translate table must " - "be strings or integers"); - goto err; - } - } - } - return new; - err: - Py_DECREF(new); - return NULL; + return fixup(self, fixswapcase); } -/*[clinic input] -str.translate as unicode_translate - - table: object - Translation table, which must be a mapping of Unicode ordinals to - Unicode ordinals, strings, or None. - / - -Replace each character in the string using the given translation table. - -The table must implement lookup/indexing via __getitem__, for instance a -dictionary or list. If this operation raises LookupError, the character is -left untouched. Characters mapped to None are deleted. -[clinic start generated code]*/ +PyDoc_STRVAR(translate__doc__, + "S.translate(table) -> unicode\n\ +\n\ +Return a copy of the string S, where all characters have been mapped\n\ +through the given translation table, which must be a mapping of\n\ +Unicode ordinals to Unicode ordinals, Unicode strings or None.\n\ +Unmapped characters are left untouched. Characters mapped to None\n\ +are deleted."); -static PyObject * -unicode_translate(PyObject *self, PyObject *table) -/*[clinic end generated code: output=3cb448ff2fd96bf3 input=6d38343db63d8eb0]*/ +static PyObject* +unicode_translate(PyUnicodeObject *self, PyObject *table) { - return _PyUnicode_TranslateCharmap(self, table, "ignore"); + return PyUnicode_TranslateCharmap(self->str, + self->length, + table, + "ignore"); } -/*[clinic input] -str.upper as unicode_upper - -Return a copy of the string converted to uppercase. -[clinic start generated code]*/ +PyDoc_STRVAR(upper__doc__, + "S.upper() -> unicode\n\ +\n\ +Return a copy of S converted to uppercase."); -static PyObject * -unicode_upper_impl(PyObject *self) -/*[clinic end generated code: output=1b7ddd16bbcdc092 input=db3d55682dfe2e6c]*/ +static PyObject* +unicode_upper(PyUnicodeObject *self) { - if (PyUnicode_READY(self) == -1) - return NULL; - if (PyUnicode_IS_ASCII(self)) - return ascii_upper_or_lower(self, 0); - return case_operation(self, do_upper); + return fixup(self, fixupper); } -/*[clinic input] -str.zfill as unicode_zfill - - width: Py_ssize_t - / - -Pad a numeric string with zeros on the left, to fill a field of the given width. - -The string is never truncated. -[clinic start generated code]*/ +PyDoc_STRVAR(zfill__doc__, + "S.zfill(width) -> unicode\n\ +\n\ +Pad a numeric string S with zeros on the left, to fill a field\n\ +of the specified width. The string S is never truncated."); static PyObject * -unicode_zfill_impl(PyObject *self, Py_ssize_t width) -/*[clinic end generated code: output=e13fb6bdf8e3b9df input=c6b2f772c6f27799]*/ +unicode_zfill(PyUnicodeObject *self, PyObject *args) { Py_ssize_t fill; - PyObject *u; - int kind; - void *data; - Py_UCS4 chr; + PyUnicodeObject *u; - if (PyUnicode_READY(self) == -1) + Py_ssize_t width; + if (!PyArg_ParseTuple(args, "n:zfill", &width)) return NULL; - if (PyUnicode_GET_LENGTH(self) >= width) - return unicode_result_unchanged(self); + if (self->length >= width) { + if (PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return (PyObject*) self; + } + else + return PyUnicode_FromUnicode( + PyUnicode_AS_UNICODE(self), + PyUnicode_GET_SIZE(self) + ); + } - fill = width - PyUnicode_GET_LENGTH(self); + fill = width - self->length; u = pad(self, fill, 0, '0'); if (u == NULL) return NULL; - kind = PyUnicode_KIND(u); - data = PyUnicode_DATA(u); - chr = PyUnicode_READ(kind, data, fill); - - if (chr == '+' || chr == '-') { + if (u->str[fill] == '+' || u->str[fill] == '-') { /* move sign to beginning of string */ - PyUnicode_WRITE(kind, data, 0, chr); - PyUnicode_WRITE(kind, data, fill, '0'); + u->str[0] = u->str[fill]; + u->str[fill] = '0'; } - assert(_PyUnicode_CheckConsistency(u, 1)); - return u; + return (PyObject*) u; } #if 0 -static PyObject * -unicode__decimal2ascii(PyObject *self) +static PyObject* +free_listsize(PyUnicodeObject *self) { - return PyUnicode_TransformDecimalAndSpaceToASCII(self); + return PyInt_FromLong(numfree); } #endif @@ -13457,11 +7771,11 @@ With optional end, stop comparing S at that position.\n\ prefix can also be a tuple of strings to try."); static PyObject * -unicode_startswith(PyObject *self, +unicode_startswith(PyUnicodeObject *self, PyObject *args) { PyObject *subobj; - PyObject *substring; + PyUnicodeObject *substring; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; int result; @@ -13471,17 +7785,12 @@ unicode_startswith(PyObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); - if (!PyUnicode_Check(substring)) { - PyErr_Format(PyExc_TypeError, - "tuple for startswith must only contain str, " - "not %.100s", - Py_TYPE(substring)->tp_name); + substring = (PyUnicodeObject *)PyUnicode_FromObject( + PyTuple_GET_ITEM(subobj, i)); + if (substring == NULL) return NULL; - } result = tailmatch(self, substring, start, end, -1); - if (result == -1) - return NULL; + Py_DECREF(substring); if (result) { Py_RETURN_TRUE; } @@ -13489,15 +7798,15 @@ unicode_startswith(PyObject *self, /* nothing matched */ Py_RETURN_FALSE; } - if (!PyUnicode_Check(subobj)) { - PyErr_Format(PyExc_TypeError, - "startswith first arg must be str or " - "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); + substring = (PyUnicodeObject *)PyUnicode_FromObject(subobj); + if (substring == NULL) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) + PyErr_Format(PyExc_TypeError, "startswith first arg must be str, " + "unicode, or tuple, not %s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, -1); - if (result == -1) - return NULL; + result = tailmatch(self, substring, start, end, -1); + Py_DECREF(substring); return PyBool_FromLong(result); } @@ -13511,11 +7820,11 @@ With optional end, stop comparing S at that position.\n\ suffix can also be a tuple of strings to try."); static PyObject * -unicode_endswith(PyObject *self, +unicode_endswith(PyUnicodeObject *self, PyObject *args) { PyObject *subobj; - PyObject *substring; + PyUnicodeObject *substring; Py_ssize_t start = 0; Py_ssize_t end = PY_SSIZE_T_MAX; int result; @@ -13525,526 +7834,159 @@ unicode_endswith(PyObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyTuple_GET_ITEM(subobj, i); - if (!PyUnicode_Check(substring)) { - PyErr_Format(PyExc_TypeError, - "tuple for endswith must only contain str, " - "not %.100s", - Py_TYPE(substring)->tp_name); + substring = (PyUnicodeObject *)PyUnicode_FromObject( + PyTuple_GET_ITEM(subobj, i)); + if (substring == NULL) return NULL; - } result = tailmatch(self, substring, start, end, +1); - if (result == -1) - return NULL; + Py_DECREF(substring); if (result) { Py_RETURN_TRUE; } } Py_RETURN_FALSE; } - if (!PyUnicode_Check(subobj)) { - PyErr_Format(PyExc_TypeError, - "endswith first arg must be str or " - "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); + substring = (PyUnicodeObject *)PyUnicode_FromObject(subobj); + if (substring == NULL) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) + PyErr_Format(PyExc_TypeError, "endswith first arg must be str, " + "unicode, or tuple, not %s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, subobj, start, end, +1); - if (result == -1) - return NULL; + result = tailmatch(self, substring, start, end, +1); + Py_DECREF(substring); return PyBool_FromLong(result); } -static inline void -_PyUnicodeWriter_Update(_PyUnicodeWriter *writer) -{ - writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); - writer->data = PyUnicode_DATA(writer->buffer); - - if (!writer->readonly) { - writer->kind = PyUnicode_KIND(writer->buffer); - writer->size = PyUnicode_GET_LENGTH(writer->buffer); - } - else { - /* use a value smaller than PyUnicode_1BYTE_KIND() so - _PyUnicodeWriter_PrepareKind() will copy the buffer. */ - writer->kind = PyUnicode_WCHAR_KIND; - assert(writer->kind <= PyUnicode_1BYTE_KIND); - - /* Copy-on-write mode: set buffer size to 0 so - * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on - * next write. */ - writer->size = 0; - } -} - -void -_PyUnicodeWriter_Init(_PyUnicodeWriter *writer) -{ - memset(writer, 0, sizeof(*writer)); - - /* ASCII is the bare minimum */ - writer->min_char = 127; - - /* use a value smaller than PyUnicode_1BYTE_KIND() so - _PyUnicodeWriter_PrepareKind() will copy the buffer. */ - writer->kind = PyUnicode_WCHAR_KIND; - assert(writer->kind <= PyUnicode_1BYTE_KIND); -} - -// Initialize _PyUnicodeWriter with initial buffer -static inline void -_PyUnicodeWriter_InitWithBuffer(_PyUnicodeWriter *writer, PyObject *buffer) -{ - memset(writer, 0, sizeof(*writer)); - writer->buffer = buffer; - _PyUnicodeWriter_Update(writer); - writer->min_length = writer->size; -} - -int -_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar) -{ - Py_ssize_t newlen; - PyObject *newbuffer; - - assert(maxchar <= MAX_UNICODE); - - /* ensure that the _PyUnicodeWriter_Prepare macro was used */ - assert((maxchar > writer->maxchar && length >= 0) - || length > 0); - - if (length > PY_SSIZE_T_MAX - writer->pos) { - PyErr_NoMemory(); - return -1; - } - newlen = writer->pos + length; - - maxchar = Py_MAX(maxchar, writer->min_char); - - if (writer->buffer == NULL) { - assert(!writer->readonly); - if (writer->overallocate - && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { - /* overallocate to limit the number of realloc() */ - newlen += newlen / OVERALLOCATE_FACTOR; - } - if (newlen < writer->min_length) - newlen = writer->min_length; - - writer->buffer = PyUnicode_New(newlen, maxchar); - if (writer->buffer == NULL) - return -1; - } - else if (newlen > writer->size) { - if (writer->overallocate - && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { - /* overallocate to limit the number of realloc() */ - newlen += newlen / OVERALLOCATE_FACTOR; - } - if (newlen < writer->min_length) - newlen = writer->min_length; - - if (maxchar > writer->maxchar || writer->readonly) { - /* resize + widen */ - maxchar = Py_MAX(maxchar, writer->maxchar); - newbuffer = PyUnicode_New(newlen, maxchar); - if (newbuffer == NULL) - return -1; - _PyUnicode_FastCopyCharacters(newbuffer, 0, - writer->buffer, 0, writer->pos); - Py_DECREF(writer->buffer); - writer->readonly = 0; - } - else { - newbuffer = resize_compact(writer->buffer, newlen); - if (newbuffer == NULL) - return -1; - } - writer->buffer = newbuffer; - } - else if (maxchar > writer->maxchar) { - assert(!writer->readonly); - newbuffer = PyUnicode_New(writer->size, maxchar); - if (newbuffer == NULL) - return -1; - _PyUnicode_FastCopyCharacters(newbuffer, 0, - writer->buffer, 0, writer->pos); - Py_SETREF(writer->buffer, newbuffer); - } - _PyUnicodeWriter_Update(writer); - return 0; - -#undef OVERALLOCATE_FACTOR -} - -int -_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, - enum PyUnicode_Kind kind) -{ - Py_UCS4 maxchar; - - /* ensure that the _PyUnicodeWriter_PrepareKind macro was used */ - assert(writer->kind < kind); - - switch (kind) - { - case PyUnicode_1BYTE_KIND: maxchar = 0xff; break; - case PyUnicode_2BYTE_KIND: maxchar = 0xffff; break; - case PyUnicode_4BYTE_KIND: maxchar = 0x10ffff; break; - default: - Py_UNREACHABLE(); - } - - return _PyUnicodeWriter_PrepareInternal(writer, 0, maxchar); -} - -static inline int -_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) -{ - assert(ch <= MAX_UNICODE); - if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0) - return -1; - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch); - writer->pos++; - return 0; -} - -int -_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, Py_UCS4 ch) -{ - return _PyUnicodeWriter_WriteCharInline(writer, ch); -} - -int -_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) -{ - Py_UCS4 maxchar; - Py_ssize_t len; - - if (PyUnicode_READY(str) == -1) - return -1; - len = PyUnicode_GET_LENGTH(str); - if (len == 0) - return 0; - maxchar = PyUnicode_MAX_CHAR_VALUE(str); - if (maxchar > writer->maxchar || len > writer->size - writer->pos) { - if (writer->buffer == NULL && !writer->overallocate) { - assert(_PyUnicode_CheckConsistency(str, 1)); - writer->readonly = 1; - Py_INCREF(str); - writer->buffer = str; - _PyUnicodeWriter_Update(writer); - writer->pos += len; - return 0; - } - if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1) - return -1; - } - _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, - str, 0, len); - writer->pos += len; - return 0; -} - -int -_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, - Py_ssize_t start, Py_ssize_t end) -{ - Py_UCS4 maxchar; - Py_ssize_t len; - - if (PyUnicode_READY(str) == -1) - return -1; - - assert(0 <= start); - assert(end <= PyUnicode_GET_LENGTH(str)); - assert(start <= end); - - if (end == 0) - return 0; - - if (start == 0 && end == PyUnicode_GET_LENGTH(str)) - return _PyUnicodeWriter_WriteStr(writer, str); - - if (PyUnicode_MAX_CHAR_VALUE(str) > writer->maxchar) - maxchar = _PyUnicode_FindMaxChar(str, start, end); - else - maxchar = writer->maxchar; - len = end - start; - - if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) - return -1; - - _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, - str, start, len); - writer->pos += len; - return 0; -} - -int -_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, - const char *ascii, Py_ssize_t len) -{ - if (len == -1) - len = strlen(ascii); - - assert(ucs1lib_find_max_char((Py_UCS1*)ascii, (Py_UCS1*)ascii + len) < 128); - - if (writer->buffer == NULL && !writer->overallocate) { - PyObject *str; - - str = _PyUnicode_FromASCII(ascii, len); - if (str == NULL) - return -1; - - writer->readonly = 1; - writer->buffer = str; - _PyUnicodeWriter_Update(writer); - writer->pos += len; - return 0; - } - - if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1) - return -1; - - switch (writer->kind) - { - case PyUnicode_1BYTE_KIND: - { - const Py_UCS1 *str = (const Py_UCS1 *)ascii; - Py_UCS1 *data = writer->data; - - memcpy(data + writer->pos, str, len); - break; - } - case PyUnicode_2BYTE_KIND: - { - _PyUnicode_CONVERT_BYTES( - Py_UCS1, Py_UCS2, - ascii, ascii + len, - (Py_UCS2 *)writer->data + writer->pos); - break; - } - case PyUnicode_4BYTE_KIND: - { - _PyUnicode_CONVERT_BYTES( - Py_UCS1, Py_UCS4, - ascii, ascii + len, - (Py_UCS4 *)writer->data + writer->pos); - break; - } - default: - Py_UNREACHABLE(); - } - - writer->pos += len; - return 0; -} - -int -_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, - const char *str, Py_ssize_t len) -{ - Py_UCS4 maxchar; - - maxchar = ucs1lib_find_max_char((Py_UCS1*)str, (Py_UCS1*)str + len); - if (_PyUnicodeWriter_Prepare(writer, len, maxchar) == -1) - return -1; - unicode_write_cstr(writer->buffer, writer->pos, str, len); - writer->pos += len; - return 0; -} - -PyObject * -_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) -{ - PyObject *str; - - if (writer->pos == 0) { - Py_CLEAR(writer->buffer); - _Py_RETURN_UNICODE_EMPTY(); - } - - str = writer->buffer; - writer->buffer = NULL; - - if (writer->readonly) { - assert(PyUnicode_GET_LENGTH(str) == writer->pos); - return str; - } - - if (PyUnicode_GET_LENGTH(str) != writer->pos) { - PyObject *str2; - str2 = resize_compact(str, writer->pos); - if (str2 == NULL) { - Py_DECREF(str); - return NULL; - } - str = str2; - } - - assert(_PyUnicode_CheckConsistency(str, 1)); - return unicode_result_ready(str); -} - -void -_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer) -{ - Py_CLEAR(writer->buffer); -} -#include "stringlib/unicode_format.h" +/* Implements do_string_format, which is unicode because of stringlib */ +#include "stringlib/string_format.h" PyDoc_STRVAR(format__doc__, - "S.format(*args, **kwargs) -> str\n\ + "S.format(*args, **kwargs) -> unicode\n\ \n\ Return a formatted version of S, using substitutions from args and kwargs.\n\ The substitutions are identified by braces ('{' and '}')."); -PyDoc_STRVAR(format_map__doc__, - "S.format_map(mapping) -> str\n\ -\n\ -Return a formatted version of S, using substitutions from mapping.\n\ -The substitutions are identified by braces ('{' and '}')."); - -/*[clinic input] -str.__format__ as unicode___format__ - - format_spec: unicode - / - -Return a formatted version of the string as described by format_spec. -[clinic start generated code]*/ - static PyObject * -unicode___format___impl(PyObject *self, PyObject *format_spec) -/*[clinic end generated code: output=45fceaca6d2ba4c8 input=5e135645d167a214]*/ +unicode__format__(PyObject *self, PyObject *args) { - _PyUnicodeWriter writer; - int ret; + PyObject *format_spec; + PyObject *result = NULL; + PyObject *tmp = NULL; - if (PyUnicode_READY(self) == -1) - return NULL; - _PyUnicodeWriter_Init(&writer); - ret = _PyUnicode_FormatAdvancedWriter(&writer, - self, format_spec, 0, - PyUnicode_GET_LENGTH(format_spec)); - if (ret == -1) { - _PyUnicodeWriter_Dealloc(&writer); - return NULL; + /* If 2.x, convert format_spec to the same type as value */ + /* This is to allow things like u''.format('') */ + if (!PyArg_ParseTuple(args, "O:__format__", &format_spec)) + goto done; + if (!(PyBytes_Check(format_spec) || PyUnicode_Check(format_spec))) { + PyErr_Format(PyExc_TypeError, "__format__ arg must be str " + "or unicode, not %s", Py_TYPE(format_spec)->tp_name); + goto done; } - return _PyUnicodeWriter_Finish(&writer); -} + tmp = PyObject_Unicode(format_spec); + if (tmp == NULL) + goto done; + format_spec = tmp; -/*[clinic input] -str.__sizeof__ as unicode_sizeof + result = _PyUnicode_FormatAdvanced(self, + PyUnicode_AS_UNICODE(format_spec), + PyUnicode_GET_SIZE(format_spec)); + done: + Py_XDECREF(tmp); + return result; +} -Return the size of the string in memory, in bytes. -[clinic start generated code]*/ +PyDoc_STRVAR(p_format__doc__, + "S.__format__(format_spec) -> unicode\n\ +\n\ +Return a formatted version of S as described by format_spec."); static PyObject * -unicode_sizeof_impl(PyObject *self) -/*[clinic end generated code: output=6dbc2f5a408b6d4f input=6dd011c108e33fb0]*/ -{ - Py_ssize_t size; - - /* If it's a compact object, account for base structure + - character data. */ - if (PyUnicode_IS_COMPACT_ASCII(self)) - size = sizeof(PyASCIIObject) + PyUnicode_GET_LENGTH(self) + 1; - else if (PyUnicode_IS_COMPACT(self)) - size = sizeof(PyCompactUnicodeObject) + - (PyUnicode_GET_LENGTH(self) + 1) * PyUnicode_KIND(self); - else { - /* If it is a two-block object, account for base object, and - for character block if present. */ - size = sizeof(PyUnicodeObject); - if (_PyUnicode_DATA_ANY(self)) - size += (PyUnicode_GET_LENGTH(self) + 1) * - PyUnicode_KIND(self); - } - /* If the wstr pointer is present, account for it unless it is shared - with the data pointer. Check if the data is not shared. */ - if (_PyUnicode_HAS_WSTR_MEMORY(self)) - size += (PyUnicode_WSTR_LENGTH(self) + 1) * sizeof(wchar_t); - if (_PyUnicode_HAS_UTF8_MEMORY(self)) - size += PyUnicode_UTF8_LENGTH(self) + 1; - - return PyLong_FromSsize_t(size); +unicode__sizeof__(PyUnicodeObject *v) +{ + return PyInt_FromSsize_t(sizeof(PyUnicodeObject) + + sizeof(Py_UNICODE) * (v->length + 1)); } +PyDoc_STRVAR(sizeof__doc__, + "S.__sizeof__() -> size of S in memory, in bytes\n\ +\n\ +"); + static PyObject * -unicode_getnewargs(PyObject *v, PyObject *Py_UNUSED(ignored)) +unicode_getnewargs(PyUnicodeObject *v) { - PyObject *copy = _PyUnicode_Copy(v); - if (!copy) - return NULL; - return Py_BuildValue("(N)", copy); + return Py_BuildValue("(u#)", v->str, v->length); } + static PyMethodDef unicode_methods[] = { - UNICODE_ENCODE_METHODDEF - UNICODE_REPLACE_METHODDEF - UNICODE_SPLIT_METHODDEF - UNICODE_RSPLIT_METHODDEF - UNICODE_JOIN_METHODDEF - UNICODE_CAPITALIZE_METHODDEF - UNICODE_CASEFOLD_METHODDEF - UNICODE_TITLE_METHODDEF - UNICODE_CENTER_METHODDEF + {"encode", (PyCFunction) unicode_encode, METH_VARARGS | METH_KEYWORDS, encode__doc__}, + {"replace", (PyCFunction) unicode_replace, METH_VARARGS, replace__doc__}, + {"split", (PyCFunction) unicode_split, METH_VARARGS, split__doc__}, + {"rsplit", (PyCFunction) unicode_rsplit, METH_VARARGS, rsplit__doc__}, + {"join", (PyCFunction) unicode_join, METH_O, join__doc__}, + {"capitalize", (PyCFunction) unicode_capitalize, METH_NOARGS, capitalize__doc__}, + {"title", (PyCFunction) unicode_title, METH_NOARGS, title__doc__}, + {"center", (PyCFunction) unicode_center, METH_VARARGS, center__doc__}, {"count", (PyCFunction) unicode_count, METH_VARARGS, count__doc__}, - UNICODE_EXPANDTABS_METHODDEF + {"expandtabs", (PyCFunction) unicode_expandtabs, METH_VARARGS, expandtabs__doc__}, {"find", (PyCFunction) unicode_find, METH_VARARGS, find__doc__}, - UNICODE_PARTITION_METHODDEF + {"partition", (PyCFunction) unicode_partition, METH_O, partition__doc__}, {"index", (PyCFunction) unicode_index, METH_VARARGS, index__doc__}, - UNICODE_LJUST_METHODDEF - UNICODE_LOWER_METHODDEF - UNICODE_LSTRIP_METHODDEF + {"ljust", (PyCFunction) unicode_ljust, METH_VARARGS, ljust__doc__}, + {"lower", (PyCFunction) unicode_lower, METH_NOARGS, lower__doc__}, + {"lstrip", (PyCFunction) unicode_lstrip, METH_VARARGS, lstrip__doc__}, + {"decode", (PyCFunction) unicode_decode, METH_VARARGS | METH_KEYWORDS, decode__doc__}, +/* {"maketrans", (PyCFunction) unicode_maketrans, METH_VARARGS, maketrans__doc__}, */ {"rfind", (PyCFunction) unicode_rfind, METH_VARARGS, rfind__doc__}, {"rindex", (PyCFunction) unicode_rindex, METH_VARARGS, rindex__doc__}, - UNICODE_RJUST_METHODDEF - UNICODE_RSTRIP_METHODDEF - UNICODE_RPARTITION_METHODDEF - UNICODE_SPLITLINES_METHODDEF - UNICODE_STRIP_METHODDEF - UNICODE_SWAPCASE_METHODDEF - UNICODE_TRANSLATE_METHODDEF - UNICODE_UPPER_METHODDEF + {"rjust", (PyCFunction) unicode_rjust, METH_VARARGS, rjust__doc__}, + {"rstrip", (PyCFunction) unicode_rstrip, METH_VARARGS, rstrip__doc__}, + {"rpartition", (PyCFunction) unicode_rpartition, METH_O, rpartition__doc__}, + {"splitlines", (PyCFunction) unicode_splitlines, METH_VARARGS, splitlines__doc__}, + {"strip", (PyCFunction) unicode_strip, METH_VARARGS, strip__doc__}, + {"swapcase", (PyCFunction) unicode_swapcase, METH_NOARGS, swapcase__doc__}, + {"translate", (PyCFunction) unicode_translate, METH_O, translate__doc__}, + {"upper", (PyCFunction) unicode_upper, METH_NOARGS, upper__doc__}, {"startswith", (PyCFunction) unicode_startswith, METH_VARARGS, startswith__doc__}, {"endswith", (PyCFunction) unicode_endswith, METH_VARARGS, endswith__doc__}, - UNICODE_ISASCII_METHODDEF - UNICODE_ISLOWER_METHODDEF - UNICODE_ISUPPER_METHODDEF - UNICODE_ISTITLE_METHODDEF - UNICODE_ISSPACE_METHODDEF - UNICODE_ISDECIMAL_METHODDEF - UNICODE_ISDIGIT_METHODDEF - UNICODE_ISNUMERIC_METHODDEF - UNICODE_ISALPHA_METHODDEF - UNICODE_ISALNUM_METHODDEF - UNICODE_ISIDENTIFIER_METHODDEF - UNICODE_ISPRINTABLE_METHODDEF - UNICODE_ZFILL_METHODDEF - {"format", (PyCFunction)(void(*)(void)) do_string_format, METH_VARARGS | METH_KEYWORDS, format__doc__}, - {"format_map", (PyCFunction) do_string_format_map, METH_O, format_map__doc__}, - UNICODE___FORMAT___METHODDEF - UNICODE_MAKETRANS_METHODDEF - UNICODE_SIZEOF_METHODDEF + {"islower", (PyCFunction) unicode_islower, METH_NOARGS, islower__doc__}, + {"isupper", (PyCFunction) unicode_isupper, METH_NOARGS, isupper__doc__}, + {"istitle", (PyCFunction) unicode_istitle, METH_NOARGS, istitle__doc__}, + {"isspace", (PyCFunction) unicode_isspace, METH_NOARGS, isspace__doc__}, + {"isdecimal", (PyCFunction) unicode_isdecimal, METH_NOARGS, isdecimal__doc__}, + {"isdigit", (PyCFunction) unicode_isdigit, METH_NOARGS, isdigit__doc__}, + {"isnumeric", (PyCFunction) unicode_isnumeric, METH_NOARGS, isnumeric__doc__}, + {"isalpha", (PyCFunction) unicode_isalpha, METH_NOARGS, isalpha__doc__}, + {"isalnum", (PyCFunction) unicode_isalnum, METH_NOARGS, isalnum__doc__}, + {"zfill", (PyCFunction) unicode_zfill, METH_VARARGS, zfill__doc__}, + {"format", (PyCFunction) do_string_format, METH_VARARGS | METH_KEYWORDS, format__doc__}, + {"__format__", (PyCFunction) unicode__format__, METH_VARARGS, p_format__doc__}, + {"_formatter_field_name_split", (PyCFunction) formatter_field_name_split, METH_NOARGS}, + {"_formatter_parser", (PyCFunction) formatter_parser, METH_NOARGS}, + {"__sizeof__", (PyCFunction) unicode__sizeof__, METH_NOARGS, sizeof__doc__}, +#if 0 + {"capwords", (PyCFunction) unicode_capwords, METH_NOARGS, capwords__doc__}, +#endif + #if 0 - /* These methods are just used for debugging the implementation. */ - {"_decimal2ascii", (PyCFunction) unicode__decimal2ascii, METH_NOARGS}, + /* This one is just used for debugging the implementation. */ + {"freelistsize", (PyCFunction) free_listsize, METH_NOARGS}, #endif - {"__getnewargs__", unicode_getnewargs, METH_NOARGS}, + {"__getnewargs__", (PyCFunction)unicode_getnewargs, METH_NOARGS}, {NULL, NULL} }; static PyObject * unicode_mod(PyObject *v, PyObject *w) { - if (!PyUnicode_Check(v)) - Py_RETURN_NOTIMPLEMENTED; + if (!PyUnicode_Check(v)) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } return PyUnicode_Format(v, w); } @@ -14052,6 +7994,7 @@ static PyNumberMethods unicode_as_number = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ + 0, /*nb_divide*/ unicode_mod, /*nb_remainder*/ }; @@ -14060,77 +8003,58 @@ static PySequenceMethods unicode_as_sequence = { PyUnicode_Concat, /* sq_concat */ (ssizeargfunc) unicode_repeat, /* sq_repeat */ (ssizeargfunc) unicode_getitem, /* sq_item */ - 0, /* sq_slice */ + (ssizessizeargfunc) unicode_slice, /* sq_slice */ 0, /* sq_ass_item */ 0, /* sq_ass_slice */ PyUnicode_Contains, /* sq_contains */ }; static PyObject* -unicode_subscript(PyObject* self, PyObject* item) +unicode_subscript(PyUnicodeObject* self, PyObject* item) { - if (PyUnicode_READY(self) == -1) - return NULL; - if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; if (i < 0) - i += PyUnicode_GET_LENGTH(self); + i += PyUnicode_GET_SIZE(self); return unicode_getitem(self, i); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength, i; - size_t cur; - PyObject *result; - void *src_data, *dest_data; - int src_kind, dest_kind; - Py_UCS4 ch, max_char, kind_limit; - - if (PySlice_Unpack(item, &start, &stop, &step) < 0) { + Py_ssize_t start, stop, step, slicelength, cur, i; + Py_UNICODE* source_buf; + Py_UNICODE* result_buf; + PyObject* result; + + if (_PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } - slicelength = PySlice_AdjustIndices(PyUnicode_GET_LENGTH(self), - &start, &stop, step); + slicelength = _PySlice_AdjustIndices(PyUnicode_GET_SIZE(self), &start, + &stop, step); if (slicelength <= 0) { - _Py_RETURN_UNICODE_EMPTY(); - } else if (start == 0 && step == 1 && - slicelength == PyUnicode_GET_LENGTH(self)) { - return unicode_result_unchanged(self); + return PyUnicode_FromUnicode(NULL, 0); + } else if (start == 0 && step == 1 && slicelength == self->length && + PyUnicode_CheckExact(self)) { + Py_INCREF(self); + return (PyObject *)self; } else if (step == 1) { - return PyUnicode_Substring(self, - start, start + slicelength); - } - /* General case */ - src_kind = PyUnicode_KIND(self); - src_data = PyUnicode_DATA(self); - if (!PyUnicode_IS_ASCII(self)) { - kind_limit = kind_maxchar_limit(src_kind); - max_char = 0; + return PyUnicode_FromUnicode(self->str + start, slicelength); + } else { + source_buf = PyUnicode_AS_UNICODE((PyObject*)self); + result_buf = (Py_UNICODE *)PyObject_MALLOC(slicelength* + sizeof(Py_UNICODE)); + + if (result_buf == NULL) + return PyErr_NoMemory(); + for (cur = start, i = 0; i < slicelength; cur += step, i++) { - ch = PyUnicode_READ(src_kind, src_data, cur); - if (ch > max_char) { - max_char = ch; - if (max_char >= kind_limit) - break; - } + result_buf[i] = source_buf[cur]; } - } - else - max_char = 127; - result = PyUnicode_New(slicelength, max_char); - if (result == NULL) - return NULL; - dest_kind = PyUnicode_KIND(result); - dest_data = PyUnicode_DATA(result); - for (cur = start, i = 0; i < slicelength; cur += step, i++) { - Py_UCS4 ch = PyUnicode_READ(src_kind, src_data, cur); - PyUnicode_WRITE(dest_kind, dest_data, i, ch); + result = PyUnicode_FromUnicode(result_buf, slicelength); + PyObject_FREE(result_buf); + return result; } - assert(_PyUnicode_CheckConsistency(result, 1)); - return result; } else { PyErr_SetString(PyExc_TypeError, "string indices must be integers"); return NULL; @@ -14143,968 +8067,776 @@ static PyMappingMethods unicode_as_mapping = { (objobjargproc)0, /* mp_ass_subscript */ }; +static Py_ssize_t +unicode_buffer_getreadbuf(PyUnicodeObject *self, + Py_ssize_t index, + const void **ptr) +{ + if (index != 0) { + PyErr_SetString(PyExc_SystemError, + "accessing non-existent unicode segment"); + return -1; + } + *ptr = (void *) self->str; + return PyUnicode_GET_DATA_SIZE(self); +} -/* Helpers for PyUnicode_Format() */ +static Py_ssize_t +unicode_buffer_getwritebuf(PyUnicodeObject *self, Py_ssize_t index, + const void **ptr) +{ + PyErr_SetString(PyExc_TypeError, + "cannot use unicode as modifiable buffer"); + return -1; +} -struct unicode_formatter_t { - PyObject *args; - int args_owned; - Py_ssize_t arglen, argidx; - PyObject *dict; +static int +unicode_buffer_getsegcount(PyUnicodeObject *self, + Py_ssize_t *lenp) +{ + if (lenp) + *lenp = PyUnicode_GET_DATA_SIZE(self); + return 1; +} - enum PyUnicode_Kind fmtkind; - Py_ssize_t fmtcnt, fmtpos; - void *fmtdata; - PyObject *fmtstr; +static Py_ssize_t +unicode_buffer_getcharbuf(PyUnicodeObject *self, + Py_ssize_t index, + const void **ptr) +{ + PyObject *str; - _PyUnicodeWriter writer; -}; + if (index != 0) { + PyErr_SetString(PyExc_SystemError, + "accessing non-existent unicode segment"); + return -1; + } + str = _PyUnicode_AsDefaultEncodedString((PyObject *)self, NULL); + if (str == NULL) + return -1; + *ptr = (void *) PyString_AS_STRING(str); + return PyString_GET_SIZE(str); +} -struct unicode_format_arg_t { - Py_UCS4 ch; - int flags; - Py_ssize_t width; - int prec; - int sign; -}; +/* Helpers for PyUnicode_Format() */ static PyObject * -unicode_format_getnextarg(struct unicode_formatter_t *ctx) +getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) { - Py_ssize_t argidx = ctx->argidx; - - if (argidx < ctx->arglen) { - ctx->argidx++; - if (ctx->arglen < 0) - return ctx->args; + Py_ssize_t argidx = *p_argidx; + if (argidx < arglen) { + (*p_argidx)++; + if (arglen < 0) + return args; else - return PyTuple_GetItem(ctx->args, argidx); + return PyTuple_GetItem(args, argidx); } PyErr_SetString(PyExc_TypeError, "not enough arguments for format string"); return NULL; } -/* Returns a new reference to a PyUnicode object, or NULL on failure. */ +#define F_LJUST (1<<0) +#define F_SIGN (1<<1) +#define F_BLANK (1<<2) +#define F_ALT (1<<3) +#define F_ZERO (1<<4) + +static Py_ssize_t +strtounicode(Py_UNICODE *buffer, const char *charbuffer) +{ + register Py_ssize_t i; + Py_ssize_t len = strlen(charbuffer); + for (i = len - 1; i >= 0; i--) + buffer[i] = (Py_UNICODE) charbuffer[i]; -/* Format a float into the writer if the writer is not NULL, or into *p_output - otherwise. + return len; +} - Return 0 on success, raise an exception and return -1 on error. */ static int -formatfloat(PyObject *v, struct unicode_format_arg_t *arg, - PyObject **p_output, - _PyUnicodeWriter *writer) +longtounicode(Py_UNICODE *buffer, size_t len, const char *format, long x) +{ + Py_ssize_t result; + + PyOS_snprintf((char *)buffer, len, format, x); + result = strtounicode(buffer, (char *)buffer); + return Py_SAFE_DOWNCAST(result, Py_ssize_t, int); +} + +/* XXX To save some code duplication, formatfloat/long/int could have been + shared with stringobject.c, converting from 8-bit to Unicode after the + formatting is done. */ + +/* Returns a new reference to a PyUnicode object, or NULL on failure. */ + +static PyObject * +formatfloat(PyObject *v, int flags, int prec, int type) { char *p; + PyObject *result; double x; - Py_ssize_t len; - int prec; - int dtoa_flags; x = PyFloat_AsDouble(v); if (x == -1.0 && PyErr_Occurred()) - return -1; + return NULL; - prec = arg->prec; if (prec < 0) prec = 6; - if (arg->flags & F_ALT) - dtoa_flags = Py_DTSF_ALT; - else - dtoa_flags = 0; - p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL); + p = PyOS_double_to_string(x, type, prec, + (flags & F_ALT) ? Py_DTSF_ALT : 0, NULL); if (p == NULL) - return -1; - len = strlen(p); - if (writer) { - if (_PyUnicodeWriter_WriteASCIIString(writer, p, len) < 0) { - PyMem_Free(p); - return -1; - } - } - else - *p_output = _PyUnicode_FromASCII(p, len); + return NULL; + result = PyUnicode_FromStringAndSize(p, strlen(p)); PyMem_Free(p); - return 0; + return result; } -/* formatlong() emulates the format codes d, u, o, x and X, and - * the F_ALT flag, for Python's long (unbounded) ints. It's not used for - * Python's regular ints. - * Return value: a new PyUnicodeObject*, or NULL if error. - * The output string is of the form - * "-"? ("0x" | "0X")? digit+ - * "0x"/"0X" are present only for x and X conversions, with F_ALT - * set in flags. The case of hex digits will be correct, - * There will be at least prec digits, zero-filled on the left if - * necessary to get that many. - * val object to be converted - * flags bitmask of format flags; only F_ALT is looked at - * prec minimum number of digits; 0-fill on left if needed - * type a character in [duoxX]; u acts the same as d - * - * CAUTION: o, x and X conversions on regular ints can never - * produce a '-' sign, but can for Python's unbounded ints. - */ -PyObject * -_PyUnicode_FormatLong(PyObject *val, int alt, int prec, int type) +static PyObject* +formatlong(PyObject *val, int flags, int prec, int type) { - PyObject *result = NULL; char *buf; - Py_ssize_t i; - int sign; /* 1 if '-', else 0 */ - int len; /* number of characters */ - Py_ssize_t llen; - int numdigits; /* len == numnondigits + numdigits */ - int numnondigits = 0; - - /* Avoid exceeding SSIZE_T_MAX */ - if (prec > INT_MAX-3) { - PyErr_SetString(PyExc_OverflowError, - "precision too large"); - return NULL; - } - - assert(PyLong_Check(val)); + int i, len; + PyObject *str; /* temporary string object. */ + PyUnicodeObject *result; - switch (type) { - default: - Py_UNREACHABLE(); - case 'd': - case 'i': - case 'u': - /* int and int subclasses should print numerically when a numeric */ - /* format code is used (see issue18780) */ - result = PyNumber_ToBase(val, 10); - break; - case 'o': - numnondigits = 2; - result = PyNumber_ToBase(val, 8); - break; - case 'x': - case 'X': - numnondigits = 2; - result = PyNumber_ToBase(val, 16); - break; - } - if (!result) - return NULL; - - assert(unicode_modifiable(result)); - assert(PyUnicode_IS_READY(result)); - assert(PyUnicode_IS_ASCII(result)); - - /* To modify the string in-place, there can only be one reference. */ - if (Py_REFCNT(result) != 1) { - Py_DECREF(result); - PyErr_BadInternalCall(); + str = _PyString_FormatLong(val, flags, prec, type, &buf, &len); + if (!str) return NULL; - } - buf = PyUnicode_DATA(result); - llen = PyUnicode_GET_LENGTH(result); - if (llen > INT_MAX) { - Py_DECREF(result); - PyErr_SetString(PyExc_ValueError, - "string too large in _PyUnicode_FormatLong"); + result = _PyUnicode_New(len); + if (!result) { + Py_DECREF(str); return NULL; } - len = (int)llen; - sign = buf[0] == '-'; - numnondigits += sign; - numdigits = len - numnondigits; - assert(numdigits > 0); - - /* Get rid of base marker unless F_ALT */ - if (((alt) == 0 && - (type == 'o' || type == 'x' || type == 'X'))) { - assert(buf[sign] == '0'); - assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' || - buf[sign+1] == 'o'); - numnondigits -= 2; - buf += 2; - len -= 2; - if (sign) - buf[0] = '-'; - assert(len == numnondigits + numdigits); - assert(numdigits > 0); - } - - /* Fill with leading zeroes to meet minimum width. */ - if (prec > numdigits) { - PyObject *r1 = PyBytes_FromStringAndSize(NULL, - numnondigits + prec); - char *b1; - if (!r1) { - Py_DECREF(result); - return NULL; - } - b1 = PyBytes_AS_STRING(r1); - for (i = 0; i < numnondigits; ++i) - *b1++ = *buf++; - for (i = 0; i < prec - numdigits; i++) - *b1++ = '0'; - for (i = 0; i < numdigits; i++) - *b1++ = *buf++; - *b1 = '\0'; - Py_DECREF(result); - result = r1; - buf = PyBytes_AS_STRING(result); - len = numnondigits + prec; - } - - /* Fix up case for hex conversions. */ - if (type == 'X') { - /* Need to convert all lower case letters to upper case. - and need to convert 0x to 0X (and -0x to -0X). */ - for (i = 0; i < len; i++) - if (buf[i] >= 'a' && buf[i] <= 'x') - buf[i] -= 'a'-'A'; - } - if (!PyUnicode_Check(result) - || buf != PyUnicode_DATA(result)) { - PyObject *unicode; - unicode = _PyUnicode_FromASCII(buf, len); - Py_DECREF(result); - result = unicode; - } - else if (len != PyUnicode_GET_LENGTH(result)) { - if (PyUnicode_Resize(&result, len) < 0) - Py_CLEAR(result); - } - return result; + for (i = 0; i < len; i++) + result->str[i] = buf[i]; + result->str[len] = 0; + Py_DECREF(str); + return (PyObject*)result; } -/* Format an integer or a float as an integer. - * Return 1 if the number has been formatted into the writer, - * 0 if the number has been formatted into *p_output - * -1 and raise an exception on error */ static int -mainformatlong(PyObject *v, - struct unicode_format_arg_t *arg, - PyObject **p_output, - _PyUnicodeWriter *writer) -{ - PyObject *iobj, *res; - char type = (char)arg->ch; - - if (!PyNumber_Check(v)) - goto wrongtype; - - /* make sure number is a type of integer for o, x, and X */ - if (!PyLong_Check(v)) { - if (type == 'o' || type == 'x' || type == 'X') { - iobj = PyNumber_Index(v); - if (iobj == NULL) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - goto wrongtype; - return -1; - } - } - else { - iobj = PyNumber_Long(v); - if (iobj == NULL ) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - goto wrongtype; - return -1; - } - } - assert(PyLong_Check(iobj)); - } - else { - iobj = v; - Py_INCREF(iobj); - } - - if (PyLong_CheckExact(v) - && arg->width == -1 && arg->prec == -1 - && !(arg->flags & (F_SIGN | F_BLANK)) - && type != 'X') - { - /* Fast path */ - int alternate = arg->flags & F_ALT; - int base; - - switch(type) - { - default: - Py_UNREACHABLE(); - case 'd': - case 'i': - case 'u': - base = 10; - break; - case 'o': - base = 8; - break; - case 'x': - case 'X': - base = 16; - break; - } +formatint(Py_UNICODE *buf, + size_t buflen, + int flags, + int prec, + int type, + PyObject *v) +{ + /* fmt = '%#.' + `prec` + 'l' + `type` + * worst case length = 3 + 19 (worst len of INT_MAX on 64-bit machine) + * + 1 + 1 + * = 24 + */ + char fmt[64]; /* plenty big enough! */ + char *sign; + long x; - if (_PyLong_FormatWriter(writer, v, base, alternate) == -1) { - Py_DECREF(iobj); - return -1; - } - Py_DECREF(iobj); - return 1; + x = PyInt_AsLong(v); + if (x == -1 && PyErr_Occurred()) + return -1; + if (x < 0 && type == 'u') { + type = 'd'; } + if (x < 0 && (type == 'x' || type == 'X' || type == 'o')) + sign = "-"; + else + sign = ""; + if (prec < 0) + prec = 1; - res = _PyUnicode_FormatLong(iobj, arg->flags & F_ALT, arg->prec, type); - Py_DECREF(iobj); - if (res == NULL) + /* buf = '+'/'-'/'' + '0'/'0x'/'' + '[0-9]'*max(prec, len(x in octal)) + * worst case buf = '-0x' + [0-9]*prec, where prec >= 11 + */ + if (buflen <= 14 || buflen <= (size_t)3 + (size_t)prec) { + PyErr_SetString(PyExc_OverflowError, + "formatted integer is too long (precision too large?)"); return -1; - *p_output = res; - return 0; + } -wrongtype: - switch(type) - { - case 'o': - case 'x': - case 'X': - PyErr_Format(PyExc_TypeError, - "%%%c format: an integer is required, " - "not %.200s", - type, Py_TYPE(v)->tp_name); - break; - default: - PyErr_Format(PyExc_TypeError, - "%%%c format: a number is required, " - "not %.200s", - type, Py_TYPE(v)->tp_name); - break; + if ((flags & F_ALT) && + (type == 'x' || type == 'X')) { + /* When converting under %#x or %#X, there are a number + * of issues that cause pain: + * - when 0 is being converted, the C standard leaves off + * the '0x' or '0X', which is inconsistent with other + * %#x/%#X conversions and inconsistent with Python's + * hex() function + * - there are platforms that violate the standard and + * convert 0 with the '0x' or '0X' + * (Metrowerks, Compaq Tru64) + * - there are platforms that give '0x' when converting + * under %#X, but convert 0 in accordance with the + * standard (OS/2 EMX) + * + * We can achieve the desired consistency by inserting our + * own '0x' or '0X' prefix, and substituting %x/%X in place + * of %#x/%#X. + * + * Note that this is the same approach as used in + * formatint() in stringobject.c + */ + PyOS_snprintf(fmt, sizeof(fmt), "%s0%c%%.%dl%c", + sign, type, prec, type); } - return -1; + else { + PyOS_snprintf(fmt, sizeof(fmt), "%s%%%s.%dl%c", + sign, (flags&F_ALT) ? "#" : "", + prec, type); + } + if (sign[0]) + return longtounicode(buf, buflen, fmt, -x); + else + return longtounicode(buf, buflen, fmt, x); } -static Py_UCS4 -formatchar(PyObject *v) +static int +formatchar(Py_UNICODE *buf, + size_t buflen, + PyObject *v) { - /* presume that the buffer is at least 3 characters long */ + PyObject *unistr; + char *str; + /* presume that the buffer is at least 2 characters long */ if (PyUnicode_Check(v)) { - if (PyUnicode_GET_LENGTH(v) == 1) { - return PyUnicode_READ_CHAR(v, 0); + if (PyUnicode_GET_SIZE(v) != 1) + goto onError; + buf[0] = PyUnicode_AS_UNICODE(v)[0]; + } + + else if (PyString_Check(v)) { + if (PyString_GET_SIZE(v) != 1) + goto onError; + /* #7649: "u'%c' % char" should behave like "u'%s' % char" and fail + with a UnicodeDecodeError if 'char' is not decodable with the + default encoding (usually ASCII, but it might be something else) */ + str = PyString_AS_STRING(v); + if ((unsigned char)str[0] > 0x7F) { + /* the char is not ASCII; try to decode the string using the + default encoding and return -1 to let the UnicodeDecodeError + be raised if the string can't be decoded */ + unistr = PyUnicode_Decode(str, 1, NULL, "strict"); + if (unistr == NULL) + return -1; + buf[0] = PyUnicode_AS_UNICODE(unistr)[0]; + Py_DECREF(unistr); } - goto onError; + else + buf[0] = (Py_UNICODE)str[0]; } + else { - PyObject *iobj; + /* Integer input truncated to a character */ long x; - /* make sure number is a type of integer */ - if (!PyLong_Check(v)) { - iobj = PyNumber_Index(v); - if (iobj == NULL) { - goto onError; - } - x = PyLong_AsLong(iobj); - Py_DECREF(iobj); - } - else { - x = PyLong_AsLong(v); - } + x = PyInt_AsLong(v); if (x == -1 && PyErr_Occurred()) goto onError; - - if (x < 0 || x > MAX_UNICODE) { +#ifdef Py_UNICODE_WIDE + if (x < 0 || x > 0x10ffff) { PyErr_SetString(PyExc_OverflowError, - "%c arg not in range(0x110000)"); - return (Py_UCS4) -1; + "%c arg not in range(0x110000) " + "(wide Python build)"); + return -1; } - - return (Py_UCS4) x; +#else + if (x < 0 || x > 0xffff) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x10000) " + "(narrow Python build)"); + return -1; + } +#endif + buf[0] = (Py_UNICODE) x; } + buf[1] = '\0'; + return 1; onError: PyErr_SetString(PyExc_TypeError, "%c requires int or char"); - return (Py_UCS4) -1; + return -1; } -/* Parse options of an argument: flags, width, precision. - Handle also "%(name)" syntax. - - Return 0 if the argument has been formatted into arg->str. - Return 1 if the argument has been written into ctx->writer, - Raise an exception and return -1 on error. */ -static int -unicode_format_arg_parse(struct unicode_formatter_t *ctx, - struct unicode_format_arg_t *arg) -{ -#define FORMAT_READ(ctx) \ - PyUnicode_READ((ctx)->fmtkind, (ctx)->fmtdata, (ctx)->fmtpos) +/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) - PyObject *v; + FORMATBUFLEN is the length of the buffer in which the ints & + chars are formatted. XXX This is a magic number. Each formatting + routine does bounds checking to ensure no overflow, but a better + solution may be to malloc a buffer of appropriate size for each + format. For now, the current solution is sufficient. +*/ +#define FORMATBUFLEN (size_t)120 - if (arg->ch == '(') { - /* Get argument value from a dictionary. Example: "%(name)s". */ - Py_ssize_t keystart; - Py_ssize_t keylen; - PyObject *key; - int pcount = 1; +PyObject *PyUnicode_Format(PyObject *format, + PyObject *args) +{ + Py_UNICODE *fmt, *res; + Py_ssize_t fmtcnt, rescnt, reslen, arglen, argidx; + int args_owned = 0; + PyUnicodeObject *result = NULL; + PyObject *dict = NULL; + PyObject *uformat; - if (ctx->dict == NULL) { - PyErr_SetString(PyExc_TypeError, - "format requires a mapping"); - return -1; - } - ++ctx->fmtpos; - --ctx->fmtcnt; - keystart = ctx->fmtpos; - /* Skip over balanced parentheses */ - while (pcount > 0 && --ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - if (arg->ch == ')') - --pcount; - else if (arg->ch == '(') - ++pcount; - ctx->fmtpos++; - } - keylen = ctx->fmtpos - keystart - 1; - if (ctx->fmtcnt < 0 || pcount > 0) { - PyErr_SetString(PyExc_ValueError, - "incomplete format key"); - return -1; - } - key = PyUnicode_Substring(ctx->fmtstr, - keystart, keystart + keylen); - if (key == NULL) - return -1; - if (ctx->args_owned) { - ctx->args_owned = 0; - Py_DECREF(ctx->args); - } - ctx->args = PyObject_GetItem(ctx->dict, key); - Py_DECREF(key); - if (ctx->args == NULL) - return -1; - ctx->args_owned = 1; - ctx->arglen = -1; - ctx->argidx = -2; - } - - /* Parse flags. Example: "%+i" => flags=F_SIGN. */ - while (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - switch (arg->ch) { - case '-': arg->flags |= F_LJUST; continue; - case '+': arg->flags |= F_SIGN; continue; - case ' ': arg->flags |= F_BLANK; continue; - case '#': arg->flags |= F_ALT; continue; - case '0': arg->flags |= F_ZERO; continue; - } - break; + if (format == NULL || args == NULL) { + PyErr_BadInternalCall(); + return NULL; } + uformat = PyUnicode_FromObject(format); + if (uformat == NULL) + return NULL; + fmt = PyUnicode_AS_UNICODE(uformat); + fmtcnt = PyUnicode_GET_SIZE(uformat); - /* Parse width. Example: "%10s" => width=10 */ - if (arg->ch == '*') { - v = unicode_format_getnextarg(ctx); - if (v == NULL) - return -1; - if (!PyLong_Check(v)) { - PyErr_SetString(PyExc_TypeError, - "* wants int"); - return -1; - } - arg->width = PyLong_AsSsize_t(v); - if (arg->width == -1 && PyErr_Occurred()) - return -1; - if (arg->width < 0) { - arg->flags |= F_LJUST; - arg->width = -arg->width; - } - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - } + reslen = rescnt = fmtcnt + 100; + result = _PyUnicode_New(reslen); + if (result == NULL) + goto onError; + res = PyUnicode_AS_UNICODE(result); + + if (PyTuple_Check(args)) { + arglen = PyTuple_Size(args); + argidx = 0; } - else if (arg->ch >= '0' && arg->ch <= '9') { - arg->width = arg->ch - '0'; - while (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - if (arg->ch < '0' || arg->ch > '9') - break; - /* Since arg->ch is unsigned, the RHS would end up as unsigned, - mixing signed and unsigned comparison. Since arg->ch is between - '0' and '9', casting to int is safe. */ - if (arg->width > (PY_SSIZE_T_MAX - ((int)arg->ch - '0')) / 10) { - PyErr_SetString(PyExc_ValueError, - "width too big"); - return -1; + else { + arglen = -1; + argidx = -2; + } + if (Py_TYPE(args)->tp_as_mapping && Py_TYPE(args)->tp_as_mapping->mp_subscript && + !PyTuple_Check(args) && !PyObject_TypeCheck(args, &PyBaseString_Type)) + dict = args; + + while (--fmtcnt >= 0) { + if (*fmt != '%') { + if (--rescnt < 0) { + rescnt = fmtcnt + 100; + reslen += rescnt; + if (_PyUnicode_Resize(&result, reslen) < 0) + goto onError; + res = PyUnicode_AS_UNICODE(result) + reslen - rescnt; + --rescnt; } - arg->width = arg->width*10 + (arg->ch - '0'); - } - } - - /* Parse precision. Example: "%.3f" => prec=3 */ - if (arg->ch == '.') { - arg->prec = 0; - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; + *res++ = *fmt++; } - if (arg->ch == '*') { - v = unicode_format_getnextarg(ctx); - if (v == NULL) - return -1; - if (!PyLong_Check(v)) { - PyErr_SetString(PyExc_TypeError, - "* wants int"); - return -1; + else { + /* Got a format specifier */ + int flags = 0; + Py_ssize_t width = -1; + int prec = -1; + Py_UNICODE c = '\0'; + Py_UNICODE fill; + int isnumok; + PyObject *v = NULL; + PyObject *temp = NULL; + Py_UNICODE *pbuf; + Py_UNICODE sign; + Py_ssize_t len; + Py_UNICODE formatbuf[FORMATBUFLEN]; /* For format{int,char}() */ + + fmt++; + if (*fmt == '(') { + Py_UNICODE *keystart; + Py_ssize_t keylen; + PyObject *key; + int pcount = 1; + + if (dict == NULL) { + PyErr_SetString(PyExc_TypeError, + "format requires a mapping"); + goto onError; + } + ++fmt; + --fmtcnt; + keystart = fmt; + /* Skip over balanced parentheses */ + while (pcount > 0 && --fmtcnt >= 0) { + if (*fmt == ')') + --pcount; + else if (*fmt == '(') + ++pcount; + fmt++; + } + keylen = fmt - keystart - 1; + if (fmtcnt < 0 || pcount > 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format key"); + goto onError; + } +#if 0 + /* keys are converted to strings using UTF-8 and + then looked up since Python uses strings to hold + variables names etc. in its namespaces and we + wouldn't want to break common idioms. */ + key = PyUnicode_EncodeUTF8(keystart, + keylen, + NULL); +#else + key = PyUnicode_FromUnicode(keystart, keylen); +#endif + if (key == NULL) + goto onError; + if (args_owned) { + Py_DECREF(args); + args_owned = 0; + } + args = PyObject_GetItem(dict, key); + Py_DECREF(key); + if (args == NULL) { + goto onError; + } + args_owned = 1; + arglen = -1; + argidx = -2; + } + while (--fmtcnt >= 0) { + switch (c = *fmt++) { + case '-': flags |= F_LJUST; continue; + case '+': flags |= F_SIGN; continue; + case ' ': flags |= F_BLANK; continue; + case '#': flags |= F_ALT; continue; + case '0': flags |= F_ZERO; continue; + } + break; } - arg->prec = _PyLong_AsInt(v); - if (arg->prec == -1 && PyErr_Occurred()) - return -1; - if (arg->prec < 0) - arg->prec = 0; - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; + if (c == '*') { + v = getnextarg(args, arglen, &argidx); + if (v == NULL) + goto onError; + if (!PyInt_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "* wants int"); + goto onError; + } + width = PyInt_AsSsize_t(v); + if (width == -1 && PyErr_Occurred()) + goto onError; + if (width < 0) { + flags |= F_LJUST; + width = -width; + } + if (--fmtcnt >= 0) + c = *fmt++; + } + else if (c >= '0' && c <= '9') { + width = c - '0'; + while (--fmtcnt >= 0) { + c = *fmt++; + if (c < '0' || c > '9') + break; + if (width > (PY_SSIZE_T_MAX - ((int)c - '0')) / 10) { + PyErr_SetString(PyExc_ValueError, + "width too big"); + goto onError; + } + width = width*10 + (c - '0'); + } } - } - else if (arg->ch >= '0' && arg->ch <= '9') { - arg->prec = arg->ch - '0'; - while (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; - if (arg->ch < '0' || arg->ch > '9') - break; - if (arg->prec > (INT_MAX - ((int)arg->ch - '0')) / 10) { - PyErr_SetString(PyExc_ValueError, - "precision too big"); - return -1; + if (c == '.') { + prec = 0; + if (--fmtcnt >= 0) + c = *fmt++; + if (c == '*') { + v = getnextarg(args, arglen, &argidx); + if (v == NULL) + goto onError; + if (!PyInt_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "* wants int"); + goto onError; + } + prec = _PyInt_AsInt(v); + if (prec == -1 && PyErr_Occurred()) + goto onError; + if (prec < 0) + prec = 0; + if (--fmtcnt >= 0) + c = *fmt++; + } + else if (c >= '0' && c <= '9') { + prec = c - '0'; + while (--fmtcnt >= 0) { + c = *fmt++; + if (c < '0' || c > '9') + break; + if (prec > (INT_MAX - ((int)c - '0')) / 10) { + PyErr_SetString(PyExc_ValueError, + "prec too big"); + goto onError; + } + prec = prec*10 + (c - '0'); + } + } + } /* prec */ + if (fmtcnt >= 0) { + if (c == 'h' || c == 'l' || c == 'L') { + if (--fmtcnt >= 0) + c = *fmt++; } - arg->prec = arg->prec*10 + (arg->ch - '0'); } - } - } - - /* Ignore "h", "l" and "L" format prefix (ex: "%hi" or "%ls") */ - if (ctx->fmtcnt >= 0) { - if (arg->ch == 'h' || arg->ch == 'l' || arg->ch == 'L') { - if (--ctx->fmtcnt >= 0) { - arg->ch = FORMAT_READ(ctx); - ctx->fmtpos++; + if (fmtcnt < 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format"); + goto onError; } - } - } - if (ctx->fmtcnt < 0) { - PyErr_SetString(PyExc_ValueError, - "incomplete format"); - return -1; - } - return 0; - -#undef FORMAT_READ -} - -/* Format one argument. Supported conversion specifiers: - - - "s", "r", "a": any type - - "i", "d", "u": int or float - - "o", "x", "X": int - - "e", "E", "f", "F", "g", "G": float - - "c": int or str (1 character) - - When possible, the output is written directly into the Unicode writer - (ctx->writer). A string is created when padding is required. - - Return 0 if the argument has been formatted into *p_str, - 1 if the argument has been written into ctx->writer, - -1 on error. */ -static int -unicode_format_arg_format(struct unicode_formatter_t *ctx, - struct unicode_format_arg_t *arg, - PyObject **p_str) -{ - PyObject *v; - _PyUnicodeWriter *writer = &ctx->writer; - - if (ctx->fmtcnt == 0) - ctx->writer.overallocate = 0; - - v = unicode_format_getnextarg(ctx); - if (v == NULL) - return -1; - - - switch (arg->ch) { - case 's': - case 'r': - case 'a': - if (PyLong_CheckExact(v) && arg->width == -1 && arg->prec == -1) { - /* Fast path */ - if (_PyLong_FormatWriter(writer, v, 10, arg->flags & F_ALT) == -1) - return -1; - return 1; - } - - if (PyUnicode_CheckExact(v) && arg->ch == 's') { - *p_str = v; - Py_INCREF(*p_str); - } - else { - if (arg->ch == 's') - *p_str = PyObject_Str(v); - else if (arg->ch == 'r') - *p_str = PyObject_Repr(v); - else - *p_str = PyObject_ASCII(v); - } - break; - - case 'i': - case 'd': - case 'u': - case 'o': - case 'x': - case 'X': - { - int ret = mainformatlong(v, arg, p_str, writer); - if (ret != 0) - return ret; - arg->sign = 1; - break; - } - - case 'e': - case 'E': - case 'f': - case 'F': - case 'g': - case 'G': - if (arg->width == -1 && arg->prec == -1 - && !(arg->flags & (F_SIGN | F_BLANK))) - { - /* Fast path */ - if (formatfloat(v, arg, NULL, writer) == -1) - return -1; - return 1; - } - - arg->sign = 1; - if (formatfloat(v, arg, p_str, NULL) == -1) - return -1; - break; - - case 'c': - { - Py_UCS4 ch = formatchar(v); - if (ch == (Py_UCS4) -1) - return -1; - if (arg->width == -1 && arg->prec == -1) { - /* Fast path */ - if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0) - return -1; - return 1; - } - *p_str = PyUnicode_FromOrdinal(ch); - break; - } - - default: - PyErr_Format(PyExc_ValueError, - "unsupported format character '%c' (0x%x) " - "at index %zd", - (31<=arg->ch && arg->ch<=126) ? (char)arg->ch : '?', - (int)arg->ch, - ctx->fmtpos - 1); - return -1; - } - if (*p_str == NULL) - return -1; - assert (PyUnicode_Check(*p_str)); - return 0; -} - -static int -unicode_format_arg_output(struct unicode_formatter_t *ctx, - struct unicode_format_arg_t *arg, - PyObject *str) -{ - Py_ssize_t len; - enum PyUnicode_Kind kind; - void *pbuf; - Py_ssize_t pindex; - Py_UCS4 signchar; - Py_ssize_t buflen; - Py_UCS4 maxchar; - Py_ssize_t sublen; - _PyUnicodeWriter *writer = &ctx->writer; - Py_UCS4 fill; - - fill = ' '; - if (arg->sign && arg->flags & F_ZERO) - fill = '0'; - - if (PyUnicode_READY(str) == -1) - return -1; - - len = PyUnicode_GET_LENGTH(str); - if ((arg->width == -1 || arg->width <= len) - && (arg->prec == -1 || arg->prec >= len) - && !(arg->flags & (F_SIGN | F_BLANK))) - { - /* Fast path */ - if (_PyUnicodeWriter_WriteStr(writer, str) == -1) - return -1; - return 0; - } - - /* Truncate the string for "s", "r" and "a" formats - if the precision is set */ - if (arg->ch == 's' || arg->ch == 'r' || arg->ch == 'a') { - if (arg->prec >= 0 && len > arg->prec) - len = arg->prec; - } - - /* Adjust sign and width */ - kind = PyUnicode_KIND(str); - pbuf = PyUnicode_DATA(str); - pindex = 0; - signchar = '\0'; - if (arg->sign) { - Py_UCS4 ch = PyUnicode_READ(kind, pbuf, pindex); - if (ch == '-' || ch == '+') { - signchar = ch; - len--; - pindex++; - } - else if (arg->flags & F_SIGN) - signchar = '+'; - else if (arg->flags & F_BLANK) - signchar = ' '; - else - arg->sign = 0; - } - if (arg->width < len) - arg->width = len; - - /* Prepare the writer */ - maxchar = writer->maxchar; - if (!(arg->flags & F_LJUST)) { - if (arg->sign) { - if ((arg->width-1) > len) - maxchar = Py_MAX(maxchar, fill); - } - else { - if (arg->width > len) - maxchar = Py_MAX(maxchar, fill); - } - } - if (PyUnicode_MAX_CHAR_VALUE(str) > maxchar) { - Py_UCS4 strmaxchar = _PyUnicode_FindMaxChar(str, 0, pindex+len); - maxchar = Py_MAX(maxchar, strmaxchar); - } - - buflen = arg->width; - if (arg->sign && len == arg->width) - buflen++; - if (_PyUnicodeWriter_Prepare(writer, buflen, maxchar) == -1) - return -1; - - /* Write the sign if needed */ - if (arg->sign) { - if (fill != ' ') { - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); - writer->pos += 1; - } - if (arg->width > len) - arg->width--; - } - - /* Write the numeric prefix for "x", "X" and "o" formats - if the alternate form is used. - For example, write "0x" for the "%#x" format. */ - if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { - assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); - assert(PyUnicode_READ(kind, pbuf, pindex + 1) == arg->ch); - if (fill != ' ') { - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); - PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); - writer->pos += 2; - pindex += 2; - } - arg->width -= 2; - if (arg->width < 0) - arg->width = 0; - len -= 2; - } - - /* Pad left with the fill character if needed */ - if (arg->width > len && !(arg->flags & F_LJUST)) { - sublen = arg->width - len; - unicode_fill(writer->kind, writer->data, fill, writer->pos, sublen); - writer->pos += sublen; - arg->width = len; - } - - /* If padding with spaces: write sign if needed and/or numeric prefix if - the alternate form is used */ - if (fill == ' ') { - if (arg->sign) { - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar); - writer->pos += 1; - } - if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) { - assert(PyUnicode_READ(kind, pbuf, pindex) == '0'); - assert(PyUnicode_READ(kind, pbuf, pindex+1) == arg->ch); - PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0'); - PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch); - writer->pos += 2; - pindex += 2; - } - } - - /* Write characters */ - if (len) { - _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, - str, pindex, len); - writer->pos += len; - } - - /* Pad right with the fill character if needed */ - if (arg->width > len) { - sublen = arg->width - len; - unicode_fill(writer->kind, writer->data, ' ', writer->pos, sublen); - writer->pos += sublen; - } - return 0; -} - -/* Helper of PyUnicode_Format(): format one arg. - Return 0 on success, raise an exception and return -1 on error. */ -static int -unicode_format_arg(struct unicode_formatter_t *ctx) -{ - struct unicode_format_arg_t arg; - PyObject *str; - int ret; - - arg.ch = PyUnicode_READ(ctx->fmtkind, ctx->fmtdata, ctx->fmtpos); - if (arg.ch == '%') { - ctx->fmtpos++; - ctx->fmtcnt--; - if (_PyUnicodeWriter_WriteCharInline(&ctx->writer, '%') < 0) - return -1; - return 0; - } - arg.flags = 0; - arg.width = -1; - arg.prec = -1; - arg.sign = 0; - str = NULL; - - ret = unicode_format_arg_parse(ctx, &arg); - if (ret == -1) - return -1; - - ret = unicode_format_arg_format(ctx, &arg, &str); - if (ret == -1) - return -1; - - if (ret != 1) { - ret = unicode_format_arg_output(ctx, &arg, str); - Py_DECREF(str); - if (ret == -1) - return -1; - } - - if (ctx->dict && (ctx->argidx < ctx->arglen)) { - PyErr_SetString(PyExc_TypeError, - "not all arguments converted during string formatting"); - return -1; - } - return 0; -} + if (c != '%') { + v = getnextarg(args, arglen, &argidx); + if (v == NULL) + goto onError; + } + sign = 0; + fill = ' '; + switch (c) { -PyObject * -PyUnicode_Format(PyObject *format, PyObject *args) -{ - struct unicode_formatter_t ctx; + case '%': + pbuf = formatbuf; + /* presume that buffer length is at least 1 */ + pbuf[0] = '%'; + len = 1; + break; - if (format == NULL || args == NULL) { - PyErr_BadInternalCall(); - return NULL; - } + case 's': + case 'r': + if (PyUnicode_CheckExact(v) && c == 's') { + temp = v; + Py_INCREF(temp); + } + else { + PyObject *unicode; + if (c == 's') + temp = PyObject_Unicode(v); + else + temp = PyObject_Repr(v); + if (temp == NULL) + goto onError; + if (PyUnicode_Check(temp)) + /* nothing to do */; + else if (PyString_Check(temp)) { + /* convert to string to Unicode */ + unicode = PyUnicode_Decode(PyString_AS_STRING(temp), + PyString_GET_SIZE(temp), + NULL, + "strict"); + Py_DECREF(temp); + temp = unicode; + if (temp == NULL) + goto onError; + } + else { + Py_DECREF(temp); + PyErr_SetString(PyExc_TypeError, + "%s argument has non-string str()"); + goto onError; + } + } + pbuf = PyUnicode_AS_UNICODE(temp); + len = PyUnicode_GET_SIZE(temp); + if (prec >= 0 && len > prec) + len = prec; + break; - if (ensure_unicode(format) < 0) - return NULL; + case 'i': + case 'd': + case 'u': + case 'o': + case 'x': + case 'X': + if (c == 'i') + c = 'd'; + isnumok = 0; + if (PyNumber_Check(v)) { + PyObject *iobj=NULL; + + if (_PyAnyInt_Check(v)) { + iobj = v; + Py_INCREF(iobj); + } + else { + iobj = PyNumber_Int(v); + if (iobj==NULL) { + PyErr_Clear(); + iobj = PyNumber_Long(v); + } + } + if (iobj!=NULL) { + if (PyInt_Check(iobj)) { + isnumok = 1; + pbuf = formatbuf; + len = formatint(pbuf, sizeof(formatbuf)/sizeof(Py_UNICODE), + flags, prec, c, iobj); + Py_DECREF(iobj); + if (len < 0) + goto onError; + sign = 1; + } + else if (PyLong_Check(iobj)) { + isnumok = 1; + temp = formatlong(iobj, flags, prec, c); + Py_DECREF(iobj); + if (!temp) + goto onError; + pbuf = PyUnicode_AS_UNICODE(temp); + len = PyUnicode_GET_SIZE(temp); + sign = 1; + } + else { + Py_DECREF(iobj); + } + } + } + if (!isnumok) { + PyErr_Format(PyExc_TypeError, + "%%%c format: a number is required, " + "not %.200s", (char)c, Py_TYPE(v)->tp_name); + goto onError; + } + if (flags & F_ZERO) + fill = '0'; + break; - ctx.fmtstr = format; - ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr); - ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr); - ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr); - ctx.fmtpos = 0; + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + temp = formatfloat(v, flags, prec, c); + if (temp == NULL) + goto onError; + pbuf = PyUnicode_AS_UNICODE(temp); + len = PyUnicode_GET_SIZE(temp); + sign = 1; + if (flags & F_ZERO) + fill = '0'; + break; - _PyUnicodeWriter_Init(&ctx.writer); - ctx.writer.min_length = ctx.fmtcnt + 100; - ctx.writer.overallocate = 1; + case 'c': + pbuf = formatbuf; + len = formatchar(pbuf, sizeof(formatbuf)/sizeof(Py_UNICODE), v); + if (len < 0) + goto onError; + break; - if (PyTuple_Check(args)) { - ctx.arglen = PyTuple_Size(args); - ctx.argidx = 0; - } - else { - ctx.arglen = -1; - ctx.argidx = -2; - } - ctx.args_owned = 0; - if (PyMapping_Check(args) && !PyTuple_Check(args) && !PyUnicode_Check(args)) - ctx.dict = args; - else - ctx.dict = NULL; - ctx.args = args; - - while (--ctx.fmtcnt >= 0) { - if (PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { - Py_ssize_t nonfmtpos; - - nonfmtpos = ctx.fmtpos++; - while (ctx.fmtcnt >= 0 && - PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') { - ctx.fmtpos++; - ctx.fmtcnt--; + default: + PyErr_Format(PyExc_ValueError, + "unsupported format character '%c' (0x%x) " + "at index %zd", + (31<=c && c<=126) ? (char)c : '?', + (int)c, + (Py_ssize_t)(fmt - 1 - + PyUnicode_AS_UNICODE(uformat))); + goto onError; + } + if (sign) { + if (*pbuf == '-' || *pbuf == '+') { + sign = *pbuf++; + len--; + } + else if (flags & F_SIGN) + sign = '+'; + else if (flags & F_BLANK) + sign = ' '; + else + sign = 0; + } + if (width < len) + width = len; + if (rescnt - (sign != 0) < width) { + reslen -= rescnt; + rescnt = width + fmtcnt + 100; + reslen += rescnt; + if (reslen < 0) { + Py_XDECREF(temp); + PyErr_NoMemory(); + goto onError; + } + if (_PyUnicode_Resize(&result, reslen) < 0) { + Py_XDECREF(temp); + goto onError; + } + res = PyUnicode_AS_UNICODE(result) + + reslen - rescnt; + } + if (sign) { + if (fill != ' ') + *res++ = sign; + rescnt--; + if (width > len) + width--; + } + if ((flags & F_ALT) && (c == 'x' || c == 'X')) { + assert(pbuf[0] == '0'); + assert(pbuf[1] == c); + if (fill != ' ') { + *res++ = *pbuf++; + *res++ = *pbuf++; + } + rescnt -= 2; + width -= 2; + if (width < 0) + width = 0; + len -= 2; } - if (ctx.fmtcnt < 0) { - ctx.fmtpos--; - ctx.writer.overallocate = 0; + if (width > len && !(flags & F_LJUST)) { + do { + --rescnt; + *res++ = fill; + } while (--width > len); + } + if (fill == ' ') { + if (sign) + *res++ = sign; + if ((flags & F_ALT) && (c == 'x' || c == 'X')) { + assert(pbuf[0] == '0'); + assert(pbuf[1] == c); + *res++ = *pbuf++; + *res++ = *pbuf++; + } } - - if (_PyUnicodeWriter_WriteSubstring(&ctx.writer, ctx.fmtstr, - nonfmtpos, ctx.fmtpos) < 0) - goto onError; - } - else { - ctx.fmtpos++; - if (unicode_format_arg(&ctx) == -1) + Py_UNICODE_COPY(res, pbuf, len); + res += len; + rescnt -= len; + while (--width >= len) { + --rescnt; + *res++ = ' '; + } + if (dict && (argidx < arglen) && c != '%') { + PyErr_SetString(PyExc_TypeError, + "not all arguments converted during string formatting"); + Py_XDECREF(temp); goto onError; - } - } - - if (ctx.argidx < ctx.arglen && !ctx.dict) { + } + Py_XDECREF(temp); + } /* '%' */ + } /* until end */ + if (argidx < arglen && !dict) { PyErr_SetString(PyExc_TypeError, "not all arguments converted during string formatting"); goto onError; } - if (ctx.args_owned) { - Py_DECREF(ctx.args); + if (_PyUnicode_Resize(&result, reslen - rescnt) < 0) + goto onError; + if (args_owned) { + Py_DECREF(args); } - return _PyUnicodeWriter_Finish(&ctx.writer); + Py_DECREF(uformat); + return (PyObject *)result; onError: - _PyUnicodeWriter_Dealloc(&ctx.writer); - if (ctx.args_owned) { - Py_DECREF(ctx.args); + Py_XDECREF(result); + Py_DECREF(uformat); + if (args_owned) { + Py_DECREF(args); } return NULL; } +static PyBufferProcs unicode_as_buffer = { + (readbufferproc) unicode_buffer_getreadbuf, + (writebufferproc) unicode_buffer_getwritebuf, + (segcountproc) unicode_buffer_getsegcount, + (charbufferproc) unicode_buffer_getcharbuf, +}; + static PyObject * unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -15112,19 +8844,19 @@ static PyObject * unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyObject *x = NULL; - static char *kwlist[] = {"object", "encoding", "errors", 0}; + static char *kwlist[] = {"string", "encoding", "errors", 0}; char *encoding = NULL; char *errors = NULL; if (type != &PyUnicode_Type) return unicode_subtype_new(type, args, kwds); - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:str", + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:unicode", kwlist, &x, &encoding, &errors)) return NULL; if (x == NULL) - _Py_RETURN_UNICODE_EMPTY(); + return (PyObject *)_PyUnicode_New(0); if (encoding == NULL && errors == NULL) - return PyObject_Str(x); + return PyObject_Unicode(x); else return PyUnicode_FromEncodedObject(x, encoding, errors); } @@ -15132,168 +8864,91 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *unicode, *self; - Py_ssize_t length, char_size; - int share_wstr, share_utf8; - unsigned int kind; - void *data; + PyUnicodeObject *tmp, *pnew; + Py_ssize_t n; assert(PyType_IsSubtype(type, &PyUnicode_Type)); - - unicode = unicode_new(&PyUnicode_Type, args, kwds); - if (unicode == NULL) + tmp = (PyUnicodeObject *)unicode_new(&PyUnicode_Type, args, kwds); + if (tmp == NULL) return NULL; - assert(_PyUnicode_CHECK(unicode)); - if (PyUnicode_READY(unicode) == -1) { - Py_DECREF(unicode); + assert(PyUnicode_Check(tmp)); + pnew = (PyUnicodeObject *) type->tp_alloc(type, n = tmp->length); + if (pnew == NULL) { + Py_DECREF(tmp); return NULL; } - - self = type->tp_alloc(type, 0); - if (self == NULL) { - Py_DECREF(unicode); - return NULL; - } - kind = PyUnicode_KIND(unicode); - length = PyUnicode_GET_LENGTH(unicode); - - _PyUnicode_LENGTH(self) = length; -#ifdef Py_DEBUG - _PyUnicode_HASH(self) = -1; -#else - _PyUnicode_HASH(self) = _PyUnicode_HASH(unicode); -#endif - _PyUnicode_STATE(self).interned = 0; - _PyUnicode_STATE(self).kind = kind; - _PyUnicode_STATE(self).compact = 0; - _PyUnicode_STATE(self).ascii = _PyUnicode_STATE(unicode).ascii; - _PyUnicode_STATE(self).ready = 1; - _PyUnicode_WSTR(self) = NULL; - _PyUnicode_UTF8_LENGTH(self) = 0; - _PyUnicode_UTF8(self) = NULL; - _PyUnicode_WSTR_LENGTH(self) = 0; - _PyUnicode_DATA_ANY(self) = NULL; - - share_utf8 = 0; - share_wstr = 0; - if (kind == PyUnicode_1BYTE_KIND) { - char_size = 1; - if (PyUnicode_MAX_CHAR_VALUE(unicode) < 128) - share_utf8 = 1; - } - else if (kind == PyUnicode_2BYTE_KIND) { - char_size = 2; - if (sizeof(wchar_t) == 2) - share_wstr = 1; - } - else { - assert(kind == PyUnicode_4BYTE_KIND); - char_size = 4; - if (sizeof(wchar_t) == 4) - share_wstr = 1; - } - - /* Ensure we won't overflow the length. */ - if (length > (PY_SSIZE_T_MAX / char_size - 1)) { - PyErr_NoMemory(); - goto onError; - } - data = PyObject_MALLOC((length + 1) * char_size); - if (data == NULL) { - PyErr_NoMemory(); - goto onError; - } - - _PyUnicode_DATA_ANY(self) = data; - if (share_utf8) { - _PyUnicode_UTF8_LENGTH(self) = length; - _PyUnicode_UTF8(self) = data; - } - if (share_wstr) { - _PyUnicode_WSTR_LENGTH(self) = length; - _PyUnicode_WSTR(self) = (wchar_t *)data; + pnew->str = (Py_UNICODE*) PyObject_MALLOC(sizeof(Py_UNICODE) * (n+1)); + if (pnew->str == NULL) { + _Py_ForgetReference((PyObject *)pnew); + PyObject_Del(pnew); + Py_DECREF(tmp); + return PyErr_NoMemory(); } - - memcpy(data, PyUnicode_DATA(unicode), - kind * (length + 1)); - assert(_PyUnicode_CheckConsistency(self, 1)); -#ifdef Py_DEBUG - _PyUnicode_HASH(self) = _PyUnicode_HASH(unicode); -#endif - Py_DECREF(unicode); - return self; - -onError: - Py_DECREF(unicode); - Py_DECREF(self); - return NULL; + Py_UNICODE_COPY(pnew->str, tmp->str, n+1); + pnew->length = n; + pnew->hash = tmp->hash; + Py_DECREF(tmp); + return (PyObject *)pnew; } PyDoc_STRVAR(unicode_doc, -"str(object='') -> str\n\ -str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ + "unicode(object='') -> unicode object\n\ +unicode(string[, encoding[, errors]]) -> unicode object\n\ \n\ -Create a new string object from the given object. If encoding or\n\ -errors is specified, then the object must expose a data buffer\n\ -that will be decoded using the given encoding and error handler.\n\ -Otherwise, returns the result of object.__str__() (if defined)\n\ -or repr(object).\n\ -encoding defaults to sys.getdefaultencoding().\n\ -errors defaults to 'strict'."); - -static PyObject *unicode_iter(PyObject *seq); +Create a new Unicode object from the given encoded string.\n\ +encoding defaults to the current default string encoding.\n\ +errors can be 'strict', 'replace' or 'ignore' and defaults to 'strict'."); PyTypeObject PyUnicode_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "str", /* tp_name */ - sizeof(PyUnicodeObject), /* tp_basicsize */ - 0, /* tp_itemsize */ + "unicode", /* tp_name */ + sizeof(PyUnicodeObject), /* tp_size */ + 0, /* tp_itemsize */ /* Slots */ - (destructor)unicode_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - unicode_repr, /* tp_repr */ - &unicode_as_number, /* tp_as_number */ - &unicode_as_sequence, /* tp_as_sequence */ - &unicode_as_mapping, /* tp_as_mapping */ - (hashfunc) unicode_hash, /* tp_hash*/ - 0, /* tp_call*/ - (reprfunc) unicode_str, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_UNICODE_SUBCLASS, /* tp_flags */ - unicode_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - PyUnicode_RichCompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - unicode_iter, /* tp_iter */ - 0, /* tp_iternext */ - unicode_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyBaseObject_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - unicode_new, /* tp_new */ - PyObject_Del, /* tp_free */ + (destructor)unicode_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + unicode_repr, /* tp_repr */ + &unicode_as_number, /* tp_as_number */ + &unicode_as_sequence, /* tp_as_sequence */ + &unicode_as_mapping, /* tp_as_mapping */ + (hashfunc) unicode_hash, /* tp_hash*/ + 0, /* tp_call*/ + (reprfunc) unicode_str, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &unicode_as_buffer, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES | + Py_TPFLAGS_BASETYPE | Py_TPFLAGS_UNICODE_SUBCLASS, /* tp_flags */ + unicode_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + PyUnicode_RichCompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + unicode_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyBaseString_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + unicode_new, /* tp_new */ + PyObject_Del, /* tp_free */ }; /* Initialize the Unicode implementation */ -PyStatus -_PyUnicode_Init(void) +void _PyUnicode_Init(void) { /* XXX - move this array to unicodectype.c ? */ - Py_UCS2 linebreak[] = { + Py_UNICODE linebreak[] = { 0x000A, /* LINE FEED */ 0x000D, /* CARRIAGE RETURN */ 0x001C, /* FILE SEPARATOR */ @@ -15305,685 +8960,64 @@ _PyUnicode_Init(void) }; /* Init the implementation */ - _Py_INCREF_UNICODE_EMPTY(); if (!unicode_empty) { - return _PyStatus_ERR("Can't create empty string"); + unicode_empty = _PyUnicode_New(0); + if (!unicode_empty) + return; } - Py_DECREF(unicode_empty); - if (PyType_Ready(&PyUnicode_Type) < 0) { - return _PyStatus_ERR("Can't initialize unicode type"); - } + if (PyType_Ready(&PyUnicode_Type) < 0) + Py_FatalError("Can't initialize 'unicode'"); /* initialize the linebreak bloom filter */ bloom_linebreak = make_bloom_mask( - PyUnicode_2BYTE_KIND, linebreak, - Py_ARRAY_LENGTH(linebreak)); - - if (PyType_Ready(&EncodingMapType) < 0) { - return _PyStatus_ERR("Can't initialize encoding map type"); - } - if (PyType_Ready(&PyFieldNameIter_Type) < 0) { - return _PyStatus_ERR("Can't initialize field name iterator type"); - } - if (PyType_Ready(&PyFormatterIter_Type) < 0) { - return _PyStatus_ERR("Can't initialize formatter iter type"); - } - return _PyStatus_OK(); -} - - -void -PyUnicode_InternInPlace(PyObject **p) -{ - PyObject *s = *p; - PyObject *t; -#ifdef Py_DEBUG - assert(s != NULL); - assert(_PyUnicode_CHECK(s)); -#else - if (s == NULL || !PyUnicode_Check(s)) - return; -#endif - /* If it's a subclass, we don't really know what putting - it in the interned dict might do. */ - if (!PyUnicode_CheckExact(s)) - return; - if (PyUnicode_CHECK_INTERNED(s)) - return; - if (interned == NULL) { - interned = PyDict_New(); - if (interned == NULL) { - PyErr_Clear(); /* Don't leave an exception */ - return; - } - } - Py_ALLOW_RECURSION - t = PyDict_SetDefault(interned, s, s); - Py_END_ALLOW_RECURSION - if (t == NULL) { - PyErr_Clear(); - return; - } - if (t != s) { - Py_INCREF(t); - Py_SETREF(*p, t); - return; - } - /* The two references in interned are not counted by refcnt. - The deallocator will take care of this */ - Py_REFCNT(s) -= 2; - _PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL; -} - -void -PyUnicode_InternImmortal(PyObject **p) -{ - PyUnicode_InternInPlace(p); - if (PyUnicode_CHECK_INTERNED(*p) != SSTATE_INTERNED_IMMORTAL) { - _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL; - Py_INCREF(*p); - } -} - -PyObject * -PyUnicode_InternFromString(const char *cp) -{ - PyObject *s = PyUnicode_FromString(cp); - if (s == NULL) - return NULL; - PyUnicode_InternInPlace(&s); - return s; -} - - -#if defined(WITH_VALGRIND) || defined(__INSURE__) -static void -unicode_release_interned(void) -{ - PyObject *keys; - PyObject *s; - Py_ssize_t i, n; - Py_ssize_t immortal_size = 0, mortal_size = 0; - - if (interned == NULL || !PyDict_Check(interned)) - return; - keys = PyDict_Keys(interned); - if (keys == NULL || !PyList_Check(keys)) { - PyErr_Clear(); - return; - } - - /* Since unicode_release_interned() is intended to help a leak - detector, interned unicode strings are not forcibly deallocated; - rather, we give them their stolen references back, and then clear - and DECREF the interned dict. */ - - n = PyList_GET_SIZE(keys); -#ifdef INTERNED_STATS - fprintf(stderr, "releasing %" PY_FORMAT_SIZE_T "d interned strings\n", - n); -#endif - for (i = 0; i < n; i++) { - s = PyList_GET_ITEM(keys, i); - if (PyUnicode_READY(s) == -1) { - Py_UNREACHABLE(); - } - switch (PyUnicode_CHECK_INTERNED(s)) { - case SSTATE_NOT_INTERNED: - /* XXX Shouldn't happen */ - break; - case SSTATE_INTERNED_IMMORTAL: - Py_REFCNT(s) += 1; - immortal_size += PyUnicode_GET_LENGTH(s); - break; - case SSTATE_INTERNED_MORTAL: - Py_REFCNT(s) += 2; - mortal_size += PyUnicode_GET_LENGTH(s); - break; - default: - Py_FatalError("Inconsistent interned string state."); - } - _PyUnicode_STATE(s).interned = SSTATE_NOT_INTERNED; - } -#ifdef INTERNED_STATS - fprintf(stderr, "total size of all interned strings: " - "%" PY_FORMAT_SIZE_T "d/%" PY_FORMAT_SIZE_T "d " - "mortal/immortal\n", mortal_size, immortal_size); -#endif - Py_DECREF(keys); - PyDict_Clear(interned); - Py_CLEAR(interned); -} -#endif - - -/********************* Unicode Iterator **************************/ - -typedef struct { - PyObject_HEAD - Py_ssize_t it_index; - PyObject *it_seq; /* Set to NULL when iterator is exhausted */ -} unicodeiterobject; - -static void -unicodeiter_dealloc(unicodeiterobject *it) -{ - _PyObject_GC_UNTRACK(it); - Py_XDECREF(it->it_seq); - PyObject_GC_Del(it); -} - -static int -unicodeiter_traverse(unicodeiterobject *it, visitproc visit, void *arg) -{ - Py_VISIT(it->it_seq); - return 0; -} - -static PyObject * -unicodeiter_next(unicodeiterobject *it) -{ - PyObject *seq, *item; - - assert(it != NULL); - seq = it->it_seq; - if (seq == NULL) - return NULL; - assert(_PyUnicode_CHECK(seq)); - - if (it->it_index < PyUnicode_GET_LENGTH(seq)) { - int kind = PyUnicode_KIND(seq); - void *data = PyUnicode_DATA(seq); - Py_UCS4 chr = PyUnicode_READ(kind, data, it->it_index); - item = PyUnicode_FromOrdinal(chr); - if (item != NULL) - ++it->it_index; - return item; - } - - it->it_seq = NULL; - Py_DECREF(seq); - return NULL; -} - -static PyObject * -unicodeiter_len(unicodeiterobject *it, PyObject *Py_UNUSED(ignored)) -{ - Py_ssize_t len = 0; - if (it->it_seq) - len = PyUnicode_GET_LENGTH(it->it_seq) - it->it_index; - return PyLong_FromSsize_t(len); -} - -PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it))."); - -static PyObject * -unicodeiter_reduce(unicodeiterobject *it, PyObject *Py_UNUSED(ignored)) -{ - _Py_IDENTIFIER(iter); - if (it->it_seq != NULL) { - return Py_BuildValue("N(O)n", _PyEval_GetBuiltinId(&PyId_iter), - it->it_seq, it->it_index); - } else { - PyObject *u = (PyObject *)_PyUnicode_New(0); - if (u == NULL) - return NULL; - return Py_BuildValue("N(N)", _PyEval_GetBuiltinId(&PyId_iter), u); - } -} - -PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); - -static PyObject * -unicodeiter_setstate(unicodeiterobject *it, PyObject *state) -{ - Py_ssize_t index = PyLong_AsSsize_t(state); - if (index == -1 && PyErr_Occurred()) - return NULL; - if (it->it_seq != NULL) { - if (index < 0) - index = 0; - else if (index > PyUnicode_GET_LENGTH(it->it_seq)) - index = PyUnicode_GET_LENGTH(it->it_seq); /* iterator truncated */ - it->it_index = index; - } - Py_RETURN_NONE; -} - -PyDoc_STRVAR(setstate_doc, "Set state information for unpickling."); - -static PyMethodDef unicodeiter_methods[] = { - {"__length_hint__", (PyCFunction)unicodeiter_len, METH_NOARGS, - length_hint_doc}, - {"__reduce__", (PyCFunction)unicodeiter_reduce, METH_NOARGS, - reduce_doc}, - {"__setstate__", (PyCFunction)unicodeiter_setstate, METH_O, - setstate_doc}, - {NULL, NULL} /* sentinel */ -}; - -PyTypeObject PyUnicodeIter_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "str_iterator", /* tp_name */ - sizeof(unicodeiterobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)unicodeiter_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ - 0, /* tp_doc */ - (traverseproc)unicodeiter_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)unicodeiter_next, /* tp_iternext */ - unicodeiter_methods, /* tp_methods */ - 0, -}; - -static PyObject * -unicode_iter(PyObject *seq) -{ - unicodeiterobject *it; - - if (!PyUnicode_Check(seq)) { - PyErr_BadInternalCall(); - return NULL; - } - if (PyUnicode_READY(seq) == -1) - return NULL; - it = PyObject_GC_New(unicodeiterobject, &PyUnicodeIter_Type); - if (it == NULL) - return NULL; - it->it_index = 0; - Py_INCREF(seq); - it->it_seq = seq; - _PyObject_GC_TRACK(it); - return (PyObject *)it; -} - - -size_t -Py_UNICODE_strlen(const Py_UNICODE *u) -{ - return wcslen(u); -} - -Py_UNICODE* -Py_UNICODE_strcpy(Py_UNICODE *s1, const Py_UNICODE *s2) -{ - Py_UNICODE *u = s1; - while ((*u++ = *s2++)); - return s1; -} - -Py_UNICODE* -Py_UNICODE_strncpy(Py_UNICODE *s1, const Py_UNICODE *s2, size_t n) -{ - Py_UNICODE *u = s1; - while ((*u++ = *s2++)) - if (n-- == 0) - break; - return s1; -} - -Py_UNICODE* -Py_UNICODE_strcat(Py_UNICODE *s1, const Py_UNICODE *s2) -{ - Py_UNICODE *u1 = s1; - u1 += wcslen(u1); - while ((*u1++ = *s2++)); - return s1; -} - -int -Py_UNICODE_strcmp(const Py_UNICODE *s1, const Py_UNICODE *s2) -{ - while (*s1 && *s2 && *s1 == *s2) - s1++, s2++; - if (*s1 && *s2) - return (*s1 < *s2) ? -1 : +1; - if (*s1) - return 1; - if (*s2) - return -1; - return 0; -} - -int -Py_UNICODE_strncmp(const Py_UNICODE *s1, const Py_UNICODE *s2, size_t n) -{ - Py_UNICODE u1, u2; - for (; n != 0; n--) { - u1 = *s1; - u2 = *s2; - if (u1 != u2) - return (u1 < u2) ? -1 : +1; - if (u1 == '\0') - return 0; - s1++; - s2++; - } - return 0; -} - -Py_UNICODE* -Py_UNICODE_strchr(const Py_UNICODE *s, Py_UNICODE c) -{ - const Py_UNICODE *p; - for (p = s; *p; p++) - if (*p == c) - return (Py_UNICODE*)p; - return NULL; -} - -Py_UNICODE* -Py_UNICODE_strrchr(const Py_UNICODE *s, Py_UNICODE c) -{ - const Py_UNICODE *p; - p = s + wcslen(s); - while (p != s) { - p--; - if (*p == c) - return (Py_UNICODE*)p; - } - return NULL; -} - -Py_UNICODE* -PyUnicode_AsUnicodeCopy(PyObject *unicode) -{ - Py_UNICODE *u, *copy; - Py_ssize_t len, size; - - if (!PyUnicode_Check(unicode)) { - PyErr_BadArgument(); - return NULL; - } - u = PyUnicode_AsUnicodeAndSize(unicode, &len); - if (u == NULL) - return NULL; - /* Ensure we won't overflow the size. */ - if (len > ((PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(Py_UNICODE)) - 1)) { - PyErr_NoMemory(); - return NULL; - } - size = len + 1; /* copy the null character */ - size *= sizeof(Py_UNICODE); - copy = PyMem_Malloc(size); - if (copy == NULL) { - PyErr_NoMemory(); - return NULL; - } - memcpy(copy, u, size); - return copy; -} - - -static int -encode_wstr_utf8(wchar_t *wstr, char **str, const char *name) -{ - int res; - res = _Py_EncodeUTF8Ex(wstr, str, NULL, NULL, 1, _Py_ERROR_STRICT); - if (res == -2) { - PyErr_Format(PyExc_RuntimeWarning, "cannot decode %s", name); - return -1; - } - if (res < 0) { - PyErr_NoMemory(); - return -1; - } - return 0; -} - - -static int -config_get_codec_name(wchar_t **config_encoding) -{ - char *encoding; - if (encode_wstr_utf8(*config_encoding, &encoding, "stdio_encoding") < 0) { - return -1; - } - - PyObject *name_obj = NULL; - PyObject *codec = _PyCodec_Lookup(encoding); - PyMem_RawFree(encoding); - - if (!codec) - goto error; - - name_obj = PyObject_GetAttrString(codec, "name"); - Py_CLEAR(codec); - if (!name_obj) { - goto error; - } - - wchar_t *wname = PyUnicode_AsWideCharString(name_obj, NULL); - Py_DECREF(name_obj); - if (wname == NULL) { - goto error; - } - - wchar_t *raw_wname = _PyMem_RawWcsdup(wname); - if (raw_wname == NULL) { - PyMem_Free(wname); - PyErr_NoMemory(); - goto error; - } - - PyMem_RawFree(*config_encoding); - *config_encoding = raw_wname; - - PyMem_Free(wname); - return 0; - -error: - Py_XDECREF(codec); - Py_XDECREF(name_obj); - return -1; -} - - -static PyStatus -init_stdio_encoding(PyThreadState *tstate) -{ - /* Update the stdio encoding to the normalized Python codec name. */ - PyConfig *config = &tstate->interp->config; - if (config_get_codec_name(&config->stdio_encoding) < 0) { - return _PyStatus_ERR("failed to get the Python codec name " - "of the stdio encoding"); - } - return _PyStatus_OK(); -} - - -static int -init_fs_codec(PyInterpreterState *interp) -{ - PyConfig *config = &interp->config; - - _Py_error_handler error_handler; - error_handler = get_error_handler_wide(config->filesystem_errors); - if (error_handler == _Py_ERROR_UNKNOWN) { - PyErr_SetString(PyExc_RuntimeError, "unknow filesystem error handler"); - return -1; - } - - char *encoding, *errors; - if (encode_wstr_utf8(config->filesystem_encoding, - &encoding, - "filesystem_encoding") < 0) { - return -1; - } - - if (encode_wstr_utf8(config->filesystem_errors, - &errors, - "filesystem_errors") < 0) { - PyMem_RawFree(encoding); - return -1; - } - - PyMem_RawFree(interp->fs_codec.encoding); - interp->fs_codec.encoding = encoding; - PyMem_RawFree(interp->fs_codec.errors); - interp->fs_codec.errors = errors; - interp->fs_codec.error_handler = error_handler; - - /* At this point, PyUnicode_EncodeFSDefault() and - PyUnicode_DecodeFSDefault() can now use the Python codec rather than - the C implementation of the filesystem encoding. */ - - /* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors - global configuration variables. */ - if (_Py_SetFileSystemEncoding(interp->fs_codec.encoding, - interp->fs_codec.errors) < 0) { - PyErr_NoMemory(); - return -1; - } - return 0; -} + linebreak, sizeof(linebreak) / sizeof(linebreak[0]) + ); + PyType_Ready(&EncodingMapType); -static PyStatus -init_fs_encoding(PyThreadState *tstate) -{ - PyInterpreterState *interp = tstate->interp; - - /* Update the filesystem encoding to the normalized Python codec name. - For example, replace "ANSI_X3.4-1968" (locale encoding) with "ascii" - (Python codec name). */ - PyConfig *config = &interp->config; - if (config_get_codec_name(&config->filesystem_encoding) < 0) { - _Py_DumpPathConfig(tstate); - return _PyStatus_ERR("failed to get the Python codec " - "of the filesystem encoding"); - } + if (PyType_Ready(&PyFieldNameIter_Type) < 0) + Py_FatalError("Can't initialize field name iterator type"); - if (init_fs_codec(interp) < 0) { - return _PyStatus_ERR("cannot initialize filesystem codec"); - } - return _PyStatus_OK(); + if (PyType_Ready(&PyFormatterIter_Type) < 0) + Py_FatalError("Can't initialize formatter iter type"); } +/* Finalize the Unicode implementation */ -PyStatus -_PyUnicode_InitEncodings(PyThreadState *tstate) -{ - PyStatus status = init_fs_encoding(tstate); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - - return init_stdio_encoding(tstate); -} - - -#ifdef MS_WINDOWS int -_PyUnicode_EnableLegacyWindowsFSEncoding(void) +PyUnicode_ClearFreeList(void) { - PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); - PyConfig *config = &interp->config; + int freelist_size = numfree; + PyUnicodeObject *u; - /* Set the filesystem encoding to mbcs/replace (PEP 529) */ - wchar_t *encoding = _PyMem_RawWcsdup(L"mbcs"); - wchar_t *errors = _PyMem_RawWcsdup(L"replace"); - if (encoding == NULL || errors == NULL) { - PyMem_RawFree(encoding); - PyMem_RawFree(errors); - PyErr_NoMemory(); - return -1; + for (u = free_list; u != NULL;) { + PyUnicodeObject *v = u; + u = *(PyUnicodeObject **)u; + if (v->str) + PyObject_DEL(v->str); + Py_XDECREF(v->defenc); + PyObject_Del(v); + numfree--; } - - PyMem_RawFree(config->filesystem_encoding); - config->filesystem_encoding = encoding; - PyMem_RawFree(config->filesystem_errors); - config->filesystem_errors = errors; - - return init_fs_codec(interp); + free_list = NULL; + assert(numfree == 0); + return freelist_size; } -#endif - void -_PyUnicode_Fini(PyThreadState *tstate) -{ - if (_Py_IsMainInterpreter(tstate)) { -#if defined(WITH_VALGRIND) || defined(__INSURE__) - /* Insure++ is a memory analysis tool that aids in discovering - * memory leaks and other memory problems. On Python exit, the - * interned string dictionaries are flagged as being in use at exit - * (which it is). Under normal circumstances, this is fine because - * the memory will be automatically reclaimed by the system. Under - * memory debugging, it's a huge source of useless noise, so we - * trade off slower shutdown for less distraction in the memory - * reports. -baw - */ - unicode_release_interned(); -#endif /* __INSURE__ */ - - Py_CLEAR(unicode_empty); - - for (Py_ssize_t i = 0; i < 256; i++) { - Py_CLEAR(unicode_latin1[i]); - } - _PyUnicode_ClearStaticStrings(); - } - - PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); - PyMem_RawFree(interp->fs_codec.encoding); - interp->fs_codec.encoding = NULL; - PyMem_RawFree(interp->fs_codec.errors); - interp->fs_codec.errors = NULL; - interp->config.filesystem_errors = (wchar_t *)_Py_ERROR_UNKNOWN; -} +_PyUnicode_Fini(void) +{ + int i; + Py_CLEAR(unicode_empty); -/* A _string module, to export formatter_parser and formatter_field_name_split - to the string.Formatter class implemented in Python. */ + for (i = 0; i < 256; i++) + Py_CLEAR(unicode_latin1[i]); -static PyMethodDef _string_methods[] = { - {"formatter_field_name_split", (PyCFunction) formatter_field_name_split, - METH_O, PyDoc_STR("split the argument as a field name")}, - {"formatter_parser", (PyCFunction) formatter_parser, - METH_O, PyDoc_STR("parse the argument as a format string")}, - {NULL, NULL} -}; - -static struct PyModuleDef _string_module = { - PyModuleDef_HEAD_INIT, - "_string", - PyDoc_STR("string helper module"), - 0, - _string_methods, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit__string(void) -{ - return PyModule_Create(&_string_module); + (void)PyUnicode_ClearFreeList(); } - #ifdef __cplusplus } #endif |