diff options
| -rw-r--r-- | Include/unicodeobject.h | 6 | ||||
| -rw-r--r-- | Modules/_codecsmodule.c | 6 | ||||
| -rw-r--r-- | Objects/unicodeobject.c | 211 |
3 files changed, 134 insertions, 89 deletions
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 6e613eb..dc3bad7 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1425,6 +1425,12 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap( (unicode ordinal -> char ordinal) */ const char *errors /* error handling */ ); +PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( + PyObject *unicode, /* Unicode object */ + PyObject *mapping, /* character mapping + (unicode ordinal -> char ordinal) */ + const char *errors /* error handling */ + ); #endif /* Translate a Py_UNICODE buffer of the given length by applying a diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index be31fd2..c9409cc 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -992,11 +992,7 @@ charmap_encode(PyObject *self, str = PyUnicode_FromObject(str); if (str == NULL) return NULL; - v = codec_tuple(PyUnicode_EncodeCharmap( - PyUnicode_AS_UNICODE(str), - PyUnicode_GET_SIZE(str), - mapping, - errors), + v = codec_tuple(_PyUnicode_EncodeCharmap(str, mapping, errors), PyUnicode_GET_SIZE(str)); Py_DECREF(str); return v; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0a33ece..0ecbd1d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -248,7 +248,7 @@ _PyUnicode_FromUCS4(const Py_UCS4 *s, Py_ssize_t size); static PyObject * unicode_encode_call_errorhandler(const char *errors, PyObject **errorHandler,const char *encoding, const char *reason, - const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, + PyObject *unicode, PyObject **exceptionObject, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos); static void @@ -4745,8 +4745,7 @@ _PyUnicode_AsUTF8String(PyObject *obj, const char *errors) #endif rep = unicode_encode_call_errorhandler( errors, &errorHandler, "utf-8", "surrogates not allowed", - PyUnicode_AS_UNICODE(unicode), PyUnicode_GET_SIZE(unicode), - &exc, startpos, startpos+1, &newpos); + obj, &exc, startpos, startpos+1, &newpos); if (!rep) goto error; @@ -6450,7 +6449,7 @@ make_encode_exception_obj(PyObject **exceptionObject, { if (*exceptionObject == NULL) { *exceptionObject = PyObject_CallFunction( - PyExc_UnicodeEncodeError, "sUnns", + PyExc_UnicodeEncodeError, "sOnns", encoding, unicode, startpos, endpos, reason); } else { @@ -6502,12 +6501,12 @@ static PyObject * unicode_encode_call_errorhandler(const char *errors, PyObject **errorHandler, const char *encoding, const char *reason, - const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, + PyObject *unicode, PyObject **exceptionObject, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos) { static char *argparse = "On;encoding error handler must return (str/bytes, int) tuple"; - + Py_ssize_t len; PyObject *restuple; PyObject *resunicode; @@ -6517,8 +6516,12 @@ unicode_encode_call_errorhandler(const char *errors, return NULL; } - make_encode_exception(exceptionObject, - encoding, unicode, size, startpos, endpos, reason); + if (PyUnicode_READY(unicode) < 0) + return NULL; + len = PyUnicode_GET_LENGTH(unicode); + + make_encode_exception_obj(exceptionObject, + encoding, unicode, startpos, endpos, reason); if (*exceptionObject == NULL) return NULL; @@ -6542,8 +6545,8 @@ unicode_encode_call_errorhandler(const char *errors, return NULL; } if (*newpos<0) - *newpos = size+*newpos; - if (*newpos<0 || *newpos>size) { + *newpos = len + *newpos; + if (*newpos<0 || *newpos>len) { PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos); Py_DECREF(restuple); return NULL; @@ -6554,18 +6557,16 @@ unicode_encode_call_errorhandler(const char *errors, } static PyObject * -unicode_encode_ucs1(const Py_UNICODE *p, - Py_ssize_t size, +unicode_encode_ucs1(PyObject *unicode, const char *errors, int limit) { + /* input state */ + Py_ssize_t pos=0, size; + int kind; + void *data; /* output object */ PyObject *res; - /* pointers to the beginning and end+1 of input */ - const Py_UNICODE *startp = p; - const Py_UNICODE *endp = p + size; - /* pointer to the beginning of the unencodable characters */ - /* const Py_UNICODE *badp = NULL; */ /* pointer into the output */ char *str; /* current output position */ @@ -6578,6 +6579,11 @@ unicode_encode_ucs1(const Py_UNICODE *p, * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */ int known_errorHandler = -1; + if (PyUnicode_READY(unicode) < 0) + return NULL; + size = PyUnicode_GET_LENGTH(unicode); + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); /* allocate enough for a simple encoding without replacements, if we need more, we'll resize */ if (size == 0) @@ -6588,28 +6594,24 @@ unicode_encode_ucs1(const Py_UNICODE *p, str = PyBytes_AS_STRING(res); ressize = size; - while (p<endp) { - Py_UNICODE c = *p; + while (pos < size) { + Py_UCS4 c = PyUnicode_READ(kind, data, pos); /* can we encode this? */ if (c<limit) { /* no overflow check, because we know that the space is enough */ *str++ = (char)c; - ++p; + ++pos; } else { - Py_ssize_t unicodepos = p-startp; Py_ssize_t requiredsize; PyObject *repunicode; - Py_ssize_t repsize; - Py_ssize_t newpos; - Py_ssize_t respos; - Py_UNICODE *uni2; + Py_ssize_t repsize, newpos, respos, i; /* startpos for collecting unencodable chars */ - const Py_UNICODE *collstart = p; - const Py_UNICODE *collend = p; + Py_ssize_t collstart = pos; + Py_ssize_t collend = pos; /* find all unecodable characters */ - while ((collend < endp) && ((*collend)>=limit)) + while ((collend < size) && (PyUnicode_READ(kind, data, collend)>=limit)) ++collend; /* cache callback name lookup (if not done yet, i.e. it's the first error) */ if (known_errorHandler==-1) { @@ -6626,39 +6628,40 @@ unicode_encode_ucs1(const Py_UNICODE *p, } switch (known_errorHandler) { case 1: /* strict */ - raise_encode_exception(&exc, encoding, startp, size, collstart-startp, collend-startp, reason); + raise_encode_exception_obj(&exc, encoding, unicode, collstart, collend, reason); goto onError; case 2: /* replace */ while (collstart++<collend) *str++ = '?'; /* fall through */ case 3: /* ignore */ - p = collend; + pos = collend; break; case 4: /* xmlcharrefreplace */ respos = str - PyBytes_AS_STRING(res); - /* determine replacement size (temporarily (mis)uses p) */ - for (p = collstart, repsize = 0; p < collend; ++p) { - if (*p<10) + /* determine replacement size */ + for (i = collstart, repsize = 0; i < collend; ++i) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch < 10) repsize += 2+1+1; - else if (*p<100) + else if (ch < 100) repsize += 2+2+1; - else if (*p<1000) + else if (ch < 1000) repsize += 2+3+1; - else if (*p<10000) + else if (ch < 10000) repsize += 2+4+1; #ifndef Py_UNICODE_WIDE else repsize += 2+5+1; #else - else if (*p<100000) + else if (ch < 100000) repsize += 2+5+1; - else if (*p<1000000) + else if (ch < 1000000) repsize += 2+6+1; else repsize += 2+7+1; #endif } - requiredsize = respos+repsize+(endp-collend); + requiredsize = respos+repsize+(size-collend); if (requiredsize > ressize) { if (requiredsize<2*ressize) requiredsize = 2*ressize; @@ -6667,17 +6670,18 @@ unicode_encode_ucs1(const Py_UNICODE *p, str = PyBytes_AS_STRING(res) + respos; ressize = requiredsize; } - /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) { - str += sprintf(str, "&#%d;", (int)*p); + /* generate replacement */ + for (i = collstart; i < collend; ++i) { + str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); } - p = collend; + pos = collend; break; default: repunicode = unicode_encode_call_errorhandler(errors, &errorHandler, - encoding, reason, startp, size, &exc, - collstart-startp, collend-startp, &newpos); - if (repunicode == NULL) + encoding, reason, unicode, &exc, + collstart, collend, &newpos); + if (repunicode == NULL || (PyUnicode_Check(repunicode) && + PyUnicode_READY(repunicode) < 0)) goto onError; if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ @@ -6694,7 +6698,7 @@ unicode_encode_ucs1(const Py_UNICODE *p, } memcpy(str, PyBytes_AsString(repunicode), repsize); str += repsize; - p = startp + newpos; + pos = newpos; Py_DECREF(repunicode); break; } @@ -6702,8 +6706,8 @@ unicode_encode_ucs1(const Py_UNICODE *p, have+the replacement+the rest of the string, so we won't have to check space for encodable characters) */ respos = str - PyBytes_AS_STRING(res); - repsize = PyUnicode_GET_SIZE(repunicode); - requiredsize = respos+repsize+(endp-collend); + repsize = PyUnicode_GET_LENGTH(repunicode); + requiredsize = respos+repsize+(size-collend); if (requiredsize > ressize) { if (requiredsize<2*ressize) requiredsize = 2*ressize; @@ -6716,17 +6720,17 @@ unicode_encode_ucs1(const Py_UNICODE *p, } /* check if there is anything unencodable in the replacement and copy it to the output */ - for (uni2 = PyUnicode_AS_UNICODE(repunicode);repsize-->0; ++uni2, ++str) { - c = *uni2; + for (i = 0; repsize-->0; ++i, ++str) { + c = PyUnicode_READ_CHAR(repunicode, i); if (c >= limit) { - raise_encode_exception(&exc, encoding, startp, size, - unicodepos, unicodepos+1, reason); + raise_encode_exception_obj(&exc, encoding, unicode, + pos, pos+1, reason); Py_DECREF(repunicode); goto onError; } *str = (char)c; } - p = startp + newpos; + pos = newpos; Py_DECREF(repunicode); } } @@ -6750,12 +6754,19 @@ unicode_encode_ucs1(const Py_UNICODE *p, return NULL; } +/* Deprecated */ PyObject * PyUnicode_EncodeLatin1(const Py_UNICODE *p, Py_ssize_t size, const char *errors) { - return unicode_encode_ucs1(p, size, errors, 256); + PyObject *result; + PyObject *unicode = PyUnicode_FromUnicode(p, size); + if (unicode == NULL) + return NULL; + result = unicode_encode_ucs1(unicode, errors, 256); + Py_DECREF(unicode); + return result; } PyObject * @@ -6774,9 +6785,7 @@ _PyUnicode_AsLatin1String(PyObject *unicode, const char *errors) PyUnicode_GET_LENGTH(unicode)); /* Non-Latin-1 characters present. Defer to above function to raise the exception. */ - return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); + return unicode_encode_ucs1(unicode, errors, 256); } PyObject* @@ -6888,12 +6897,19 @@ PyUnicode_DecodeASCII(const char *s, return NULL; } +/* Deprecated */ PyObject * PyUnicode_EncodeASCII(const Py_UNICODE *p, Py_ssize_t size, const char *errors) { - return unicode_encode_ucs1(p, size, errors, 128); + PyObject *result; + PyObject *unicode = PyUnicode_FromUnicode(p, size); + if (unicode == NULL) + return NULL; + result = unicode_encode_ucs1(unicode, errors, 128); + Py_DECREF(unicode); + return result; } PyObject * @@ -6910,9 +6926,7 @@ _PyUnicode_AsASCIIString(PyObject *unicode, const char *errors) if (PyUnicode_MAX_CHAR_VALUE(unicode) < 128) return PyBytes_FromStringAndSize(PyUnicode_DATA(unicode), PyUnicode_GET_LENGTH(unicode)); - return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - errors); + return unicode_encode_ucs1(unicode, errors, 128); } PyObject * @@ -8182,13 +8196,13 @@ charmapencode_output(Py_UNICODE c, PyObject *mapping, Return 0 on success, -1 on error */ static int charmap_encoding_error( - const Py_UNICODE *p, Py_ssize_t size, Py_ssize_t *inpos, PyObject *mapping, + PyObject *unicode, Py_ssize_t *inpos, PyObject *mapping, PyObject **exceptionObject, int *known_errorHandler, PyObject **errorHandler, const char *errors, PyObject **res, Py_ssize_t *respos) { PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ - Py_ssize_t repsize; + Py_ssize_t size, repsize; Py_ssize_t newpos; Py_UNICODE *uni2; /* startpos for collecting unencodable chars */ @@ -8198,19 +8212,25 @@ charmap_encoding_error( char *encoding = "charmap"; char *reason = "character maps to <undefined>"; charmapencode_result x; + Py_UCS4 ch; + if (PyUnicode_READY(unicode) < 0) + return -1; + size = PyUnicode_GET_LENGTH(unicode); /* find all unencodable characters */ while (collendpos < size) { PyObject *rep; if (Py_TYPE(mapping) == &EncodingMapType) { - int res = encoding_map_lookup(p[collendpos], mapping); + ch = PyUnicode_READ_CHAR(unicode, collendpos); + int res = encoding_map_lookup(ch, mapping); if (res != -1) break; ++collendpos; continue; } - rep = charmapencode_lookup(p[collendpos], mapping); + ch = PyUnicode_READ_CHAR(unicode, collendpos); + rep = charmapencode_lookup(ch, mapping); if (rep==NULL) return -1; else if (rep!=Py_None) { @@ -8236,7 +8256,7 @@ charmap_encoding_error( } switch (*known_errorHandler) { case 1: /* strict */ - raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); + raise_encode_exception_obj(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; case 2: /* replace */ for (collpos = collstartpos; collpos<collendpos; ++collpos) { @@ -8245,7 +8265,7 @@ charmap_encoding_error( return -1; } else if (x==enc_FAILED) { - raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); + raise_encode_exception_obj(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; } } @@ -8258,13 +8278,13 @@ charmap_encoding_error( for (collpos = collstartpos; collpos < collendpos; ++collpos) { char buffer[2+29+1+1]; char *cp; - sprintf(buffer, "&#%d;", (int)p[collpos]); + sprintf(buffer, "&#%d;", (int)PyUnicode_READ_CHAR(unicode, collpos)); for (cp = buffer; *cp; ++cp) { x = charmapencode_output(*cp, mapping, res, respos); if (x==enc_EXCEPTION) return -1; else if (x==enc_FAILED) { - raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); + raise_encode_exception_obj(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; } } @@ -8273,7 +8293,7 @@ charmap_encoding_error( break; default: repunicode = unicode_encode_call_errorhandler(errors, errorHandler, - encoding, reason, p, size, exceptionObject, + encoding, reason, unicode, exceptionObject, collstartpos, collendpos, &newpos); if (repunicode == NULL) return -1; @@ -8305,7 +8325,7 @@ charmap_encoding_error( } else if (x==enc_FAILED) { Py_DECREF(repunicode); - raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason); + raise_encode_exception_obj(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; } } @@ -8316,15 +8336,15 @@ charmap_encoding_error( } PyObject * -PyUnicode_EncodeCharmap(const Py_UNICODE *p, - Py_ssize_t size, - PyObject *mapping, - const char *errors) +_PyUnicode_EncodeCharmap(PyObject *unicode, + PyObject *mapping, + const char *errors) { /* output object */ PyObject *res = NULL; /* current input position */ Py_ssize_t inpos = 0; + Py_ssize_t size; /* current output position */ Py_ssize_t respos = 0; PyObject *errorHandler = NULL; @@ -8334,9 +8354,13 @@ PyUnicode_EncodeCharmap(const Py_UNICODE *p, * 3=ignore, 4=xmlcharrefreplace */ int known_errorHandler = -1; + if (PyUnicode_READY(unicode) < 0) + return NULL; + size = PyUnicode_GET_LENGTH(unicode); + /* Default to Latin-1 */ if (mapping == NULL) - return PyUnicode_EncodeLatin1(p, size, errors); + return unicode_encode_ucs1(unicode, errors, 256); /* allocate enough for a simple encoding without replacements, if we need more, we'll resize */ @@ -8347,12 +8371,13 @@ PyUnicode_EncodeCharmap(const Py_UNICODE *p, return res; while (inpos<size) { + Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, inpos); /* try to encode it */ - charmapencode_result x = charmapencode_output(p[inpos], mapping, &res, &respos); + charmapencode_result x = charmapencode_output(ch, mapping, &res, &respos); if (x==enc_EXCEPTION) /* error */ goto onError; if (x==enc_FAILED) { /* unencodable character */ - if (charmap_encoding_error(p, size, &inpos, mapping, + if (charmap_encoding_error(unicode, &inpos, mapping, &exc, &known_errorHandler, &errorHandler, errors, &res, &respos)) { @@ -8380,6 +8405,22 @@ PyUnicode_EncodeCharmap(const Py_UNICODE *p, return NULL; } +/* Deprecated */ +PyObject * +PyUnicode_EncodeCharmap(const Py_UNICODE *p, + Py_ssize_t size, + PyObject *mapping, + const char *errors) +{ + PyObject *result; + PyObject *unicode = PyUnicode_FromUnicode(p, size); + if (unicode == NULL) + return NULL; + result = _PyUnicode_EncodeCharmap(unicode, mapping, errors); + Py_DECREF(unicode); + return NULL; +} + PyObject * PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) @@ -8388,10 +8429,7 @@ PyUnicode_AsCharmapString(PyObject *unicode, PyErr_BadArgument(); return NULL; } - return PyUnicode_EncodeCharmap(PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - mapping, - NULL); + return _PyUnicode_EncodeCharmap(unicode, mapping, NULL); } /* create or adjust a UnicodeTranslateError */ @@ -8893,6 +8931,7 @@ PyUnicode_EncodeDecimal(Py_UNICODE *s, Py_UNICODE *p, *end; PyObject *errorHandler = NULL; PyObject *exc = NULL; + PyObject *unicode; const char *encoding = "decimal"; const char *reason = "invalid decimal Unicode string"; /* the following variable is used for caching string comparisons @@ -8973,9 +9012,13 @@ PyUnicode_EncodeDecimal(Py_UNICODE *s, p = collend; break; default: + unicode = PyUnicode_FromUnicode(s, length); + if (unicode == NULL) + goto onError; repunicode = unicode_encode_call_errorhandler(errors, &errorHandler, - encoding, reason, s, length, &exc, + encoding, reason, unicode, &exc, collstart-s, collend-s, &newpos); + Py_DECREF(unicode); if (repunicode == NULL) goto onError; if (!PyUnicode_Check(repunicode)) { |