From 158701d8866b0cb53953bf0ae0fd2834be766da1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 22 Apr 2010 19:41:01 +0000 Subject: Merged revisions 80382 via svnmerge from svn+ssh://pythondev@svn.python.org/python/branches/py3k ........ r80382 | victor.stinner | 2010-04-22 21:38:16 +0200 (jeu., 22 avril 2010) | 3 lines Issue #8092: Fix PyUnicode_EncodeUTF8() to support error handler producing unicode string (eg. backslashreplace) ........ --- Lib/test/test_codecs.py | 10 ++++ Misc/NEWS | 3 ++ Objects/unicodeobject.c | 127 ++++++++++++++++++++++++++++++------------------ 3 files changed, 93 insertions(+), 47 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 2c646ba..227e375 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -571,6 +571,16 @@ class UTF8Test(ReadTest): def test_lone_surrogates(self): self.assertRaises(UnicodeEncodeError, "\ud800".encode, "utf-8") self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "utf-8") + self.assertEqual("[\uDC80]".encode("utf-8", "backslashreplace"), + b'[\\udc80]') + self.assertEqual("[\uDC80]".encode("utf-8", "xmlcharrefreplace"), + b'[�]') + self.assertEqual("[\uDC80]".encode("utf-8", "surrogateescape"), + b'[\x80]') + self.assertEqual("[\uDC80]".encode("utf-8", "ignore"), + b'[]') + self.assertEqual("[\uDC80]".encode("utf-8", "replace"), + b'[?]') def test_surrogatepass_handler(self): self.assertEquals("abc\ud800def".encode("utf-8", "surrogatepass"), diff --git a/Misc/NEWS b/Misc/NEWS index ab687ff..764e3a5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ What's New in Python 3.1.3? Core and Builtins ----------------- +- Issue #8092: Fix PyUnicode_EncodeUTF8() to support error handler producing + unicode string (eg. backslashreplace) + - Issue #8014: Setting a T_UINT or T_PYSSIZET attribute of an object with PyMemberDefs could produce an internal error; raise TypeError instead. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index c9ff059..4a0746f 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -159,6 +159,12 @@ static PyObject *unicode_encode_call_errorhandler(const char *errors, const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos); +static void raise_encode_exception(PyObject **exceptionObject, + const char *encoding, + const Py_UNICODE *unicode, Py_ssize_t size, + Py_ssize_t startpos, Py_ssize_t endpos, + const char *reason); + /* Same for linebreaks */ static unsigned char ascii_linebreak[] = { 0, 0, 0, 0, 0, 0, 0, 0, @@ -2461,61 +2467,88 @@ PyUnicode_EncodeUTF8(const Py_UNICODE *s, /* Encode Latin-1 */ *p++ = (char)(0xc0 | (ch >> 6)); *p++ = (char)(0x80 | (ch & 0x3f)); - } - else { - /* Encode UCS2 Unicode ordinals */ - if (ch < 0x10000) { + } else if (0xD800 <= ch && ch <= 0xDFFF) { #ifndef Py_UNICODE_WIDE - /* Special case: check for high surrogate */ - if (0xD800 <= ch && ch <= 0xDBFF && i != size) { - Py_UCS4 ch2 = s[i]; - /* Check for low surrogate and combine the two to - form a UCS4 value */ - if (0xDC00 <= ch2 && ch2 <= 0xDFFF) { - ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000; - i++; - goto encodeUCS4; - } - /* Fall through: handles isolated high surrogates */ - } + /* Special case: check for high and low surrogate */ + if (ch <= 0xDBFF && i != size && 0xDC00 <= s[i] && s[i] <= 0xDFFF) { + Py_UCS4 ch2 = s[i]; + /* Combine the two surrogates to form a UCS4 value */ + ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000; + i++; + + /* Encode UCS4 Unicode ordinals */ + *p++ = (char)(0xf0 | (ch >> 18)); + *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + #endif - if (ch >= 0xd800 && ch <= 0xdfff) { - Py_ssize_t newpos; - PyObject *rep; - char *prep; - int k; - rep = unicode_encode_call_errorhandler - (errors, &errorHandler, "utf-8", "surrogates not allowed", - s, size, &exc, i-1, i, &newpos); - if (!rep) - goto error; - /* Implementation limitations: only support error handler that return - bytes, and only support up to four replacement bytes. */ - if (!PyBytes_Check(rep)) { - PyErr_SetString(PyExc_TypeError, "error handler should have returned bytes"); - Py_DECREF(rep); + } else { + Py_ssize_t newpos; + PyObject *rep; + Py_ssize_t repsize, k; + rep = unicode_encode_call_errorhandler + (errors, &errorHandler, "utf-8", "surrogates not allowed", + s, size, &exc, i-1, i, &newpos); + if (!rep) + goto error; + + if (PyBytes_Check(rep)) + repsize = PyBytes_GET_SIZE(rep); + else + repsize = PyUnicode_GET_SIZE(rep); + + if (repsize > 4) { + Py_ssize_t offset; + + if (result == NULL) + offset = p - stackbuf; + else + offset = p - PyBytes_AS_STRING(result); + + if (nallocated > PY_SSIZE_T_MAX - repsize + 4) { + /* integer overflow */ + PyErr_NoMemory(); goto error; } - if (PyBytes_Size(rep) > 4) { - PyErr_SetString(PyExc_TypeError, "error handler returned too many bytes"); - Py_DECREF(rep); - goto error; + nallocated += repsize - 4; + if (result != NULL) { + if (_PyBytes_Resize(&result, nallocated) < 0) + goto error; + } else { + result = PyBytes_FromStringAndSize(NULL, nallocated); + if (result == NULL) + goto error; + Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); } - prep = PyBytes_AsString(rep); - for(k = PyBytes_Size(rep); k > 0; k--) + p = PyBytes_AS_STRING(result) + offset; + } + + if (PyBytes_Check(rep)) { + char *prep = PyBytes_AS_STRING(rep); + for(k = repsize; k > 0; k--) *p++ = *prep++; - Py_DECREF(rep); - continue; - + } else /* rep is unicode */ { + Py_UNICODE *prep = PyUnicode_AS_UNICODE(rep); + Py_UNICODE c; + + for(k=0; k> 12)); - *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); - *p++ = (char)(0x80 | (ch & 0x3f)); - continue; + Py_DECREF(rep); } -#ifndef Py_UNICODE_WIDE - encodeUCS4: -#endif + } else if (ch < 0x10000) { + *p++ = (char)(0xe0 | (ch >> 12)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } else /* ch >= 0x10000 */ { /* Encode UCS4 Unicode ordinals */ *p++ = (char)(0xf0 | (ch >> 18)); *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); -- cgit v0.12