From 58cf607d13c178f41aed05458296b68e985c5fff Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 19 Nov 2013 11:32:41 +0200 Subject: Issue #12892: The utf-16* and utf-32* codecs now reject (lone) surrogates. The utf-16* and utf-32* encoders no longer allow surrogate code points (U+D800-U+DFFF) to be encoded. The utf-32* decoders no longer decode byte sequences that correspond to surrogate code points. The surrogatepass error handler now works with the utf-16* and utf-32* codecs. Based on patches by Victor Stinner and Kang-Hao (Kenny) Lu. --- Doc/library/codecs.rst | 25 +++-- Doc/whatsnew/3.4.rst | 7 ++ Lib/test/test_codecs.py | 68 ++++++++++--- Misc/ACKS | 1 + Misc/NEWS | 6 ++ Objects/stringlib/codecs.h | 198 +++++++++++++++++++++++++++++++++--- Objects/unicodeobject.c | 245 ++++++++++++++++++++++++++++++++++++++++----- Python/codecs.c | 163 ++++++++++++++++++++++++++---- 8 files changed, 637 insertions(+), 76 deletions(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 0925e82..358fde7 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -365,18 +365,23 @@ and implemented by all standard Python codecs: | | in :pep:`383`. | +-------------------------+-----------------------------------------------+ -In addition, the following error handlers are specific to a single codec: +In addition, the following error handlers are specific to Unicode encoding +schemes: -+-------------------+---------+-------------------------------------------+ -| Value | Codec | Meaning | -+===================+=========+===========================================+ -|``'surrogatepass'``| utf-8 | Allow encoding and decoding of surrogate | -| | | codes in UTF-8. | -+-------------------+---------+-------------------------------------------+ ++-------------------+------------------------+-------------------------------------------+ +| Value | Codec | Meaning | ++===================+========================+===========================================+ +|``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding of surrogate | +| | utf-16-be, utf-16-le, | codes in all the Unicode encoding schemes.| +| | utf-32-be, utf-32-le | | ++-------------------+------------------------+-------------------------------------------+ .. versionadded:: 3.1 The ``'surrogateescape'`` and ``'surrogatepass'`` error handlers. +.. versionchanged:: 3.4 + The ``'surrogatepass'`` error handlers now works with utf-16\* and utf-32\* codecs. + The set of allowed values can be extended via :meth:`register_error`. @@ -1167,6 +1172,12 @@ particular, the following variants typically exist: | utf_8_sig | | all languages | +-----------------+--------------------------------+--------------------------------+ +.. versionchanged:: 3.4 + The utf-16\* and utf-32\* encoders no longer allow surrogate code points + (U+D800--U+DFFF) to be encoded. The utf-32\* decoders no longer decode + byte sequences that correspond to surrogate code points. + + Python Specific Encodings ------------------------- diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 197702a..f72f544 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -253,6 +253,13 @@ Some smaller changes made to the core Python language are: ``__main__.__file__`` when a script has been executed directly using a relative path (Contributed by Brett Cannon in :issue:`18416`). +* Now all the UTF-\* codecs (except UTF-7) reject surrogates during both + encoding and decoding unless the ``surrogatepass`` error handler is used, + with the exception of the UTF-16 decoder that accepts valid surrogate pairs, + and the UTF-16 encoder that produces them while encoding non-BMP characters. + Contributed by Victor Stinner, Kang-Hao (Kenny) Lu and Serhiy Storchaka in + :issue:`12892`. + New Modules =========== diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 55becf4..31bd089 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -300,8 +300,46 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual(reader.readline(), s5) self.assertEqual(reader.readline(), "") + ill_formed_sequence_replace = "\ufffd" + + def test_lone_surrogates(self): + self.assertRaises(UnicodeEncodeError, "\ud800".encode, self.encoding) + self.assertEqual("[\uDC80]".encode(self.encoding, "backslashreplace"), + "[\\udc80]".encode(self.encoding)) + self.assertEqual("[\uDC80]".encode(self.encoding, "xmlcharrefreplace"), + "[�]".encode(self.encoding)) + self.assertEqual("[\uDC80]".encode(self.encoding, "ignore"), + "[]".encode(self.encoding)) + self.assertEqual("[\uDC80]".encode(self.encoding, "replace"), + "[?]".encode(self.encoding)) + + bom = "".encode(self.encoding) + for before, after in [("\U00010fff", "A"), ("[", "]"), + ("A", "\U00010fff")]: + before_sequence = before.encode(self.encoding)[len(bom):] + after_sequence = after.encode(self.encoding)[len(bom):] + test_string = before + "\uDC80" + after + test_sequence = (bom + before_sequence + + self.ill_formed_sequence + after_sequence) + self.assertRaises(UnicodeDecodeError, test_sequence.decode, + self.encoding) + self.assertEqual(test_string.encode(self.encoding, + "surrogatepass"), + test_sequence) + self.assertEqual(test_sequence.decode(self.encoding, + "surrogatepass"), + test_string) + self.assertEqual(test_sequence.decode(self.encoding, "ignore"), + before + after) + self.assertEqual(test_sequence.decode(self.encoding, "replace"), + before + self.ill_formed_sequence_replace + after) + class UTF32Test(ReadTest, unittest.TestCase): encoding = "utf-32" + if sys.byteorder == 'little': + ill_formed_sequence = b"\x80\xdc\x00\x00" + else: + ill_formed_sequence = b"\x00\x00\xdc\x80" spamle = (b'\xff\xfe\x00\x00' b's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00' @@ -393,6 +431,7 @@ class UTF32Test(ReadTest, unittest.TestCase): class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" + ill_formed_sequence = b"\x80\xdc\x00\x00" def test_partial(self): self.check_partial( @@ -437,6 +476,7 @@ class UTF32LETest(ReadTest, unittest.TestCase): class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" + ill_formed_sequence = b"\x00\x00\xdc\x80" def test_partial(self): self.check_partial( @@ -482,6 +522,10 @@ class UTF32BETest(ReadTest, unittest.TestCase): class UTF16Test(ReadTest, unittest.TestCase): encoding = "utf-16" + if sys.byteorder == 'little': + ill_formed_sequence = b"\x80\xdc" + else: + ill_formed_sequence = b"\xdc\x80" spamle = b'\xff\xfes\x00p\x00a\x00m\x00s\x00p\x00a\x00m\x00' spambe = b'\xfe\xff\x00s\x00p\x00a\x00m\x00s\x00p\x00a\x00m' @@ -562,6 +606,7 @@ class UTF16Test(ReadTest, unittest.TestCase): class UTF16LETest(ReadTest, unittest.TestCase): encoding = "utf-16-le" + ill_formed_sequence = b"\x80\xdc" def test_partial(self): self.check_partial( @@ -605,6 +650,7 @@ class UTF16LETest(ReadTest, unittest.TestCase): class UTF16BETest(ReadTest, unittest.TestCase): encoding = "utf-16-be" + ill_formed_sequence = b"\xdc\x80" def test_partial(self): self.check_partial( @@ -648,6 +694,8 @@ class UTF16BETest(ReadTest, unittest.TestCase): class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" + ill_formed_sequence = b"\xed\xb2\x80" + ill_formed_sequence_replace = "\ufffd" * 3 def test_partial(self): self.check_partial( @@ -677,18 +725,11 @@ class UTF8Test(ReadTest, unittest.TestCase): u, u.encode(self.encoding)) def test_lone_surrogates(self): - self.assertRaises(UnicodeEncodeError, "\ud800".encode, "utf-8") - self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "utf-8") - self.assertEqual("[\uDC80]".encode("utf-8", "backslashreplace"), - b'[\\udc80]') - self.assertEqual("[\uDC80]".encode("utf-8", "xmlcharrefreplace"), - b'[�]') - self.assertEqual("[\uDC80]".encode("utf-8", "surrogateescape"), + super().test_lone_surrogates() + # not sure if this is making sense for + # UTF-16 and UTF-32 + self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"), b'[\x80]') - self.assertEqual("[\uDC80]".encode("utf-8", "ignore"), - b'[]') - self.assertEqual("[\uDC80]".encode("utf-8", "replace"), - b'[?]') def test_surrogatepass_handler(self): self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"), @@ -851,6 +892,9 @@ class UTF7Test(ReadTest, unittest.TestCase): self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-') self.assertEqual(b'+2AHcoA-'.decode(self.encoding), '\U000104A0') + test_lone_surrogates = None + + class UTF16ExTest(unittest.TestCase): def test_errors(self): @@ -875,7 +919,7 @@ class ReadBufferTest(unittest.TestCase): self.assertRaises(TypeError, codecs.readbuffer_encode) self.assertRaises(TypeError, codecs.readbuffer_encode, 42) -class UTF8SigTest(ReadTest, unittest.TestCase): +class UTF8SigTest(UTF8Test, unittest.TestCase): encoding = "utf-8-sig" def test_partial(self): diff --git a/Misc/ACKS b/Misc/ACKS index 34fac02..436b9ac 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -783,6 +783,7 @@ Ned Jackson Lovely Jason Lowe Tony Lownds Ray Loyzaga +Kang-Hao (Kenny) Lu Lukas Lueg Loren Luke Fredrik Lundh diff --git a/Misc/NEWS b/Misc/NEWS index 7025410..57f5d99 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,12 @@ Projected release date: 2013-11-24 Core and Builtins ----------------- +- Issue #12892: The utf-16* and utf-32* encoders no longer allow surrogate code + points (U+D800-U+DFFF) to be encoded. The utf-32* decoders no longer decode + byte sequences that correspond to surrogate code points. The surrogatepass + error handler now works with the utf-16* and utf-32* codecs. Based on + patches by Victor Stinner and Kang-Hao (Kenny) Lu. + - Issue #17806: Added keyword-argument support for "tabsize" to str/bytes.expandtabs(). diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 57319c6..14fdc6c 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -596,66 +596,232 @@ IllegalSurrogate: #undef SWAB -Py_LOCAL_INLINE(void) -STRINGLIB(utf16_encode)(unsigned short *out, - const STRINGLIB_CHAR *in, +#if STRINGLIB_MAX_CHAR >= 0x80 +Py_LOCAL_INLINE(Py_ssize_t) +STRINGLIB(utf16_encode_)(const STRINGLIB_CHAR *in, Py_ssize_t len, + unsigned short **outptr, int native_ordering) { + unsigned short *out = *outptr; const STRINGLIB_CHAR *end = in + len; #if STRINGLIB_SIZEOF_CHAR == 1 # define SWAB2(CH) ((CH) << 8) #else # define SWAB2(CH) (((CH) << 8) | ((CH) >> 8)) #endif -#if STRINGLIB_MAX_CHAR < 0x10000 if (native_ordering) { -# if STRINGLIB_SIZEOF_CHAR == 2 - Py_MEMCPY(out, in, 2 * len); -# else - _PyUnicode_CONVERT_BYTES(STRINGLIB_CHAR, unsigned short, in, end, out); +#if STRINGLIB_MAX_CHAR < 0x10000 + const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4); + while (in < unrolled_end) { +# if STRINGLIB_MAX_CHAR >= 0xd800 + if (((in[0] ^ 0xd800) & + (in[1] ^ 0xd800) & + (in[2] ^ 0xd800) & + (in[3] ^ 0xd800) & 0xf800) == 0) + break; # endif + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; + in += 4; out += 4; + } +#endif + while (in < end) { + Py_UCS4 ch; + ch = *in++; +#if STRINGLIB_MAX_CHAR >= 0xd800 + if (ch < 0xd800) + *out++ = ch; + else if (ch < 0xe000) + /* reject surrogate characters (U+DC800-U+DFFF) */ + goto fail; +# if STRINGLIB_MAX_CHAR >= 0x10000 + else if (ch >= 0x10000) { + out[0] = Py_UNICODE_HIGH_SURROGATE(ch); + out[1] = Py_UNICODE_LOW_SURROGATE(ch); + out += 2; + } +# endif + else +#endif + *out++ = ch; + } } else { +#if STRINGLIB_MAX_CHAR < 0x10000 const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4); while (in < unrolled_end) { +# if STRINGLIB_MAX_CHAR >= 0xd800 + if (((in[0] ^ 0xd800) & + (in[1] ^ 0xd800) & + (in[2] ^ 0xd800) & + (in[3] ^ 0xd800) & 0xf800) == 0) + break; +# endif out[0] = SWAB2(in[0]); out[1] = SWAB2(in[1]); out[2] = SWAB2(in[2]); out[3] = SWAB2(in[3]); in += 4; out += 4; } +#endif while (in < end) { - *out++ = SWAB2(*in); - ++in; + Py_UCS4 ch = *in++; +#if STRINGLIB_MAX_CHAR >= 0xd800 + if (ch < 0xd800) + *out++ = SWAB2((Py_UCS2)ch); + else if (ch < 0xe000) + /* reject surrogate characters (U+DC800-U+DFFF) */ + goto fail; +# if STRINGLIB_MAX_CHAR >= 0x10000 + else if (ch >= 0x10000) { + Py_UCS2 ch1 = Py_UNICODE_HIGH_SURROGATE(ch); + Py_UCS2 ch2 = Py_UNICODE_LOW_SURROGATE(ch); + out[0] = SWAB2(ch1); + out[1] = SWAB2(ch2); + out += 2; + } +# endif + else +#endif + *out++ = SWAB2((Py_UCS2)ch); } } -#else + *outptr = out; + return len; +#if STRINGLIB_MAX_CHAR >= 0xd800 + fail: +#endif + *outptr = out; + return len - (end - in + 1); +} +#endif + +#undef SWAB2 + +#if STRINGLIB_MAX_CHAR >= 0x80 +Py_LOCAL_INLINE(Py_ssize_t) +STRINGLIB(utf16_encode)(const STRINGLIB_CHAR *in, + Py_ssize_t len, + unsigned short **outptr, + int native_ordering) +{ + unsigned short *out = *outptr; + const STRINGLIB_CHAR *end = in + len; +#if STRINGLIB_SIZEOF_CHAR == 1 if (native_ordering) { + const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4); + while (in < unrolled_end) { + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; + in += 4; out += 4; + } + while (in < end) { + *out++ = *in++; + } + } else { +# define SWAB2(CH) ((CH) << 8) /* high byte is zero */ + const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4); + while (in < unrolled_end) { + out[0] = SWAB2(in[0]); + out[1] = SWAB2(in[1]); + out[2] = SWAB2(in[2]); + out[3] = SWAB2(in[3]); + in += 4; out += 4; + } while (in < end) { Py_UCS4 ch = *in++; - if (ch < 0x10000) + *out++ = SWAB2((Py_UCS2)ch); + } +#undef SWAB2 + } + *outptr = out; + return len; +#else + if (native_ordering) { +#if STRINGLIB_MAX_CHAR < 0x10000 + const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4); + while (in < unrolled_end) { + /* check if any character is a surrogate character */ + if (((in[0] ^ 0xd800) & + (in[1] ^ 0xd800) & + (in[2] ^ 0xd800) & + (in[3] ^ 0xd800) & 0xf800) == 0) + break; + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; + in += 4; out += 4; + } +#endif + while (in < end) { + Py_UCS4 ch; + ch = *in++; + if (ch < 0xd800) *out++ = ch; - else { + else if (ch < 0xe000) + /* reject surrogate characters (U+DC800-U+DFFF) */ + goto fail; +#if STRINGLIB_MAX_CHAR >= 0x10000 + else if (ch >= 0x10000) { out[0] = Py_UNICODE_HIGH_SURROGATE(ch); out[1] = Py_UNICODE_LOW_SURROGATE(ch); out += 2; } +#endif + else + *out++ = ch; } } else { +#define SWAB2(CH) (((CH) << 8) | ((CH) >> 8)) +#if STRINGLIB_MAX_CHAR < 0x10000 + const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4); + while (in < unrolled_end) { + /* check if any character is a surrogate character */ + if (((in[0] ^ 0xd800) & + (in[1] ^ 0xd800) & + (in[2] ^ 0xd800) & + (in[3] ^ 0xd800) & 0xf800) == 0) + break; + out[0] = SWAB2(in[0]); + out[1] = SWAB2(in[1]); + out[2] = SWAB2(in[2]); + out[3] = SWAB2(in[3]); + in += 4; out += 4; + } +#endif while (in < end) { Py_UCS4 ch = *in++; - if (ch < 0x10000) + if (ch < 0xd800) *out++ = SWAB2((Py_UCS2)ch); - else { + else if (ch < 0xe000) + /* reject surrogate characters (U+DC800-U+DFFF) */ + goto fail; +#if STRINGLIB_MAX_CHAR >= 0x10000 + else if (ch >= 0x10000) { Py_UCS2 ch1 = Py_UNICODE_HIGH_SURROGATE(ch); Py_UCS2 ch2 = Py_UNICODE_LOW_SURROGATE(ch); out[0] = SWAB2(ch1); out[1] = SWAB2(ch2); out += 2; } +#endif + else + *out++ = SWAB2((Py_UCS2)ch); } +#undef SWAB2 } + *outptr = out; + return len; + fail: + *outptr = out; + return len - (end - in + 1); #endif -#undef SWAB2 } +#endif + #endif /* STRINGLIB_IS_UNICODE */ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index bddfafd..7114006 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4963,6 +4963,7 @@ PyUnicode_DecodeUTF32Stateful(const char *s, _PyUnicodeWriter writer; const unsigned char *q, *e; int le, bo = 0; /* assume native ordering by default */ + const char *encoding; const char *errmsg = ""; PyObject *errorHandler = NULL; PyObject *exc = NULL; @@ -5002,6 +5003,7 @@ PyUnicode_DecodeUTF32Stateful(const char *s, #else le = bo <= 0; #endif + encoding = le ? "utf-32-le" : "utf-32-be"; _PyUnicodeWriter_Init(&writer); writer.min_length = (e - q + 3) / 4; @@ -5022,6 +5024,9 @@ PyUnicode_DecodeUTF32Stateful(const char *s, ch = (q[3] << 24) | (q[2] << 16) | (q[1] << 8) | q[0]; if (ch > maxch) break; + if (kind != PyUnicode_1BYTE_KIND && + Py_UNICODE_IS_SURROGATE(ch)) + break; PyUnicode_WRITE(kind, data, pos++, ch); q += 4; } while (q <= last); @@ -5031,6 +5036,9 @@ PyUnicode_DecodeUTF32Stateful(const char *s, ch = (q[0] << 24) | (q[1] << 16) | (q[2] << 8) | q[3]; if (ch > maxch) break; + if (kind != PyUnicode_1BYTE_KIND && + Py_UNICODE_IS_SURROGATE(ch)) + break; PyUnicode_WRITE(kind, data, pos++, ch); q += 4; } while (q <= last); @@ -5038,7 +5046,12 @@ PyUnicode_DecodeUTF32Stateful(const char *s, writer.pos = pos; } - if (ch <= maxch) { + if (Py_UNICODE_IS_SURROGATE(ch)) { + errmsg = "codepoint in surrogate code point range(0xd800, 0xe000)"; + startinpos = ((const char *)q) - starts; + endinpos = startinpos + 4; + } + else if (ch <= maxch) { if (q == e || consumed) break; /* remaining bytes at the end? (size should be divisible by 4) */ @@ -5062,7 +5075,7 @@ PyUnicode_DecodeUTF32Stateful(const char *s, chooses to skip the input */ if (unicode_decode_call_errorhandler_writer( errors, &errorHandler, - "utf32", errmsg, + encoding, errmsg, &starts, (const char **)&e, &startinpos, &endinpos, &exc, (const char **)&q, &writer)) goto onError; @@ -5099,6 +5112,10 @@ _PyUnicode_EncodeUTF32(PyObject *str, #else int iorder[] = {3, 2, 1, 0}; #endif + const char *encoding; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + PyObject *rep = NULL; #define STORECHAR(CH) \ do { \ @@ -5130,7 +5147,7 @@ _PyUnicode_EncodeUTF32(PyObject *str, if (byteorder == 0) STORECHAR(0xFEFF); if (len == 0) - goto done; + return v; if (byteorder == -1) { /* force LE */ @@ -5138,6 +5155,7 @@ _PyUnicode_EncodeUTF32(PyObject *str, iorder[1] = 1; iorder[2] = 2; iorder[3] = 3; + encoding = "utf-32-le"; } else if (byteorder == 1) { /* force BE */ @@ -5145,13 +5163,103 @@ _PyUnicode_EncodeUTF32(PyObject *str, iorder[1] = 2; iorder[2] = 1; iorder[3] = 0; + encoding = "utf-32-be"; } + else + encoding = "utf-32"; - for (i = 0; i < len; i++) - STORECHAR(PyUnicode_READ(kind, data, i)); + if (kind == PyUnicode_1BYTE_KIND) { + for (i = 0; i < len; i++) + STORECHAR(PyUnicode_READ(kind, data, i)); + return v; + } - done: + for (i = 0; i < len;) { + Py_ssize_t repsize, moreunits; + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + i++; + assert(ch <= MAX_UNICODE); + if (!Py_UNICODE_IS_SURROGATE(ch)) { + STORECHAR(ch); + continue; + } + + rep = unicode_encode_call_errorhandler( + errors, &errorHandler, + encoding, "surrogates not allowed", + str, &exc, i-1, i, &i); + + if (!rep) + goto error; + + if (PyBytes_Check(rep)) { + repsize = PyBytes_GET_SIZE(rep); + if (repsize & 3) { + raise_encode_exception(&exc, encoding, + str, i - 1, i, + "surrogates not allowed"); + goto error; + } + moreunits = repsize / 4; + } + else { + assert(PyUnicode_Check(rep)); + if (PyUnicode_READY(rep) < 0) + goto error; + moreunits = repsize = PyUnicode_GET_LENGTH(rep); + if (!PyUnicode_IS_ASCII(rep)) { + raise_encode_exception(&exc, encoding, + str, i - 1, i, + "surrogates not allowed"); + goto error; + } + } + + /* four bytes are reserved for each surrogate */ + if (moreunits > 1) { + Py_ssize_t outpos = p - (unsigned char*) PyBytes_AS_STRING(v); + Py_ssize_t morebytes = 4 * (moreunits - 1); + if (PyBytes_GET_SIZE(v) > PY_SSIZE_T_MAX - morebytes) { + /* integer overflow */ + PyErr_NoMemory(); + goto error; + } + if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + morebytes) < 0) + goto error; + p = (unsigned char*) PyBytes_AS_STRING(v) + outpos; + } + + if (PyBytes_Check(rep)) { + Py_MEMCPY(p, PyBytes_AS_STRING(rep), repsize); + p += repsize; + } else /* rep is unicode */ { + const Py_UCS1 *repdata; + assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); + repdata = PyUnicode_1BYTE_DATA(rep); + while (repsize--) { + Py_UCS4 ch = *repdata++; + STORECHAR(ch); + } + } + + Py_CLEAR(rep); + } + + /* Cut back to size actually needed. This is necessary for, for example, + encoding of a string containing isolated surrogates and the 'ignore' + handler is used. */ + nsize = p - (unsigned char*) PyBytes_AS_STRING(v); + if (nsize != PyBytes_GET_SIZE(v)) + _PyBytes_Resize(&v, nsize); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); return v; + error: + Py_XDECREF(rep); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + Py_XDECREF(v); + return NULL; #undef STORECHAR } @@ -5204,6 +5312,7 @@ PyUnicode_DecodeUTF16Stateful(const char *s, const char *errmsg = ""; PyObject *errorHandler = NULL; PyObject *exc = NULL; + const char *encoding; q = (unsigned char *)s; e = q + size; @@ -5237,8 +5346,10 @@ PyUnicode_DecodeUTF16Stateful(const char *s, #if PY_LITTLE_ENDIAN native_ordering = bo <= 0; + encoding = bo <= 0 ? "utf-16-le" : "utf-16-be"; #else native_ordering = bo >= 0; + encoding = bo >= 0 ? "utf-16-be" : "utf-16-le"; #endif /* Note: size will always be longer than the resulting Unicode @@ -5312,7 +5423,7 @@ PyUnicode_DecodeUTF16Stateful(const char *s, if (unicode_decode_call_errorhandler_writer( errors, &errorHandler, - "utf16", errmsg, + encoding, errmsg, &starts, (const char **)&e, &startinpos, @@ -5348,13 +5459,17 @@ _PyUnicode_EncodeUTF16(PyObject *str, Py_ssize_t len; PyObject *v; unsigned short *out; - Py_ssize_t bytesize; Py_ssize_t pairs; #if PY_BIG_ENDIAN int native_ordering = byteorder >= 0; #else int native_ordering = byteorder <= 0; #endif + const char *encoding; + Py_ssize_t nsize, pos; + PyObject *errorHandler = NULL; + PyObject *exc = NULL; + PyObject *rep = NULL; if (!PyUnicode_Check(str)) { PyErr_BadArgument(); @@ -5376,8 +5491,8 @@ _PyUnicode_EncodeUTF16(PyObject *str, } if (len > PY_SSIZE_T_MAX / 2 - pairs - (byteorder == 0)) return PyErr_NoMemory(); - bytesize = (len + pairs + (byteorder == 0)) * 2; - v = PyBytes_FromStringAndSize(NULL, bytesize); + nsize = len + pairs + (byteorder == 0); + v = PyBytes_FromStringAndSize(NULL, nsize * 2); if (v == NULL) return NULL; @@ -5389,25 +5504,107 @@ _PyUnicode_EncodeUTF16(PyObject *str, if (len == 0) goto done; - switch (kind) { - case PyUnicode_1BYTE_KIND: { - ucs1lib_utf16_encode(out, (const Py_UCS1 *)data, len, native_ordering); - break; - } - case PyUnicode_2BYTE_KIND: { - ucs2lib_utf16_encode(out, (const Py_UCS2 *)data, len, native_ordering); - break; - } - case PyUnicode_4BYTE_KIND: { - ucs4lib_utf16_encode(out, (const Py_UCS4 *)data, len, native_ordering); - break; + if (kind == PyUnicode_1BYTE_KIND) { + ucs1lib_utf16_encode((const Py_UCS1 *)data, len, &out, native_ordering); + goto done; } - default: - assert(0); + + if (byteorder < 0) + encoding = "utf-16-le"; + else if (byteorder > 0) + encoding = "utf-16-be"; + else + encoding = "utf-16"; + + pos = 0; + while (pos < len) { + Py_ssize_t repsize, moreunits; + + if (kind == PyUnicode_2BYTE_KIND) { + pos += ucs2lib_utf16_encode((const Py_UCS2 *)data + pos, len - pos, + &out, native_ordering); + } + else { + assert(kind == PyUnicode_4BYTE_KIND); + pos += ucs4lib_utf16_encode((const Py_UCS4 *)data + pos, len - pos, + &out, native_ordering); + } + if (pos == len) + break; + + rep = unicode_encode_call_errorhandler( + errors, &errorHandler, + encoding, "surrogates not allowed", + str, &exc, pos, pos + 1, &pos); + if (!rep) + goto error; + + if (PyBytes_Check(rep)) { + repsize = PyBytes_GET_SIZE(rep); + if (repsize & 1) { + raise_encode_exception(&exc, encoding, + str, pos - 1, pos, + "surrogates not allowed"); + goto error; + } + moreunits = repsize / 2; + } + else { + assert(PyUnicode_Check(rep)); + if (PyUnicode_READY(rep) < 0) + goto error; + moreunits = repsize = PyUnicode_GET_LENGTH(rep); + if (!PyUnicode_IS_ASCII(rep)) { + raise_encode_exception(&exc, encoding, + str, pos - 1, pos, + "surrogates not allowed"); + goto error; + } + } + + /* two bytes are reserved for each surrogate */ + if (moreunits > 1) { + Py_ssize_t outpos = out - (unsigned short*) PyBytes_AS_STRING(v); + Py_ssize_t morebytes = 2 * (moreunits - 1); + if (PyBytes_GET_SIZE(v) > PY_SSIZE_T_MAX - morebytes) { + /* integer overflow */ + PyErr_NoMemory(); + goto error; + } + if (_PyBytes_Resize(&v, PyBytes_GET_SIZE(v) + morebytes) < 0) + goto error; + out = (unsigned short*) PyBytes_AS_STRING(v) + outpos; + } + + if (PyBytes_Check(rep)) { + Py_MEMCPY(out, PyBytes_AS_STRING(rep), repsize); + out += moreunits; + } else /* rep is unicode */ { + assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); + ucs1lib_utf16_encode(PyUnicode_1BYTE_DATA(rep), repsize, + &out, native_ordering); + } + + Py_CLEAR(rep); } + /* Cut back to size actually needed. This is necessary for, for example, + encoding of a string containing isolated surrogates and the 'ignore' handler + is used. */ + nsize = (unsigned char*) out - (unsigned char*) PyBytes_AS_STRING(v); + if (nsize != PyBytes_GET_SIZE(v)) + _PyBytes_Resize(&v, nsize); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); done: return v; + error: + Py_XDECREF(rep); + Py_XDECREF(errorHandler); + Py_XDECREF(exc); + Py_XDECREF(v); + return NULL; +#undef STORECHAR } PyObject * diff --git a/Python/codecs.c b/Python/codecs.c index fe0cab4..8fe0af7 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -753,6 +753,65 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) } } +#define ENC_UTF8 0 +#define ENC_UTF16BE 1 +#define ENC_UTF16LE 2 +#define ENC_UTF32BE 3 +#define ENC_UTF32LE 4 + +static int +get_standard_encoding(const char *encoding, int *bytelength) +{ + if (Py_TOLOWER(encoding[0]) == 'u' && + Py_TOLOWER(encoding[1]) == 't' && + Py_TOLOWER(encoding[2]) == 'f') { + encoding += 3; + if (*encoding == '-' || *encoding == '_' ) + encoding++; + if (encoding[0] == '1' && encoding[1] == '6') { + encoding += 2; + *bytelength = 2; + if (*encoding == '\0') { +#ifdef WORDS_BIGENDIAN + return ENC_UTF16BE; +#else + return ENC_UTF16LE; +#endif + } + if (*encoding == '-' || *encoding == '_' ) + encoding++; + if (Py_TOLOWER(encoding[1]) == 'e' && encoding[2] == '\0') { + if (Py_TOLOWER(encoding[0]) == 'b') + return ENC_UTF16BE; + if (Py_TOLOWER(encoding[0]) == 'l') + return ENC_UTF16LE; + } + } + else if (encoding[0] == '3' && encoding[1] == '2') { + encoding += 2; + *bytelength = 4; + if (*encoding == '\0') { +#ifdef WORDS_BIGENDIAN + return ENC_UTF32BE; +#else + return ENC_UTF32LE; +#endif + } + if (*encoding == '-' || *encoding == '_' ) + encoding++; + if (Py_TOLOWER(encoding[1]) == 'e' && encoding[2] == '\0') { + if (Py_TOLOWER(encoding[0]) == 'b') + return ENC_UTF32BE; + if (Py_TOLOWER(encoding[0]) == 'l') + return ENC_UTF32LE; + } + } + } + /* utf-8 */ + *bytelength = 3; + return ENC_UTF8; +} + /* This handler is declared static until someone demonstrates a need to call it directly. */ static PyObject * @@ -760,24 +819,40 @@ PyCodec_SurrogatePassErrors(PyObject *exc) { PyObject *restuple; PyObject *object; + PyObject *encode; + char *encoding; + int code; + int bytelength; Py_ssize_t i; Py_ssize_t start; Py_ssize_t end; PyObject *res; if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) { - char *outp; + unsigned char *outp; if (PyUnicodeEncodeError_GetStart(exc, &start)) return NULL; if (PyUnicodeEncodeError_GetEnd(exc, &end)) return NULL; if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; - res = PyBytes_FromStringAndSize(NULL, 3*(end-start)); + if (!(encode = PyUnicodeEncodeError_GetEncoding(exc))) { + Py_DECREF(object); + return NULL; + } + if (!(encoding = PyUnicode_AsUTF8(encode))) { + Py_DECREF(object); + Py_DECREF(encode); + return NULL; + } + code = get_standard_encoding(encoding, &bytelength); + Py_DECREF(encode); + + res = PyBytes_FromStringAndSize(NULL, bytelength*(end-start)); if (!res) { Py_DECREF(object); return NULL; } - outp = PyBytes_AsString(res); + outp = (unsigned char*)PyBytes_AsString(res); for (i = start; i < end; i++) { /* object is guaranteed to be "ready" */ Py_UCS4 ch = PyUnicode_READ_CHAR(object, i); @@ -788,9 +863,33 @@ PyCodec_SurrogatePassErrors(PyObject *exc) Py_DECREF(object); return NULL; } - *outp++ = (char)(0xe0 | (ch >> 12)); - *outp++ = (char)(0x80 | ((ch >> 6) & 0x3f)); - *outp++ = (char)(0x80 | (ch & 0x3f)); + switch (code) { + case ENC_UTF8: + *outp++ = (unsigned char)(0xe0 | (ch >> 12)); + *outp++ = (unsigned char)(0x80 | ((ch >> 6) & 0x3f)); + *outp++ = (unsigned char)(0x80 | (ch & 0x3f)); + break; + case ENC_UTF16LE: + *outp++ = (unsigned char) ch; + *outp++ = (unsigned char)(ch >> 8); + break; + case ENC_UTF16BE: + *outp++ = (unsigned char)(ch >> 8); + *outp++ = (unsigned char) ch; + break; + case ENC_UTF32LE: + *outp++ = (unsigned char) ch; + *outp++ = (unsigned char)(ch >> 8); + *outp++ = (unsigned char)(ch >> 16); + *outp++ = (unsigned char)(ch >> 24); + break; + case ENC_UTF32BE: + *outp++ = (unsigned char)(ch >> 24); + *outp++ = (unsigned char)(ch >> 16); + *outp++ = (unsigned char)(ch >> 8); + *outp++ = (unsigned char) ch; + break; + } } restuple = Py_BuildValue("(On)", res, end); Py_DECREF(res); @@ -802,34 +901,64 @@ PyCodec_SurrogatePassErrors(PyObject *exc) Py_UCS4 ch = 0; if (PyUnicodeDecodeError_GetStart(exc, &start)) return NULL; + if (PyUnicodeDecodeError_GetEnd(exc, &end)) + return NULL; if (!(object = PyUnicodeDecodeError_GetObject(exc))) return NULL; if (!(p = (unsigned char*)PyBytes_AsString(object))) { Py_DECREF(object); return NULL; } + if (!(encode = PyUnicodeDecodeError_GetEncoding(exc))) { + Py_DECREF(object); + return NULL; + } + if (!(encoding = PyUnicode_AsUTF8(encode))) { + Py_DECREF(object); + Py_DECREF(encode); + return NULL; + } + code = get_standard_encoding(encoding, &bytelength); + Py_DECREF(encode); + /* Try decoding a single surrogate character. If there are more, let the codec call us again. */ p += start; - if (PyBytes_GET_SIZE(object) - start >= 3 && - (p[0] & 0xf0) == 0xe0 && - (p[1] & 0xc0) == 0x80 && - (p[2] & 0xc0) == 0x80) { - /* it's a three-byte code */ - ch = ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + (p[2] & 0x3f); - if (!Py_UNICODE_IS_SURROGATE(ch)) - /* it's not a surrogate - fail */ - ch = 0; + if (PyBytes_GET_SIZE(object) - start >= bytelength) { + switch (code) { + case ENC_UTF8: + if ((p[0] & 0xf0) == 0xe0 && + (p[1] & 0xc0) == 0x80 && + (p[2] & 0xc0) == 0x80) { + /* it's a three-byte code */ + ch = ((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) << 6) + (p[2] & 0x3f); + } + break; + case ENC_UTF16LE: + ch = p[1] << 8 | p[0]; + break; + case ENC_UTF16BE: + ch = p[0] << 8 | p[1]; + break; + case ENC_UTF32LE: + ch = (p[3] << 24) | (p[2] << 16) | (p[1] << 8) | p[0]; + break; + case ENC_UTF32BE: + ch = (p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]; + break; + } } + Py_DECREF(object); - if (ch == 0) { + if (!Py_UNICODE_IS_SURROGATE(ch)) { + /* it's not a surrogate - fail */ PyErr_SetObject(PyExceptionInstance_Class(exc), exc); return NULL; } res = PyUnicode_FromOrdinal(ch); if (res == NULL) return NULL; - return Py_BuildValue("(Nn)", res, start+3); + return Py_BuildValue("(Nn)", res, start + bytelength); } else { wrong_exception_type(exc); -- cgit v0.12