From 4fb8caee87fea1707f9a754365d5ec30c9220f6f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 15 Jan 2013 14:43:21 +0200 Subject: Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping" in any mapping, not only in an unicode string. --- Lib/test/test_codecs.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Objects/unicodeobject.c | 46 +++++++++++++++++++++++++--------------------- 3 files changed, 74 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index acf9f64..1fa9ee0 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1586,6 +1586,10 @@ class CharmapTest(unittest.TestCase): codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab" ) + self.assertRaises(UnicodeDecodeError, + codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab\ufffe" + ) + self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"), ("ab\ufffd", 3) @@ -1642,6 +1646,17 @@ class CharmapTest(unittest.TestCase): {0: 'a', 1: 'b'} ) + self.assertRaises(UnicodeDecodeError, + codecs.charmap_decode, b"\x00\x01\x02", "strict", + {0: 'a', 1: 'b', 2: None} + ) + + # Issue #14850 + self.assertRaises(UnicodeDecodeError, + codecs.charmap_decode, b"\x00\x01\x02", "strict", + {0: 'a', 1: 'b', 2: '\ufffe'} + ) + self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "replace", {0: 'a', 1: 'b'}), @@ -1654,6 +1669,13 @@ class CharmapTest(unittest.TestCase): ("ab\ufffd", 3) ) + # Issue #14850 + self.assertEqual( + codecs.charmap_decode(b"\x00\x01\x02", "replace", + {0: 'a', 1: 'b', 2: '\ufffe'}), + ("ab\ufffd", 3) + ) + self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "ignore", {0: 'a', 1: 'b'}), @@ -1666,6 +1688,13 @@ class CharmapTest(unittest.TestCase): ("ab", 3) ) + # Issue #14850 + self.assertEqual( + codecs.charmap_decode(b"\x00\x01\x02", "ignore", + {0: 'a', 1: 'b', 2: '\ufffe'}), + ("ab", 3) + ) + allbytes = bytes(range(256)) self.assertEqual( codecs.charmap_decode(allbytes, "ignore", {}), @@ -1700,6 +1729,11 @@ class CharmapTest(unittest.TestCase): {0: a, 1: b}, ) + self.assertRaises(UnicodeDecodeError, + codecs.charmap_decode, b"\x00\x01\x02", "strict", + {0: a, 1: b, 2: 0xFFFE}, + ) + self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "replace", {0: a, 1: b}), @@ -1707,11 +1741,23 @@ class CharmapTest(unittest.TestCase): ) self.assertEqual( + codecs.charmap_decode(b"\x00\x01\x02", "replace", + {0: a, 1: b, 2: 0xFFFE}), + ("ab\ufffd", 3) + ) + + self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "ignore", {0: a, 1: b}), ("ab", 3) ) + self.assertEqual( + codecs.charmap_decode(b"\x00\x01\x02", "ignore", + {0: a, 1: b, 2: 0xFFFE}), + ("ab", 3) + ) + class WithStmtTest(unittest.TestCase): def test_encodedfile(self): diff --git a/Misc/NEWS b/Misc/NEWS index 150f7fc..45c0598 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.2.4 Core and Builtins ----------------- +- Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping" + in any mapping, not only in a string. + - Issue #11461: Fix the incremental UTF-16 decoder. Original patch by Amaury Forgeot d'Arc. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7f86bfd..e1df874 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5245,15 +5245,18 @@ PyObject *PyUnicode_DecodeCharmap(const char *s, if (PyErr_ExceptionMatches(PyExc_LookupError)) { /* No mapping found means: mapping is undefined. */ PyErr_Clear(); - x = Py_None; - Py_INCREF(x); + goto Undefined; } else goto onError; } /* Apply mapping */ + if (x == Py_None) + goto Undefined; if (PyLong_Check(x)) { long value = PyLong_AS_LONG(x); + if (value == 0xFFFE) + goto Undefined; if (value < 0 || value > 0x10FFFF) { PyErr_SetString(PyExc_TypeError, "character mapping must be in range(0x110000)"); @@ -5286,29 +5289,16 @@ PyObject *PyUnicode_DecodeCharmap(const char *s, #endif *p++ = (Py_UNICODE)value; } - else if (x == Py_None) { - /* undefined mapping */ - outpos = p-PyUnicode_AS_UNICODE(v); - startinpos = s-starts; - endinpos = startinpos+1; - if (unicode_decode_call_errorhandler( - errors, &errorHandler, - "charmap", "character maps to ", - &starts, &e, &startinpos, &endinpos, &exc, &s, - &v, &outpos, &p)) { - Py_DECREF(x); - goto onError; - } - Py_DECREF(x); - continue; - } else if (PyUnicode_Check(x)) { Py_ssize_t targetsize = PyUnicode_GET_SIZE(x); - if (targetsize == 1) + if (targetsize == 1) { /* 1-1 mapping */ - *p++ = *PyUnicode_AS_UNICODE(x); - + Py_UNICODE value = *PyUnicode_AS_UNICODE(x); + if (value == 0xFFFE) + goto Undefined; + *p++ = value; + } else if (targetsize > 1) { /* 1-n mapping */ if (targetsize > extrachars) { @@ -5342,6 +5332,20 @@ PyObject *PyUnicode_DecodeCharmap(const char *s, } Py_DECREF(x); ++s; + continue; +Undefined: + /* undefined mapping */ + Py_XDECREF(x); + outpos = p-PyUnicode_AS_UNICODE(v); + startinpos = s-starts; + endinpos = startinpos+1; + if (unicode_decode_call_errorhandler( + errors, &errorHandler, + "charmap", "character maps to ", + &starts, &e, &startinpos, &endinpos, &exc, &s, + &v, &outpos, &p)) { + goto onError; + } } } if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v)) -- cgit v0.12