diff options
| -rw-r--r-- | Lib/test/test_codecs.py | 18 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst | 2 | ||||
| -rw-r--r-- | Objects/unicodeobject.c | 9 |
3 files changed, 24 insertions, 5 deletions
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 8c92556..79cddb8 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3190,6 +3190,24 @@ class CodePageTest(unittest.TestCase): codec = codecs.lookup('cp123') self.assertEqual(codec.name, 'mbcs') + @support.bigmemtest(size=2**31, memuse=7, dry_run=False) + def test_large_input(self): + # Test input longer than INT_MAX. + # Input should contain undecodable bytes before and after + # the INT_MAX limit. + encoded = (b'01234567' * (2**28-1) + + b'\x85\x86\xea\xeb\xec\xef\xfc\xfd\xfe\xff') + self.assertEqual(len(encoded), 2**31+2) + decoded = codecs.code_page_decode(932, encoded, 'surrogateescape', True) + self.assertEqual(decoded[1], len(encoded)) + del encoded + self.assertEqual(len(decoded[0]), decoded[1]) + self.assertEqual(decoded[0][:10], '0123456701') + self.assertEqual(decoded[0][-20:], + '6701234567' + '\udc85\udc86\udcea\udceb\udcec' + '\udcef\udcfc\udcfd\udcfe\udcff') + class ASCIITest(unittest.TestCase): def test_encode(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst b/Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst new file mode 100644 index 0000000..dc2de44 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-12-01-19-20-53.bpo-35372.RwVJjZ.rst @@ -0,0 +1,2 @@ +Fixed the code page decoder for input longer than 2 GiB containing +undecodable bytes. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index bc98c44..1351eec 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -7211,7 +7211,7 @@ decode_code_page_errors(UINT code_page, "in the target code page."; /* each step cannot decode more than 1 character, but a character can be represented as a surrogate pair */ - wchar_t buffer[2], *startout, *out; + wchar_t buffer[2], *out; int insize; Py_ssize_t outsize; PyObject *errorHandler = NULL; @@ -7248,7 +7248,7 @@ decode_code_page_errors(UINT code_page, *v = (PyObject*)_PyUnicode_New(size * Py_ARRAY_LENGTH(buffer)); if (*v == NULL) goto error; - startout = PyUnicode_AS_UNICODE(*v); + out = PyUnicode_AS_UNICODE(*v); } else { /* Extend unicode object */ @@ -7259,11 +7259,10 @@ decode_code_page_errors(UINT code_page, } if (unicode_resize(v, n + size * Py_ARRAY_LENGTH(buffer)) < 0) goto error; - startout = PyUnicode_AS_UNICODE(*v) + n; + out = PyUnicode_AS_UNICODE(*v) + n; } /* Decode the byte string character per character */ - out = startout; while (in < endin) { /* Decode a character */ @@ -7318,7 +7317,7 @@ decode_code_page_errors(UINT code_page, *out = 0; /* Extend unicode object */ - outsize = out - startout; + outsize = out - PyUnicode_AS_UNICODE(*v); assert(outsize <= PyUnicode_WSTR_LENGTH(*v)); if (unicode_resize(v, outsize) < 0) goto error; |