diff options
| -rw-r--r-- | Lib/test/test_multibytecodec.py | 7 | ||||
| -rw-r--r-- | Misc/NEWS | 4 | ||||
| -rw-r--r-- | Modules/cjkcodecs/multibytecodec.c | 19 |
3 files changed, 22 insertions, 8 deletions
diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index ce267dd..2929f98 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -44,6 +44,13 @@ class Test_MultibyteCodec(unittest.TestCase): self.assertRaises(IndexError, dec, b'apple\x92ham\x93spam', 'test.cjktest') + def test_errorcallback_custom_ignore(self): + # Issue #23215: MemoryError with custom error handlers and multibyte codecs + data = 100 * "\udc00" + codecs.register_error("test.ignore", codecs.ignore_errors) + for enc in ALL_CJKENCODINGS: + self.assertEqual(data.encode(enc, "test.ignore"), b'') + def test_codingspec(self): try: for enc in ALL_CJKENCODINGS: @@ -13,6 +13,10 @@ Core and Builtins Library ------- +- Issue #23215: Multibyte codecs with custom error handlers that ignores errors + consumed too much memory and raised SystemError or MemoryError. + Original patch by Aleksi Torhamo. + - Issue #5700: io.FileIO() called flush() after closing the file. flush() was not called in close() if closefd=False. diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 31bb35f..fe5b362 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -174,8 +174,10 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) orgsize = PyBytes_GET_SIZE(buf->outobj); incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize); - if (orgsize > PY_SSIZE_T_MAX - incsize) + if (orgsize > PY_SSIZE_T_MAX - incsize) { + PyErr_NoMemory(); return -1; + } if (_PyBytes_Resize(&buf->outobj, orgsize + incsize) == -1) return -1; @@ -186,11 +188,11 @@ expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize) return 0; } -#define REQUIRE_ENCODEBUFFER(buf, s) { \ - if ((s) < 1 || (buf)->outbuf + (s) > (buf)->outbuf_end) \ +#define REQUIRE_ENCODEBUFFER(buf, s) do { \ + if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \ if (expand_encodebuffer(buf, s) == -1) \ goto errorexit; \ -} +} while(0) /** @@ -324,10 +326,11 @@ multibytecodec_encerror(MultibyteCodec *codec, assert(PyBytes_Check(retstr)); retstrsize = PyBytes_GET_SIZE(retstr); - REQUIRE_ENCODEBUFFER(buf, retstrsize); - - memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize); - buf->outbuf += retstrsize; + if (retstrsize > 0) { + REQUIRE_ENCODEBUFFER(buf, retstrsize); + memcpy(buf->outbuf, PyBytes_AS_STRING(retstr), retstrsize); + buf->outbuf += retstrsize; + } newpos = PyLong_AsSsize_t(PyTuple_GET_ITEM(retobj, 1)); if (newpos < 0 && !PyErr_Occurred()) |