bpo-24214: Fixed the UTF-8 incremental decoder. (GH-12603)

The bug occurred when the encoded surrogate character is passed
to the incremental decoder in two chunks.

1 files changed, 9 insertions, 0 deletions

diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 3314493..05843c5 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -406,6 +406,15 @@ class ReadTest(MixInCheckStateHandling):
             self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"),
                              before + backslashreplace + after)
 
+    def test_incremental_surrogatepass(self):
+        # Test incremental decoder for surrogatepass handler:
+        # see issue #24214
+        data = '\uD901'.encode(self.encoding, 'surrogatepass')
+        for i in range(1, len(data)):
+            dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass')
+            self.assertEqual(dec.decode(data[:i]), '')
+            self.assertEqual(dec.decode(data[i:], True), '\uD901')
+
 
 class UTF32Test(ReadTest, unittest.TestCase):
     encoding = "utf-32"