Issue #11461: Fix the incremental UTF-16 decoder. Original patch by

Amaury Forgeot d'Arc. Added tests for partial decoding of non-BMP
characters.

3 files changed, 47 insertions, 9 deletions

diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 0f7c23e..4c58b2d 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -313,7 +313,7 @@ class UTF32Test(ReadTest):
 
     def test_partial(self):
         self.check_partial(
-            "\x00\xff\u0100\uffff",
+            "\x00\xff\u0100\uffff\U00010000",
             [
                 "", # first byte of BOM read
                 "", # second byte of BOM read
@@ -335,6 +335,10 @@ class UTF32Test(ReadTest):
                 "\x00\xff\u0100",
                 "\x00\xff\u0100",
                 "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff\U00010000",
             ]
         )
 
@@ -369,7 +373,7 @@ class UTF32LETest(ReadTest):
 
     def test_partial(self):
         self.check_partial(
-            "\x00\xff\u0100\uffff",
+            "\x00\xff\u0100\uffff\U00010000",
             [
                 "",
                 "",
@@ -387,6 +391,10 @@ class UTF32LETest(ReadTest):
                 "\x00\xff\u0100",
                 "\x00\xff\u0100",
                 "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff\U00010000",
             ]
         )
 
@@ -409,7 +417,7 @@ class UTF32BETest(ReadTest):
 
     def test_partial(self):
         self.check_partial(
-            "\x00\xff\u0100\uffff",
+            "\x00\xff\u0100\uffff\U00010000",
             [
                 "",
                 "",
@@ -427,6 +435,10 @@ class UTF32BETest(ReadTest):
                 "\x00\xff\u0100",
                 "\x00\xff\u0100",
                 "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff\U00010000",
             ]
         )
 
@@ -477,7 +489,7 @@ class UTF16Test(ReadTest):
 
     def test_partial(self):
         self.check_partial(
-            "\x00\xff\u0100\uffff",
+            "\x00\xff\u0100\uffff\U00010000",
             [
                 "", # first byte of BOM read
                 "", # second byte of BOM read => byteorder known
@@ -489,6 +501,10 @@ class UTF16Test(ReadTest):
                 "\x00\xff\u0100",
                 "\x00\xff\u0100",
                 "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff\U00010000",
             ]
         )
 
@@ -526,7 +542,7 @@ class UTF16LETest(ReadTest):
 
     def test_partial(self):
         self.check_partial(
-            "\x00\xff\u0100\uffff",
+            "\x00\xff\u0100\uffff\U00010000",
             [
                 "",
                 "\x00",
@@ -536,6 +552,10 @@ class UTF16LETest(ReadTest):
                 "\x00\xff\u0100",
                 "\x00\xff\u0100",
                 "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff\U00010000",
             ]
         )
 
@@ -565,7 +585,7 @@ class UTF16BETest(ReadTest):
 
     def test_partial(self):
         self.check_partial(
-            "\x00\xff\u0100\uffff",
+            "\x00\xff\u0100\uffff\U00010000",
             [
                 "",
                 "\x00",
@@ -575,6 +595,10 @@ class UTF16BETest(ReadTest):
                 "\x00\xff\u0100",
                 "\x00\xff\u0100",
                 "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff",
+                "\x00\xff\u0100\uffff\U00010000",
             ]
         )
 
@@ -604,7 +628,7 @@ class UTF8Test(ReadTest):
 
     def test_partial(self):
         self.check_partial(
-            "\x00\xff\u07ff\u0800\uffff",
+            "\x00\xff\u07ff\u0800\uffff\U00010000",
             [
                 "\x00",
                 "\x00",
@@ -617,6 +641,10 @@ class UTF8Test(ReadTest):
                 "\x00\xff\u07ff\u0800",
                 "\x00\xff\u07ff\u0800",
                 "\x00\xff\u07ff\u0800\uffff",
+                "\x00\xff\u07ff\u0800\uffff",
+                "\x00\xff\u07ff\u0800\uffff",
+                "\x00\xff\u07ff\u0800\uffff",
+                "\x00\xff\u07ff\u0800\uffff\U00010000",
             ]
         )
 
@@ -694,7 +722,7 @@ class UTF8SigTest(ReadTest):
 
     def test_partial(self):
         self.check_partial(
-            "\ufeff\x00\xff\u07ff\u0800\uffff",
+            "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
             [
                 "",
                 "",
@@ -713,6 +741,10 @@ class UTF8SigTest(ReadTest):
                 "\ufeff\x00\xff\u07ff\u0800",
                 "\ufeff\x00\xff\u07ff\u0800",
                 "\ufeff\x00\xff\u07ff\u0800\uffff",
+                "\ufeff\x00\xff\u07ff\u0800\uffff",
+                "\ufeff\x00\xff\u07ff\u0800\uffff",
+                "\ufeff\x00\xff\u07ff\u0800\uffff",
+                "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
             ]
         )
 
diff --git a/Misc/NEWS b/Misc/NEWS
index 8ada15a..f51476a 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@ What's New in Python 3.2.4
 Core and Builtins
 -----------------
 
+- Issue #11461: Fix the incremental UTF-16 decoder. Original patch by
+  Amaury Forgeot d'Arc.
+
 - Issue #16367: Fix FileIO.readall() on Windows for files larger than 2 GB.
 
 - Issue #16455: On FreeBSD and Solaris, if the locale is C, the
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 7cd0399..7f86bfd 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -3573,8 +3573,11 @@ PyUnicode_DecodeUTF16Stateful(const char *s,
 
         /* UTF-16 code pair: */
         if (e - q < 2) {
+            q -= 2;
+            if (consumed)
+                break;
             errmsg = "unexpected end of data";
-            startinpos = (((const char *)q) - 2) - starts;
+            startinpos = ((const char *)q) - starts;
             endinpos = ((const char *)e) - starts;
             goto utf16Error;
         }