From d32594ad27f48a898d42a0ea30b9d007b1c57de9 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Tue, 25 Jun 2019 02:12:16 -0700
Subject: bpo-24214: Fixed the UTF-8 and UTF-16 incremental decoders.
 (GH-14304)

* The UTF-8 incremental decoders fails now fast if encounter
  a sequence that can't be handled by the error handler.
* The UTF-16 incremental decoders with the surrogatepass error
  handler decodes now a lone low surrogate with final=False.
(cherry picked from commit 894263ba80af4b7733c2df95b527e96953922656)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
---
 Lib/test/test_codecs.py                            | 24 ++++++++++++++++++++++
 .../2019-06-22-12-45-20.bpo-24214.hIiHeD.rst       |  2 ++
 Objects/stringlib/codecs.h                         |  6 +++---
 Objects/unicodeobject.c                            | 10 ++++++---
 4 files changed, 36 insertions(+), 6 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-06-22-12-45-20.bpo-24214.hIiHeD.rst

diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index d98f178..4317dfc 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -429,11 +429,18 @@ class ReadTest(MixInCheckStateHandling):
     def test_incremental_surrogatepass(self):
         # Test incremental decoder for surrogatepass handler:
         # see issue #24214
+        # High surrogate
         data = '\uD901'.encode(self.encoding, 'surrogatepass')
         for i in range(1, len(data)):
             dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass')
             self.assertEqual(dec.decode(data[:i]), '')
             self.assertEqual(dec.decode(data[i:], True), '\uD901')
+        # Low surrogate
+        data = '\uDC02'.encode(self.encoding, 'surrogatepass')
+        for i in range(1, len(data)):
+            dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass')
+            self.assertEqual(dec.decode(data[:i]), '')
+            self.assertEqual(dec.decode(data[i:]), '\uDC02')
 
 
 class UTF32Test(ReadTest, unittest.TestCase):
@@ -874,6 +881,23 @@ class UTF8Test(ReadTest, unittest.TestCase):
         with self.assertRaises(UnicodeDecodeError):
             b"abc\xed\xa0z".decode(self.encoding, "surrogatepass")
 
+    def test_incremental_errors(self):
+        # Test that the incremental decoder can fail with final=False.
+        # See issue #24214
+        cases = [b'\x80', b'\xBF', b'\xC0', b'\xC1', b'\xF5', b'\xF6', b'\xFF']
+        for prefix in (b'\xC2', b'\xDF', b'\xE0', b'\xE0\xA0', b'\xEF',
+                       b'\xEF\xBF', b'\xF0', b'\xF0\x90', b'\xF0\x90\x80',
+                       b'\xF4', b'\xF4\x8F', b'\xF4\x8F\xBF'):
+            for suffix in b'\x7F', b'\xC0':
+                cases.append(prefix + suffix)
+        cases.extend((b'\xE0\x80', b'\xE0\x9F', b'\xED\xA0\x80',
+                      b'\xED\xBF\xBF', b'\xF0\x80', b'\xF0\x8F', b'\xF4\x90'))
+
+        for data in cases:
+            with self.subTest(data=data):
+                dec = codecs.getincrementaldecoder(self.encoding)()
+                self.assertRaises(UnicodeDecodeError, dec.decode, data)
+
 
 class UTF7Test(ReadTest, unittest.TestCase):
     encoding = "utf-7"
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-06-22-12-45-20.bpo-24214.hIiHeD.rst b/Misc/NEWS.d/next/Core and Builtins/2019-06-22-12-45-20.bpo-24214.hIiHeD.rst
new file mode 100644
index 0000000..2d70ce0
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-06-22-12-45-20.bpo-24214.hIiHeD.rst	
@@ -0,0 +1,2 @@
+Improved support of the surrogatepass error handler in the UTF-8 and UTF-16
+incremental decoders.
diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h
index 8645bc2..d6f2b98 100644
--- a/Objects/stringlib/codecs.h
+++ b/Objects/stringlib/codecs.h
@@ -207,7 +207,7 @@ STRINGLIB(utf8_decode)(const char **inptr, const char *end,
                     goto InvalidContinuation1;
             } else if (ch == 0xF4 && ch2 >= 0x90) {
                 /* invalid sequence
-                   \xF4\x90\x80\80- -- 110000- overflow */
+                   \xF4\x90\x80\x80- -- 110000- overflow */
                 goto InvalidContinuation1;
             }
             if (!IS_CONTINUATION_BYTE(ch3)) {
@@ -573,10 +573,10 @@ STRINGLIB(utf16_decode)(const unsigned char **inptr, const unsigned char *e,
         }
 
         /* UTF-16 code pair: */
-        if (q >= e)
-            goto UnexpectedEnd;
         if (!Py_UNICODE_IS_HIGH_SURROGATE(ch))
             goto IllegalEncoding;
+        if (q >= e)
+            goto UnexpectedEnd;
         ch2 = (q[ihi] << 8) | q[ilo];
         q += 2;
         if (!Py_UNICODE_IS_LOW_SURROGATE(ch2))
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 6ec4127..132f57a 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4937,11 +4937,15 @@ unicode_decode_utf8(const char *s, Py_ssize_t size,
             endinpos = startinpos + 1;
             break;
         case 2:
-        case 3:
-        case 4:
-            if (s == end || consumed) {
+            if (consumed && (unsigned char)s[0] == 0xED && end - s == 2
+                && (unsigned char)s[1] >= 0xA0 && (unsigned char)s[1] <= 0xBF)
+            {
+                /* Truncated surrogate code in range D800-DFFF */
                 goto End;
             }
+            /* fall through */
+        case 3:
+        case 4:
             errmsg = "invalid continuation byte";
             startinpos = s - starts;
             endinpos = startinpos + ch - 1;
-- 
cgit v0.12