[3.10] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) (GH-107224) (#107230)

Previously *consumed was not set in this case.

(cherry picked from commit b8b3e6afc0a48c3cbb7c36d2f73e332edcd6058c)
(cherry picked from commit f08e52ccb027f6f703302b8c1a82db9fd3934270)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

1 files changed, 54 insertions, 0 deletions

diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py
new file mode 100644
index 0000000..e467261
--- /dev/null
+++ b/Lib/test/test_capi/test_codecs.py
@@ -0,0 +1,54 @@
+import unittest
+from test.support import import_helper
+
+_testcapi = import_helper.import_module('_testcapi')
+
+
+class CAPITest(unittest.TestCase):
+
+    def test_decodeutf8(self):
+        """Test PyUnicode_DecodeUTF8()"""
+        decodeutf8 = _testcapi.unicode_decodeutf8
+
+        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
+            b = s.encode('utf-8')
+            self.assertEqual(decodeutf8(b), s)
+            self.assertEqual(decodeutf8(b, 'strict'), s)
+
+        self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80')
+        self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0')
+        self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff')
+        self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f')
+        self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd')
+        self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb')
+
+        self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo')
+        # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and
+        # negative size.
+
+    def test_decodeutf8stateful(self):
+        """Test PyUnicode_DecodeUTF8Stateful()"""
+        decodeutf8stateful = _testcapi.unicode_decodeutf8stateful
+
+        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
+            b = s.encode('utf-8')
+            self.assertEqual(decodeutf8stateful(b), (s, len(b)))
+            self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b)))
+
+        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80')
+        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0')
+        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff')
+        self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1))
+        self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1))
+        self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb')
+        self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4))
+
+        self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo')
+        # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and
+        # negative size.
+        # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of
+        # "consumed".
+
+
+if __name__ == "__main__":
+    unittest.main()