Issue #8478: Untokenizer.compat now processes first token from iterator input.

Patch based on lines from Georg Brandl, Eric Snow, and Gareth Rees.

1 files changed, 13 insertions, 0 deletions

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 476ed76..7008d0e 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1165,6 +1165,19 @@ class UntokenizeTest(TestCase):
                 'start (1,3) precedes previous end (2,2)')
         self.assertRaises(ValueError, u.add_whitespace, (2,1))
 
+    def test_iter_compat(self):
+        u = Untokenizer()
+        token = (NAME, 'Hello')
+        tokens = [(ENCODING, 'utf-8'), token]
+        u.compat(token, iter([]))
+        self.assertEqual(u.tokens, ["Hello "])
+        u = Untokenizer()
+        self.assertEqual(u.untokenize(iter([token])), 'Hello ')
+        u = Untokenizer()
+        self.assertEqual(u.untokenize(iter(tokens)), 'Hello ')
+        self.assertEqual(u.encoding, 'utf-8')
+        self.assertEqual(untokenize(iter(tokens)), b'Hello ')
+
 
 __test__ = {"doctests" : doctests, 'decistmt': decistmt}