summaryrefslogtreecommitdiffstats
path: root/Lib/test/test_tokenize.py
diff options
context:
space:
mode:
Diffstat (limited to 'Lib/test/test_tokenize.py')
-rw-r--r--Lib/test/test_tokenize.py13
1 files changed, 13 insertions, 0 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 476ed76..7008d0e 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1165,6 +1165,19 @@ class UntokenizeTest(TestCase):
'start (1,3) precedes previous end (2,2)')
self.assertRaises(ValueError, u.add_whitespace, (2,1))
+ def test_iter_compat(self):
+ u = Untokenizer()
+ token = (NAME, 'Hello')
+ tokens = [(ENCODING, 'utf-8'), token]
+ u.compat(token, iter([]))
+ self.assertEqual(u.tokens, ["Hello "])
+ u = Untokenizer()
+ self.assertEqual(u.untokenize(iter([token])), 'Hello ')
+ u = Untokenizer()
+ self.assertEqual(u.untokenize(iter(tokens)), 'Hello ')
+ self.assertEqual(u.encoding, 'utf-8')
+ self.assertEqual(untokenize(iter(tokens)), b'Hello ')
+
__test__ = {"doctests" : doctests, 'decistmt': decistmt}