diff options
| author | Pablo Galindo Salgado <Pablogsal@gmail.com> | 2021-07-31 01:17:09 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-07-31 01:17:09 (GMT) |
| commit | b6bde9fc42aecad5be0457198d17cfe7b481ad79 (patch) | |
| tree | ec5e777de20072b7bf98d8101adf07e942cde4e8 /Lib/test | |
| parent | e63e6311aa258a5f3f49a7aed9fdde445fd384d6 (diff) | |
| download | cpython-b6bde9fc42aecad5be0457198d17cfe7b481ad79.zip cpython-b6bde9fc42aecad5be0457198d17cfe7b481ad79.tar.gz cpython-b6bde9fc42aecad5be0457198d17cfe7b481ad79.tar.bz2 | |
bpo-44667: Treat correctly lines ending with comments and no newlines in the Python tokenizer (GH-27499)
Diffstat (limited to 'Lib/test')
| -rw-r--r-- | Lib/test/test_tokenize.py | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 681f2c7..4bce1ca 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1458,6 +1458,16 @@ class TestTokenize(TestCase): # See http://bugs.python.org/issue16152 self.assertExactTypeEqual('@ ', token.AT) + def test_comment_at_the_end_of_the_source_without_newline(self): + # See http://bugs.python.org/issue44667 + source = 'b = 1\n\n#test' + expected_tokens = [token.NAME, token.EQUAL, token.NUMBER, token.NEWLINE, token.NL, token.COMMENT] + + tokens = list(tokenize(BytesIO(source.encode('utf-8')).readline)) + self.assertEqual(tok_name[tokens[0].exact_type], tok_name[ENCODING]) + for i in range(6): + self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]]) + self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER]) class UntokenizeTest(TestCase): |