diff options
| -rw-r--r-- | Lib/test/test_tokenize.py | 17 | ||||
| -rw-r--r-- | Lib/tokenize.py | 6 |
2 files changed, 22 insertions, 1 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 7a472b2..38611a7 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2,7 +2,7 @@ doctests = """ Tests for the tokenize module. The tests can be really simple. Given a small fragment of source -code, print out a table with tokens. The ENDMARK is omitted for +code, print out a table with tokens. The ENDMARKER is omitted for brevity. >>> dump_tokens("1 + 1") @@ -1180,6 +1180,7 @@ class TestTokenize(TestCase): class UntokenizeTest(TestCase): def test_bad_input_order(self): + # raise if previous row u = Untokenizer() u.prev_row = 2 u.prev_col = 2 @@ -1187,8 +1188,22 @@ class UntokenizeTest(TestCase): u.add_whitespace((1,3)) self.assertEqual(cm.exception.args[0], 'start (1,3) precedes previous end (2,2)') + # raise if previous column in row self.assertRaises(ValueError, u.add_whitespace, (2,1)) + def test_backslash_continuation(self): + # The problem is that <whitespace>\<newline> leaves no token + u = Untokenizer() + u.prev_row = 1 + u.prev_col = 1 + u.tokens = [] + u.add_whitespace((2, 0)) + self.assertEqual(u.tokens, ['\\\n']) + u.prev_row = 2 + u.add_whitespace((4, 4)) + self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', ' ']) + self.assertTrue(roundtrip('a\n b\n c\n \\\n c\n')) + def test_iter_compat(self): u = Untokenizer() token = (NAME, 'Hello') diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 74bd15c..e320d39 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -234,6 +234,10 @@ class Untokenizer: if row < self.prev_row or row == self.prev_row and col < self.prev_col: raise ValueError("start ({},{}) precedes previous end ({},{})" .format(row, col, self.prev_row, self.prev_col)) + row_offset = row - self.prev_row + if row_offset: + self.tokens.append("\\\n" * row_offset) + self.prev_col = 0 col_offset = col - self.prev_col if col_offset: self.tokens.append(" " * col_offset) @@ -248,6 +252,8 @@ class Untokenizer: if tok_type == ENCODING: self.encoding = token continue + if tok_type == ENDMARKER: + break self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end |