diff options
| author | Ezio Melotti <ezio.melotti@gmail.com> | 2012-11-03 15:51:25 (GMT) |
|---|---|---|
| committer | Ezio Melotti <ezio.melotti@gmail.com> | 2012-11-03 15:51:25 (GMT) |
| commit | 4e0f36b1565ba07a33d6b68f80c52be44cf916b2 (patch) | |
| tree | 2de2ae931889427a4ef662bbb3c65ec80ba8f076 /Lib | |
| parent | 76bcff27b2429d15805003ebeac8452698a61870 (diff) | |
| parent | fafa8b7797034aa799157172392a5b7721e1854a (diff) | |
| download | cpython-4e0f36b1565ba07a33d6b68f80c52be44cf916b2.zip cpython-4e0f36b1565ba07a33d6b68f80c52be44cf916b2.tar.gz cpython-4e0f36b1565ba07a33d6b68f80c52be44cf916b2.tar.bz2 | |
#16152: merge with 3.3.
Diffstat (limited to 'Lib')
| -rw-r--r-- | Lib/test/test_tokenize.py | 4 | ||||
| -rw-r--r-- | Lib/tokenize.py | 4 |
2 files changed, 7 insertions, 1 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 213e9b4..b4a58f0 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1109,6 +1109,10 @@ class TestTokenize(TestCase): token.NAME, token.AMPER, token.NUMBER, token.RPAR) + def test_pathological_trailing_whitespace(self): + # See http://bugs.python.org/issue16152 + self.assertExactTypeEqual('@ ', token.AT) + __test__ = {"doctests" : doctests, 'decistmt': decistmt} def test_main(): diff --git a/Lib/tokenize.py b/Lib/tokenize.py index d669412..cbf91ef 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -162,7 +162,7 @@ ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) def _compile(expr): @@ -555,6 +555,8 @@ def _tokenize(readline, encoding): if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue token, initial = line[start:end], line[start] if (initial in numchars or # ordinary number |