diff options
Diffstat (limited to 'Lib')
| -rw-r--r-- | Lib/test/test_tokenize.py | 5 | ||||
| -rw-r--r-- | Lib/tokenize.py | 4 |
2 files changed, 8 insertions, 1 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index b6a9ca1..f9652ce 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -552,6 +552,11 @@ Evil tabs DEDENT '' (4, 0) (4, 0) DEDENT '' (4, 0) (4, 0) +Pathological whitespace (http://bugs.python.org/issue16152) + >>> dump_tokens("@ ") + ENCODING 'utf-8' (0, 0) (0, 0) + OP '@' (1, 0) (1, 1) + Non-ascii identifiers >>> dump_tokens("Örter = 'places'\\ngrün = 'green'") diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 59081d3..29c9e29 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -108,7 +108,7 @@ ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) def _compile(expr): @@ -473,6 +473,8 @@ def _tokenize(readline, encoding): if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue token, initial = line[start:end], line[start] if (initial in numchars or # ordinary number |