diff options
| author | Georg Brandl <georg@python.org> | 2007-03-18 19:01:53 (GMT) |
|---|---|---|
| committer | Georg Brandl <georg@python.org> | 2007-03-18 19:01:53 (GMT) |
| commit | dde002899db8d04ac25d630fcc3a27e8bbf282ea (patch) | |
| tree | 336d26b7a0e0da705cc729688de862bea896b251 /Lib/tokenize.py | |
| parent | 428f0641ec34902b0cce2cfdca833c79e6fdab7c (diff) | |
| download | cpython-dde002899db8d04ac25d630fcc3a27e8bbf282ea.zip cpython-dde002899db8d04ac25d630fcc3a27e8bbf282ea.tar.gz cpython-dde002899db8d04ac25d630fcc3a27e8bbf282ea.tar.bz2 | |
Make ELLIPSIS a separate token. This makes it a syntax error to write ". . ." for Ellipsis.
Diffstat (limited to 'Lib/tokenize.py')
| -rw-r--r-- | Lib/tokenize.py | 6 |
1 files changed, 3 insertions, 3 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index e502da9..cda82ca 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -83,7 +83,7 @@ Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", r"~") Bracket = '[][(){}]' -Special = group(r'\r?\n', r'[:;.,@]') +Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]') Funny = group(Operator, Bracket, Special) PlainToken = group(Number, Funny, String, Name) @@ -334,8 +334,8 @@ def generate_tokens(readline): spos, epos, pos = (lnum, start), (lnum, end), end token, initial = line[start:end], line[start] - if initial in numchars or \ - (initial == '.' and token != '.'): # ordinary number + if (initial in numchars or # ordinary number + (initial == '.' and token != '.' and token != '...')): yield (NUMBER, token, spos, epos, line) elif initial in '\r\n': yield (NL if parenlev > 0 else NEWLINE, |