diff options
| author | Guido van Rossum <guido@python.org> | 1997-03-10 23:17:01 (GMT) |
|---|---|---|
| committer | Guido van Rossum <guido@python.org> | 1997-03-10 23:17:01 (GMT) |
| commit | b5dc5e3d7ea44ee4d029d26c98bc99deeffee346 (patch) | |
| tree | 734dfd927ef498cf77a902e3386d03e000998453 /Lib/tokenize.py | |
| parent | c43b6850544af69c1d489d23c5dbc9cab7aea5e8 (diff) | |
| download | cpython-b5dc5e3d7ea44ee4d029d26c98bc99deeffee346.zip cpython-b5dc5e3d7ea44ee4d029d26c98bc99deeffee346.tar.gz cpython-b5dc5e3d7ea44ee4d029d26c98bc99deeffee346.tar.bz2 | |
Added support for imaginary constants (e.g. 0j, 1j, 1.0j).
Diffstat (limited to 'Lib/tokenize.py')
| -rw-r--r-- | Lib/tokenize.py | 9 |
1 files changed, 5 insertions, 4 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index d742899..06ed746 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -14,7 +14,7 @@ tokenizer exactly. """ -__version__ = "Ka-Ping Yee, 4 March 1997, updated by GvR, 6 March 1997" +__version__ = "Ka-Ping Yee, 4 March 1997, updated by GvR, 10 March 1997" import string, regex from token import * @@ -24,14 +24,15 @@ def group(*choices): return '\(' + string.join(choices, '\|') + '\)' Ignore = '[ \f\t]*\([\]\r?\n[ \t]*\)*\(#.*\)?' Name = '[a-zA-Z_][a-zA-Z0-9_]*' +ImagZero = '0[jJ]' # This is not caught by any of the following Hexnumber = '0[xX][0-9a-fA-F]*[lL]?' Octnumber = '0[0-7]*[lL]?' -Decnumber = '[1-9][0-9]*[lL]?' -Intnumber = group(Hexnumber, Octnumber, Decnumber) +Decnumber = '[1-9][0-9]*[lLjJ]?' +Intnumber = group(ImagZero, Hexnumber, Octnumber, Decnumber) Exponent = '[eE][-+]?[0-9]+' Pointfloat = group('[0-9]+\.[0-9]*', '\.[0-9]+') + group(Exponent) + '?' Expfloat = '[0-9]+' + Exponent -Floatnumber = group(Pointfloat, Expfloat) +Floatnumber = group(Pointfloat, Expfloat) + "[jJ]?" Number = group(Floatnumber, Intnumber) Single = group('^\'', '[^\]\'') |