diff options
| author | Nevada Sanchez <me@nevadasanchez.com> | 2017-04-13 17:32:54 (GMT) |
|---|---|---|
| committer | Mariatta <Mariatta@users.noreply.github.com> | 2017-04-13 17:32:54 (GMT) |
| commit | a6e395dffadf8c5124903c01ad69fefa36b1a935 (patch) | |
| tree | 82fc605d0a1b44e9442a735f37771c4b4cfb3c86 /Lib/lib2to3/pgen2/tokenize.py | |
| parent | 873ef20d0007b4b120933473e6252d2309a70102 (diff) | |
| download | cpython-a6e395dffadf8c5124903c01ad69fefa36b1a935.zip cpython-a6e395dffadf8c5124903c01ad69fefa36b1a935.tar.gz cpython-a6e395dffadf8c5124903c01ad69fefa36b1a935.tar.bz2 | |
bpo-29869: Allow underscores in numeric literals in lib2to3. (GH-1119)
* Allow underscores in numeric literals in lib2to3.
* Stricter literal parsing for Python 3.6 in lib2to3.pgen2.tokenize.
* Add test case for underscores in literals in Python 3.
Diffstat (limited to 'Lib/lib2to3/pgen2/tokenize.py')
| -rw-r--r-- | Lib/lib2to3/pgen2/tokenize.py | 16 |
1 files changed, 8 insertions, 8 deletions
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index d14db60..fba0fa2 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ Comment = r'#[^\r\n]*' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB][01]*' -Hexnumber = r'0[xX][\da-fA-F]*[lL]?' -Octnumber = r'0[oO]?[0-7]*[lL]?' -Decnumber = r'[1-9]\d*[lL]?' +Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' +Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' +Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' +Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+' -Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) -Expfloat = r'\d+' + Exponent +Exponent = r'[eE][-+]?\d+(?:_\d+)*' +Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) +Expfloat = r'\d+(?:_\d+)*' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. |