diff options
| -rw-r--r-- | Lib/lib2to3/pgen2/tokenize.py | 16 | ||||
| -rw-r--r-- | Lib/lib2to3/tests/data/py3_test_grammar.py | 22 |
2 files changed, 8 insertions, 30 deletions
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index fba0fa2..d14db60 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -54,16 +54,16 @@ Comment = r'#[^\r\n]*' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' -Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' -Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' -Octnumber = r'0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?' -Decnumber = group(r'[1-9]\d*(?:_\d+)*[lL]?', '0[lL]?') +Binnumber = r'0[bB][01]*' +Hexnumber = r'0[xX][\da-fA-F]*[lL]?' +Octnumber = r'0[oO]?[0-7]*[lL]?' +Decnumber = r'[1-9]\d*[lL]?' Intnumber = group(Binnumber, Hexnumber, Octnumber, Decnumber) -Exponent = r'[eE][-+]?\d+(?:_\d+)*' -Pointfloat = group(r'\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?', r'\.\d+(?:_\d+)*') + maybe(Exponent) -Expfloat = r'\d+(?:_\d+)*' + Exponent +Exponent = r'[eE][-+]?\d+' +Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) +Expfloat = r'\d+' + Exponent Floatnumber = group(Pointfloat, Expfloat) -Imagnumber = group(r'\d+(?:_\d+)*[jJ]', Floatnumber + r'[jJ]') +Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) # Tail end of ' string. diff --git a/Lib/lib2to3/tests/data/py3_test_grammar.py b/Lib/lib2to3/tests/data/py3_test_grammar.py index 0b9bee0..cf31a54 100644 --- a/Lib/lib2to3/tests/data/py3_test_grammar.py +++ b/Lib/lib2to3/tests/data/py3_test_grammar.py @@ -72,28 +72,6 @@ class TokenTests(unittest.TestCase): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 - def testUnderscoresInNumbers(self): - # Integers - x = 1_0 - x = 123_456_7_89 - x = 0xabc_123_4_5 - x = 0X_abc_123 - x = 0B11_01 - x = 0b_11_01 - x = 0o45_67 - x = 0O_45_67 - - # Floats - x = 3_1.4 - x = 03_1.4 - x = 3_1. - x = .3_1 - x = 3.1_4 - x = 0_3.1_4 - x = 3e1_4 - x = 3_1e+4_1 - x = 3_1E-4_1 - def testFloats(self): x = 3.14 x = 314. |