diff options
| -rw-r--r-- | Lib/lib2to3/pgen2/tokenize.py | 11 | ||||
| -rw-r--r-- | Lib/lib2to3/tests/test_parser.py | 10 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst | 1 |
3 files changed, 16 insertions, 6 deletions
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index bf83ab8..c07b34f 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -56,7 +56,7 @@ def _combinations(*l): Whitespace = r'[ \f\t]*' Comment = r'#[^\r\n]*' Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) -Name = r'[a-zA-Z_]\w*' +Name = r'\w+' Binnumber = r'0[bB]_?[01]+(?:_[01]+)*' Hexnumber = r'0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?' @@ -107,8 +107,8 @@ ContStr = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + PseudoExtras = group(r'\\\r?\n', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) -tokenprog, pseudoprog, single3prog, double3prog = list(map( - re.compile, (Token, PseudoToken, Single3, Double3))) +tokenprog, pseudoprog, single3prog, double3prog = map( + re.compile, (Token, PseudoToken, Single3, Double3)) _strprefixes = ( _combinations('r', 'R', 'f', 'F') | @@ -349,7 +349,6 @@ def generate_tokens(readline): logical line; continuation lines are included. """ lnum = parenlev = continued = 0 - namechars, numchars = string.ascii_letters + '_', '0123456789' contstr, needcont = '', 0 contline = None indents = [0] @@ -451,7 +450,7 @@ def generate_tokens(readline): spos, epos, pos = (lnum, start), (lnum, end), end token, initial = line[start:end], line[start] - if initial in numchars or \ + if initial in string.digits or \ (initial == '.' and token != '.'): # ordinary number yield (NUMBER, token, spos, epos, line) elif initial in '\r\n': @@ -501,7 +500,7 @@ def generate_tokens(readline): yield stashed stashed = None yield (STRING, token, spos, epos, line) - elif initial in namechars: # ordinary name + elif initial.isidentifier(): # ordinary name if token in ('async', 'await'): if async_def: yield (ASYNC if token == 'async' else AWAIT, diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 7eb9afa..829e5a7 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -529,6 +529,16 @@ class TestSetLiteral(GrammarTest): self.validate("""x = {2, 3, 4,}""") +# Adapted from Python 3's Lib/test/test_unicode_identifiers.py and +# Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers +class TestIdentfier(GrammarTest): + def test_non_ascii_identifiers(self): + self.validate("Örter = 'places'\ngrün = 'green'") + self.validate("蟒 = a蟒 = 锦蛇 = 1") + self.validate("µ = aµ = µµ = 1") + self.validate("𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = a_𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1") + + class TestNumericLiterals(GrammarTest): def test_new_octal_notation(self): self.validate("""0o7777777777777""") diff --git a/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst b/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst new file mode 100644 index 0000000..7ace7ba --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-08-27-16-01-22.bpo-34515.S0Irst.rst @@ -0,0 +1 @@ +Fix parsing non-ASCII identifiers in :mod:`lib2to3.pgen2.tokenize` (PEP 3131). |