diff options
| -rw-r--r-- | Lib/test/test_tokenize.py | 21 | ||||
| -rw-r--r-- | Lib/tokenize.py | 4 |
2 files changed, 24 insertions, 1 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 5a81a5f..6b57777 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -408,6 +408,25 @@ def"', """\ OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) """) + self.check_tokenize("def d23(a: str, b: int=3) -> int: pass", """\ + NAME 'def' (1, 0) (1, 3) + NAME 'd23' (1, 4) (1, 7) + OP '(' (1, 7) (1, 8) + NAME 'a' (1, 8) (1, 9) + OP ':' (1, 9) (1, 10) + NAME 'str' (1, 11) (1, 14) + OP ',' (1, 14) (1, 15) + NAME 'b' (1, 16) (1, 17) + OP ':' (1, 17) (1, 18) + NAME 'int' (1, 19) (1, 22) + OP '=' (1, 22) (1, 23) + NUMBER '3' (1, 23) (1, 24) + OP ')' (1, 24) (1, 25) + OP '->' (1, 26) (1, 28) + NAME 'int' (1, 29) (1, 32) + OP ':' (1, 32) (1, 33) + NAME 'pass' (1, 34) (1, 38) + """) def test_comparison(self): # Comparison @@ -1371,6 +1390,8 @@ class TestTokenize(TestCase): self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL) self.assertExactTypeEqual('//', token.DOUBLESLASH) self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL) + self.assertExactTypeEqual('...', token.ELLIPSIS) + self.assertExactTypeEqual('->', token.RARROW) self.assertExactTypeEqual('@', token.AT) self.assertExactTypeEqual('@=', token.ATEQUAL) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 825aa90..eea88b7 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -86,12 +86,14 @@ EXACT_TOKEN_TYPES = { '%=': PERCENTEQUAL, '&=': AMPEREQUAL, '|=': VBAREQUAL, - '^=': CIRCUMFLEXEQUAL, + '^=': CIRCUMFLEXEQUAL, '<<=': LEFTSHIFTEQUAL, '>>=': RIGHTSHIFTEQUAL, '**=': DOUBLESTAREQUAL, '//': DOUBLESLASH, '//=': DOUBLESLASHEQUAL, + '...': ELLIPSIS, + '->': RARROW, '@': AT, '@=': ATEQUAL, } |