diff options
| author | Jim Fasarakis-Hilliard <d.f.hilliard@gmail.com> | 2017-03-14 20:16:15 (GMT) |
|---|---|---|
| committer | Stefan Krah <skrah@bytereef.org> | 2017-03-14 20:16:15 (GMT) |
| commit | d4914e9041cb9455592facba2a1afa6d905f1c01 (patch) | |
| tree | a079685cdebad7f9b8bb3ac5c84e7a0f15a3584b | |
| parent | 9135275cba680902e6caf29461f0423dc570190d (diff) | |
| download | cpython-d4914e9041cb9455592facba2a1afa6d905f1c01.zip cpython-d4914e9041cb9455592facba2a1afa6d905f1c01.tar.gz cpython-d4914e9041cb9455592facba2a1afa6d905f1c01.tar.bz2 | |
Add ELLIPSIS and RARROW. Add tests (#666)
| -rw-r--r-- | Lib/test/test_tokenize.py | 21 | ||||
| -rw-r--r-- | Lib/tokenize.py | 4 |
2 files changed, 24 insertions, 1 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 5a81a5f..6b57777 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -408,6 +408,25 @@ def"', """\ OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) """) + self.check_tokenize("def d23(a: str, b: int=3) -> int: pass", """\ + NAME 'def' (1, 0) (1, 3) + NAME 'd23' (1, 4) (1, 7) + OP '(' (1, 7) (1, 8) + NAME 'a' (1, 8) (1, 9) + OP ':' (1, 9) (1, 10) + NAME 'str' (1, 11) (1, 14) + OP ',' (1, 14) (1, 15) + NAME 'b' (1, 16) (1, 17) + OP ':' (1, 17) (1, 18) + NAME 'int' (1, 19) (1, 22) + OP '=' (1, 22) (1, 23) + NUMBER '3' (1, 23) (1, 24) + OP ')' (1, 24) (1, 25) + OP '->' (1, 26) (1, 28) + NAME 'int' (1, 29) (1, 32) + OP ':' (1, 32) (1, 33) + NAME 'pass' (1, 34) (1, 38) + """) def test_comparison(self): # Comparison @@ -1371,6 +1390,8 @@ class TestTokenize(TestCase): self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL) self.assertExactTypeEqual('//', token.DOUBLESLASH) self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL) + self.assertExactTypeEqual('...', token.ELLIPSIS) + self.assertExactTypeEqual('->', token.RARROW) self.assertExactTypeEqual('@', token.AT) self.assertExactTypeEqual('@=', token.ATEQUAL) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 825aa90..eea88b7 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -86,12 +86,14 @@ EXACT_TOKEN_TYPES = { '%=': PERCENTEQUAL, '&=': AMPEREQUAL, '|=': VBAREQUAL, - '^=': CIRCUMFLEXEQUAL, + '^=': CIRCUMFLEXEQUAL, '<<=': LEFTSHIFTEQUAL, '>>=': RIGHTSHIFTEQUAL, '**=': DOUBLESTAREQUAL, '//': DOUBLESLASH, '//=': DOUBLESLASHEQUAL, + '...': ELLIPSIS, + '->': RARROW, '@': AT, '@=': ATEQUAL, } |