diff options
Diffstat (limited to 'Lib/test/test_tokenize.py')
| -rw-r--r-- | Lib/test/test_tokenize.py | 186 |
1 files changed, 186 insertions, 0 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 03f6148..ed75171 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -641,6 +641,192 @@ Legacy unicode literals: NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "U'green'" (2, 7) (2, 15) + +Async/await extension: + + >>> dump_tokens("async = 1") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'async' (1, 0) (1, 5) + OP '=' (1, 6) (1, 7) + NUMBER '1' (1, 8) (1, 9) + + >>> dump_tokens("a = (async = 1)") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'a' (1, 0) (1, 1) + OP '=' (1, 2) (1, 3) + OP '(' (1, 4) (1, 5) + NAME 'async' (1, 5) (1, 10) + OP '=' (1, 11) (1, 12) + NUMBER '1' (1, 13) (1, 14) + OP ')' (1, 14) (1, 15) + + >>> dump_tokens("async()") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'async' (1, 0) (1, 5) + OP '(' (1, 5) (1, 6) + OP ')' (1, 6) (1, 7) + + >>> dump_tokens("class async(Bar):pass") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'class' (1, 0) (1, 5) + NAME 'async' (1, 6) (1, 11) + OP '(' (1, 11) (1, 12) + NAME 'Bar' (1, 12) (1, 15) + OP ')' (1, 15) (1, 16) + OP ':' (1, 16) (1, 17) + NAME 'pass' (1, 17) (1, 21) + + >>> dump_tokens("class async:pass") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'class' (1, 0) (1, 5) + NAME 'async' (1, 6) (1, 11) + OP ':' (1, 11) (1, 12) + NAME 'pass' (1, 12) (1, 16) + + >>> dump_tokens("await = 1") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'await' (1, 0) (1, 5) + OP '=' (1, 6) (1, 7) + NUMBER '1' (1, 8) (1, 9) + + >>> dump_tokens("foo.async") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'foo' (1, 0) (1, 3) + OP '.' (1, 3) (1, 4) + NAME 'async' (1, 4) (1, 9) + + >>> dump_tokens("async for a in b: pass") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'async' (1, 0) (1, 5) + NAME 'for' (1, 6) (1, 9) + NAME 'a' (1, 10) (1, 11) + NAME 'in' (1, 12) (1, 14) + NAME 'b' (1, 15) (1, 16) + OP ':' (1, 16) (1, 17) + NAME 'pass' (1, 18) (1, 22) + + >>> dump_tokens("async with a as b: pass") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'async' (1, 0) (1, 5) + NAME 'with' (1, 6) (1, 10) + NAME 'a' (1, 11) (1, 12) + NAME 'as' (1, 13) (1, 15) + NAME 'b' (1, 16) (1, 17) + OP ':' (1, 17) (1, 18) + NAME 'pass' (1, 19) (1, 23) + + >>> dump_tokens("async.foo") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'async' (1, 0) (1, 5) + OP '.' (1, 5) (1, 6) + NAME 'foo' (1, 6) (1, 9) + + >>> dump_tokens("async") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'async' (1, 0) (1, 5) + + >>> dump_tokens("async\\n#comment\\nawait") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'async' (1, 0) (1, 5) + NEWLINE '\\n' (1, 5) (1, 6) + COMMENT '#comment' (2, 0) (2, 8) + NL '\\n' (2, 8) (2, 9) + NAME 'await' (3, 0) (3, 5) + + >>> dump_tokens("async\\n...\\nawait") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'async' (1, 0) (1, 5) + NEWLINE '\\n' (1, 5) (1, 6) + OP '...' (2, 0) (2, 3) + NEWLINE '\\n' (2, 3) (2, 4) + NAME 'await' (3, 0) (3, 5) + + >>> dump_tokens("async\\nawait") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'async' (1, 0) (1, 5) + NEWLINE '\\n' (1, 5) (1, 6) + NAME 'await' (2, 0) (2, 5) + + >>> dump_tokens("foo.async + 1") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'foo' (1, 0) (1, 3) + OP '.' (1, 3) (1, 4) + NAME 'async' (1, 4) (1, 9) + OP '+' (1, 10) (1, 11) + NUMBER '1' (1, 12) (1, 13) + + >>> dump_tokens("async def foo(): pass") + ENCODING 'utf-8' (0, 0) (0, 0) + ASYNC 'async' (1, 0) (1, 5) + NAME 'def' (1, 6) (1, 9) + NAME 'foo' (1, 10) (1, 13) + OP '(' (1, 13) (1, 14) + OP ')' (1, 14) (1, 15) + OP ':' (1, 15) (1, 16) + NAME 'pass' (1, 17) (1, 21) + + >>> dump_tokens('''async def foo(): + ... def foo(await): + ... await = 1 + ... if 1: + ... await + ... async += 1 + ... ''') + ENCODING 'utf-8' (0, 0) (0, 0) + ASYNC 'async' (1, 0) (1, 5) + NAME 'def' (1, 6) (1, 9) + NAME 'foo' (1, 10) (1, 13) + OP '(' (1, 13) (1, 14) + OP ')' (1, 14) (1, 15) + OP ':' (1, 15) (1, 16) + NEWLINE '\\n' (1, 16) (1, 17) + INDENT ' ' (2, 0) (2, 2) + NAME 'def' (2, 2) (2, 5) + NAME 'foo' (2, 6) (2, 9) + OP '(' (2, 9) (2, 10) + NAME 'await' (2, 10) (2, 15) + OP ')' (2, 15) (2, 16) + OP ':' (2, 16) (2, 17) + NEWLINE '\\n' (2, 17) (2, 18) + INDENT ' ' (3, 0) (3, 4) + NAME 'await' (3, 4) (3, 9) + OP '=' (3, 10) (3, 11) + NUMBER '1' (3, 12) (3, 13) + NEWLINE '\\n' (3, 13) (3, 14) + DEDENT '' (4, 2) (4, 2) + NAME 'if' (4, 2) (4, 4) + NUMBER '1' (4, 5) (4, 6) + OP ':' (4, 6) (4, 7) + NEWLINE '\\n' (4, 7) (4, 8) + INDENT ' ' (5, 0) (5, 4) + AWAIT 'await' (5, 4) (5, 9) + NEWLINE '\\n' (5, 9) (5, 10) + DEDENT '' (6, 0) (6, 0) + DEDENT '' (6, 0) (6, 0) + NAME 'async' (6, 0) (6, 5) + OP '+=' (6, 6) (6, 8) + NUMBER '1' (6, 9) (6, 10) + NEWLINE '\\n' (6, 10) (6, 11) + + >>> dump_tokens('''async def foo(): + ... async for i in 1: pass''') + ENCODING 'utf-8' (0, 0) (0, 0) + ASYNC 'async' (1, 0) (1, 5) + NAME 'def' (1, 6) (1, 9) + NAME 'foo' (1, 10) (1, 13) + OP '(' (1, 13) (1, 14) + OP ')' (1, 14) (1, 15) + OP ':' (1, 15) (1, 16) + NEWLINE '\\n' (1, 16) (1, 17) + INDENT ' ' (2, 0) (2, 2) + ASYNC 'async' (2, 2) (2, 7) + NAME 'for' (2, 8) (2, 11) + NAME 'i' (2, 12) (2, 13) + NAME 'in' (2, 14) (2, 16) + NUMBER '1' (2, 17) (2, 18) + OP ':' (2, 18) (2, 19) + NAME 'pass' (2, 20) (2, 24) + DEDENT '' (3, 0) (3, 0) """ from test import support |