PEP 0492 -- Coroutines with async and await syntax. Issue #24017.

4 files changed, 101 insertions, 8 deletions

diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
index ce95d26..c954669 100644
--- a/Lib/lib2to3/Grammar.txt
+++ b/Lib/lib2to3/Grammar.txt
@@ -33,7 +33,8 @@ eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
-decorated: decorators (classdef | funcdef)
+decorated: decorators (classdef | funcdef | async_funcdef)
+async_funcdef: ASYNC funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 parameters: '(' [typedargslist] ')'
 typedargslist: ((tfpdef ['=' test] ',')*
@@ -82,7 +83,8 @@ global_stmt: ('global' | 'nonlocal') NAME (',' NAME)*
 exec_stmt: 'exec' expr ['in' test [',' test]]
 assert_stmt: 'assert' test [',' test]
 
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
 if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
 while_stmt: 'while' test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -121,7 +123,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 arith_expr: term (('+'|'-') term)*
 term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 factor: ('+'|'-'|'~') factor | power
-power: atom trailer* ['**' factor]
+power: [AWAIT] atom trailer* ['**' factor]
 atom: ('(' [yield_expr|testlist_gexp] ')' |
        '[' [listmaker] ']' |
        '{' [dictsetmaker] '}' |
diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py
index 7599396..1a67955 100755
--- a/Lib/lib2to3/pgen2/token.py
+++ b/Lib/lib2to3/pgen2/token.py
@@ -62,8 +62,10 @@ OP = 52
 COMMENT = 53
 NL = 54
 RARROW = 55
-ERRORTOKEN = 56
-N_TOKENS = 57
+AWAIT = 56
+ASYNC = 57
+ERRORTOKEN = 58
+N_TOKENS = 59
 NT_OFFSET = 256
 #--end constants--
 
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 3dd1ee9..690fec4 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -220,7 +220,7 @@ class Untokenizer:
         for tok in iterable:
             toknum, tokval = tok[:2]
 
-            if toknum in (NAME, NUMBER):
+            if toknum in (NAME, NUMBER, ASYNC, AWAIT):
                 tokval += ' '
 
             if toknum == INDENT:
@@ -366,6 +366,10 @@ def generate_tokens(readline):
     contline = None
     indents = [0]
 
+    # 'stashed' and 'ctx' are used for async/await parsing
+    stashed = None
+    ctx = [('sync', 0)]
+
     while 1:                                   # loop over lines in stream
         try:
             line = readline()
@@ -406,6 +410,10 @@ def generate_tokens(readline):
                 pos = pos + 1
             if pos == max: break
 
+            if stashed:
+                yield stashed
+                stashed = None
+
             if line[pos] in '#\r\n':           # skip comments or blank lines
                 if line[pos] == '#':
                     comment_token = line[pos:].rstrip('\r\n')
@@ -449,9 +457,15 @@ def generate_tokens(readline):
                     newline = NEWLINE
                     if parenlev > 0:
                         newline = NL
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield (newline, token, spos, epos, line)
                 elif initial == '#':
                     assert not token.endswith("\n")
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield (COMMENT, token, spos, epos, line)
                 elif token in triple_quoted:
                     endprog = endprogs[token]
@@ -459,6 +473,9 @@ def generate_tokens(readline):
                     if endmatch:                           # all on one line
                         pos = endmatch.end(0)
                         token = line[start:pos]
+                        if stashed:
+                            yield stashed
+                            stashed = None
                         yield (STRING, token, spos, (lnum, pos), line)
                     else:
                         strstart = (lnum, start)           # multiple lines
@@ -476,22 +493,64 @@ def generate_tokens(readline):
                         contline = line
                         break
                     else:                                  # ordinary string
+                        if stashed:
+                            yield stashed
+                            stashed = None
                         yield (STRING, token, spos, epos, line)
                 elif initial in namechars:                 # ordinary name
-                    yield (NAME, token, spos, epos, line)
+                    if token in ('async', 'await'):
+                        if ctx[-1][0] == 'async' and ctx[-1][1] < indents[-1]:
+                            yield (ASYNC if token == 'async' else AWAIT,
+                                   token, spos, epos, line)
+                            continue
+
+                    tok = (NAME, token, spos, epos, line)
+                    if token == 'async' and not stashed:
+                        stashed = tok
+                        continue
+
+                    if token == 'def':
+                        if (stashed
+                                and stashed[0] == NAME
+                                and stashed[1] == 'async'):
+
+                            ctx.append(('async', indents[-1]))
+
+                            yield (ASYNC, stashed[1],
+                                   stashed[2], stashed[3],
+                                   stashed[4])
+                            stashed = None
+                        else:
+                            ctx.append(('sync', indents[-1]))
+
+                    if stashed:
+                        yield stashed
+                        stashed = None
+
+                    yield tok
                 elif initial == '\\':                      # continued stmt
                     # This yield is new; needed for better idempotency:
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield (NL, token, spos, (lnum, pos), line)
                     continued = 1
                 else:
                     if initial in '([{': parenlev = parenlev + 1
                     elif initial in ')]}': parenlev = parenlev - 1
+                    if stashed:
+                        yield stashed
+                        stashed = None
                     yield (OP, token, spos, epos, line)
             else:
                 yield (ERRORTOKEN, line[pos],
                            (lnum, pos), (lnum, pos+1), line)
                 pos = pos + 1
 
+    if stashed:
+        yield stashed
+        stashed = None
+
     for indent in indents[1:]:                 # pop remaining indent levels
         yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
     yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 5bb9d2b..107b5ab 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -55,12 +55,42 @@ class TestMatrixMultiplication(GrammarTest):
 
 
 class TestYieldFrom(GrammarTest):
-    def test_matrix_multiplication_operator(self):
+    def test_yield_from(self):
         self.validate("yield from x")
         self.validate("(yield from x) + y")
         self.invalid_syntax("yield from")
 
 
+class TestAsyncAwait(GrammarTest):
+    def test_await_expr(self):
+        self.validate("""async def foo():
+                             await x
+                      """)
+
+        self.invalid_syntax("await x")
+        self.invalid_syntax("""def foo():
+                                   await x""")
+
+    def test_async_var(self):
+        self.validate("""async = 1""")
+        self.validate("""await = 1""")
+        self.validate("""def async(): pass""")
+
+    def test_async_with(self):
+        self.validate("""async def foo():
+                             async for a in b: pass""")
+
+        self.invalid_syntax("""def foo():
+                                   async for a in b: pass""")
+
+    def test_async_for(self):
+        self.validate("""async def foo():
+                             async with a: pass""")
+
+        self.invalid_syntax("""def foo():
+                                   async with a: pass""")
+
+
 class TestRaiseChanges(GrammarTest):
     def test_2x_style_1(self):
         self.validate("raise")