bpo-34616: Add PyCF_ALLOW_TOP_LEVEL_AWAIT to allow top-level await (GH-13148)

Co-Authored-By: Yury Selivanov <yury@magic.io>

7 files changed, 109 insertions, 8 deletions

diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 613e4f7..1a9a8b5 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -257,6 +257,12 @@ are always available.  They are listed here in alphabetical order.
    can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on
    the :class:`~__future__._Feature` instance in the :mod:`__future__` module.
 
+   The optional argument *flags* also controls whether the compiled source is
+   allowed to contain top-level ``await``, ``async for`` and ``async with``.
+   When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code
+   object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively
+   executed via ``await eval(code_object)``.
+
    The argument *optimize* specifies the optimization level of the compiler; the
    default value of ``-1`` selects the optimization level of the interpreter as
    given by :option:`-O` options.  Explicit levels are ``0`` (no optimization;
@@ -290,6 +296,10 @@ are always available.  They are listed here in alphabetical order.
       Previously, :exc:`TypeError` was raised when null bytes were encountered
       in *source*.
 
+   .. versionadded:: 3.8
+      ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
+      support for top-level ``await``, ``async for``, and ``async with``.
+
 
 .. class:: complex([real[, imag]])
 
diff --git a/Include/compile.h b/Include/compile.h
index 1370867..a833caa 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -23,6 +23,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
 #define PyCF_ONLY_AST 0x0400
 #define PyCF_IGNORE_COOKIE 0x0800
 #define PyCF_TYPE_COMMENTS 0x1000
+#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
 
 #ifndef Py_LIMITED_API
 typedef struct {
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 5674ea8..4a358e8 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -1,6 +1,7 @@
 # Python test set -- built-in functions
 
 import ast
+import asyncio
 import builtins
 import collections
 import decimal
@@ -18,9 +19,14 @@ import types
 import unittest
 import warnings
 from contextlib import ExitStack
+from inspect import CO_COROUTINE
+from itertools import product
+from textwrap import dedent
+from types import AsyncGeneratorType, FunctionType
 from operator import neg
 from test.support import (
-    EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink)
+    EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink,
+    maybe_get_event_loop_policy)
 from test.support.script_helper import assert_python_ok
 from unittest.mock import MagicMock, patch
 try:
@@ -358,6 +364,71 @@ class BuiltinTest(unittest.TestCase):
                 rv = ns['f']()
                 self.assertEqual(rv, tuple(expected))
 
+    def test_compile_top_level_await(self):
+        """Test whether code some top level await can be compiled.
+
+        Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set,
+        and make sure the generated code object has the CO_COROUTINE flag set in
+        order to execute it with  `await eval(.....)` instead of exec, or via a
+        FunctionType.
+        """
+
+        # helper function just to check we can run top=level async-for
+        async def arange(n):
+            for i in range(n):
+                yield i
+
+        modes = ('single', 'exec')
+        code_samples = ['''a = await asyncio.sleep(0, result=1)''',
+        '''async for i in arange(1):
+               a = 1''',
+        '''async with asyncio.Lock() as l:
+               a = 1''']
+        policy = maybe_get_event_loop_policy()
+        try:
+            for mode, code_sample in product(modes,code_samples):
+                source = dedent(code_sample)
+                with self.assertRaises(SyntaxError, msg=f"{source=} {mode=}"):
+                    compile(source, '?' , mode)
+
+                co = compile(source,
+                             '?',
+                             mode,
+                             flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+
+                self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
+                                 msg=f"{source=} {mode=}")
+
+
+                # test we can create and  advance a function type
+                globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
+                async_f = FunctionType(co, globals_)
+                asyncio.run(async_f())
+                self.assertEqual(globals_['a'], 1)
+
+                # test we can await-eval,
+                globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
+                asyncio.run(eval(co, globals_))
+                self.assertEqual(globals_['a'], 1)
+        finally:
+            asyncio.set_event_loop_policy(policy)
+
+    def test_compile_async_generator(self):
+        """
+        With the PyCF_ALLOW_TOP_LEVEL_AWAIT flag added in 3.8, we want to
+        make sure AsyncGenerators are still properly not marked with CO_COROUTINE
+        """
+        code = dedent("""async def ticker():
+                for i in range(10):
+                    yield i
+                    await asyncio.sleep(0)""")
+
+        co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+        glob = {}
+        exec(co, glob)
+        self.assertEqual(type(glob['ticker']()), AsyncGeneratorType)
+
+
     def test_delattr(self):
         sys.spam = 1
         delattr(sys, 'spam')
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst
new file mode 100644
index 0000000..c264d21
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst
@@ -0,0 +1 @@
+The ``compile()`` builtin functions now support the ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` flag,  which allow to compile sources that  contains top-level ``await``, ``async with`` or ``async for``. This is useful to evaluate async-code from with an already async functions; for example in a custom REPL.
\ No newline at end of file
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 4091b6d..cb0e6d7 100644
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1000,6 +1000,8 @@ class ASTModuleVisitor(PickleVisitor):
         self.emit("if (!m) return NULL;", 1)
         self.emit("d = PyModule_GetDict(m);", 1)
         self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1)
+        self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)', 1)
+        self.emit("return NULL;", 2)
         self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1)
         self.emit("return NULL;", 2)
         self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)', 1)
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index cb53a41..5527505 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -8776,6 +8776,8 @@ PyInit__ast(void)
     if (!m) return NULL;
     d = PyModule_GetDict(m);
     if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;
+    if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)
+        return NULL;
     if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)
         return NULL;
     if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)
diff --git a/Python/compile.c b/Python/compile.c
index 63b2456..734e840 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -2609,7 +2609,9 @@ static int
 compiler_async_for(struct compiler *c, stmt_ty s)
 {
     basicblock *start, *except, *end;
-    if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
+    if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
+        c->u->u_ste->ste_coroutine = 1;
+    } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
         return compiler_error(c, "'async for' outside async function");
     }
 
@@ -4564,7 +4566,9 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
     withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
 
     assert(s->kind == AsyncWith_kind);
-    if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
+    if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
+        c->u->u_ste->ste_coroutine = 1;
+    } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){
         return compiler_error(c, "'async with' outside async function");
     }
 
@@ -4773,12 +4777,16 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
         ADDOP(c, YIELD_FROM);
         break;
     case Await_kind:
-        if (c->u->u_ste->ste_type != FunctionBlock)
-            return compiler_error(c, "'await' outside function");
+        if (!(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT)){
+            if (c->u->u_ste->ste_type != FunctionBlock){
+                return compiler_error(c, "'await' outside function");
+            }
 
-        if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
-                c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION)
-            return compiler_error(c, "'await' outside async function");
+            if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && 
+                    c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION){
+                return compiler_error(c, "'await' outside async function");
+            }
+        }
 
         VISIT(c, expr, e->v.Await.value);
         ADDOP(c, GET_AWAITABLE);
@@ -5712,6 +5720,12 @@ compute_code_flags(struct compiler *c)
     /* (Only) inherit compilerflags in PyCF_MASK */
     flags |= (c->c_flags->cf_flags & PyCF_MASK);
 
+    if ((c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) &&
+         ste->ste_coroutine &&
+         !ste->ste_generator) {
+        flags |= CO_COROUTINE;
+    }
+
     return flags;
 }