Issue #28003: Implement PEP 525 -- Asynchronous Generators.

12 files changed, 937 insertions, 13 deletions

diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
index 918b869..b420586 100644
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -13,7 +13,6 @@ conscious design decision, leaving the door open for keyword arguments
 to modify the meaning of the API call itself.
 """
 
-
 import collections
 import concurrent.futures
 import heapq
@@ -28,6 +27,7 @@ import time
 import traceback
 import sys
 import warnings
+import weakref
 
 from . import compat
 from . import coroutines
@@ -242,6 +242,13 @@ class BaseEventLoop(events.AbstractEventLoop):
         self._task_factory = None
         self._coroutine_wrapper_set = False
 
+        # A weak set of all asynchronous generators that are being iterated
+        # by the loop.
+        self._asyncgens = weakref.WeakSet()
+
+        # Set to True when `loop.shutdown_asyncgens` is called.
+        self._asyncgens_shutdown_called = False
+
     def __repr__(self):
         return ('<%s running=%s closed=%s debug=%s>'
                 % (self.__class__.__name__, self.is_running(),
@@ -333,6 +340,46 @@ class BaseEventLoop(events.AbstractEventLoop):
         if self._closed:
             raise RuntimeError('Event loop is closed')
 
+    def _asyncgen_finalizer_hook(self, agen):
+        self._asyncgens.discard(agen)
+        if not self.is_closed():
+            self.create_task(agen.aclose())
+
+    def _asyncgen_firstiter_hook(self, agen):
+        if self._asyncgens_shutdown_called:
+            warnings.warn(
+                "asynchronous generator {!r} was scheduled after "
+                "loop.shutdown_asyncgens() call".format(agen),
+                ResourceWarning, source=self)
+
+        self._asyncgens.add(agen)
+
+    @coroutine
+    def shutdown_asyncgens(self):
+        """Shutdown all active asynchronous generators."""
+        self._asyncgens_shutdown_called = True
+
+        if not len(self._asyncgens):
+            return
+
+        closing_agens = list(self._asyncgens)
+        self._asyncgens.clear()
+
+        shutdown_coro = tasks.gather(
+            *[ag.aclose() for ag in closing_agens],
+            return_exceptions=True,
+            loop=self)
+
+        results = yield from shutdown_coro
+        for result, agen in zip(results, closing_agens):
+            if isinstance(result, Exception):
+                self.call_exception_handler({
+                    'message': 'an error occurred during closing of '
+                               'asynchronous generator {!r}'.format(agen),
+                    'exception': result,
+                    'asyncgen': agen
+                })
+
     def run_forever(self):
         """Run until stop() is called."""
         self._check_closed()
@@ -340,6 +387,9 @@ class BaseEventLoop(events.AbstractEventLoop):
             raise RuntimeError('Event loop is running.')
         self._set_coroutine_wrapper(self._debug)
         self._thread_id = threading.get_ident()
+        old_agen_hooks = sys.get_asyncgen_hooks()
+        sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook,
+                               finalizer=self._asyncgen_finalizer_hook)
         try:
             while True:
                 self._run_once()
@@ -349,6 +399,7 @@ class BaseEventLoop(events.AbstractEventLoop):
             self._stopping = False
             self._thread_id = None
             self._set_coroutine_wrapper(False)
+            sys.set_asyncgen_hooks(*old_agen_hooks)
 
     def run_until_complete(self, future):
         """Run until the Future is done.
@@ -1179,7 +1230,9 @@ class BaseEventLoop(events.AbstractEventLoop):
         - 'handle' (optional): Handle instance;
         - 'protocol' (optional): Protocol instance;
         - 'transport' (optional): Transport instance;
-        - 'socket' (optional): Socket instance.
+        - 'socket' (optional): Socket instance;
+        - 'asyncgen' (optional): Asynchronous generator that caused
+                                 the exception.
 
         New keys maybe introduced in the future.
 
diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py
index 71bc6fb..9c338b0 100644
--- a/Lib/asyncio/coroutines.py
+++ b/Lib/asyncio/coroutines.py
@@ -276,7 +276,10 @@ def _format_coroutine(coro):
     try:
         coro_code = coro.gi_code
     except AttributeError:
-        coro_code = coro.cr_code
+        try:
+            coro_code = coro.cr_code
+        except AttributeError:
+            return repr(coro)
 
     try:
         coro_frame = coro.gi_frame
diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py
index c48c5be..cc9a986 100644
--- a/Lib/asyncio/events.py
+++ b/Lib/asyncio/events.py
@@ -248,6 +248,10 @@ class AbstractEventLoop:
         """
         raise NotImplementedError
 
+    def shutdown_asyncgens(self):
+        """Shutdown all active asynchronous generators."""
+        raise NotImplementedError
+
     # Methods scheduling callbacks.  All these return Handles.
 
     def _timer_handle_cancelled(self, handle):
diff --git a/Lib/dis.py b/Lib/dis.py
index 59886f1..556d84e 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -87,6 +87,7 @@ COMPILER_FLAG_NAMES = {
     64: "NOFREE",
    128: "COROUTINE",
    256: "ITERABLE_COROUTINE",
+   512: "ASYNC_GENERATOR",
 }
 
 def pretty_flags(flags):
diff --git a/Lib/inspect.py b/Lib/inspect.py
index 72c1691..2380095 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -185,6 +185,13 @@ def iscoroutinefunction(object):
     return bool((isfunction(object) or ismethod(object)) and
                 object.__code__.co_flags & CO_COROUTINE)
 
+def isasyncgenfunction(object):
+    return bool((isfunction(object) or ismethod(object)) and
+                object.__code__.co_flags & CO_ASYNC_GENERATOR)
+
+def isasyncgen(object):
+    return isinstance(object, types.AsyncGeneratorType)
+
 def isgenerator(object):
     """Return true if the object is a generator.
 
diff --git a/Lib/test/badsyntax_async6.py b/Lib/test/badsyntax_async6.py
deleted file mode 100644
index cb0a23d..0000000
--- a/Lib/test/badsyntax_async6.py
+++ /dev/null
@@ -1,2 +0,0 @@
-async def foo():
-    yield
diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py
new file mode 100644
index 0000000..41b1b4f
--- /dev/null
+++ b/Lib/test/test_asyncgen.py
@@ -0,0 +1,823 @@
+import asyncio
+import inspect
+import sys
+import types
+import unittest
+
+from unittest import mock
+
+
+class AwaitException(Exception):
+    pass
+
+
+@types.coroutine
+def awaitable(*, throw=False):
+    if throw:
+        yield ('throw',)
+    else:
+        yield ('result',)
+
+
+def run_until_complete(coro):
+    exc = False
+    while True:
+        try:
+            if exc:
+                exc = False
+                fut = coro.throw(AwaitException)
+            else:
+                fut = coro.send(None)
+        except StopIteration as ex:
+            return ex.args[0]
+
+        if fut == ('throw',):
+            exc = True
+
+
+def to_list(gen):
+    async def iterate():
+        res = []
+        async for i in gen:
+            res.append(i)
+        return res
+
+    return run_until_complete(iterate())
+
+
+class AsyncGenSyntaxTest(unittest.TestCase):
+
+    def test_async_gen_syntax_01(self):
+        code = '''async def foo():
+            await abc
+            yield from 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_02(self):
+        code = '''async def foo():
+            yield from 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_03(self):
+        code = '''async def foo():
+            await abc
+            yield
+            return 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_04(self):
+        code = '''async def foo():
+            yield
+            return 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_05(self):
+        code = '''async def foo():
+            if 0:
+                yield
+            return 12
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
+            exec(code, {}, {})
+
+
+class AsyncGenTest(unittest.TestCase):
+
+    def compare_generators(self, sync_gen, async_gen):
+        def sync_iterate(g):
+            res = []
+            while True:
+                try:
+                    res.append(g.__next__())
+                except StopIteration:
+                    res.append('STOP')
+                    break
+                except Exception as ex:
+                    res.append(str(type(ex)))
+            return res
+
+        def async_iterate(g):
+            res = []
+            while True:
+                try:
+                    g.__anext__().__next__()
+                except StopAsyncIteration:
+                    res.append('STOP')
+                    break
+                except StopIteration as ex:
+                    if ex.args:
+                        res.append(ex.args[0])
+                    else:
+                        res.append('EMPTY StopIteration')
+                        break
+                except Exception as ex:
+                    res.append(str(type(ex)))
+            return res
+
+        sync_gen_result = sync_iterate(sync_gen)
+        async_gen_result = async_iterate(async_gen)
+        self.assertEqual(sync_gen_result, async_gen_result)
+        return async_gen_result
+
+    def test_async_gen_iteration_01(self):
+        async def gen():
+            await awaitable()
+            a = yield 123
+            self.assertIs(a, None)
+            await awaitable()
+            yield 456
+            await awaitable()
+            yield 789
+
+        self.assertEqual(to_list(gen()), [123, 456, 789])
+
+    def test_async_gen_iteration_02(self):
+        async def gen():
+            await awaitable()
+            yield 123
+            await awaitable()
+
+        g = gen()
+        ai = g.__aiter__()
+        self.assertEqual(ai.__anext__().__next__(), ('result',))
+
+        try:
+            ai.__anext__().__next__()
+        except StopIteration as ex:
+            self.assertEqual(ex.args[0], 123)
+        else:
+            self.fail('StopIteration was not raised')
+
+        self.assertEqual(ai.__anext__().__next__(), ('result',))
+
+        try:
+            ai.__anext__().__next__()
+        except StopAsyncIteration as ex:
+            self.assertFalse(ex.args)
+        else:
+            self.fail('StopAsyncIteration was not raised')
+
+    def test_async_gen_exception_03(self):
+        async def gen():
+            await awaitable()
+            yield 123
+            await awaitable(throw=True)
+            yield 456
+
+        with self.assertRaises(AwaitException):
+            to_list(gen())
+
+    def test_async_gen_exception_04(self):
+        async def gen():
+            await awaitable()
+            yield 123
+            1 / 0
+
+        g = gen()
+        ai = g.__aiter__()
+        self.assertEqual(ai.__anext__().__next__(), ('result',))
+
+        try:
+            ai.__anext__().__next__()
+        except StopIteration as ex:
+            self.assertEqual(ex.args[0], 123)
+        else:
+            self.fail('StopIteration was not raised')
+
+        with self.assertRaises(ZeroDivisionError):
+            ai.__anext__().__next__()
+
+    def test_async_gen_exception_05(self):
+        async def gen():
+            yield 123
+            raise StopAsyncIteration
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'async generator.*StopAsyncIteration'):
+            to_list(gen())
+
+    def test_async_gen_exception_06(self):
+        async def gen():
+            yield 123
+            raise StopIteration
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'async generator.*StopIteration'):
+            to_list(gen())
+
+    def test_async_gen_exception_07(self):
+        def sync_gen():
+            try:
+                yield 1
+                1 / 0
+            finally:
+                yield 2
+                yield 3
+
+            yield 100
+
+        async def async_gen():
+            try:
+                yield 1
+                1 / 0
+            finally:
+                yield 2
+                yield 3
+
+            yield 100
+
+        self.compare_generators(sync_gen(), async_gen())
+
+    def test_async_gen_exception_08(self):
+        def sync_gen():
+            try:
+                yield 1
+            finally:
+                yield 2
+                1 / 0
+                yield 3
+
+            yield 100
+
+        async def async_gen():
+            try:
+                yield 1
+                await awaitable()
+            finally:
+                await awaitable()
+                yield 2
+                1 / 0
+                yield 3
+
+            yield 100
+
+        self.compare_generators(sync_gen(), async_gen())
+
+    def test_async_gen_exception_09(self):
+        def sync_gen():
+            try:
+                yield 1
+                1 / 0
+            finally:
+                yield 2
+                yield 3
+
+            yield 100
+
+        async def async_gen():
+            try:
+                await awaitable()
+                yield 1
+                1 / 0
+            finally:
+                yield 2
+                await awaitable()
+                yield 3
+
+            yield 100
+
+        self.compare_generators(sync_gen(), async_gen())
+
+    def test_async_gen_exception_10(self):
+        async def gen():
+            yield 123
+        with self.assertRaisesRegex(TypeError,
+                                    "non-None value .* async generator"):
+            gen().__anext__().send(100)
+
+    def test_async_gen_api_01(self):
+        async def gen():
+            yield 123
+
+        g = gen()
+
+        self.assertEqual(g.__name__, 'gen')
+        g.__name__ = '123'
+        self.assertEqual(g.__name__, '123')
+
+        self.assertIn('.gen', g.__qualname__)
+        g.__qualname__ = '123'
+        self.assertEqual(g.__qualname__, '123')
+
+        self.assertIsNone(g.ag_await)
+        self.assertIsInstance(g.ag_frame, types.FrameType)
+        self.assertFalse(g.ag_running)
+        self.assertIsInstance(g.ag_code, types.CodeType)
+
+        self.assertTrue(inspect.isawaitable(g.aclose()))
+
+
+class AsyncGenAsyncioTest(unittest.TestCase):
+
+    def setUp(self):
+        self.loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(None)
+
+    def tearDown(self):
+        self.loop.close()
+        self.loop = None
+
+    async def to_list(self, gen):
+        res = []
+        async for i in gen:
+            res.append(i)
+        return res
+
+    def test_async_gen_asyncio_01(self):
+        async def gen():
+            yield 1
+            await asyncio.sleep(0.01, loop=self.loop)
+            yield 2
+            await asyncio.sleep(0.01, loop=self.loop)
+            return
+            yield 3
+
+        res = self.loop.run_until_complete(self.to_list(gen()))
+        self.assertEqual(res, [1, 2])
+
+    def test_async_gen_asyncio_02(self):
+        async def gen():
+            yield 1
+            await asyncio.sleep(0.01, loop=self.loop)
+            yield 2
+            1 / 0
+            yield 3
+
+        with self.assertRaises(ZeroDivisionError):
+            self.loop.run_until_complete(self.to_list(gen()))
+
+    def test_async_gen_asyncio_03(self):
+        loop = self.loop
+
+        class Gen:
+            async def __aiter__(self):
+                yield 1
+                await asyncio.sleep(0.01, loop=loop)
+                yield 2
+
+        res = loop.run_until_complete(self.to_list(Gen()))
+        self.assertEqual(res, [1, 2])
+
+    def test_async_gen_asyncio_anext_04(self):
+        async def foo():
+            yield 1
+            await asyncio.sleep(0.01, loop=self.loop)
+            try:
+                yield 2
+                yield 3
+            except ZeroDivisionError:
+                yield 1000
+            await asyncio.sleep(0.01, loop=self.loop)
+            yield 4
+
+        async def run1():
+            it = foo().__aiter__()
+
+            self.assertEqual(await it.__anext__(), 1)
+            self.assertEqual(await it.__anext__(), 2)
+            self.assertEqual(await it.__anext__(), 3)
+            self.assertEqual(await it.__anext__(), 4)
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        async def run2():
+            it = foo().__aiter__()
+
+            self.assertEqual(await it.__anext__(), 1)
+            self.assertEqual(await it.__anext__(), 2)
+            try:
+                it.__anext__().throw(ZeroDivisionError)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 1000)
+            else:
+                self.fail('StopIteration was not raised')
+            self.assertEqual(await it.__anext__(), 4)
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run1())
+        self.loop.run_until_complete(run2())
+
+    def test_async_gen_asyncio_anext_05(self):
+        async def foo():
+            v = yield 1
+            v = yield v
+            yield v * 100
+
+        async def run():
+            it = foo().__aiter__()
+
+            try:
+                it.__anext__().send(None)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 1)
+            else:
+                self.fail('StopIteration was not raised')
+
+            try:
+                it.__anext__().send(10)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 10)
+            else:
+                self.fail('StopIteration was not raised')
+
+            try:
+                it.__anext__().send(12)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 1200)
+            else:
+                self.fail('StopIteration was not raised')
+
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_aclose_06(self):
+        async def foo():
+            try:
+                yield 1
+                1 / 0
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                yield 12
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            await it.__anext__()
+            await gen.aclose()
+
+        with self.assertRaisesRegex(
+                RuntimeError,
+                "async generator ignored GeneratorExit"):
+            self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_aclose_07(self):
+        DONE = 0
+
+        async def foo():
+            nonlocal DONE
+            try:
+                yield 1
+                1 / 0
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE += 1
+            DONE += 1000
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            await it.__anext__()
+            await gen.aclose()
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_aclose_08(self):
+        DONE = 0
+
+        fut = asyncio.Future(loop=self.loop)
+
+        async def foo():
+            nonlocal DONE
+            try:
+                yield 1
+                await fut
+                DONE += 1000
+                yield 2
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE += 1
+            DONE += 1000
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            self.assertEqual(await it.__anext__(), 1)
+            t = self.loop.create_task(it.__anext__())
+            await asyncio.sleep(0.01, loop=self.loop)
+            await gen.aclose()
+            return t
+
+        t = self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+        # Silence ResourceWarnings
+        fut.cancel()
+        t.cancel()
+        self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop))
+
+    def test_async_gen_asyncio_gc_aclose_09(self):
+        DONE = 0
+
+        async def gen():
+            nonlocal DONE
+            try:
+                while True:
+                    yield 1
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+            await g.__anext__()
+            await g.__anext__()
+            del g
+
+            await asyncio.sleep(0.1, loop=self.loop)
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_asend_01(self):
+        DONE = 0
+
+        # Sanity check:
+        def sgen():
+            v = yield 1
+            yield v * 2
+        sg = sgen()
+        v = sg.send(None)
+        self.assertEqual(v, 1)
+        v = sg.send(100)
+        self.assertEqual(v, 200)
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                v = yield 1
+                await asyncio.sleep(0.01, loop=self.loop)
+                yield v * 2
+                await asyncio.sleep(0.01, loop=self.loop)
+                return
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            v = await g.asend(100)
+            self.assertEqual(v, 200)
+
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_asend_02(self):
+        DONE = 0
+
+        async def sleep_n_crash(delay):
+            await asyncio.sleep(delay, loop=self.loop)
+            1 / 0
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                v = yield 1
+                await sleep_n_crash(0.01)
+                DONE += 1000
+                yield v * 2
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            await g.asend(100)
+
+        with self.assertRaises(ZeroDivisionError):
+            self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_asend_03(self):
+        DONE = 0
+
+        async def sleep_n_crash(delay):
+            fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop),
+                                        loop=self.loop)
+            self.loop.call_later(delay / 2, lambda: fut.cancel())
+            return await fut
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                v = yield 1
+                await sleep_n_crash(0.01)
+                DONE += 1000
+                yield v * 2
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            await g.asend(100)
+
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_athrow_01(self):
+        DONE = 0
+
+        class FooEr(Exception):
+            pass
+
+        # Sanity check:
+        def sgen():
+            try:
+                v = yield 1
+            except FooEr:
+                v = 1000
+            yield v * 2
+        sg = sgen()
+        v = sg.send(None)
+        self.assertEqual(v, 1)
+        v = sg.throw(FooEr)
+        self.assertEqual(v, 2000)
+        with self.assertRaises(StopIteration):
+            sg.send(None)
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                try:
+                    v = yield 1
+                except FooEr:
+                    v = 1000
+                    await asyncio.sleep(0.01, loop=self.loop)
+                yield v * 2
+                await asyncio.sleep(0.01, loop=self.loop)
+                # return
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            v = await g.athrow(FooEr)
+            self.assertEqual(v, 2000)
+
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_athrow_02(self):
+        DONE = 0
+
+        class FooEr(Exception):
+            pass
+
+        async def sleep_n_crash(delay):
+            fut = asyncio.ensure_future(asyncio.sleep(delay, loop=self.loop),
+                                        loop=self.loop)
+            self.loop.call_later(delay / 2, lambda: fut.cancel())
+            return await fut
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01, loop=self.loop)
+                try:
+                    v = yield 1
+                except FooEr:
+                    await sleep_n_crash(0.01)
+                yield v * 2
+                await asyncio.sleep(0.01, loop=self.loop)
+                # return
+            finally:
+                await asyncio.sleep(0.01, loop=self.loop)
+                await asyncio.sleep(0.01, loop=self.loop)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            try:
+                await g.athrow(FooEr)
+            except asyncio.CancelledError:
+                self.assertEqual(DONE, 1)
+                raise
+            else:
+                self.fail('CancelledError was not raised')
+
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_shutdown_01(self):
+        finalized = 0
+
+        async def waiter(timeout):
+            nonlocal finalized
+            try:
+                await asyncio.sleep(timeout, loop=self.loop)
+                yield 1
+            finally:
+                await asyncio.sleep(0, loop=self.loop)
+                finalized += 1
+
+        async def wait():
+            async for _ in waiter(1):
+                pass
+
+        t1 = self.loop.create_task(wait())
+        t2 = self.loop.create_task(wait())
+
+        self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
+
+        self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+        self.assertEqual(finalized, 2)
+
+        # Silence warnings
+        t1.cancel()
+        t2.cancel()
+        self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
+
+    def test_async_gen_asyncio_shutdown_02(self):
+        logged = 0
+
+        def logger(loop, context):
+            nonlocal logged
+            self.assertIn('asyncgen', context)
+            expected = 'an error occurred during closing of asynchronous'
+            if expected in context['message']:
+                logged += 1
+
+        async def waiter(timeout):
+            try:
+                await asyncio.sleep(timeout, loop=self.loop)
+                yield 1
+            finally:
+                1 / 0
+
+        async def wait():
+            async for _ in waiter(1):
+                pass
+
+        t = self.loop.create_task(wait())
+        self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
+
+        self.loop.set_exception_handler(logger)
+        self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+
+        self.assertEqual(logged, 1)
+
+        # Silence warnings
+        t.cancel()
+        self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py
index ee2482d..fee9ae3 100644
--- a/Lib/test/test_coroutines.py
+++ b/Lib/test/test_coroutines.py
@@ -88,12 +88,6 @@ class AsyncBadSyntaxTest(unittest.TestCase):
         with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
             import test.badsyntax_async5
 
-    def test_badsyntax_6(self):
-        with self.assertRaisesRegex(
-            SyntaxError, "'yield' inside async function"):
-
-            import test.badsyntax_async6
-
     def test_badsyntax_7(self):
         with self.assertRaisesRegex(
             SyntaxError, "'yield from' inside async function"):
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index 6081073..21b8cb7 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -574,7 +574,7 @@ Argument count:    0
 Kw-only arguments: 0
 Number of locals:  2
 Stack size:        17
-Flags:             OPTIMIZED, NEWLOCALS, GENERATOR, NOFREE, COROUTINE
+Flags:             OPTIMIZED, NEWLOCALS, NOFREE, COROUTINE
 Constants:
    0: None
    1: 1"""
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index 47244ae..97634e5 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -65,7 +65,8 @@ class IsTestBase(unittest.TestCase):
                       inspect.isframe, inspect.isfunction, inspect.ismethod,
                       inspect.ismodule, inspect.istraceback,
                       inspect.isgenerator, inspect.isgeneratorfunction,
-                      inspect.iscoroutine, inspect.iscoroutinefunction])
+                      inspect.iscoroutine, inspect.iscoroutinefunction,
+                      inspect.isasyncgen, inspect.isasyncgenfunction])
 
     def istest(self, predicate, exp):
         obj = eval(exp)
@@ -73,6 +74,7 @@ class IsTestBase(unittest.TestCase):
 
         for other in self.predicates - set([predicate]):
             if (predicate == inspect.isgeneratorfunction or \
+               predicate == inspect.isasyncgenfunction or \
                predicate == inspect.iscoroutinefunction) and \
                other == inspect.isfunction:
                 continue
@@ -82,6 +84,10 @@ def generator_function_example(self):
     for i in range(2):
         yield i
 
+async def async_generator_function_example(self):
+    async for i in range(2):
+        yield i
+
 async def coroutine_function_example(self):
     return 'spam'
 
@@ -122,6 +128,10 @@ class TestPredicates(IsTestBase):
         self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory')
         self.istest(inspect.isgenerator, '(x for x in range(2))')
         self.istest(inspect.isgeneratorfunction, 'generator_function_example')
+        self.istest(inspect.isasyncgen,
+                    'async_generator_function_example(1)')
+        self.istest(inspect.isasyncgenfunction,
+                    'async_generator_function_example')
 
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index a6cd95e..37ee0b0 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -1192,6 +1192,32 @@ class SizeofTest(unittest.TestCase):
         # sys.flags
         check(sys.flags, vsize('') + self.P * len(sys.flags))
 
+    def test_asyncgen_hooks(self):
+        old = sys.get_asyncgen_hooks()
+        self.assertIsNone(old.firstiter)
+        self.assertIsNone(old.finalizer)
+
+        firstiter = lambda *a: None
+        sys.set_asyncgen_hooks(firstiter=firstiter)
+        hooks = sys.get_asyncgen_hooks()
+        self.assertIs(hooks.firstiter, firstiter)
+        self.assertIs(hooks[0], firstiter)
+        self.assertIs(hooks.finalizer, None)
+        self.assertIs(hooks[1], None)
+
+        finalizer = lambda *a: None
+        sys.set_asyncgen_hooks(finalizer=finalizer)
+        hooks = sys.get_asyncgen_hooks()
+        self.assertIs(hooks.firstiter, firstiter)
+        self.assertIs(hooks[0], firstiter)
+        self.assertIs(hooks.finalizer, finalizer)
+        self.assertIs(hooks[1], finalizer)
+
+        sys.set_asyncgen_hooks(*old)
+        cur = sys.get_asyncgen_hooks()
+        self.assertIsNone(cur.firstiter)
+        self.assertIsNone(cur.finalizer)
+
 
 def test_main():
     test.support.run_unittest(SysModuleTest, SizeofTest)
diff --git a/Lib/types.py b/Lib/types.py
index 48891cd..d8d8470 100644
--- a/Lib/types.py
+++ b/Lib/types.py
@@ -24,6 +24,11 @@ _c = _c()
 CoroutineType = type(_c)
 _c.close()  # Prevent ResourceWarning
 
+async def _ag():
+    yield
+_ag = _ag()
+AsyncGeneratorType = type(_ag)
+
 class _C:
     def _m(self): pass
 MethodType = type(_C()._m)