summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Doc/glossary.rst7
-rw-r--r--Doc/reference/compound_stmts.rst2
-rw-r--r--Doc/reference/datamodel.rst48
-rw-r--r--Doc/whatsnew/3.5.rst13
-rw-r--r--Include/genobject.h3
-rw-r--r--Lib/_collections_abc.py4
-rw-r--r--Lib/asyncio/compat.py1
-rw-r--r--Lib/asyncio/streams.py6
-rw-r--r--Lib/test/test_coroutines.py98
-rw-r--r--Lib/test/test_grammar.py2
-rw-r--r--Misc/NEWS5
-rw-r--r--Objects/genobject.c94
-rw-r--r--Python/ceval.c40
13 files changed, 291 insertions, 32 deletions
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 75b380b..e7bcb6a 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -76,13 +76,12 @@ Glossary
asynchronous iterable
An object, that can be used in an :keyword:`async for` statement.
- Must return an :term:`awaitable` from its :meth:`__aiter__` method,
- which should in turn be resolved in an :term:`asynchronous iterator`
- object. Introduced by :pep:`492`.
+ Must return an :term:`asyncronous iterator` from its
+ :meth:`__aiter__` method. Introduced by :pep:`492`.
asynchronous iterator
An object that implements :meth:`__aiter__` and :meth:`__anext__`
- methods, that must return :term:`awaitable` objects.
+ methods. ``__anext__`` must return an :term:`awaitable` object.
:keyword:`async for` resolves awaitable returned from asynchronous
iterator's :meth:`__anext__` method until it raises
:exc:`StopAsyncIteration` exception. Introduced by :pep:`492`.
diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst
index 8047673..2469422 100644
--- a/Doc/reference/compound_stmts.rst
+++ b/Doc/reference/compound_stmts.rst
@@ -726,7 +726,7 @@ The following code::
Is semantically equivalent to::
iter = (ITER)
- iter = await type(iter).__aiter__(iter)
+ iter = type(iter).__aiter__(iter)
running = True
while running:
try:
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 3ddbd62..493acaa 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -2359,6 +2359,7 @@ generators, coroutines do not directly support iteration.
Coroutine objects are automatically closed using the above process when
they are about to be destroyed.
+.. _async-iterators:
Asynchronous Iterators
----------------------
@@ -2371,7 +2372,7 @@ Asynchronous iterators can be used in an :keyword:`async for` statement.
.. method:: object.__aiter__(self)
- Must return an *awaitable* resulting in an *asynchronous iterator* object.
+ Must return an *asynchronous iterator* object.
.. method:: object.__anext__(self)
@@ -2384,7 +2385,7 @@ An example of an asynchronous iterable object::
async def readline(self):
...
- async def __aiter__(self):
+ def __aiter__(self):
return self
async def __anext__(self):
@@ -2395,6 +2396,49 @@ An example of an asynchronous iterable object::
.. versionadded:: 3.5
+.. note::
+
+ .. versionchanged:: 3.5.2
+ Starting with CPython 3.5.2, ``__aiter__`` can directly return
+ :term:`asynchronous iterators <asynchronous iterator>`. Returning
+ an :term:`awaitable` object will result in a
+ :exc:`PendingDeprecationWarning`.
+
+ The recommended way of writing backwards compatible code in
+ CPython 3.5.x is to continue returning awaitables from
+ ``__aiter__``. If you want to avoid the PendingDeprecationWarning
+ and keep the code backwards compatible, the following decorator
+ can be used::
+
+ import functools
+ import sys
+
+ if sys.version_info < (3, 5, 2):
+ def aiter_compat(func):
+ @functools.wraps(func)
+ async def wrapper(self):
+ return func(self)
+ return wrapper
+ else:
+ def aiter_compat(func):
+ return func
+
+ Example::
+
+ class AsyncIterator:
+
+ @aiter_compat
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ ...
+
+ Starting with CPython 3.6, the :exc:`PendingDeprecationWarning`
+ will be replaced with the :exc:`DeprecationWarning`.
+ In CPython 3.7, returning an awaitable from ``__aiter__`` will
+ result in a :exc:`RuntimeError`.
+
Asynchronous Context Managers
-----------------------------
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
index 83d5ce6..2d7f8a4 100644
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -247,6 +247,19 @@ be used inside a coroutine function declared with :keyword:`async def`.
Coroutine functions are intended to be run inside a compatible event loop,
such as the :ref:`asyncio loop <asyncio-event-loop>`.
+
+.. note::
+
+ .. versionchanged:: 3.5.2
+ Starting with CPython 3.5.2, ``__aiter__`` can directly return
+ :term:`asynchronous iterators <asynchronous iterator>`. Returning
+ an :term:`awaitable` object will result in a
+ :exc:`PendingDeprecationWarning`.
+
+ See more details in the :ref:`async-iterators` documentation
+ section.
+
+
.. seealso::
:pep:`492` -- Coroutines with async and await syntax
diff --git a/Include/genobject.h b/Include/genobject.h
index 30cb023..1ff32a8 100644
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -54,6 +54,9 @@ typedef struct {
PyAPI_DATA(PyTypeObject) PyCoro_Type;
PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type;
+PyAPI_DATA(PyTypeObject) _PyAIterWrapper_Type;
+PyObject *_PyAIterWrapper_New(PyObject *aiter);
+
#define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type)
PyObject *_PyCoro_GetAwaitableIter(PyObject *o);
PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *,
diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py
index f89bb6f..fc9c9f1 100644
--- a/Lib/_collections_abc.py
+++ b/Lib/_collections_abc.py
@@ -156,7 +156,7 @@ class AsyncIterable(metaclass=ABCMeta):
__slots__ = ()
@abstractmethod
- async def __aiter__(self):
+ def __aiter__(self):
return AsyncIterator()
@classmethod
@@ -176,7 +176,7 @@ class AsyncIterator(AsyncIterable):
"""Return the next item or raise StopAsyncIteration when exhausted."""
raise StopAsyncIteration
- async def __aiter__(self):
+ def __aiter__(self):
return self
@classmethod
diff --git a/Lib/asyncio/compat.py b/Lib/asyncio/compat.py
index 660b7e7..4790bb4 100644
--- a/Lib/asyncio/compat.py
+++ b/Lib/asyncio/compat.py
@@ -4,6 +4,7 @@ import sys
PY34 = sys.version_info >= (3, 4)
PY35 = sys.version_info >= (3, 5)
+PY352 = sys.version_info >= (3, 5, 2)
def flatten_list_bytes(list_of_data):
diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index 6f465af..c88a87c 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -689,3 +689,9 @@ class StreamReader:
if val == b'':
raise StopAsyncIteration
return val
+
+ if compat.PY352:
+ # In Python 3.5.2 and greater, __aiter__ should return
+ # the asynchronous iterator directly.
+ def __aiter__(self):
+ return self
diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py
index 187348d..4f725ae 100644
--- a/Lib/test/test_coroutines.py
+++ b/Lib/test/test_coroutines.py
@@ -1255,8 +1255,9 @@ class CoroutineTest(unittest.TestCase):
buffer = []
async def test1():
- async for i1, i2 in AsyncIter():
- buffer.append(i1 + i2)
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i1, i2 in AsyncIter():
+ buffer.append(i1 + i2)
yielded, _ = run_async(test1())
# Make sure that __aiter__ was called only once
@@ -1268,12 +1269,13 @@ class CoroutineTest(unittest.TestCase):
buffer = []
async def test2():
nonlocal buffer
- async for i in AsyncIter():
- buffer.append(i[0])
- if i[0] == 20:
- break
- else:
- buffer.append('what?')
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i in AsyncIter():
+ buffer.append(i[0])
+ if i[0] == 20:
+ break
+ else:
+ buffer.append('what?')
buffer.append('end')
yielded, _ = run_async(test2())
@@ -1286,12 +1288,13 @@ class CoroutineTest(unittest.TestCase):
buffer = []
async def test3():
nonlocal buffer
- async for i in AsyncIter():
- if i[0] > 20:
- continue
- buffer.append(i[0])
- else:
- buffer.append('what?')
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i in AsyncIter():
+ if i[0] > 20:
+ continue
+ buffer.append(i[0])
+ else:
+ buffer.append('what?')
buffer.append('end')
yielded, _ = run_async(test3())
@@ -1338,7 +1341,7 @@ class CoroutineTest(unittest.TestCase):
def test_for_4(self):
class I:
- async def __aiter__(self):
+ def __aiter__(self):
return self
def __anext__(self):
@@ -1368,8 +1371,9 @@ class CoroutineTest(unittest.TestCase):
return 123
async def foo():
- async for i in I():
- print('never going to happen')
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i in I():
+ print('never going to happen')
with self.assertRaisesRegex(
TypeError,
@@ -1393,7 +1397,7 @@ class CoroutineTest(unittest.TestCase):
def __init__(self):
self.i = 0
- async def __aiter__(self):
+ def __aiter__(self):
return self
async def __anext__(self):
@@ -1417,7 +1421,11 @@ class CoroutineTest(unittest.TestCase):
I += 1
I += 1000
- run_async(main())
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ # Test that __aiter__ that returns an asyncronous iterator
+ # directly does not throw any warnings.
+ run_async(main())
self.assertEqual(I, 111011)
self.assertEqual(sys.getrefcount(manager), mrefs_before)
@@ -1472,13 +1480,63 @@ class CoroutineTest(unittest.TestCase):
1/0
async def foo():
nonlocal CNT
+ with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+ async for i in AI():
+ CNT += 1
+ CNT += 10
+ with self.assertRaises(ZeroDivisionError):
+ run_async(foo())
+ self.assertEqual(CNT, 0)
+
+ def test_for_8(self):
+ CNT = 0
+ class AI:
+ def __aiter__(self):
+ 1/0
+ async def foo():
+ nonlocal CNT
async for i in AI():
CNT += 1
CNT += 10
with self.assertRaises(ZeroDivisionError):
- run_async(foo())
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ # Test that if __aiter__ raises an exception it propagates
+ # without any kind of warning.
+ run_async(foo())
self.assertEqual(CNT, 0)
+ def test_for_9(self):
+ # Test that PendingDeprecationWarning can safely be converted into
+ # an exception (__aiter__ should not have a chance to raise
+ # a ZeroDivisionError.)
+ class AI:
+ async def __aiter__(self):
+ 1/0
+ async def foo():
+ async for i in AI():
+ pass
+
+ with self.assertRaises(PendingDeprecationWarning):
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ run_async(foo())
+
+ def test_for_10(self):
+ # Test that PendingDeprecationWarning can safely be converted into
+ # an exception.
+ class AI:
+ async def __aiter__(self):
+ pass
+ async def foo():
+ async for i in AI():
+ pass
+
+ with self.assertRaises(PendingDeprecationWarning):
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ run_async(foo())
+
def test_copy(self):
async def func(): pass
coro = func()
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index d68cc7d..154e3b6 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -1076,7 +1076,7 @@ class GrammarTests(unittest.TestCase):
class Done(Exception): pass
class AIter:
- async def __aiter__(self):
+ def __aiter__(self):
return self
async def __anext__(self):
raise StopAsyncIteration
diff --git a/Misc/NEWS b/Misc/NEWS
index e16fa19..e7abe73 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -130,6 +130,11 @@ Core and Builtins
- Issue #25887: Raise a RuntimeError when a coroutine object is awaited
more than once.
+- Issue #27243: Update the __aiter__ protocol: instead of returning
+ an awaitable that resolves to an asynchronous iterator, the asynchronous
+ iterator should be returned directly. Doing the former will trigger a
+ PendingDeprecationWarning.
+
Library
-------
diff --git a/Objects/genobject.c b/Objects/genobject.c
index f74d044..b3e0a46 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -992,3 +992,97 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
{
return gen_new_with_qualname(&PyCoro_Type, f, name, qualname);
}
+
+
+/* __aiter__ wrapper; see http://bugs.python.org/issue27243 for details. */
+
+typedef struct {
+ PyObject_HEAD
+ PyObject *aw_aiter;
+} PyAIterWrapper;
+
+
+static PyObject *
+aiter_wrapper_iternext(PyAIterWrapper *aw)
+{
+ PyErr_SetObject(PyExc_StopIteration, aw->aw_aiter);
+ return NULL;
+}
+
+static int
+aiter_wrapper_traverse(PyAIterWrapper *aw, visitproc visit, void *arg)
+{
+ Py_VISIT((PyObject *)aw->aw_aiter);
+ return 0;
+}
+
+static void
+aiter_wrapper_dealloc(PyAIterWrapper *aw)
+{
+ _PyObject_GC_UNTRACK((PyObject *)aw);
+ Py_CLEAR(aw->aw_aiter);
+ PyObject_GC_Del(aw);
+}
+
+static PyAsyncMethods aiter_wrapper_as_async = {
+ PyObject_SelfIter, /* am_await */
+ 0, /* am_aiter */
+ 0 /* am_anext */
+};
+
+PyTypeObject _PyAIterWrapper_Type = {
+ PyVarObject_HEAD_INIT(&PyType_Type, 0)
+ "aiter_wrapper",
+ sizeof(PyAIterWrapper), /* tp_basicsize */
+ 0, /* tp_itemsize */
+ (destructor)aiter_wrapper_dealloc, /* destructor tp_dealloc */
+ 0, /* tp_print */
+ 0, /* tp_getattr */
+ 0, /* tp_setattr */
+ &aiter_wrapper_as_async, /* tp_as_async */
+ 0, /* tp_repr */
+ 0, /* tp_as_number */
+ 0, /* tp_as_sequence */
+ 0, /* tp_as_mapping */
+ 0, /* tp_hash */
+ 0, /* tp_call */
+ 0, /* tp_str */
+ PyObject_GenericGetAttr, /* tp_getattro */
+ 0, /* tp_setattro */
+ 0, /* tp_as_buffer */
+ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
+ "A wrapper object for __aiter__ bakwards compatibility.",
+ (traverseproc)aiter_wrapper_traverse, /* tp_traverse */
+ 0, /* tp_clear */
+ 0, /* tp_richcompare */
+ 0, /* tp_weaklistoffset */
+ PyObject_SelfIter, /* tp_iter */
+ (iternextfunc)aiter_wrapper_iternext, /* tp_iternext */
+ 0, /* tp_methods */
+ 0, /* tp_members */
+ 0, /* tp_getset */
+ 0, /* tp_base */
+ 0, /* tp_dict */
+ 0, /* tp_descr_get */
+ 0, /* tp_descr_set */
+ 0, /* tp_dictoffset */
+ 0, /* tp_init */
+ 0, /* tp_alloc */
+ 0, /* tp_new */
+ PyObject_Del, /* tp_free */
+};
+
+
+PyObject *
+_PyAIterWrapper_New(PyObject *aiter)
+{
+ PyAIterWrapper *aw = PyObject_GC_New(PyAIterWrapper,
+ &_PyAIterWrapper_Type);
+ if (aw == NULL) {
+ return NULL;
+ }
+ Py_INCREF(aiter);
+ aw->aw_aiter = aiter;
+ _PyObject_GC_TRACK(aw);
+ return (PyObject *)aw;
+}
diff --git a/Python/ceval.c b/Python/ceval.c
index 3758b09..3d69038 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1933,8 +1933,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
PyObject *obj = TOP();
PyTypeObject *type = Py_TYPE(obj);
- if (type->tp_as_async != NULL)
+ if (type->tp_as_async != NULL) {
getter = type->tp_as_async->am_aiter;
+ }
if (getter != NULL) {
iter = (*getter)(obj);
@@ -1955,6 +1956,27 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
goto error;
}
+ if (Py_TYPE(iter)->tp_as_async != NULL &&
+ Py_TYPE(iter)->tp_as_async->am_anext != NULL) {
+
+ /* Starting with CPython 3.5.2 __aiter__ should return
+ asynchronous iterators directly (not awaitables that
+ resolve to asynchronous iterators.)
+
+ Therefore, we check if the object that was returned
+ from __aiter__ has an __anext__ method. If it does,
+ we wrap it in an awaitable that resolves to `iter`.
+
+ See http://bugs.python.org/issue27243 for more
+ details.
+ */
+
+ PyObject *wrapper = _PyAIterWrapper_New(iter);
+ Py_DECREF(iter);
+ SET_TOP(wrapper);
+ DISPATCH();
+ }
+
awaitable = _PyCoro_GetAwaitableIter(iter);
if (awaitable == NULL) {
SET_TOP(NULL);
@@ -1966,9 +1988,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
Py_DECREF(iter);
goto error;
- } else
+ } else {
Py_DECREF(iter);
+ if (PyErr_WarnFormat(
+ PyExc_PendingDeprecationWarning, 1,
+ "'%.100s' implements legacy __aiter__ protocol; "
+ "__aiter__ should return an asynchronous "
+ "iterator, not awaitable",
+ type->tp_name))
+ {
+ /* Warning was converted to an error. */
+ Py_DECREF(awaitable);
+ SET_TOP(NULL);
+ goto error;
+ }
+ }
+
SET_TOP(awaitable);
DISPATCH();
}